1 /*
2  * Copyright (c) 2016 Hisilicon Limited.
3  *
4  * This software is available to you under a choice of one of two
5  * licenses.  You may choose to be licensed under the terms of the GNU
6  * General Public License (GPL) Version 2, available from the file
7  * COPYING in the main directory of this source tree, or the
8  * OpenIB.org BSD license below:
9  *
10  *     Redistribution and use in source and binary forms, with or
11  *     without modification, are permitted provided that the following
12  *     conditions are met:
13  *
14  *      - Redistributions of source code must retain the above
15  *        copyright notice, this list of conditions and the following
16  *        disclaimer.
17  *
18  *      - Redistributions in binary form must reproduce the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer in the documentation and/or other materials
21  *        provided with the distribution.
22  *
23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30  * SOFTWARE.
31  */
32 
33 #ifndef _HNS_ROCE_DEVICE_H
34 #define _HNS_ROCE_DEVICE_H
35 
36 #include <rdma/ib_verbs.h>
37 
38 #define DRV_NAME "hns_roce"
39 
40 #define HNS_ROCE_HW_VER1	('h' << 24 | 'i' << 16 | '0' << 8 | '6')
41 
42 #define MAC_ADDR_OCTET_NUM			6
43 #define HNS_ROCE_MAX_MSG_LEN			0x80000000
44 
45 #define HNS_ROCE_ALOGN_UP(a, b) ((((a) + (b) - 1) / (b)) * (b))
46 
47 #define HNS_ROCE_IB_MIN_SQ_STRIDE		6
48 
49 #define HNS_ROCE_BA_SIZE			(32 * 4096)
50 
51 /* Hardware specification only for v1 engine */
52 #define HNS_ROCE_MIN_CQE_NUM			0x40
53 #define HNS_ROCE_MIN_WQE_NUM			0x20
54 
55 /* Hardware specification only for v1 engine */
56 #define HNS_ROCE_MAX_INNER_MTPT_NUM		0x7
57 #define HNS_ROCE_MAX_MTPT_PBL_NUM		0x100000
58 
59 #define HNS_ROCE_EACH_FREE_CQ_WAIT_MSECS	20
60 #define HNS_ROCE_MAX_FREE_CQ_WAIT_CNT	\
61 	(5000 / HNS_ROCE_EACH_FREE_CQ_WAIT_MSECS)
62 #define HNS_ROCE_CQE_WCMD_EMPTY_BIT		0x2
63 #define HNS_ROCE_MIN_CQE_CNT			16
64 
65 #define HNS_ROCE_MAX_IRQ_NUM			34
66 
67 #define HNS_ROCE_COMP_VEC_NUM			32
68 
69 #define HNS_ROCE_AEQE_VEC_NUM			1
70 #define HNS_ROCE_AEQE_OF_VEC_NUM		1
71 
72 /* 4G/4K = 1M */
73 #define HNS_ROCE_SL_SHIFT			28
74 #define HNS_ROCE_TCLASS_SHIFT			20
75 #define HNS_ROCE_FLOW_LABLE_MASK		0xfffff
76 
77 #define HNS_ROCE_MAX_PORTS			6
78 #define HNS_ROCE_MAX_GID_NUM			16
79 #define HNS_ROCE_GID_SIZE			16
80 
81 #define HNS_ROCE_HOP_NUM_0			0xff
82 
83 #define BITMAP_NO_RR				0
84 #define BITMAP_RR				1
85 
86 #define MR_TYPE_MR				0x00
87 #define MR_TYPE_DMA				0x03
88 
89 #define PKEY_ID					0xffff
90 #define GUID_LEN				8
91 #define NODE_DESC_SIZE				64
92 #define DB_REG_OFFSET				0x1000
93 
94 #define SERV_TYPE_RC				0
95 #define SERV_TYPE_RD				1
96 #define SERV_TYPE_UC				2
97 #define SERV_TYPE_UD				3
98 
99 #define PAGES_SHIFT_8				8
100 #define PAGES_SHIFT_16				16
101 #define PAGES_SHIFT_24				24
102 #define PAGES_SHIFT_32				32
103 
104 enum hns_roce_qp_state {
105 	HNS_ROCE_QP_STATE_RST,
106 	HNS_ROCE_QP_STATE_INIT,
107 	HNS_ROCE_QP_STATE_RTR,
108 	HNS_ROCE_QP_STATE_RTS,
109 	HNS_ROCE_QP_STATE_SQD,
110 	HNS_ROCE_QP_STATE_ERR,
111 	HNS_ROCE_QP_NUM_STATE,
112 };
113 
114 enum hns_roce_event {
115 	HNS_ROCE_EVENT_TYPE_PATH_MIG                  = 0x01,
116 	HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED           = 0x02,
117 	HNS_ROCE_EVENT_TYPE_COMM_EST                  = 0x03,
118 	HNS_ROCE_EVENT_TYPE_SQ_DRAINED                = 0x04,
119 	HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR            = 0x05,
120 	HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR    = 0x06,
121 	HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR     = 0x07,
122 	HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH           = 0x08,
123 	HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH        = 0x09,
124 	HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR           = 0x0a,
125 	HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR           = 0x0b,
126 	HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW               = 0x0c,
127 	HNS_ROCE_EVENT_TYPE_CQ_ID_INVALID             = 0x0d,
128 	HNS_ROCE_EVENT_TYPE_PORT_CHANGE               = 0x0f,
129 	/* 0x10 and 0x11 is unused in currently application case */
130 	HNS_ROCE_EVENT_TYPE_DB_OVERFLOW               = 0x12,
131 	HNS_ROCE_EVENT_TYPE_MB                        = 0x13,
132 	HNS_ROCE_EVENT_TYPE_CEQ_OVERFLOW              = 0x14,
133 };
134 
135 /* Local Work Queue Catastrophic Error,SUBTYPE 0x5 */
136 enum {
137 	HNS_ROCE_LWQCE_QPC_ERROR		= 1,
138 	HNS_ROCE_LWQCE_MTU_ERROR		= 2,
139 	HNS_ROCE_LWQCE_WQE_BA_ADDR_ERROR	= 3,
140 	HNS_ROCE_LWQCE_WQE_ADDR_ERROR		= 4,
141 	HNS_ROCE_LWQCE_SQ_WQE_SHIFT_ERROR	= 5,
142 	HNS_ROCE_LWQCE_SL_ERROR			= 6,
143 	HNS_ROCE_LWQCE_PORT_ERROR		= 7,
144 };
145 
146 /* Local Access Violation Work Queue Error,SUBTYPE 0x7 */
147 enum {
148 	HNS_ROCE_LAVWQE_R_KEY_VIOLATION		= 1,
149 	HNS_ROCE_LAVWQE_LENGTH_ERROR		= 2,
150 	HNS_ROCE_LAVWQE_VA_ERROR		= 3,
151 	HNS_ROCE_LAVWQE_PD_ERROR		= 4,
152 	HNS_ROCE_LAVWQE_RW_ACC_ERROR		= 5,
153 	HNS_ROCE_LAVWQE_KEY_STATE_ERROR		= 6,
154 	HNS_ROCE_LAVWQE_MR_OPERATION_ERROR	= 7,
155 };
156 
157 /* DOORBELL overflow subtype */
158 enum {
159 	HNS_ROCE_DB_SUBTYPE_SDB_OVF		= 1,
160 	HNS_ROCE_DB_SUBTYPE_SDB_ALM_OVF		= 2,
161 	HNS_ROCE_DB_SUBTYPE_ODB_OVF		= 3,
162 	HNS_ROCE_DB_SUBTYPE_ODB_ALM_OVF		= 4,
163 	HNS_ROCE_DB_SUBTYPE_SDB_ALM_EMP		= 5,
164 	HNS_ROCE_DB_SUBTYPE_ODB_ALM_EMP		= 6,
165 };
166 
167 enum {
168 	/* RQ&SRQ related operations */
169 	HNS_ROCE_OPCODE_SEND_DATA_RECEIVE	= 0x06,
170 	HNS_ROCE_OPCODE_RDMA_WITH_IMM_RECEIVE	= 0x07,
171 };
172 
173 enum {
174 	HNS_ROCE_CAP_FLAG_REREG_MR		= BIT(0),
175 	HNS_ROCE_CAP_FLAG_ROCE_V1_V2		= BIT(1),
176 };
177 
178 enum hns_roce_mtt_type {
179 	MTT_TYPE_WQE,
180 	MTT_TYPE_CQE,
181 };
182 
183 #define HNS_ROCE_CMD_SUCCESS			1
184 
185 #define HNS_ROCE_PORT_DOWN			0
186 #define HNS_ROCE_PORT_UP			1
187 
188 #define HNS_ROCE_MTT_ENTRY_PER_SEG		8
189 
190 #define PAGE_ADDR_SHIFT				12
191 
192 struct hns_roce_uar {
193 	u64		pfn;
194 	unsigned long	index;
195 };
196 
197 struct hns_roce_ucontext {
198 	struct ib_ucontext	ibucontext;
199 	struct hns_roce_uar	uar;
200 };
201 
202 struct hns_roce_pd {
203 	struct ib_pd		ibpd;
204 	unsigned long		pdn;
205 };
206 
207 struct hns_roce_bitmap {
208 	/* Bitmap Traversal last a bit which is 1 */
209 	unsigned long		last;
210 	unsigned long		top;
211 	unsigned long		max;
212 	unsigned long		reserved_top;
213 	unsigned long		mask;
214 	spinlock_t		lock;
215 	unsigned long		*table;
216 };
217 
218 /* Order bitmap length -- bit num compute formula: 1 << (max_order - order) */
219 /* Order = 0: bitmap is biggest, order = max bitmap is least (only a bit) */
220 /* Every bit repesent to a partner free/used status in bitmap */
221 /*
222  * Initial, bits of other bitmap are all 0 except that a bit of max_order is 1
223  * Bit = 1 represent to idle and available; bit = 0: not available
224  */
225 struct hns_roce_buddy {
226 	/* Members point to every order level bitmap */
227 	unsigned long **bits;
228 	/* Represent to avail bits of the order level bitmap */
229 	u32            *num_free;
230 	int             max_order;
231 	spinlock_t      lock;
232 };
233 
234 /* For Hardware Entry Memory */
235 struct hns_roce_hem_table {
236 	/* HEM type: 0 = qpc, 1 = mtt, 2 = cqc, 3 = srq, 4 = other */
237 	u32		type;
238 	/* HEM array elment num */
239 	unsigned long	num_hem;
240 	/* HEM entry record obj total num */
241 	unsigned long	num_obj;
242 	/*Single obj size */
243 	unsigned long	obj_size;
244 	unsigned long	table_chunk_size;
245 	int		lowmem;
246 	struct mutex	mutex;
247 	struct hns_roce_hem **hem;
248 	u64		**bt_l1;
249 	dma_addr_t	*bt_l1_dma_addr;
250 	u64		**bt_l0;
251 	dma_addr_t	*bt_l0_dma_addr;
252 };
253 
254 struct hns_roce_mtt {
255 	unsigned long		first_seg;
256 	int			order;
257 	int			page_shift;
258 	enum hns_roce_mtt_type	mtt_type;
259 };
260 
261 /* Only support 4K page size for mr register */
262 #define MR_SIZE_4K 0
263 
264 struct hns_roce_mr {
265 	struct ib_mr		ibmr;
266 	struct ib_umem		*umem;
267 	u64			iova; /* MR's virtual orignal addr */
268 	u64			size; /* Address range of MR */
269 	u32			key; /* Key of MR */
270 	u32			pd;   /* PD num of MR */
271 	u32			access;/* Access permission of MR */
272 	int			enabled; /* MR's active status */
273 	int			type;	/* MR's register type */
274 	u64			*pbl_buf;/* MR's PBL space */
275 	dma_addr_t		pbl_dma_addr;	/* MR's PBL space PA */
276 	u32			pbl_size;/* PA number in the PBL */
277 	u64			pbl_ba;/* page table address */
278 	u32			l0_chunk_last_num;/* L0 last number */
279 	u32			l1_chunk_last_num;/* L1 last number */
280 	u64			**pbl_bt_l2;/* PBL BT L2 */
281 	u64			**pbl_bt_l1;/* PBL BT L1 */
282 	u64			*pbl_bt_l0;/* PBL BT L0 */
283 	dma_addr_t		*pbl_l2_dma_addr;/* PBL BT L2 dma addr */
284 	dma_addr_t		*pbl_l1_dma_addr;/* PBL BT L1 dma addr */
285 	dma_addr_t		pbl_l0_dma_addr;/* PBL BT L0 dma addr */
286 	u32			pbl_ba_pg_sz;/* BT chunk page size */
287 	u32			pbl_buf_pg_sz;/* buf chunk page size */
288 	u32			pbl_hop_num;/* multi-hop number */
289 };
290 
291 struct hns_roce_mr_table {
292 	struct hns_roce_bitmap		mtpt_bitmap;
293 	struct hns_roce_buddy		mtt_buddy;
294 	struct hns_roce_hem_table	mtt_table;
295 	struct hns_roce_hem_table	mtpt_table;
296 	struct hns_roce_buddy		mtt_cqe_buddy;
297 	struct hns_roce_hem_table	mtt_cqe_table;
298 };
299 
300 struct hns_roce_wq {
301 	u64		*wrid;     /* Work request ID */
302 	spinlock_t	lock;
303 	int		wqe_cnt;  /* WQE num */
304 	u32		max_post;
305 	int		max_gs;
306 	int		offset;
307 	int		wqe_shift;/* WQE size */
308 	u32		head;
309 	u32		tail;
310 	void __iomem	*db_reg_l;
311 };
312 
313 struct hns_roce_sge {
314 	int		sge_cnt;  /* SGE num */
315 	int		offset;
316 	int		sge_shift;/* SGE size */
317 };
318 
319 struct hns_roce_buf_list {
320 	void		*buf;
321 	dma_addr_t	map;
322 };
323 
324 struct hns_roce_buf {
325 	struct hns_roce_buf_list	direct;
326 	struct hns_roce_buf_list	*page_list;
327 	int				nbufs;
328 	u32				npages;
329 	int				page_shift;
330 };
331 
332 struct hns_roce_cq_buf {
333 	struct hns_roce_buf hr_buf;
334 	struct hns_roce_mtt hr_mtt;
335 };
336 
337 struct hns_roce_cq {
338 	struct ib_cq			ib_cq;
339 	struct hns_roce_cq_buf		hr_buf;
340 	spinlock_t			lock;
341 	struct ib_umem			*umem;
342 	void (*comp)(struct hns_roce_cq *);
343 	void (*event)(struct hns_roce_cq *, enum hns_roce_event);
344 
345 	struct hns_roce_uar		*uar;
346 	u32				cq_depth;
347 	u32				cons_index;
348 	void __iomem			*cq_db_l;
349 	u16				*tptr_addr;
350 	int				arm_sn;
351 	unsigned long			cqn;
352 	u32				vector;
353 	atomic_t			refcount;
354 	struct completion		free;
355 };
356 
357 struct hns_roce_srq {
358 	struct ib_srq		ibsrq;
359 	int			srqn;
360 };
361 
362 struct hns_roce_uar_table {
363 	struct hns_roce_bitmap bitmap;
364 };
365 
366 struct hns_roce_qp_table {
367 	struct hns_roce_bitmap		bitmap;
368 	spinlock_t			lock;
369 	struct hns_roce_hem_table	qp_table;
370 	struct hns_roce_hem_table	irrl_table;
371 	struct hns_roce_hem_table	trrl_table;
372 };
373 
374 struct hns_roce_cq_table {
375 	struct hns_roce_bitmap		bitmap;
376 	spinlock_t			lock;
377 	struct radix_tree_root		tree;
378 	struct hns_roce_hem_table	table;
379 };
380 
381 struct hns_roce_raq_table {
382 	struct hns_roce_buf_list	*e_raq_buf;
383 };
384 
385 struct hns_roce_av {
386 	__le32      port_pd;
387 	u8          gid_index;
388 	u8          stat_rate;
389 	u8          hop_limit;
390 	__le32      sl_tclass_flowlabel;
391 	u8          dgid[HNS_ROCE_GID_SIZE];
392 	u8          mac[6];
393 	__le16      vlan;
394 };
395 
396 struct hns_roce_ah {
397 	struct ib_ah		ibah;
398 	struct hns_roce_av	av;
399 };
400 
401 struct hns_roce_cmd_context {
402 	struct completion	done;
403 	int			result;
404 	int			next;
405 	u64			out_param;
406 	u16			token;
407 };
408 
409 struct hns_roce_cmdq {
410 	struct dma_pool		*pool;
411 	struct mutex		hcr_mutex;
412 	struct semaphore	poll_sem;
413 	/*
414 	 * Event mode: cmd register mutex protection,
415 	 * ensure to not exceed max_cmds and user use limit region
416 	 */
417 	struct semaphore	event_sem;
418 	int			max_cmds;
419 	spinlock_t		context_lock;
420 	int			free_head;
421 	struct hns_roce_cmd_context *context;
422 	/*
423 	 * Result of get integer part
424 	 * which max_comds compute according a power of 2
425 	 */
426 	u16			token_mask;
427 	/*
428 	 * Process whether use event mode, init default non-zero
429 	 * After the event queue of cmd event ready,
430 	 * can switch into event mode
431 	 * close device, switch into poll mode(non event mode)
432 	 */
433 	u8			use_events;
434 	u8			toggle;
435 };
436 
437 struct hns_roce_cmd_mailbox {
438 	void		       *buf;
439 	dma_addr_t		dma;
440 };
441 
442 struct hns_roce_dev;
443 
444 struct hns_roce_qp {
445 	struct ib_qp		ibqp;
446 	struct hns_roce_buf	hr_buf;
447 	struct hns_roce_wq	rq;
448 	__le64			doorbell_qpn;
449 	__le32			sq_signal_bits;
450 	u32			sq_next_wqe;
451 	int			sq_max_wqes_per_wr;
452 	int			sq_spare_wqes;
453 	struct hns_roce_wq	sq;
454 
455 	struct ib_umem		*umem;
456 	struct hns_roce_mtt	mtt;
457 	u32			buff_size;
458 	struct mutex		mutex;
459 	u8			port;
460 	u8			phy_port;
461 	u8			sl;
462 	u8			resp_depth;
463 	u8			state;
464 	u32			access_flags;
465 	u32			pkey_index;
466 	void			(*event)(struct hns_roce_qp *,
467 					 enum hns_roce_event);
468 	unsigned long		qpn;
469 
470 	atomic_t		refcount;
471 	struct completion	free;
472 
473 	struct hns_roce_sge	sge;
474 	u32			next_sge;
475 };
476 
477 struct hns_roce_sqp {
478 	struct hns_roce_qp	hr_qp;
479 };
480 
481 struct hns_roce_ib_iboe {
482 	spinlock_t		lock;
483 	struct net_device      *netdevs[HNS_ROCE_MAX_PORTS];
484 	struct notifier_block	nb;
485 	u8			phy_port[HNS_ROCE_MAX_PORTS];
486 };
487 
488 struct hns_roce_eq {
489 	struct hns_roce_dev		*hr_dev;
490 	void __iomem			*doorbell;
491 
492 	int				type_flag;/* Aeq:1 ceq:0 */
493 	int				eqn;
494 	u32				entries;
495 	int				log_entries;
496 	int				eqe_size;
497 	int				irq;
498 	int				log_page_size;
499 	int				cons_index;
500 	struct hns_roce_buf_list	*buf_list;
501 };
502 
503 struct hns_roce_eq_table {
504 	struct hns_roce_eq	*eq;
505 	void __iomem		**eqc_base;
506 };
507 
508 struct hns_roce_caps {
509 	u8		num_ports;
510 	int		gid_table_len[HNS_ROCE_MAX_PORTS];
511 	int		pkey_table_len[HNS_ROCE_MAX_PORTS];
512 	int		local_ca_ack_delay;
513 	int		num_uars;
514 	u32		phy_num_uars;
515 	u32		max_sq_sg;	/* 2 */
516 	u32		max_sq_inline;	/* 32 */
517 	u32		max_rq_sg;	/* 2 */
518 	int		num_qps;	/* 256k */
519 	u32		max_wqes;	/* 16k */
520 	u32		max_sq_desc_sz;	/* 64 */
521 	u32		max_rq_desc_sz;	/* 64 */
522 	u32		max_srq_desc_sz;
523 	int		max_qp_init_rdma;
524 	int		max_qp_dest_rdma;
525 	int		num_cqs;
526 	int		max_cqes;
527 	int		min_cqes;
528 	u32		min_wqes;
529 	int		reserved_cqs;
530 	int		num_aeq_vectors;	/* 1 */
531 	int		num_comp_vectors;	/* 32 ceq */
532 	int		num_other_vectors;
533 	int		num_mtpts;
534 	u32		num_mtt_segs;
535 	u32		num_cqe_segs;
536 	int		reserved_mrws;
537 	int		reserved_uars;
538 	int		num_pds;
539 	int		reserved_pds;
540 	u32		mtt_entry_sz;
541 	u32		cq_entry_sz;
542 	u32		page_size_cap;
543 	u32		reserved_lkey;
544 	int		mtpt_entry_sz;
545 	int		qpc_entry_sz;
546 	int		irrl_entry_sz;
547 	int		trrl_entry_sz;
548 	int		cqc_entry_sz;
549 	u32		pbl_ba_pg_sz;
550 	u32		pbl_buf_pg_sz;
551 	u32		pbl_hop_num;
552 	int		aeqe_depth;
553 	int		ceqe_depth[HNS_ROCE_COMP_VEC_NUM];
554 	enum ib_mtu	max_mtu;
555 	u32		qpc_bt_num;
556 	u32		srqc_bt_num;
557 	u32		cqc_bt_num;
558 	u32		mpt_bt_num;
559 	u32		qpc_ba_pg_sz;
560 	u32		qpc_buf_pg_sz;
561 	u32		qpc_hop_num;
562 	u32		srqc_ba_pg_sz;
563 	u32		srqc_buf_pg_sz;
564 	u32		srqc_hop_num;
565 	u32		cqc_ba_pg_sz;
566 	u32		cqc_buf_pg_sz;
567 	u32		cqc_hop_num;
568 	u32		mpt_ba_pg_sz;
569 	u32		mpt_buf_pg_sz;
570 	u32		mpt_hop_num;
571 	u32		mtt_ba_pg_sz;
572 	u32		mtt_buf_pg_sz;
573 	u32		mtt_hop_num;
574 	u32		cqe_ba_pg_sz;
575 	u32		cqe_buf_pg_sz;
576 	u32		cqe_hop_num;
577 	u32		chunk_sz;	/* chunk size in non multihop mode*/
578 	u64		flags;
579 };
580 
581 struct hns_roce_hw {
582 	int (*reset)(struct hns_roce_dev *hr_dev, bool enable);
583 	int (*cmq_init)(struct hns_roce_dev *hr_dev);
584 	void (*cmq_exit)(struct hns_roce_dev *hr_dev);
585 	int (*hw_profile)(struct hns_roce_dev *hr_dev);
586 	int (*hw_init)(struct hns_roce_dev *hr_dev);
587 	void (*hw_exit)(struct hns_roce_dev *hr_dev);
588 	int (*post_mbox)(struct hns_roce_dev *hr_dev, u64 in_param,
589 			 u64 out_param, u32 in_modifier, u8 op_modifier, u16 op,
590 			 u16 token, int event);
591 	int (*chk_mbox)(struct hns_roce_dev *hr_dev, unsigned long timeout);
592 	int (*set_gid)(struct hns_roce_dev *hr_dev, u8 port, int gid_index,
593 		       union ib_gid *gid, const struct ib_gid_attr *attr);
594 	int (*set_mac)(struct hns_roce_dev *hr_dev, u8 phy_port, u8 *addr);
595 	void (*set_mtu)(struct hns_roce_dev *hr_dev, u8 phy_port,
596 			enum ib_mtu mtu);
597 	int (*write_mtpt)(void *mb_buf, struct hns_roce_mr *mr,
598 			  unsigned long mtpt_idx);
599 	int (*rereg_write_mtpt)(struct hns_roce_dev *hr_dev,
600 				struct hns_roce_mr *mr, int flags, u32 pdn,
601 				int mr_access_flags, u64 iova, u64 size,
602 				void *mb_buf);
603 	void (*write_cqc)(struct hns_roce_dev *hr_dev,
604 			  struct hns_roce_cq *hr_cq, void *mb_buf, u64 *mtts,
605 			  dma_addr_t dma_handle, int nent, u32 vector);
606 	int (*set_hem)(struct hns_roce_dev *hr_dev,
607 		       struct hns_roce_hem_table *table, int obj, int step_idx);
608 	int (*clear_hem)(struct hns_roce_dev *hr_dev,
609 			 struct hns_roce_hem_table *table, int obj,
610 			 int step_idx);
611 	int (*query_qp)(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
612 			int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr);
613 	int (*modify_qp)(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
614 			 int attr_mask, enum ib_qp_state cur_state,
615 			 enum ib_qp_state new_state);
616 	int (*destroy_qp)(struct ib_qp *ibqp);
617 	int (*post_send)(struct ib_qp *ibqp, struct ib_send_wr *wr,
618 			 struct ib_send_wr **bad_wr);
619 	int (*post_recv)(struct ib_qp *qp, struct ib_recv_wr *recv_wr,
620 			 struct ib_recv_wr **bad_recv_wr);
621 	int (*req_notify_cq)(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
622 	int (*poll_cq)(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
623 	int (*dereg_mr)(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr);
624 	int (*destroy_cq)(struct ib_cq *ibcq);
625 	int (*modify_cq)(struct ib_cq *cq, u16 cq_count, u16 cq_period);
626 };
627 
628 struct hns_roce_dev {
629 	struct ib_device	ib_dev;
630 	struct platform_device  *pdev;
631 	struct pci_dev		*pci_dev;
632 	struct device		*dev;
633 	struct hns_roce_uar     priv_uar;
634 	const char		*irq_names[HNS_ROCE_MAX_IRQ_NUM];
635 	spinlock_t		sm_lock;
636 	spinlock_t		bt_cmd_lock;
637 	struct hns_roce_ib_iboe iboe;
638 
639 	int			irq[HNS_ROCE_MAX_IRQ_NUM];
640 	u8 __iomem		*reg_base;
641 	struct hns_roce_caps	caps;
642 	struct radix_tree_root  qp_table_tree;
643 
644 	unsigned char	dev_addr[HNS_ROCE_MAX_PORTS][MAC_ADDR_OCTET_NUM];
645 	u64			sys_image_guid;
646 	u32                     vendor_id;
647 	u32                     vendor_part_id;
648 	u32                     hw_rev;
649 	void __iomem            *priv_addr;
650 
651 	struct hns_roce_cmdq	cmd;
652 	struct hns_roce_bitmap    pd_bitmap;
653 	struct hns_roce_uar_table uar_table;
654 	struct hns_roce_mr_table  mr_table;
655 	struct hns_roce_cq_table  cq_table;
656 	struct hns_roce_qp_table  qp_table;
657 	struct hns_roce_eq_table  eq_table;
658 
659 	int			cmd_mod;
660 	int			loop_idc;
661 	u32			sdb_offset;
662 	u32			odb_offset;
663 	dma_addr_t		tptr_dma_addr; /*only for hw v1*/
664 	u32			tptr_size; /*only for hw v1*/
665 	const struct hns_roce_hw *hw;
666 	void			*priv;
667 };
668 
669 static inline struct hns_roce_dev *to_hr_dev(struct ib_device *ib_dev)
670 {
671 	return container_of(ib_dev, struct hns_roce_dev, ib_dev);
672 }
673 
674 static inline struct hns_roce_ucontext
675 			*to_hr_ucontext(struct ib_ucontext *ibucontext)
676 {
677 	return container_of(ibucontext, struct hns_roce_ucontext, ibucontext);
678 }
679 
680 static inline struct hns_roce_pd *to_hr_pd(struct ib_pd *ibpd)
681 {
682 	return container_of(ibpd, struct hns_roce_pd, ibpd);
683 }
684 
685 static inline struct hns_roce_ah *to_hr_ah(struct ib_ah *ibah)
686 {
687 	return container_of(ibah, struct hns_roce_ah, ibah);
688 }
689 
690 static inline struct hns_roce_mr *to_hr_mr(struct ib_mr *ibmr)
691 {
692 	return container_of(ibmr, struct hns_roce_mr, ibmr);
693 }
694 
695 static inline struct hns_roce_qp *to_hr_qp(struct ib_qp *ibqp)
696 {
697 	return container_of(ibqp, struct hns_roce_qp, ibqp);
698 }
699 
700 static inline struct hns_roce_cq *to_hr_cq(struct ib_cq *ib_cq)
701 {
702 	return container_of(ib_cq, struct hns_roce_cq, ib_cq);
703 }
704 
705 static inline struct hns_roce_srq *to_hr_srq(struct ib_srq *ibsrq)
706 {
707 	return container_of(ibsrq, struct hns_roce_srq, ibsrq);
708 }
709 
710 static inline struct hns_roce_sqp *hr_to_hr_sqp(struct hns_roce_qp *hr_qp)
711 {
712 	return container_of(hr_qp, struct hns_roce_sqp, hr_qp);
713 }
714 
715 static inline void hns_roce_write64_k(__be32 val[2], void __iomem *dest)
716 {
717 	__raw_writeq(*(u64 *) val, dest);
718 }
719 
720 static inline struct hns_roce_qp
721 	*__hns_roce_qp_lookup(struct hns_roce_dev *hr_dev, u32 qpn)
722 {
723 	return radix_tree_lookup(&hr_dev->qp_table_tree,
724 				 qpn & (hr_dev->caps.num_qps - 1));
725 }
726 
727 static inline void *hns_roce_buf_offset(struct hns_roce_buf *buf, int offset)
728 {
729 	u32 page_size = 1 << buf->page_shift;
730 
731 	if (buf->nbufs == 1)
732 		return (char *)(buf->direct.buf) + offset;
733 	else
734 		return (char *)(buf->page_list[offset >> buf->page_shift].buf) +
735 		       (offset & (page_size - 1));
736 }
737 
738 int hns_roce_init_uar_table(struct hns_roce_dev *dev);
739 int hns_roce_uar_alloc(struct hns_roce_dev *dev, struct hns_roce_uar *uar);
740 void hns_roce_uar_free(struct hns_roce_dev *dev, struct hns_roce_uar *uar);
741 void hns_roce_cleanup_uar_table(struct hns_roce_dev *dev);
742 
743 int hns_roce_cmd_init(struct hns_roce_dev *hr_dev);
744 void hns_roce_cmd_cleanup(struct hns_roce_dev *hr_dev);
745 void hns_roce_cmd_event(struct hns_roce_dev *hr_dev, u16 token, u8 status,
746 			u64 out_param);
747 int hns_roce_cmd_use_events(struct hns_roce_dev *hr_dev);
748 void hns_roce_cmd_use_polling(struct hns_roce_dev *hr_dev);
749 
750 int hns_roce_mtt_init(struct hns_roce_dev *hr_dev, int npages, int page_shift,
751 		      struct hns_roce_mtt *mtt);
752 void hns_roce_mtt_cleanup(struct hns_roce_dev *hr_dev,
753 			  struct hns_roce_mtt *mtt);
754 int hns_roce_buf_write_mtt(struct hns_roce_dev *hr_dev,
755 			   struct hns_roce_mtt *mtt, struct hns_roce_buf *buf);
756 
757 int hns_roce_init_pd_table(struct hns_roce_dev *hr_dev);
758 int hns_roce_init_mr_table(struct hns_roce_dev *hr_dev);
759 int hns_roce_init_eq_table(struct hns_roce_dev *hr_dev);
760 int hns_roce_init_cq_table(struct hns_roce_dev *hr_dev);
761 int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev);
762 
763 void hns_roce_cleanup_pd_table(struct hns_roce_dev *hr_dev);
764 void hns_roce_cleanup_mr_table(struct hns_roce_dev *hr_dev);
765 void hns_roce_cleanup_eq_table(struct hns_roce_dev *hr_dev);
766 void hns_roce_cleanup_cq_table(struct hns_roce_dev *hr_dev);
767 void hns_roce_cleanup_qp_table(struct hns_roce_dev *hr_dev);
768 
769 int hns_roce_bitmap_alloc(struct hns_roce_bitmap *bitmap, unsigned long *obj);
770 void hns_roce_bitmap_free(struct hns_roce_bitmap *bitmap, unsigned long obj,
771 			 int rr);
772 int hns_roce_bitmap_init(struct hns_roce_bitmap *bitmap, u32 num, u32 mask,
773 			 u32 reserved_bot, u32 resetrved_top);
774 void hns_roce_bitmap_cleanup(struct hns_roce_bitmap *bitmap);
775 void hns_roce_cleanup_bitmap(struct hns_roce_dev *hr_dev);
776 int hns_roce_bitmap_alloc_range(struct hns_roce_bitmap *bitmap, int cnt,
777 				int align, unsigned long *obj);
778 void hns_roce_bitmap_free_range(struct hns_roce_bitmap *bitmap,
779 				unsigned long obj, int cnt,
780 				int rr);
781 
782 struct ib_ah *hns_roce_create_ah(struct ib_pd *pd,
783 				 struct rdma_ah_attr *ah_attr,
784 				 struct ib_udata *udata);
785 int hns_roce_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr);
786 int hns_roce_destroy_ah(struct ib_ah *ah);
787 
788 struct ib_pd *hns_roce_alloc_pd(struct ib_device *ib_dev,
789 				struct ib_ucontext *context,
790 				struct ib_udata *udata);
791 int hns_roce_dealloc_pd(struct ib_pd *pd);
792 
793 struct ib_mr *hns_roce_get_dma_mr(struct ib_pd *pd, int acc);
794 struct ib_mr *hns_roce_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
795 				   u64 virt_addr, int access_flags,
796 				   struct ib_udata *udata);
797 int hns_roce_rereg_user_mr(struct ib_mr *mr, int flags, u64 start, u64 length,
798 			   u64 virt_addr, int mr_access_flags, struct ib_pd *pd,
799 			   struct ib_udata *udata);
800 int hns_roce_dereg_mr(struct ib_mr *ibmr);
801 int hns_roce_hw2sw_mpt(struct hns_roce_dev *hr_dev,
802 		       struct hns_roce_cmd_mailbox *mailbox,
803 		       unsigned long mpt_index);
804 unsigned long key_to_hw_index(u32 key);
805 
806 void hns_roce_buf_free(struct hns_roce_dev *hr_dev, u32 size,
807 		       struct hns_roce_buf *buf);
808 int hns_roce_buf_alloc(struct hns_roce_dev *hr_dev, u32 size, u32 max_direct,
809 		       struct hns_roce_buf *buf, u32 page_shift);
810 
811 int hns_roce_ib_umem_write_mtt(struct hns_roce_dev *hr_dev,
812 			       struct hns_roce_mtt *mtt, struct ib_umem *umem);
813 
814 struct ib_qp *hns_roce_create_qp(struct ib_pd *ib_pd,
815 				 struct ib_qp_init_attr *init_attr,
816 				 struct ib_udata *udata);
817 int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
818 		       int attr_mask, struct ib_udata *udata);
819 void *get_recv_wqe(struct hns_roce_qp *hr_qp, int n);
820 void *get_send_wqe(struct hns_roce_qp *hr_qp, int n);
821 void *get_send_extend_sge(struct hns_roce_qp *hr_qp, int n);
822 bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, int nreq,
823 			  struct ib_cq *ib_cq);
824 enum hns_roce_qp_state to_hns_roce_state(enum ib_qp_state state);
825 void hns_roce_lock_cqs(struct hns_roce_cq *send_cq,
826 		       struct hns_roce_cq *recv_cq);
827 void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq,
828 			 struct hns_roce_cq *recv_cq);
829 void hns_roce_qp_remove(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp);
830 void hns_roce_qp_free(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp);
831 void hns_roce_release_range_qp(struct hns_roce_dev *hr_dev, int base_qpn,
832 			       int cnt);
833 __be32 send_ieth(struct ib_send_wr *wr);
834 int to_hr_qp_type(int qp_type);
835 
836 struct ib_cq *hns_roce_ib_create_cq(struct ib_device *ib_dev,
837 				    const struct ib_cq_init_attr *attr,
838 				    struct ib_ucontext *context,
839 				    struct ib_udata *udata);
840 
841 int hns_roce_ib_destroy_cq(struct ib_cq *ib_cq);
842 void hns_roce_free_cq(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq);
843 
844 void hns_roce_cq_completion(struct hns_roce_dev *hr_dev, u32 cqn);
845 void hns_roce_cq_event(struct hns_roce_dev *hr_dev, u32 cqn, int event_type);
846 void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type);
847 int hns_get_gid_index(struct hns_roce_dev *hr_dev, u8 port, int gid_index);
848 int hns_roce_init(struct hns_roce_dev *hr_dev);
849 void hns_roce_exit(struct hns_roce_dev *hr_dev);
850 
851 #endif /* _HNS_ROCE_DEVICE_H */
852