1 /*
2  * Copyright (c) 2016-2017 Hisilicon Limited.
3  *
4  * This software is available to you under a choice of one of two
5  * licenses.  You may choose to be licensed under the terms of the GNU
6  * General Public License (GPL) Version 2, available from the file
7  * COPYING in the main directory of this source tree, or the
8  * OpenIB.org BSD license below:
9  *
10  *     Redistribution and use in source and binary forms, with or
11  *     without modification, are permitted provided that the following
12  *     conditions are met:
13  *
14  *      - Redistributions of source code must retain the above
15  *        copyright notice, this list of conditions and the following
16  *        disclaimer.
17  *
18  *      - Redistributions in binary form must reproduce the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer in the documentation and/or other materials
21  *        provided with the distribution.
22  *
23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30  * SOFTWARE.
31  */
32 
33 #include <linux/acpi.h>
34 #include <linux/etherdevice.h>
35 #include <linux/interrupt.h>
36 #include <linux/kernel.h>
37 #include <linux/types.h>
38 #include <net/addrconf.h>
39 #include <rdma/ib_addr.h>
40 #include <rdma/ib_cache.h>
41 #include <rdma/ib_umem.h>
42 #include <rdma/uverbs_ioctl.h>
43 
44 #include "hnae3.h"
45 #include "hns_roce_common.h"
46 #include "hns_roce_device.h"
47 #include "hns_roce_cmd.h"
48 #include "hns_roce_hem.h"
49 #include "hns_roce_hw_v2.h"
50 
51 static void set_data_seg_v2(struct hns_roce_v2_wqe_data_seg *dseg,
52 			    struct ib_sge *sg)
53 {
54 	dseg->lkey = cpu_to_le32(sg->lkey);
55 	dseg->addr = cpu_to_le64(sg->addr);
56 	dseg->len  = cpu_to_le32(sg->length);
57 }
58 
59 static void set_frmr_seg(struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
60 			 struct hns_roce_wqe_frmr_seg *fseg,
61 			 const struct ib_reg_wr *wr)
62 {
63 	struct hns_roce_mr *mr = to_hr_mr(wr->mr);
64 
65 	/* use ib_access_flags */
66 	roce_set_bit(rc_sq_wqe->byte_4,
67 		     V2_RC_FRMR_WQE_BYTE_4_BIND_EN_S,
68 		     wr->access & IB_ACCESS_MW_BIND ? 1 : 0);
69 	roce_set_bit(rc_sq_wqe->byte_4,
70 		     V2_RC_FRMR_WQE_BYTE_4_ATOMIC_S,
71 		     wr->access & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0);
72 	roce_set_bit(rc_sq_wqe->byte_4,
73 		     V2_RC_FRMR_WQE_BYTE_4_RR_S,
74 		     wr->access & IB_ACCESS_REMOTE_READ ? 1 : 0);
75 	roce_set_bit(rc_sq_wqe->byte_4,
76 		     V2_RC_FRMR_WQE_BYTE_4_RW_S,
77 		     wr->access & IB_ACCESS_REMOTE_WRITE ? 1 : 0);
78 	roce_set_bit(rc_sq_wqe->byte_4,
79 		     V2_RC_FRMR_WQE_BYTE_4_LW_S,
80 		     wr->access & IB_ACCESS_LOCAL_WRITE ? 1 : 0);
81 
82 	/* Data structure reuse may lead to confusion */
83 	rc_sq_wqe->msg_len = cpu_to_le32(mr->pbl_ba & 0xffffffff);
84 	rc_sq_wqe->inv_key = cpu_to_le32(mr->pbl_ba >> 32);
85 
86 	rc_sq_wqe->byte_16 = cpu_to_le32(wr->mr->length & 0xffffffff);
87 	rc_sq_wqe->byte_20 = cpu_to_le32(wr->mr->length >> 32);
88 	rc_sq_wqe->rkey = cpu_to_le32(wr->key);
89 	rc_sq_wqe->va = cpu_to_le64(wr->mr->iova);
90 
91 	fseg->pbl_size = cpu_to_le32(mr->pbl_size);
92 	roce_set_field(fseg->mode_buf_pg_sz,
93 		       V2_RC_FRMR_WQE_BYTE_40_PBL_BUF_PG_SZ_M,
94 		       V2_RC_FRMR_WQE_BYTE_40_PBL_BUF_PG_SZ_S,
95 		       mr->pbl_buf_pg_sz + PG_SHIFT_OFFSET);
96 	roce_set_bit(fseg->mode_buf_pg_sz,
97 		     V2_RC_FRMR_WQE_BYTE_40_BLK_MODE_S, 0);
98 }
99 
100 static void set_atomic_seg(struct hns_roce_wqe_atomic_seg *aseg,
101 			   const struct ib_atomic_wr *wr)
102 {
103 	if (wr->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP) {
104 		aseg->fetchadd_swap_data = cpu_to_le64(wr->swap);
105 		aseg->cmp_data  = cpu_to_le64(wr->compare_add);
106 	} else {
107 		aseg->fetchadd_swap_data = cpu_to_le64(wr->compare_add);
108 		aseg->cmp_data  = 0;
109 	}
110 }
111 
112 static void set_extend_sge(struct hns_roce_qp *qp, const struct ib_send_wr *wr,
113 			   unsigned int *sge_ind)
114 {
115 	struct hns_roce_v2_wqe_data_seg *dseg;
116 	struct ib_sge *sg;
117 	int num_in_wqe = 0;
118 	int extend_sge_num;
119 	int fi_sge_num;
120 	int se_sge_num;
121 	int shift;
122 	int i;
123 
124 	if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC)
125 		num_in_wqe = HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE;
126 	extend_sge_num = wr->num_sge - num_in_wqe;
127 	sg = wr->sg_list + num_in_wqe;
128 	shift = qp->hr_buf.page_shift;
129 
130 	/*
131 	 * Check whether wr->num_sge sges are in the same page. If not, we
132 	 * should calculate how many sges in the first page and the second
133 	 * page.
134 	 */
135 	dseg = get_send_extend_sge(qp, (*sge_ind) & (qp->sge.sge_cnt - 1));
136 	fi_sge_num = (round_up((uintptr_t)dseg, 1 << shift) -
137 		      (uintptr_t)dseg) /
138 		      sizeof(struct hns_roce_v2_wqe_data_seg);
139 	if (extend_sge_num > fi_sge_num) {
140 		se_sge_num = extend_sge_num - fi_sge_num;
141 		for (i = 0; i < fi_sge_num; i++) {
142 			set_data_seg_v2(dseg++, sg + i);
143 			(*sge_ind)++;
144 		}
145 		dseg = get_send_extend_sge(qp,
146 					   (*sge_ind) & (qp->sge.sge_cnt - 1));
147 		for (i = 0; i < se_sge_num; i++) {
148 			set_data_seg_v2(dseg++, sg + fi_sge_num + i);
149 			(*sge_ind)++;
150 		}
151 	} else {
152 		for (i = 0; i < extend_sge_num; i++) {
153 			set_data_seg_v2(dseg++, sg + i);
154 			(*sge_ind)++;
155 		}
156 	}
157 }
158 
159 static int set_rwqe_data_seg(struct ib_qp *ibqp, const struct ib_send_wr *wr,
160 			     struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
161 			     void *wqe, unsigned int *sge_ind,
162 			     const struct ib_send_wr **bad_wr)
163 {
164 	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
165 	struct hns_roce_v2_wqe_data_seg *dseg = wqe;
166 	struct hns_roce_qp *qp = to_hr_qp(ibqp);
167 	int i;
168 
169 	if (wr->send_flags & IB_SEND_INLINE && wr->num_sge) {
170 		if (le32_to_cpu(rc_sq_wqe->msg_len) >
171 		    hr_dev->caps.max_sq_inline) {
172 			*bad_wr = wr;
173 			dev_err(hr_dev->dev, "inline len(1-%d)=%d, illegal",
174 				rc_sq_wqe->msg_len, hr_dev->caps.max_sq_inline);
175 			return -EINVAL;
176 		}
177 
178 		if (wr->opcode == IB_WR_RDMA_READ) {
179 			*bad_wr =  wr;
180 			dev_err(hr_dev->dev, "Not support inline data!\n");
181 			return -EINVAL;
182 		}
183 
184 		for (i = 0; i < wr->num_sge; i++) {
185 			memcpy(wqe, ((void *)wr->sg_list[i].addr),
186 			       wr->sg_list[i].length);
187 			wqe += wr->sg_list[i].length;
188 		}
189 
190 		roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_INLINE_S,
191 			     1);
192 	} else {
193 		if (wr->num_sge <= HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) {
194 			for (i = 0; i < wr->num_sge; i++) {
195 				if (likely(wr->sg_list[i].length)) {
196 					set_data_seg_v2(dseg, wr->sg_list + i);
197 					dseg++;
198 				}
199 			}
200 		} else {
201 			roce_set_field(rc_sq_wqe->byte_20,
202 				     V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M,
203 				     V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S,
204 				     (*sge_ind) & (qp->sge.sge_cnt - 1));
205 
206 			for (i = 0; i < HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE; i++) {
207 				if (likely(wr->sg_list[i].length)) {
208 					set_data_seg_v2(dseg, wr->sg_list + i);
209 					dseg++;
210 				}
211 			}
212 
213 			set_extend_sge(qp, wr, sge_ind);
214 		}
215 
216 		roce_set_field(rc_sq_wqe->byte_16,
217 			       V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M,
218 			       V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S, wr->num_sge);
219 	}
220 
221 	return 0;
222 }
223 
224 static int hns_roce_v2_modify_qp(struct ib_qp *ibqp,
225 				 const struct ib_qp_attr *attr,
226 				 int attr_mask, enum ib_qp_state cur_state,
227 				 enum ib_qp_state new_state);
228 
229 static int hns_roce_v2_post_send(struct ib_qp *ibqp,
230 				 const struct ib_send_wr *wr,
231 				 const struct ib_send_wr **bad_wr)
232 {
233 	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
234 	struct hns_roce_ah *ah = to_hr_ah(ud_wr(wr)->ah);
235 	struct hns_roce_v2_ud_send_wqe *ud_sq_wqe;
236 	struct hns_roce_v2_rc_send_wqe *rc_sq_wqe;
237 	struct hns_roce_qp *qp = to_hr_qp(ibqp);
238 	struct hns_roce_wqe_frmr_seg *fseg;
239 	struct device *dev = hr_dev->dev;
240 	struct hns_roce_v2_db sq_db;
241 	struct ib_qp_attr attr;
242 	unsigned int sge_ind;
243 	unsigned int owner_bit;
244 	unsigned long flags;
245 	unsigned int ind;
246 	void *wqe = NULL;
247 	bool loopback;
248 	int attr_mask;
249 	u32 tmp_len;
250 	int ret = 0;
251 	u32 hr_op;
252 	u8 *smac;
253 	int nreq;
254 	int i;
255 
256 	if (unlikely(ibqp->qp_type != IB_QPT_RC &&
257 		     ibqp->qp_type != IB_QPT_GSI &&
258 		     ibqp->qp_type != IB_QPT_UD)) {
259 		dev_err(dev, "Not supported QP(0x%x)type!\n", ibqp->qp_type);
260 		*bad_wr = wr;
261 		return -EOPNOTSUPP;
262 	}
263 
264 	if (unlikely(qp->state == IB_QPS_RESET || qp->state == IB_QPS_INIT ||
265 		     qp->state == IB_QPS_RTR)) {
266 		dev_err(dev, "Post WQE fail, QP state %d err!\n", qp->state);
267 		*bad_wr = wr;
268 		return -EINVAL;
269 	}
270 
271 	spin_lock_irqsave(&qp->sq.lock, flags);
272 	ind = qp->sq_next_wqe;
273 	sge_ind = qp->next_sge;
274 
275 	for (nreq = 0; wr; ++nreq, wr = wr->next) {
276 		if (hns_roce_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) {
277 			ret = -ENOMEM;
278 			*bad_wr = wr;
279 			goto out;
280 		}
281 
282 		if (unlikely(wr->num_sge > qp->sq.max_gs)) {
283 			dev_err(dev, "num_sge=%d > qp->sq.max_gs=%d\n",
284 				wr->num_sge, qp->sq.max_gs);
285 			ret = -EINVAL;
286 			*bad_wr = wr;
287 			goto out;
288 		}
289 
290 		wqe = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1));
291 		qp->sq.wrid[(qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1)] =
292 								      wr->wr_id;
293 
294 		owner_bit =
295 		       ~(((qp->sq.head + nreq) >> ilog2(qp->sq.wqe_cnt)) & 0x1);
296 		tmp_len = 0;
297 
298 		/* Corresponding to the QP type, wqe process separately */
299 		if (ibqp->qp_type == IB_QPT_GSI) {
300 			ud_sq_wqe = wqe;
301 			memset(ud_sq_wqe, 0, sizeof(*ud_sq_wqe));
302 
303 			roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_0_M,
304 				       V2_UD_SEND_WQE_DMAC_0_S, ah->av.mac[0]);
305 			roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_1_M,
306 				       V2_UD_SEND_WQE_DMAC_1_S, ah->av.mac[1]);
307 			roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_2_M,
308 				       V2_UD_SEND_WQE_DMAC_2_S, ah->av.mac[2]);
309 			roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_3_M,
310 				       V2_UD_SEND_WQE_DMAC_3_S, ah->av.mac[3]);
311 			roce_set_field(ud_sq_wqe->byte_48,
312 				       V2_UD_SEND_WQE_BYTE_48_DMAC_4_M,
313 				       V2_UD_SEND_WQE_BYTE_48_DMAC_4_S,
314 				       ah->av.mac[4]);
315 			roce_set_field(ud_sq_wqe->byte_48,
316 				       V2_UD_SEND_WQE_BYTE_48_DMAC_5_M,
317 				       V2_UD_SEND_WQE_BYTE_48_DMAC_5_S,
318 				       ah->av.mac[5]);
319 
320 			/* MAC loopback */
321 			smac = (u8 *)hr_dev->dev_addr[qp->port];
322 			loopback = ether_addr_equal_unaligned(ah->av.mac,
323 							      smac) ? 1 : 0;
324 
325 			roce_set_bit(ud_sq_wqe->byte_40,
326 				     V2_UD_SEND_WQE_BYTE_40_LBI_S, loopback);
327 
328 			roce_set_field(ud_sq_wqe->byte_4,
329 				       V2_UD_SEND_WQE_BYTE_4_OPCODE_M,
330 				       V2_UD_SEND_WQE_BYTE_4_OPCODE_S,
331 				       HNS_ROCE_V2_WQE_OP_SEND);
332 
333 			for (i = 0; i < wr->num_sge; i++)
334 				tmp_len += wr->sg_list[i].length;
335 
336 			ud_sq_wqe->msg_len =
337 			 cpu_to_le32(le32_to_cpu(ud_sq_wqe->msg_len) + tmp_len);
338 
339 			switch (wr->opcode) {
340 			case IB_WR_SEND_WITH_IMM:
341 			case IB_WR_RDMA_WRITE_WITH_IMM:
342 				ud_sq_wqe->immtdata =
343 				      cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
344 				break;
345 			default:
346 				ud_sq_wqe->immtdata = 0;
347 				break;
348 			}
349 
350 			/* Set sig attr */
351 			roce_set_bit(ud_sq_wqe->byte_4,
352 				   V2_UD_SEND_WQE_BYTE_4_CQE_S,
353 				   (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0);
354 
355 			/* Set se attr */
356 			roce_set_bit(ud_sq_wqe->byte_4,
357 				  V2_UD_SEND_WQE_BYTE_4_SE_S,
358 				  (wr->send_flags & IB_SEND_SOLICITED) ? 1 : 0);
359 
360 			roce_set_bit(ud_sq_wqe->byte_4,
361 				     V2_UD_SEND_WQE_BYTE_4_OWNER_S, owner_bit);
362 
363 			roce_set_field(ud_sq_wqe->byte_16,
364 				       V2_UD_SEND_WQE_BYTE_16_PD_M,
365 				       V2_UD_SEND_WQE_BYTE_16_PD_S,
366 				       to_hr_pd(ibqp->pd)->pdn);
367 
368 			roce_set_field(ud_sq_wqe->byte_16,
369 				       V2_UD_SEND_WQE_BYTE_16_SGE_NUM_M,
370 				       V2_UD_SEND_WQE_BYTE_16_SGE_NUM_S,
371 				       wr->num_sge);
372 
373 			roce_set_field(ud_sq_wqe->byte_20,
374 				     V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M,
375 				     V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S,
376 				     sge_ind & (qp->sge.sge_cnt - 1));
377 
378 			roce_set_field(ud_sq_wqe->byte_24,
379 				       V2_UD_SEND_WQE_BYTE_24_UDPSPN_M,
380 				       V2_UD_SEND_WQE_BYTE_24_UDPSPN_S, 0);
381 			ud_sq_wqe->qkey =
382 			     cpu_to_le32(ud_wr(wr)->remote_qkey & 0x80000000 ?
383 			     qp->qkey : ud_wr(wr)->remote_qkey);
384 			roce_set_field(ud_sq_wqe->byte_32,
385 				       V2_UD_SEND_WQE_BYTE_32_DQPN_M,
386 				       V2_UD_SEND_WQE_BYTE_32_DQPN_S,
387 				       ud_wr(wr)->remote_qpn);
388 
389 			roce_set_field(ud_sq_wqe->byte_36,
390 				       V2_UD_SEND_WQE_BYTE_36_VLAN_M,
391 				       V2_UD_SEND_WQE_BYTE_36_VLAN_S,
392 				       ah->av.vlan_id);
393 			roce_set_field(ud_sq_wqe->byte_36,
394 				       V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_M,
395 				       V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_S,
396 				       ah->av.hop_limit);
397 			roce_set_field(ud_sq_wqe->byte_36,
398 				       V2_UD_SEND_WQE_BYTE_36_TCLASS_M,
399 				       V2_UD_SEND_WQE_BYTE_36_TCLASS_S,
400 				       ah->av.tclass);
401 			roce_set_field(ud_sq_wqe->byte_40,
402 				       V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_M,
403 				       V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_S,
404 				       ah->av.flowlabel);
405 			roce_set_field(ud_sq_wqe->byte_40,
406 				       V2_UD_SEND_WQE_BYTE_40_SL_M,
407 				       V2_UD_SEND_WQE_BYTE_40_SL_S,
408 				       ah->av.sl);
409 			roce_set_field(ud_sq_wqe->byte_40,
410 				       V2_UD_SEND_WQE_BYTE_40_PORTN_M,
411 				       V2_UD_SEND_WQE_BYTE_40_PORTN_S,
412 				       qp->port);
413 
414 			roce_set_bit(ud_sq_wqe->byte_40,
415 				     V2_UD_SEND_WQE_BYTE_40_UD_VLAN_EN_S,
416 				     ah->av.vlan_en ? 1 : 0);
417 			roce_set_field(ud_sq_wqe->byte_48,
418 				       V2_UD_SEND_WQE_BYTE_48_SGID_INDX_M,
419 				       V2_UD_SEND_WQE_BYTE_48_SGID_INDX_S,
420 				       hns_get_gid_index(hr_dev, qp->phy_port,
421 							 ah->av.gid_index));
422 
423 			memcpy(&ud_sq_wqe->dgid[0], &ah->av.dgid[0],
424 			       GID_LEN_V2);
425 
426 			set_extend_sge(qp, wr, &sge_ind);
427 			ind++;
428 		} else if (ibqp->qp_type == IB_QPT_RC) {
429 			rc_sq_wqe = wqe;
430 			memset(rc_sq_wqe, 0, sizeof(*rc_sq_wqe));
431 			for (i = 0; i < wr->num_sge; i++)
432 				tmp_len += wr->sg_list[i].length;
433 
434 			rc_sq_wqe->msg_len =
435 			 cpu_to_le32(le32_to_cpu(rc_sq_wqe->msg_len) + tmp_len);
436 
437 			switch (wr->opcode) {
438 			case IB_WR_SEND_WITH_IMM:
439 			case IB_WR_RDMA_WRITE_WITH_IMM:
440 				rc_sq_wqe->immtdata =
441 				      cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
442 				break;
443 			case IB_WR_SEND_WITH_INV:
444 				rc_sq_wqe->inv_key =
445 					cpu_to_le32(wr->ex.invalidate_rkey);
446 				break;
447 			default:
448 				rc_sq_wqe->immtdata = 0;
449 				break;
450 			}
451 
452 			roce_set_bit(rc_sq_wqe->byte_4,
453 				     V2_RC_SEND_WQE_BYTE_4_FENCE_S,
454 				     (wr->send_flags & IB_SEND_FENCE) ? 1 : 0);
455 
456 			roce_set_bit(rc_sq_wqe->byte_4,
457 				  V2_RC_SEND_WQE_BYTE_4_SE_S,
458 				  (wr->send_flags & IB_SEND_SOLICITED) ? 1 : 0);
459 
460 			roce_set_bit(rc_sq_wqe->byte_4,
461 				   V2_RC_SEND_WQE_BYTE_4_CQE_S,
462 				   (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0);
463 
464 			roce_set_bit(rc_sq_wqe->byte_4,
465 				     V2_RC_SEND_WQE_BYTE_4_OWNER_S, owner_bit);
466 
467 			wqe += sizeof(struct hns_roce_v2_rc_send_wqe);
468 			switch (wr->opcode) {
469 			case IB_WR_RDMA_READ:
470 				hr_op = HNS_ROCE_V2_WQE_OP_RDMA_READ;
471 				rc_sq_wqe->rkey =
472 					cpu_to_le32(rdma_wr(wr)->rkey);
473 				rc_sq_wqe->va =
474 					cpu_to_le64(rdma_wr(wr)->remote_addr);
475 				break;
476 			case IB_WR_RDMA_WRITE:
477 				hr_op = HNS_ROCE_V2_WQE_OP_RDMA_WRITE;
478 				rc_sq_wqe->rkey =
479 					cpu_to_le32(rdma_wr(wr)->rkey);
480 				rc_sq_wqe->va =
481 					cpu_to_le64(rdma_wr(wr)->remote_addr);
482 				break;
483 			case IB_WR_RDMA_WRITE_WITH_IMM:
484 				hr_op = HNS_ROCE_V2_WQE_OP_RDMA_WRITE_WITH_IMM;
485 				rc_sq_wqe->rkey =
486 					cpu_to_le32(rdma_wr(wr)->rkey);
487 				rc_sq_wqe->va =
488 					cpu_to_le64(rdma_wr(wr)->remote_addr);
489 				break;
490 			case IB_WR_SEND:
491 				hr_op = HNS_ROCE_V2_WQE_OP_SEND;
492 				break;
493 			case IB_WR_SEND_WITH_INV:
494 				hr_op = HNS_ROCE_V2_WQE_OP_SEND_WITH_INV;
495 				break;
496 			case IB_WR_SEND_WITH_IMM:
497 				hr_op = HNS_ROCE_V2_WQE_OP_SEND_WITH_IMM;
498 				break;
499 			case IB_WR_LOCAL_INV:
500 				hr_op = HNS_ROCE_V2_WQE_OP_LOCAL_INV;
501 				roce_set_bit(rc_sq_wqe->byte_4,
502 					       V2_RC_SEND_WQE_BYTE_4_SO_S, 1);
503 				rc_sq_wqe->inv_key =
504 					    cpu_to_le32(wr->ex.invalidate_rkey);
505 				break;
506 			case IB_WR_REG_MR:
507 				hr_op = HNS_ROCE_V2_WQE_OP_FAST_REG_PMR;
508 				fseg = wqe;
509 				set_frmr_seg(rc_sq_wqe, fseg, reg_wr(wr));
510 				break;
511 			case IB_WR_ATOMIC_CMP_AND_SWP:
512 				hr_op = HNS_ROCE_V2_WQE_OP_ATOM_CMP_AND_SWAP;
513 				rc_sq_wqe->rkey =
514 					cpu_to_le32(atomic_wr(wr)->rkey);
515 				rc_sq_wqe->va =
516 					cpu_to_le64(atomic_wr(wr)->remote_addr);
517 				break;
518 			case IB_WR_ATOMIC_FETCH_AND_ADD:
519 				hr_op = HNS_ROCE_V2_WQE_OP_ATOM_FETCH_AND_ADD;
520 				rc_sq_wqe->rkey =
521 					cpu_to_le32(atomic_wr(wr)->rkey);
522 				rc_sq_wqe->va =
523 					cpu_to_le64(atomic_wr(wr)->remote_addr);
524 				break;
525 			case IB_WR_MASKED_ATOMIC_CMP_AND_SWP:
526 				hr_op =
527 				       HNS_ROCE_V2_WQE_OP_ATOM_MSK_CMP_AND_SWAP;
528 				break;
529 			case IB_WR_MASKED_ATOMIC_FETCH_AND_ADD:
530 				hr_op =
531 				      HNS_ROCE_V2_WQE_OP_ATOM_MSK_FETCH_AND_ADD;
532 				break;
533 			default:
534 				hr_op = HNS_ROCE_V2_WQE_OP_MASK;
535 				break;
536 			}
537 
538 			roce_set_field(rc_sq_wqe->byte_4,
539 				       V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
540 				       V2_RC_SEND_WQE_BYTE_4_OPCODE_S, hr_op);
541 
542 			if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
543 			    wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
544 				struct hns_roce_v2_wqe_data_seg *dseg;
545 
546 				dseg = wqe;
547 				set_data_seg_v2(dseg, wr->sg_list);
548 				wqe += sizeof(struct hns_roce_v2_wqe_data_seg);
549 				set_atomic_seg(wqe, atomic_wr(wr));
550 				roce_set_field(rc_sq_wqe->byte_16,
551 					       V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M,
552 					       V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S,
553 					       wr->num_sge);
554 			} else if (wr->opcode != IB_WR_REG_MR) {
555 				ret = set_rwqe_data_seg(ibqp, wr, rc_sq_wqe,
556 							wqe, &sge_ind, bad_wr);
557 				if (ret)
558 					goto out;
559 			}
560 
561 			ind++;
562 		} else {
563 			dev_err(dev, "Illegal qp_type(0x%x)\n", ibqp->qp_type);
564 			spin_unlock_irqrestore(&qp->sq.lock, flags);
565 			*bad_wr = wr;
566 			return -EOPNOTSUPP;
567 		}
568 	}
569 
570 out:
571 	if (likely(nreq)) {
572 		qp->sq.head += nreq;
573 		/* Memory barrier */
574 		wmb();
575 
576 		sq_db.byte_4 = 0;
577 		sq_db.parameter = 0;
578 
579 		roce_set_field(sq_db.byte_4, V2_DB_BYTE_4_TAG_M,
580 			       V2_DB_BYTE_4_TAG_S, qp->doorbell_qpn);
581 		roce_set_field(sq_db.byte_4, V2_DB_BYTE_4_CMD_M,
582 			       V2_DB_BYTE_4_CMD_S, HNS_ROCE_V2_SQ_DB);
583 		roce_set_field(sq_db.parameter, V2_DB_PARAMETER_IDX_M,
584 			       V2_DB_PARAMETER_IDX_S,
585 			       qp->sq.head & ((qp->sq.wqe_cnt << 1) - 1));
586 		roce_set_field(sq_db.parameter, V2_DB_PARAMETER_SL_M,
587 			       V2_DB_PARAMETER_SL_S, qp->sl);
588 
589 		hns_roce_write64(hr_dev, (__le32 *)&sq_db, qp->sq.db_reg_l);
590 
591 		qp->sq_next_wqe = ind;
592 		qp->next_sge = sge_ind;
593 
594 		if (qp->state == IB_QPS_ERR) {
595 			attr_mask = IB_QP_STATE;
596 			attr.qp_state = IB_QPS_ERR;
597 
598 			ret = hns_roce_v2_modify_qp(&qp->ibqp, &attr, attr_mask,
599 						    qp->state, IB_QPS_ERR);
600 			if (ret) {
601 				spin_unlock_irqrestore(&qp->sq.lock, flags);
602 				*bad_wr = wr;
603 				return ret;
604 			}
605 		}
606 	}
607 
608 	spin_unlock_irqrestore(&qp->sq.lock, flags);
609 
610 	return ret;
611 }
612 
613 static int hns_roce_v2_post_recv(struct ib_qp *ibqp,
614 				 const struct ib_recv_wr *wr,
615 				 const struct ib_recv_wr **bad_wr)
616 {
617 	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
618 	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
619 	struct hns_roce_v2_wqe_data_seg *dseg;
620 	struct hns_roce_rinl_sge *sge_list;
621 	struct device *dev = hr_dev->dev;
622 	struct ib_qp_attr attr;
623 	unsigned long flags;
624 	void *wqe = NULL;
625 	int attr_mask;
626 	int ret = 0;
627 	int nreq;
628 	int ind;
629 	int i;
630 
631 	spin_lock_irqsave(&hr_qp->rq.lock, flags);
632 	ind = hr_qp->rq.head & (hr_qp->rq.wqe_cnt - 1);
633 
634 	if (hr_qp->state == IB_QPS_RESET) {
635 		spin_unlock_irqrestore(&hr_qp->rq.lock, flags);
636 		*bad_wr = wr;
637 		return -EINVAL;
638 	}
639 
640 	for (nreq = 0; wr; ++nreq, wr = wr->next) {
641 		if (hns_roce_wq_overflow(&hr_qp->rq, nreq,
642 			hr_qp->ibqp.recv_cq)) {
643 			ret = -ENOMEM;
644 			*bad_wr = wr;
645 			goto out;
646 		}
647 
648 		if (unlikely(wr->num_sge > hr_qp->rq.max_gs)) {
649 			dev_err(dev, "rq:num_sge=%d > qp->sq.max_gs=%d\n",
650 				wr->num_sge, hr_qp->rq.max_gs);
651 			ret = -EINVAL;
652 			*bad_wr = wr;
653 			goto out;
654 		}
655 
656 		wqe = get_recv_wqe(hr_qp, ind);
657 		dseg = (struct hns_roce_v2_wqe_data_seg *)wqe;
658 		for (i = 0; i < wr->num_sge; i++) {
659 			if (!wr->sg_list[i].length)
660 				continue;
661 			set_data_seg_v2(dseg, wr->sg_list + i);
662 			dseg++;
663 		}
664 
665 		if (i < hr_qp->rq.max_gs) {
666 			dseg->lkey = cpu_to_le32(HNS_ROCE_INVALID_LKEY);
667 			dseg->addr = 0;
668 		}
669 
670 		/* rq support inline data */
671 		if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) {
672 			sge_list = hr_qp->rq_inl_buf.wqe_list[ind].sg_list;
673 			hr_qp->rq_inl_buf.wqe_list[ind].sge_cnt =
674 							       (u32)wr->num_sge;
675 			for (i = 0; i < wr->num_sge; i++) {
676 				sge_list[i].addr =
677 					       (void *)(u64)wr->sg_list[i].addr;
678 				sge_list[i].len = wr->sg_list[i].length;
679 			}
680 		}
681 
682 		hr_qp->rq.wrid[ind] = wr->wr_id;
683 
684 		ind = (ind + 1) & (hr_qp->rq.wqe_cnt - 1);
685 	}
686 
687 out:
688 	if (likely(nreq)) {
689 		hr_qp->rq.head += nreq;
690 		/* Memory barrier */
691 		wmb();
692 
693 		*hr_qp->rdb.db_record = hr_qp->rq.head & 0xffff;
694 
695 		if (hr_qp->state == IB_QPS_ERR) {
696 			attr_mask = IB_QP_STATE;
697 			attr.qp_state = IB_QPS_ERR;
698 
699 			ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, &attr,
700 						    attr_mask, hr_qp->state,
701 						    IB_QPS_ERR);
702 			if (ret) {
703 				spin_unlock_irqrestore(&hr_qp->rq.lock, flags);
704 				*bad_wr = wr;
705 				return ret;
706 			}
707 		}
708 	}
709 	spin_unlock_irqrestore(&hr_qp->rq.lock, flags);
710 
711 	return ret;
712 }
713 
714 static int hns_roce_v2_cmd_hw_reseted(struct hns_roce_dev *hr_dev,
715 				      unsigned long instance_stage,
716 				      unsigned long reset_stage)
717 {
718 	/* When hardware reset has been completed once or more, we should stop
719 	 * sending mailbox&cmq&doorbell to hardware. If now in .init_instance()
720 	 * function, we should exit with error. If now at HNAE3_INIT_CLIENT
721 	 * stage of soft reset process, we should exit with error, and then
722 	 * HNAE3_INIT_CLIENT related process can rollback the operation like
723 	 * notifing hardware to free resources, HNAE3_INIT_CLIENT related
724 	 * process will exit with error to notify NIC driver to reschedule soft
725 	 * reset process once again.
726 	 */
727 	hr_dev->is_reset = true;
728 	hr_dev->dis_db = true;
729 
730 	if (reset_stage == HNS_ROCE_STATE_RST_INIT ||
731 	    instance_stage == HNS_ROCE_STATE_INIT)
732 		return CMD_RST_PRC_EBUSY;
733 
734 	return CMD_RST_PRC_SUCCESS;
735 }
736 
737 static int hns_roce_v2_cmd_hw_resetting(struct hns_roce_dev *hr_dev,
738 					unsigned long instance_stage,
739 					unsigned long reset_stage)
740 {
741 	struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
742 	struct hnae3_handle *handle = priv->handle;
743 	const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
744 
745 	/* When hardware reset is detected, we should stop sending mailbox&cmq&
746 	 * doorbell to hardware. If now in .init_instance() function, we should
747 	 * exit with error. If now at HNAE3_INIT_CLIENT stage of soft reset
748 	 * process, we should exit with error, and then HNAE3_INIT_CLIENT
749 	 * related process can rollback the operation like notifing hardware to
750 	 * free resources, HNAE3_INIT_CLIENT related process will exit with
751 	 * error to notify NIC driver to reschedule soft reset process once
752 	 * again.
753 	 */
754 	hr_dev->dis_db = true;
755 	if (!ops->get_hw_reset_stat(handle))
756 		hr_dev->is_reset = true;
757 
758 	if (!hr_dev->is_reset || reset_stage == HNS_ROCE_STATE_RST_INIT ||
759 	    instance_stage == HNS_ROCE_STATE_INIT)
760 		return CMD_RST_PRC_EBUSY;
761 
762 	return CMD_RST_PRC_SUCCESS;
763 }
764 
765 static int hns_roce_v2_cmd_sw_resetting(struct hns_roce_dev *hr_dev)
766 {
767 	struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
768 	struct hnae3_handle *handle = priv->handle;
769 	const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
770 
771 	/* When software reset is detected at .init_instance() function, we
772 	 * should stop sending mailbox&cmq&doorbell to hardware, and exit
773 	 * with error.
774 	 */
775 	hr_dev->dis_db = true;
776 	if (ops->ae_dev_reset_cnt(handle) != hr_dev->reset_cnt)
777 		hr_dev->is_reset = true;
778 
779 	return CMD_RST_PRC_EBUSY;
780 }
781 
782 static int hns_roce_v2_rst_process_cmd(struct hns_roce_dev *hr_dev)
783 {
784 	struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
785 	struct hnae3_handle *handle = priv->handle;
786 	const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
787 	unsigned long instance_stage;	/* the current instance stage */
788 	unsigned long reset_stage;	/* the current reset stage */
789 	unsigned long reset_cnt;
790 	bool sw_resetting;
791 	bool hw_resetting;
792 
793 	if (hr_dev->is_reset)
794 		return CMD_RST_PRC_SUCCESS;
795 
796 	/* Get information about reset from NIC driver or RoCE driver itself,
797 	 * the meaning of the following variables from NIC driver are described
798 	 * as below:
799 	 * reset_cnt -- The count value of completed hardware reset.
800 	 * hw_resetting -- Whether hardware device is resetting now.
801 	 * sw_resetting -- Whether NIC's software reset process is running now.
802 	 */
803 	instance_stage = handle->rinfo.instance_state;
804 	reset_stage = handle->rinfo.reset_state;
805 	reset_cnt = ops->ae_dev_reset_cnt(handle);
806 	hw_resetting = ops->get_hw_reset_stat(handle);
807 	sw_resetting = ops->ae_dev_resetting(handle);
808 
809 	if (reset_cnt != hr_dev->reset_cnt)
810 		return hns_roce_v2_cmd_hw_reseted(hr_dev, instance_stage,
811 						  reset_stage);
812 	else if (hw_resetting)
813 		return hns_roce_v2_cmd_hw_resetting(hr_dev, instance_stage,
814 						    reset_stage);
815 	else if (sw_resetting && instance_stage == HNS_ROCE_STATE_INIT)
816 		return hns_roce_v2_cmd_sw_resetting(hr_dev);
817 
818 	return 0;
819 }
820 
821 static int hns_roce_cmq_space(struct hns_roce_v2_cmq_ring *ring)
822 {
823 	int ntu = ring->next_to_use;
824 	int ntc = ring->next_to_clean;
825 	int used = (ntu - ntc + ring->desc_num) % ring->desc_num;
826 
827 	return ring->desc_num - used - 1;
828 }
829 
830 static int hns_roce_alloc_cmq_desc(struct hns_roce_dev *hr_dev,
831 				   struct hns_roce_v2_cmq_ring *ring)
832 {
833 	int size = ring->desc_num * sizeof(struct hns_roce_cmq_desc);
834 
835 	ring->desc = kzalloc(size, GFP_KERNEL);
836 	if (!ring->desc)
837 		return -ENOMEM;
838 
839 	ring->desc_dma_addr = dma_map_single(hr_dev->dev, ring->desc, size,
840 					     DMA_BIDIRECTIONAL);
841 	if (dma_mapping_error(hr_dev->dev, ring->desc_dma_addr)) {
842 		ring->desc_dma_addr = 0;
843 		kfree(ring->desc);
844 		ring->desc = NULL;
845 		return -ENOMEM;
846 	}
847 
848 	return 0;
849 }
850 
851 static void hns_roce_free_cmq_desc(struct hns_roce_dev *hr_dev,
852 				   struct hns_roce_v2_cmq_ring *ring)
853 {
854 	dma_unmap_single(hr_dev->dev, ring->desc_dma_addr,
855 			 ring->desc_num * sizeof(struct hns_roce_cmq_desc),
856 			 DMA_BIDIRECTIONAL);
857 
858 	ring->desc_dma_addr = 0;
859 	kfree(ring->desc);
860 }
861 
862 static int hns_roce_init_cmq_ring(struct hns_roce_dev *hr_dev, bool ring_type)
863 {
864 	struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
865 	struct hns_roce_v2_cmq_ring *ring = (ring_type == TYPE_CSQ) ?
866 					    &priv->cmq.csq : &priv->cmq.crq;
867 
868 	ring->flag = ring_type;
869 	ring->next_to_clean = 0;
870 	ring->next_to_use = 0;
871 
872 	return hns_roce_alloc_cmq_desc(hr_dev, ring);
873 }
874 
875 static void hns_roce_cmq_init_regs(struct hns_roce_dev *hr_dev, bool ring_type)
876 {
877 	struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
878 	struct hns_roce_v2_cmq_ring *ring = (ring_type == TYPE_CSQ) ?
879 					    &priv->cmq.csq : &priv->cmq.crq;
880 	dma_addr_t dma = ring->desc_dma_addr;
881 
882 	if (ring_type == TYPE_CSQ) {
883 		roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_L_REG, (u32)dma);
884 		roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_H_REG,
885 			   upper_32_bits(dma));
886 		roce_write(hr_dev, ROCEE_TX_CMQ_DEPTH_REG,
887 			   ring->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S);
888 		roce_write(hr_dev, ROCEE_TX_CMQ_HEAD_REG, 0);
889 		roce_write(hr_dev, ROCEE_TX_CMQ_TAIL_REG, 0);
890 	} else {
891 		roce_write(hr_dev, ROCEE_RX_CMQ_BASEADDR_L_REG, (u32)dma);
892 		roce_write(hr_dev, ROCEE_RX_CMQ_BASEADDR_H_REG,
893 			   upper_32_bits(dma));
894 		roce_write(hr_dev, ROCEE_RX_CMQ_DEPTH_REG,
895 			   ring->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S);
896 		roce_write(hr_dev, ROCEE_RX_CMQ_HEAD_REG, 0);
897 		roce_write(hr_dev, ROCEE_RX_CMQ_TAIL_REG, 0);
898 	}
899 }
900 
901 static int hns_roce_v2_cmq_init(struct hns_roce_dev *hr_dev)
902 {
903 	struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
904 	int ret;
905 
906 	/* Setup the queue entries for command queue */
907 	priv->cmq.csq.desc_num = CMD_CSQ_DESC_NUM;
908 	priv->cmq.crq.desc_num = CMD_CRQ_DESC_NUM;
909 
910 	/* Setup the lock for command queue */
911 	spin_lock_init(&priv->cmq.csq.lock);
912 	spin_lock_init(&priv->cmq.crq.lock);
913 
914 	/* Setup Tx write back timeout */
915 	priv->cmq.tx_timeout = HNS_ROCE_CMQ_TX_TIMEOUT;
916 
917 	/* Init CSQ */
918 	ret = hns_roce_init_cmq_ring(hr_dev, TYPE_CSQ);
919 	if (ret) {
920 		dev_err(hr_dev->dev, "Init CSQ error, ret = %d.\n", ret);
921 		return ret;
922 	}
923 
924 	/* Init CRQ */
925 	ret = hns_roce_init_cmq_ring(hr_dev, TYPE_CRQ);
926 	if (ret) {
927 		dev_err(hr_dev->dev, "Init CRQ error, ret = %d.\n", ret);
928 		goto err_crq;
929 	}
930 
931 	/* Init CSQ REG */
932 	hns_roce_cmq_init_regs(hr_dev, TYPE_CSQ);
933 
934 	/* Init CRQ REG */
935 	hns_roce_cmq_init_regs(hr_dev, TYPE_CRQ);
936 
937 	return 0;
938 
939 err_crq:
940 	hns_roce_free_cmq_desc(hr_dev, &priv->cmq.csq);
941 
942 	return ret;
943 }
944 
945 static void hns_roce_v2_cmq_exit(struct hns_roce_dev *hr_dev)
946 {
947 	struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
948 
949 	hns_roce_free_cmq_desc(hr_dev, &priv->cmq.csq);
950 	hns_roce_free_cmq_desc(hr_dev, &priv->cmq.crq);
951 }
952 
953 static void hns_roce_cmq_setup_basic_desc(struct hns_roce_cmq_desc *desc,
954 					  enum hns_roce_opcode_type opcode,
955 					  bool is_read)
956 {
957 	memset((void *)desc, 0, sizeof(struct hns_roce_cmq_desc));
958 	desc->opcode = cpu_to_le16(opcode);
959 	desc->flag =
960 		cpu_to_le16(HNS_ROCE_CMD_FLAG_NO_INTR | HNS_ROCE_CMD_FLAG_IN);
961 	if (is_read)
962 		desc->flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_WR);
963 	else
964 		desc->flag &= cpu_to_le16(~HNS_ROCE_CMD_FLAG_WR);
965 }
966 
967 static int hns_roce_cmq_csq_done(struct hns_roce_dev *hr_dev)
968 {
969 	struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
970 	u32 head = roce_read(hr_dev, ROCEE_TX_CMQ_HEAD_REG);
971 
972 	return head == priv->cmq.csq.next_to_use;
973 }
974 
975 static int hns_roce_cmq_csq_clean(struct hns_roce_dev *hr_dev)
976 {
977 	struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
978 	struct hns_roce_v2_cmq_ring *csq = &priv->cmq.csq;
979 	struct hns_roce_cmq_desc *desc;
980 	u16 ntc = csq->next_to_clean;
981 	u32 head;
982 	int clean = 0;
983 
984 	desc = &csq->desc[ntc];
985 	head = roce_read(hr_dev, ROCEE_TX_CMQ_HEAD_REG);
986 	while (head != ntc) {
987 		memset(desc, 0, sizeof(*desc));
988 		ntc++;
989 		if (ntc == csq->desc_num)
990 			ntc = 0;
991 		desc = &csq->desc[ntc];
992 		clean++;
993 	}
994 	csq->next_to_clean = ntc;
995 
996 	return clean;
997 }
998 
999 static int __hns_roce_cmq_send(struct hns_roce_dev *hr_dev,
1000 			       struct hns_roce_cmq_desc *desc, int num)
1001 {
1002 	struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
1003 	struct hns_roce_v2_cmq_ring *csq = &priv->cmq.csq;
1004 	struct hns_roce_cmq_desc *desc_to_use;
1005 	bool complete = false;
1006 	u32 timeout = 0;
1007 	int handle = 0;
1008 	u16 desc_ret;
1009 	int ret = 0;
1010 	int ntc;
1011 
1012 	spin_lock_bh(&csq->lock);
1013 
1014 	if (num > hns_roce_cmq_space(csq)) {
1015 		spin_unlock_bh(&csq->lock);
1016 		return -EBUSY;
1017 	}
1018 
1019 	/*
1020 	 * Record the location of desc in the cmq for this time
1021 	 * which will be use for hardware to write back
1022 	 */
1023 	ntc = csq->next_to_use;
1024 
1025 	while (handle < num) {
1026 		desc_to_use = &csq->desc[csq->next_to_use];
1027 		*desc_to_use = desc[handle];
1028 		dev_dbg(hr_dev->dev, "set cmq desc:\n");
1029 		csq->next_to_use++;
1030 		if (csq->next_to_use == csq->desc_num)
1031 			csq->next_to_use = 0;
1032 		handle++;
1033 	}
1034 
1035 	/* Write to hardware */
1036 	roce_write(hr_dev, ROCEE_TX_CMQ_TAIL_REG, csq->next_to_use);
1037 
1038 	/*
1039 	 * If the command is sync, wait for the firmware to write back,
1040 	 * if multi descriptors to be sent, use the first one to check
1041 	 */
1042 	if (le16_to_cpu(desc->flag) & HNS_ROCE_CMD_FLAG_NO_INTR) {
1043 		do {
1044 			if (hns_roce_cmq_csq_done(hr_dev))
1045 				break;
1046 			udelay(1);
1047 			timeout++;
1048 		} while (timeout < priv->cmq.tx_timeout);
1049 	}
1050 
1051 	if (hns_roce_cmq_csq_done(hr_dev)) {
1052 		complete = true;
1053 		handle = 0;
1054 		while (handle < num) {
1055 			/* get the result of hardware write back */
1056 			desc_to_use = &csq->desc[ntc];
1057 			desc[handle] = *desc_to_use;
1058 			dev_dbg(hr_dev->dev, "Get cmq desc:\n");
1059 			desc_ret = le16_to_cpu(desc[handle].retval);
1060 			if (desc_ret == CMD_EXEC_SUCCESS)
1061 				ret = 0;
1062 			else
1063 				ret = -EIO;
1064 			priv->cmq.last_status = desc_ret;
1065 			ntc++;
1066 			handle++;
1067 			if (ntc == csq->desc_num)
1068 				ntc = 0;
1069 		}
1070 	}
1071 
1072 	if (!complete)
1073 		ret = -EAGAIN;
1074 
1075 	/* clean the command send queue */
1076 	handle = hns_roce_cmq_csq_clean(hr_dev);
1077 	if (handle != num)
1078 		dev_warn(hr_dev->dev, "Cleaned %d, need to clean %d\n",
1079 			 handle, num);
1080 
1081 	spin_unlock_bh(&csq->lock);
1082 
1083 	return ret;
1084 }
1085 
1086 static int hns_roce_cmq_send(struct hns_roce_dev *hr_dev,
1087 			     struct hns_roce_cmq_desc *desc, int num)
1088 {
1089 	int retval;
1090 	int ret;
1091 
1092 	ret = hns_roce_v2_rst_process_cmd(hr_dev);
1093 	if (ret == CMD_RST_PRC_SUCCESS)
1094 		return 0;
1095 	if (ret == CMD_RST_PRC_EBUSY)
1096 		return -EBUSY;
1097 
1098 	ret = __hns_roce_cmq_send(hr_dev, desc, num);
1099 	if (ret) {
1100 		retval = hns_roce_v2_rst_process_cmd(hr_dev);
1101 		if (retval == CMD_RST_PRC_SUCCESS)
1102 			return 0;
1103 		else if (retval == CMD_RST_PRC_EBUSY)
1104 			return -EBUSY;
1105 	}
1106 
1107 	return ret;
1108 }
1109 
1110 static int hns_roce_cmq_query_hw_info(struct hns_roce_dev *hr_dev)
1111 {
1112 	struct hns_roce_query_version *resp;
1113 	struct hns_roce_cmq_desc desc;
1114 	int ret;
1115 
1116 	hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_HW_VER, true);
1117 	ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1118 	if (ret)
1119 		return ret;
1120 
1121 	resp = (struct hns_roce_query_version *)desc.data;
1122 	hr_dev->hw_rev = le16_to_cpu(resp->rocee_hw_version);
1123 	hr_dev->vendor_id = hr_dev->pci_dev->vendor;
1124 
1125 	return 0;
1126 }
1127 
1128 static bool hns_roce_func_clr_chk_rst(struct hns_roce_dev *hr_dev)
1129 {
1130 	struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
1131 	struct hnae3_handle *handle = priv->handle;
1132 	const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
1133 	unsigned long reset_cnt;
1134 	bool sw_resetting;
1135 	bool hw_resetting;
1136 
1137 	reset_cnt = ops->ae_dev_reset_cnt(handle);
1138 	hw_resetting = ops->get_hw_reset_stat(handle);
1139 	sw_resetting = ops->ae_dev_resetting(handle);
1140 
1141 	if (reset_cnt != hr_dev->reset_cnt || hw_resetting || sw_resetting)
1142 		return true;
1143 
1144 	return false;
1145 }
1146 
1147 static void hns_roce_func_clr_rst_prc(struct hns_roce_dev *hr_dev, int retval,
1148 				      int flag)
1149 {
1150 	struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
1151 	struct hnae3_handle *handle = priv->handle;
1152 	const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
1153 	unsigned long instance_stage;
1154 	unsigned long reset_cnt;
1155 	unsigned long end;
1156 	bool sw_resetting;
1157 	bool hw_resetting;
1158 
1159 	instance_stage = handle->rinfo.instance_state;
1160 	reset_cnt = ops->ae_dev_reset_cnt(handle);
1161 	hw_resetting = ops->get_hw_reset_stat(handle);
1162 	sw_resetting = ops->ae_dev_resetting(handle);
1163 
1164 	if (reset_cnt != hr_dev->reset_cnt) {
1165 		hr_dev->dis_db = true;
1166 		hr_dev->is_reset = true;
1167 		dev_info(hr_dev->dev, "Func clear success after reset.\n");
1168 	} else if (hw_resetting) {
1169 		hr_dev->dis_db = true;
1170 
1171 		dev_warn(hr_dev->dev,
1172 			 "Func clear is pending, device in resetting state.\n");
1173 		end = HNS_ROCE_V2_HW_RST_TIMEOUT;
1174 		while (end) {
1175 			if (!ops->get_hw_reset_stat(handle)) {
1176 				hr_dev->is_reset = true;
1177 				dev_info(hr_dev->dev,
1178 					 "Func clear success after reset.\n");
1179 				return;
1180 			}
1181 			msleep(HNS_ROCE_V2_HW_RST_COMPLETION_WAIT);
1182 			end -= HNS_ROCE_V2_HW_RST_COMPLETION_WAIT;
1183 		}
1184 
1185 		dev_warn(hr_dev->dev, "Func clear failed.\n");
1186 	} else if (sw_resetting && instance_stage == HNS_ROCE_STATE_INIT) {
1187 		hr_dev->dis_db = true;
1188 
1189 		dev_warn(hr_dev->dev,
1190 			 "Func clear is pending, device in resetting state.\n");
1191 		end = HNS_ROCE_V2_HW_RST_TIMEOUT;
1192 		while (end) {
1193 			if (ops->ae_dev_reset_cnt(handle) !=
1194 			    hr_dev->reset_cnt) {
1195 				hr_dev->is_reset = true;
1196 				dev_info(hr_dev->dev,
1197 					 "Func clear success after sw reset\n");
1198 				return;
1199 			}
1200 			msleep(HNS_ROCE_V2_HW_RST_COMPLETION_WAIT);
1201 			end -= HNS_ROCE_V2_HW_RST_COMPLETION_WAIT;
1202 		}
1203 
1204 		dev_warn(hr_dev->dev, "Func clear failed because of unfinished sw reset\n");
1205 	} else {
1206 		if (retval && !flag)
1207 			dev_warn(hr_dev->dev,
1208 				 "Func clear read failed, ret = %d.\n", retval);
1209 
1210 		dev_warn(hr_dev->dev, "Func clear failed.\n");
1211 	}
1212 }
1213 static void hns_roce_function_clear(struct hns_roce_dev *hr_dev)
1214 {
1215 	bool fclr_write_fail_flag = false;
1216 	struct hns_roce_func_clear *resp;
1217 	struct hns_roce_cmq_desc desc;
1218 	unsigned long end;
1219 	int ret = 0;
1220 
1221 	if (hns_roce_func_clr_chk_rst(hr_dev))
1222 		goto out;
1223 
1224 	hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_FUNC_CLEAR, false);
1225 	resp = (struct hns_roce_func_clear *)desc.data;
1226 
1227 	ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1228 	if (ret) {
1229 		fclr_write_fail_flag = true;
1230 		dev_err(hr_dev->dev, "Func clear write failed, ret = %d.\n",
1231 			 ret);
1232 		goto out;
1233 	}
1234 
1235 	msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_INTERVAL);
1236 	end = HNS_ROCE_V2_FUNC_CLEAR_TIMEOUT_MSECS;
1237 	while (end) {
1238 		if (hns_roce_func_clr_chk_rst(hr_dev))
1239 			goto out;
1240 		msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT);
1241 		end -= HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT;
1242 
1243 		hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_FUNC_CLEAR,
1244 					      true);
1245 
1246 		ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1247 		if (ret)
1248 			continue;
1249 
1250 		if (roce_get_bit(resp->func_done, FUNC_CLEAR_RST_FUN_DONE_S)) {
1251 			hr_dev->is_reset = true;
1252 			return;
1253 		}
1254 	}
1255 
1256 out:
1257 	dev_err(hr_dev->dev, "Func clear fail.\n");
1258 	hns_roce_func_clr_rst_prc(hr_dev, ret, fclr_write_fail_flag);
1259 }
1260 
1261 static int hns_roce_query_fw_ver(struct hns_roce_dev *hr_dev)
1262 {
1263 	struct hns_roce_query_fw_info *resp;
1264 	struct hns_roce_cmq_desc desc;
1265 	int ret;
1266 
1267 	hns_roce_cmq_setup_basic_desc(&desc, HNS_QUERY_FW_VER, true);
1268 	ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1269 	if (ret)
1270 		return ret;
1271 
1272 	resp = (struct hns_roce_query_fw_info *)desc.data;
1273 	hr_dev->caps.fw_ver = (u64)(le32_to_cpu(resp->fw_ver));
1274 
1275 	return 0;
1276 }
1277 
1278 static int hns_roce_config_global_param(struct hns_roce_dev *hr_dev)
1279 {
1280 	struct hns_roce_cfg_global_param *req;
1281 	struct hns_roce_cmq_desc desc;
1282 
1283 	hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GLOBAL_PARAM,
1284 				      false);
1285 
1286 	req = (struct hns_roce_cfg_global_param *)desc.data;
1287 	memset(req, 0, sizeof(*req));
1288 	roce_set_field(req->time_cfg_udp_port,
1289 		       CFG_GLOBAL_PARAM_DATA_0_ROCEE_TIME_1US_CFG_M,
1290 		       CFG_GLOBAL_PARAM_DATA_0_ROCEE_TIME_1US_CFG_S, 0x3e8);
1291 	roce_set_field(req->time_cfg_udp_port,
1292 		       CFG_GLOBAL_PARAM_DATA_0_ROCEE_UDP_PORT_M,
1293 		       CFG_GLOBAL_PARAM_DATA_0_ROCEE_UDP_PORT_S, 0x12b7);
1294 
1295 	return hns_roce_cmq_send(hr_dev, &desc, 1);
1296 }
1297 
1298 static int hns_roce_query_pf_resource(struct hns_roce_dev *hr_dev)
1299 {
1300 	struct hns_roce_cmq_desc desc[2];
1301 	struct hns_roce_pf_res_a *req_a;
1302 	struct hns_roce_pf_res_b *req_b;
1303 	int ret;
1304 	int i;
1305 
1306 	for (i = 0; i < 2; i++) {
1307 		hns_roce_cmq_setup_basic_desc(&desc[i],
1308 					      HNS_ROCE_OPC_QUERY_PF_RES, true);
1309 
1310 		if (i == 0)
1311 			desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1312 		else
1313 			desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1314 	}
1315 
1316 	ret = hns_roce_cmq_send(hr_dev, desc, 2);
1317 	if (ret)
1318 		return ret;
1319 
1320 	req_a = (struct hns_roce_pf_res_a *)desc[0].data;
1321 	req_b = (struct hns_roce_pf_res_b *)desc[1].data;
1322 
1323 	hr_dev->caps.qpc_bt_num = roce_get_field(req_a->qpc_bt_idx_num,
1324 						 PF_RES_DATA_1_PF_QPC_BT_NUM_M,
1325 						 PF_RES_DATA_1_PF_QPC_BT_NUM_S);
1326 	hr_dev->caps.srqc_bt_num = roce_get_field(req_a->srqc_bt_idx_num,
1327 						PF_RES_DATA_2_PF_SRQC_BT_NUM_M,
1328 						PF_RES_DATA_2_PF_SRQC_BT_NUM_S);
1329 	hr_dev->caps.cqc_bt_num = roce_get_field(req_a->cqc_bt_idx_num,
1330 						 PF_RES_DATA_3_PF_CQC_BT_NUM_M,
1331 						 PF_RES_DATA_3_PF_CQC_BT_NUM_S);
1332 	hr_dev->caps.mpt_bt_num = roce_get_field(req_a->mpt_bt_idx_num,
1333 						 PF_RES_DATA_4_PF_MPT_BT_NUM_M,
1334 						 PF_RES_DATA_4_PF_MPT_BT_NUM_S);
1335 
1336 	hr_dev->caps.sl_num = roce_get_field(req_b->qid_idx_sl_num,
1337 					     PF_RES_DATA_3_PF_SL_NUM_M,
1338 					     PF_RES_DATA_3_PF_SL_NUM_S);
1339 	hr_dev->caps.sccc_bt_num = roce_get_field(req_b->sccc_bt_idx_num,
1340 					     PF_RES_DATA_4_PF_SCCC_BT_NUM_M,
1341 					     PF_RES_DATA_4_PF_SCCC_BT_NUM_S);
1342 
1343 	return 0;
1344 }
1345 
1346 static int hns_roce_query_pf_timer_resource(struct hns_roce_dev *hr_dev)
1347 {
1348 	struct hns_roce_pf_timer_res_a *req_a;
1349 	struct hns_roce_cmq_desc desc[2];
1350 	int ret, i;
1351 
1352 	for (i = 0; i < 2; i++) {
1353 		hns_roce_cmq_setup_basic_desc(&desc[i],
1354 					      HNS_ROCE_OPC_QUERY_PF_TIMER_RES,
1355 					      true);
1356 
1357 		if (i == 0)
1358 			desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1359 		else
1360 			desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1361 	}
1362 
1363 	ret = hns_roce_cmq_send(hr_dev, desc, 2);
1364 	if (ret)
1365 		return ret;
1366 
1367 	req_a = (struct hns_roce_pf_timer_res_a *)desc[0].data;
1368 
1369 	hr_dev->caps.qpc_timer_bt_num =
1370 				roce_get_field(req_a->qpc_timer_bt_idx_num,
1371 					PF_RES_DATA_1_PF_QPC_TIMER_BT_NUM_M,
1372 					PF_RES_DATA_1_PF_QPC_TIMER_BT_NUM_S);
1373 	hr_dev->caps.cqc_timer_bt_num =
1374 				roce_get_field(req_a->cqc_timer_bt_idx_num,
1375 					PF_RES_DATA_2_PF_CQC_TIMER_BT_NUM_M,
1376 					PF_RES_DATA_2_PF_CQC_TIMER_BT_NUM_S);
1377 
1378 	return 0;
1379 }
1380 
1381 static int hns_roce_set_vf_switch_param(struct hns_roce_dev *hr_dev,
1382 						  int vf_id)
1383 {
1384 	struct hns_roce_cmq_desc desc;
1385 	struct hns_roce_vf_switch *swt;
1386 	int ret;
1387 
1388 	swt = (struct hns_roce_vf_switch *)desc.data;
1389 	hns_roce_cmq_setup_basic_desc(&desc, HNS_SWITCH_PARAMETER_CFG, true);
1390 	swt->rocee_sel |= cpu_to_le32(HNS_ICL_SWITCH_CMD_ROCEE_SEL);
1391 	roce_set_field(swt->fun_id,
1392 			VF_SWITCH_DATA_FUN_ID_VF_ID_M,
1393 			VF_SWITCH_DATA_FUN_ID_VF_ID_S,
1394 			vf_id);
1395 	ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1396 	if (ret)
1397 		return ret;
1398 	desc.flag =
1399 		cpu_to_le16(HNS_ROCE_CMD_FLAG_NO_INTR | HNS_ROCE_CMD_FLAG_IN);
1400 	desc.flag &= cpu_to_le16(~HNS_ROCE_CMD_FLAG_WR);
1401 	roce_set_bit(swt->cfg, VF_SWITCH_DATA_CFG_ALW_LPBK_S, 1);
1402 	roce_set_bit(swt->cfg, VF_SWITCH_DATA_CFG_ALW_LCL_LPBK_S, 0);
1403 	roce_set_bit(swt->cfg, VF_SWITCH_DATA_CFG_ALW_DST_OVRD_S, 1);
1404 
1405 	return hns_roce_cmq_send(hr_dev, &desc, 1);
1406 }
1407 
1408 static int hns_roce_alloc_vf_resource(struct hns_roce_dev *hr_dev)
1409 {
1410 	struct hns_roce_cmq_desc desc[2];
1411 	struct hns_roce_vf_res_a *req_a;
1412 	struct hns_roce_vf_res_b *req_b;
1413 	int i;
1414 
1415 	req_a = (struct hns_roce_vf_res_a *)desc[0].data;
1416 	req_b = (struct hns_roce_vf_res_b *)desc[1].data;
1417 	memset(req_a, 0, sizeof(*req_a));
1418 	memset(req_b, 0, sizeof(*req_b));
1419 	for (i = 0; i < 2; i++) {
1420 		hns_roce_cmq_setup_basic_desc(&desc[i],
1421 					      HNS_ROCE_OPC_ALLOC_VF_RES, false);
1422 
1423 		if (i == 0)
1424 			desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1425 		else
1426 			desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1427 
1428 		if (i == 0) {
1429 			roce_set_field(req_a->vf_qpc_bt_idx_num,
1430 				       VF_RES_A_DATA_1_VF_QPC_BT_IDX_M,
1431 				       VF_RES_A_DATA_1_VF_QPC_BT_IDX_S, 0);
1432 			roce_set_field(req_a->vf_qpc_bt_idx_num,
1433 				       VF_RES_A_DATA_1_VF_QPC_BT_NUM_M,
1434 				       VF_RES_A_DATA_1_VF_QPC_BT_NUM_S,
1435 				       HNS_ROCE_VF_QPC_BT_NUM);
1436 
1437 			roce_set_field(req_a->vf_srqc_bt_idx_num,
1438 				       VF_RES_A_DATA_2_VF_SRQC_BT_IDX_M,
1439 				       VF_RES_A_DATA_2_VF_SRQC_BT_IDX_S, 0);
1440 			roce_set_field(req_a->vf_srqc_bt_idx_num,
1441 				       VF_RES_A_DATA_2_VF_SRQC_BT_NUM_M,
1442 				       VF_RES_A_DATA_2_VF_SRQC_BT_NUM_S,
1443 				       HNS_ROCE_VF_SRQC_BT_NUM);
1444 
1445 			roce_set_field(req_a->vf_cqc_bt_idx_num,
1446 				       VF_RES_A_DATA_3_VF_CQC_BT_IDX_M,
1447 				       VF_RES_A_DATA_3_VF_CQC_BT_IDX_S, 0);
1448 			roce_set_field(req_a->vf_cqc_bt_idx_num,
1449 				       VF_RES_A_DATA_3_VF_CQC_BT_NUM_M,
1450 				       VF_RES_A_DATA_3_VF_CQC_BT_NUM_S,
1451 				       HNS_ROCE_VF_CQC_BT_NUM);
1452 
1453 			roce_set_field(req_a->vf_mpt_bt_idx_num,
1454 				       VF_RES_A_DATA_4_VF_MPT_BT_IDX_M,
1455 				       VF_RES_A_DATA_4_VF_MPT_BT_IDX_S, 0);
1456 			roce_set_field(req_a->vf_mpt_bt_idx_num,
1457 				       VF_RES_A_DATA_4_VF_MPT_BT_NUM_M,
1458 				       VF_RES_A_DATA_4_VF_MPT_BT_NUM_S,
1459 				       HNS_ROCE_VF_MPT_BT_NUM);
1460 
1461 			roce_set_field(req_a->vf_eqc_bt_idx_num,
1462 				       VF_RES_A_DATA_5_VF_EQC_IDX_M,
1463 				       VF_RES_A_DATA_5_VF_EQC_IDX_S, 0);
1464 			roce_set_field(req_a->vf_eqc_bt_idx_num,
1465 				       VF_RES_A_DATA_5_VF_EQC_NUM_M,
1466 				       VF_RES_A_DATA_5_VF_EQC_NUM_S,
1467 				       HNS_ROCE_VF_EQC_NUM);
1468 		} else {
1469 			roce_set_field(req_b->vf_smac_idx_num,
1470 				       VF_RES_B_DATA_1_VF_SMAC_IDX_M,
1471 				       VF_RES_B_DATA_1_VF_SMAC_IDX_S, 0);
1472 			roce_set_field(req_b->vf_smac_idx_num,
1473 				       VF_RES_B_DATA_1_VF_SMAC_NUM_M,
1474 				       VF_RES_B_DATA_1_VF_SMAC_NUM_S,
1475 				       HNS_ROCE_VF_SMAC_NUM);
1476 
1477 			roce_set_field(req_b->vf_sgid_idx_num,
1478 				       VF_RES_B_DATA_2_VF_SGID_IDX_M,
1479 				       VF_RES_B_DATA_2_VF_SGID_IDX_S, 0);
1480 			roce_set_field(req_b->vf_sgid_idx_num,
1481 				       VF_RES_B_DATA_2_VF_SGID_NUM_M,
1482 				       VF_RES_B_DATA_2_VF_SGID_NUM_S,
1483 				       HNS_ROCE_VF_SGID_NUM);
1484 
1485 			roce_set_field(req_b->vf_qid_idx_sl_num,
1486 				       VF_RES_B_DATA_3_VF_QID_IDX_M,
1487 				       VF_RES_B_DATA_3_VF_QID_IDX_S, 0);
1488 			roce_set_field(req_b->vf_qid_idx_sl_num,
1489 				       VF_RES_B_DATA_3_VF_SL_NUM_M,
1490 				       VF_RES_B_DATA_3_VF_SL_NUM_S,
1491 				       HNS_ROCE_VF_SL_NUM);
1492 
1493 			roce_set_field(req_b->vf_sccc_idx_num,
1494 				       VF_RES_B_DATA_4_VF_SCCC_BT_IDX_M,
1495 				       VF_RES_B_DATA_4_VF_SCCC_BT_IDX_S, 0);
1496 			roce_set_field(req_b->vf_sccc_idx_num,
1497 				       VF_RES_B_DATA_4_VF_SCCC_BT_NUM_M,
1498 				       VF_RES_B_DATA_4_VF_SCCC_BT_NUM_S,
1499 				       HNS_ROCE_VF_SCCC_BT_NUM);
1500 		}
1501 	}
1502 
1503 	return hns_roce_cmq_send(hr_dev, desc, 2);
1504 }
1505 
1506 static int hns_roce_v2_set_bt(struct hns_roce_dev *hr_dev)
1507 {
1508 	u8 srqc_hop_num = hr_dev->caps.srqc_hop_num;
1509 	u8 qpc_hop_num = hr_dev->caps.qpc_hop_num;
1510 	u8 cqc_hop_num = hr_dev->caps.cqc_hop_num;
1511 	u8 mpt_hop_num = hr_dev->caps.mpt_hop_num;
1512 	u8 sccc_hop_num = hr_dev->caps.sccc_hop_num;
1513 	struct hns_roce_cfg_bt_attr *req;
1514 	struct hns_roce_cmq_desc desc;
1515 
1516 	hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_BT_ATTR, false);
1517 	req = (struct hns_roce_cfg_bt_attr *)desc.data;
1518 	memset(req, 0, sizeof(*req));
1519 
1520 	roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_M,
1521 		       CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_S,
1522 		       hr_dev->caps.qpc_ba_pg_sz + PG_SHIFT_OFFSET);
1523 	roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_BUF_PGSZ_M,
1524 		       CFG_BT_ATTR_DATA_0_VF_QPC_BUF_PGSZ_S,
1525 		       hr_dev->caps.qpc_buf_pg_sz + PG_SHIFT_OFFSET);
1526 	roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_HOPNUM_M,
1527 		       CFG_BT_ATTR_DATA_0_VF_QPC_HOPNUM_S,
1528 		       qpc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : qpc_hop_num);
1529 
1530 	roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_BA_PGSZ_M,
1531 		       CFG_BT_ATTR_DATA_1_VF_SRQC_BA_PGSZ_S,
1532 		       hr_dev->caps.srqc_ba_pg_sz + PG_SHIFT_OFFSET);
1533 	roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_BUF_PGSZ_M,
1534 		       CFG_BT_ATTR_DATA_1_VF_SRQC_BUF_PGSZ_S,
1535 		       hr_dev->caps.srqc_buf_pg_sz + PG_SHIFT_OFFSET);
1536 	roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_HOPNUM_M,
1537 		       CFG_BT_ATTR_DATA_1_VF_SRQC_HOPNUM_S,
1538 		       srqc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : srqc_hop_num);
1539 
1540 	roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_BA_PGSZ_M,
1541 		       CFG_BT_ATTR_DATA_2_VF_CQC_BA_PGSZ_S,
1542 		       hr_dev->caps.cqc_ba_pg_sz + PG_SHIFT_OFFSET);
1543 	roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_BUF_PGSZ_M,
1544 		       CFG_BT_ATTR_DATA_2_VF_CQC_BUF_PGSZ_S,
1545 		       hr_dev->caps.cqc_buf_pg_sz + PG_SHIFT_OFFSET);
1546 	roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_HOPNUM_M,
1547 		       CFG_BT_ATTR_DATA_2_VF_CQC_HOPNUM_S,
1548 		       cqc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : cqc_hop_num);
1549 
1550 	roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_BA_PGSZ_M,
1551 		       CFG_BT_ATTR_DATA_3_VF_MPT_BA_PGSZ_S,
1552 		       hr_dev->caps.mpt_ba_pg_sz + PG_SHIFT_OFFSET);
1553 	roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_BUF_PGSZ_M,
1554 		       CFG_BT_ATTR_DATA_3_VF_MPT_BUF_PGSZ_S,
1555 		       hr_dev->caps.mpt_buf_pg_sz + PG_SHIFT_OFFSET);
1556 	roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_M,
1557 		       CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_S,
1558 		       mpt_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : mpt_hop_num);
1559 
1560 	roce_set_field(req->vf_sccc_cfg,
1561 		       CFG_BT_ATTR_DATA_4_VF_SCCC_BA_PGSZ_M,
1562 		       CFG_BT_ATTR_DATA_4_VF_SCCC_BA_PGSZ_S,
1563 		       hr_dev->caps.sccc_ba_pg_sz + PG_SHIFT_OFFSET);
1564 	roce_set_field(req->vf_sccc_cfg,
1565 		       CFG_BT_ATTR_DATA_4_VF_SCCC_BUF_PGSZ_M,
1566 		       CFG_BT_ATTR_DATA_4_VF_SCCC_BUF_PGSZ_S,
1567 		       hr_dev->caps.sccc_buf_pg_sz + PG_SHIFT_OFFSET);
1568 	roce_set_field(req->vf_sccc_cfg,
1569 		       CFG_BT_ATTR_DATA_4_VF_SCCC_HOPNUM_M,
1570 		       CFG_BT_ATTR_DATA_4_VF_SCCC_HOPNUM_S,
1571 		       sccc_hop_num ==
1572 			      HNS_ROCE_HOP_NUM_0 ? 0 : sccc_hop_num);
1573 
1574 	return hns_roce_cmq_send(hr_dev, &desc, 1);
1575 }
1576 
1577 static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev)
1578 {
1579 	struct hns_roce_caps *caps = &hr_dev->caps;
1580 	int ret;
1581 
1582 	ret = hns_roce_cmq_query_hw_info(hr_dev);
1583 	if (ret) {
1584 		dev_err(hr_dev->dev, "Query hardware version fail, ret = %d.\n",
1585 			ret);
1586 		return ret;
1587 	}
1588 
1589 	ret = hns_roce_query_fw_ver(hr_dev);
1590 	if (ret) {
1591 		dev_err(hr_dev->dev, "Query firmware version fail, ret = %d.\n",
1592 			ret);
1593 		return ret;
1594 	}
1595 
1596 	ret = hns_roce_config_global_param(hr_dev);
1597 	if (ret) {
1598 		dev_err(hr_dev->dev, "Configure global param fail, ret = %d.\n",
1599 			ret);
1600 		return ret;
1601 	}
1602 
1603 	/* Get pf resource owned by every pf */
1604 	ret = hns_roce_query_pf_resource(hr_dev);
1605 	if (ret) {
1606 		dev_err(hr_dev->dev, "Query pf resource fail, ret = %d.\n",
1607 			ret);
1608 		return ret;
1609 	}
1610 
1611 	if (hr_dev->pci_dev->revision == 0x21) {
1612 		ret = hns_roce_query_pf_timer_resource(hr_dev);
1613 		if (ret) {
1614 			dev_err(hr_dev->dev,
1615 				"Query pf timer resource fail, ret = %d.\n",
1616 				ret);
1617 			return ret;
1618 		}
1619 	}
1620 
1621 	ret = hns_roce_alloc_vf_resource(hr_dev);
1622 	if (ret) {
1623 		dev_err(hr_dev->dev, "Allocate vf resource fail, ret = %d.\n",
1624 			ret);
1625 		return ret;
1626 	}
1627 
1628 	if (hr_dev->pci_dev->revision == 0x21) {
1629 		ret = hns_roce_set_vf_switch_param(hr_dev, 0);
1630 		if (ret) {
1631 			dev_err(hr_dev->dev,
1632 				"Set function switch param fail, ret = %d.\n",
1633 				ret);
1634 			return ret;
1635 		}
1636 	}
1637 
1638 	hr_dev->vendor_part_id = hr_dev->pci_dev->device;
1639 	hr_dev->sys_image_guid = be64_to_cpu(hr_dev->ib_dev.node_guid);
1640 
1641 	caps->num_qps		= HNS_ROCE_V2_MAX_QP_NUM;
1642 	caps->max_wqes		= HNS_ROCE_V2_MAX_WQE_NUM;
1643 	caps->num_cqs		= HNS_ROCE_V2_MAX_CQ_NUM;
1644 	caps->num_srqs		= HNS_ROCE_V2_MAX_SRQ_NUM;
1645 	caps->min_cqes		= HNS_ROCE_MIN_CQE_NUM;
1646 	caps->max_cqes		= HNS_ROCE_V2_MAX_CQE_NUM;
1647 	caps->max_srqwqes	= HNS_ROCE_V2_MAX_SRQWQE_NUM;
1648 	caps->max_sq_sg		= HNS_ROCE_V2_MAX_SQ_SGE_NUM;
1649 	caps->max_extend_sg	= HNS_ROCE_V2_MAX_EXTEND_SGE_NUM;
1650 	caps->max_rq_sg		= HNS_ROCE_V2_MAX_RQ_SGE_NUM;
1651 	caps->max_sq_inline	= HNS_ROCE_V2_MAX_SQ_INLINE;
1652 	caps->max_srq_sg	= HNS_ROCE_V2_MAX_SRQ_SGE_NUM;
1653 	caps->num_uars		= HNS_ROCE_V2_UAR_NUM;
1654 	caps->phy_num_uars	= HNS_ROCE_V2_PHY_UAR_NUM;
1655 	caps->num_aeq_vectors	= HNS_ROCE_V2_AEQE_VEC_NUM;
1656 	caps->num_comp_vectors	= HNS_ROCE_V2_COMP_VEC_NUM;
1657 	caps->num_other_vectors	= HNS_ROCE_V2_ABNORMAL_VEC_NUM;
1658 	caps->num_mtpts		= HNS_ROCE_V2_MAX_MTPT_NUM;
1659 	caps->num_mtt_segs	= HNS_ROCE_V2_MAX_MTT_SEGS;
1660 	caps->num_cqe_segs	= HNS_ROCE_V2_MAX_CQE_SEGS;
1661 	caps->num_srqwqe_segs	= HNS_ROCE_V2_MAX_SRQWQE_SEGS;
1662 	caps->num_idx_segs	= HNS_ROCE_V2_MAX_IDX_SEGS;
1663 	caps->num_pds		= HNS_ROCE_V2_MAX_PD_NUM;
1664 	caps->max_qp_init_rdma	= HNS_ROCE_V2_MAX_QP_INIT_RDMA;
1665 	caps->max_qp_dest_rdma	= HNS_ROCE_V2_MAX_QP_DEST_RDMA;
1666 	caps->max_sq_desc_sz	= HNS_ROCE_V2_MAX_SQ_DESC_SZ;
1667 	caps->max_rq_desc_sz	= HNS_ROCE_V2_MAX_RQ_DESC_SZ;
1668 	caps->max_srq_desc_sz	= HNS_ROCE_V2_MAX_SRQ_DESC_SZ;
1669 	caps->qpc_entry_sz	= HNS_ROCE_V2_QPC_ENTRY_SZ;
1670 	caps->irrl_entry_sz	= HNS_ROCE_V2_IRRL_ENTRY_SZ;
1671 	caps->trrl_entry_sz	= HNS_ROCE_V2_TRRL_ENTRY_SZ;
1672 	caps->cqc_entry_sz	= HNS_ROCE_V2_CQC_ENTRY_SZ;
1673 	caps->srqc_entry_sz	= HNS_ROCE_V2_SRQC_ENTRY_SZ;
1674 	caps->mtpt_entry_sz	= HNS_ROCE_V2_MTPT_ENTRY_SZ;
1675 	caps->mtt_entry_sz	= HNS_ROCE_V2_MTT_ENTRY_SZ;
1676 	caps->idx_entry_sz	= 4;
1677 	caps->cq_entry_sz	= HNS_ROCE_V2_CQE_ENTRY_SIZE;
1678 	caps->page_size_cap	= HNS_ROCE_V2_PAGE_SIZE_SUPPORTED;
1679 	caps->reserved_lkey	= 0;
1680 	caps->reserved_pds	= 0;
1681 	caps->reserved_mrws	= 1;
1682 	caps->reserved_uars	= 0;
1683 	caps->reserved_cqs	= 0;
1684 	caps->reserved_srqs	= 0;
1685 	caps->reserved_qps	= HNS_ROCE_V2_RSV_QPS;
1686 
1687 	caps->qpc_ba_pg_sz	= 0;
1688 	caps->qpc_buf_pg_sz	= 0;
1689 	caps->qpc_hop_num	= HNS_ROCE_CONTEXT_HOP_NUM;
1690 	caps->srqc_ba_pg_sz	= 0;
1691 	caps->srqc_buf_pg_sz	= 0;
1692 	caps->srqc_hop_num	= HNS_ROCE_CONTEXT_HOP_NUM;
1693 	caps->cqc_ba_pg_sz	= 0;
1694 	caps->cqc_buf_pg_sz	= 0;
1695 	caps->cqc_hop_num	= HNS_ROCE_CONTEXT_HOP_NUM;
1696 	caps->mpt_ba_pg_sz	= 0;
1697 	caps->mpt_buf_pg_sz	= 0;
1698 	caps->mpt_hop_num	= HNS_ROCE_CONTEXT_HOP_NUM;
1699 	caps->pbl_ba_pg_sz	= 2;
1700 	caps->pbl_buf_pg_sz	= 0;
1701 	caps->pbl_hop_num	= HNS_ROCE_PBL_HOP_NUM;
1702 	caps->mtt_ba_pg_sz	= 0;
1703 	caps->mtt_buf_pg_sz	= 0;
1704 	caps->mtt_hop_num	= HNS_ROCE_MTT_HOP_NUM;
1705 	caps->wqe_sq_hop_num	= 2;
1706 	caps->wqe_sge_hop_num	= 1;
1707 	caps->wqe_rq_hop_num	= 2;
1708 	caps->cqe_ba_pg_sz	= 6;
1709 	caps->cqe_buf_pg_sz	= 0;
1710 	caps->cqe_hop_num	= HNS_ROCE_CQE_HOP_NUM;
1711 	caps->srqwqe_ba_pg_sz	= 0;
1712 	caps->srqwqe_buf_pg_sz	= 0;
1713 	caps->srqwqe_hop_num	= HNS_ROCE_SRQWQE_HOP_NUM;
1714 	caps->idx_ba_pg_sz	= 0;
1715 	caps->idx_buf_pg_sz	= 0;
1716 	caps->idx_hop_num	= HNS_ROCE_IDX_HOP_NUM;
1717 	caps->eqe_ba_pg_sz	= 0;
1718 	caps->eqe_buf_pg_sz	= 0;
1719 	caps->eqe_hop_num	= HNS_ROCE_EQE_HOP_NUM;
1720 	caps->tsq_buf_pg_sz	= 0;
1721 	caps->chunk_sz		= HNS_ROCE_V2_TABLE_CHUNK_SIZE;
1722 
1723 	caps->flags		= HNS_ROCE_CAP_FLAG_REREG_MR |
1724 				  HNS_ROCE_CAP_FLAG_ROCE_V1_V2 |
1725 				  HNS_ROCE_CAP_FLAG_RQ_INLINE |
1726 				  HNS_ROCE_CAP_FLAG_RECORD_DB |
1727 				  HNS_ROCE_CAP_FLAG_SQ_RECORD_DB;
1728 
1729 	if (hr_dev->pci_dev->revision == 0x21)
1730 		caps->flags |= HNS_ROCE_CAP_FLAG_MW |
1731 			       HNS_ROCE_CAP_FLAG_FRMR;
1732 
1733 	caps->pkey_table_len[0] = 1;
1734 	caps->gid_table_len[0] = HNS_ROCE_V2_GID_INDEX_NUM;
1735 	caps->ceqe_depth	= HNS_ROCE_V2_COMP_EQE_NUM;
1736 	caps->aeqe_depth	= HNS_ROCE_V2_ASYNC_EQE_NUM;
1737 	caps->local_ca_ack_delay = 0;
1738 	caps->max_mtu = IB_MTU_4096;
1739 
1740 	caps->max_srqs		= HNS_ROCE_V2_MAX_SRQ;
1741 	caps->max_srq_wrs	= HNS_ROCE_V2_MAX_SRQ_WR;
1742 	caps->max_srq_sges	= HNS_ROCE_V2_MAX_SRQ_SGE;
1743 
1744 	if (hr_dev->pci_dev->revision == 0x21) {
1745 		caps->flags |= HNS_ROCE_CAP_FLAG_ATOMIC |
1746 			       HNS_ROCE_CAP_FLAG_SRQ |
1747 			       HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL;
1748 
1749 		caps->num_qpc_timer	  = HNS_ROCE_V2_MAX_QPC_TIMER_NUM;
1750 		caps->qpc_timer_entry_sz  = HNS_ROCE_V2_QPC_TIMER_ENTRY_SZ;
1751 		caps->qpc_timer_ba_pg_sz  = 0;
1752 		caps->qpc_timer_buf_pg_sz = 0;
1753 		caps->qpc_timer_hop_num   = HNS_ROCE_HOP_NUM_0;
1754 		caps->num_cqc_timer	  = HNS_ROCE_V2_MAX_CQC_TIMER_NUM;
1755 		caps->cqc_timer_entry_sz  = HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ;
1756 		caps->cqc_timer_ba_pg_sz  = 0;
1757 		caps->cqc_timer_buf_pg_sz = 0;
1758 		caps->cqc_timer_hop_num   = HNS_ROCE_HOP_NUM_0;
1759 
1760 		caps->sccc_entry_sz	= HNS_ROCE_V2_SCCC_ENTRY_SZ;
1761 		caps->sccc_ba_pg_sz	= 0;
1762 		caps->sccc_buf_pg_sz    = 0;
1763 		caps->sccc_hop_num	= HNS_ROCE_SCCC_HOP_NUM;
1764 	}
1765 
1766 	ret = hns_roce_v2_set_bt(hr_dev);
1767 	if (ret)
1768 		dev_err(hr_dev->dev, "Configure bt attribute fail, ret = %d.\n",
1769 			ret);
1770 
1771 	return ret;
1772 }
1773 
1774 static int hns_roce_config_link_table(struct hns_roce_dev *hr_dev,
1775 				      enum hns_roce_link_table_type type)
1776 {
1777 	struct hns_roce_cmq_desc desc[2];
1778 	struct hns_roce_cfg_llm_a *req_a =
1779 				(struct hns_roce_cfg_llm_a *)desc[0].data;
1780 	struct hns_roce_cfg_llm_b *req_b =
1781 				(struct hns_roce_cfg_llm_b *)desc[1].data;
1782 	struct hns_roce_v2_priv *priv = hr_dev->priv;
1783 	struct hns_roce_link_table *link_tbl;
1784 	struct hns_roce_link_table_entry *entry;
1785 	enum hns_roce_opcode_type opcode;
1786 	u32 page_num;
1787 	int i;
1788 
1789 	switch (type) {
1790 	case TSQ_LINK_TABLE:
1791 		link_tbl = &priv->tsq;
1792 		opcode = HNS_ROCE_OPC_CFG_EXT_LLM;
1793 		break;
1794 	case TPQ_LINK_TABLE:
1795 		link_tbl = &priv->tpq;
1796 		opcode = HNS_ROCE_OPC_CFG_TMOUT_LLM;
1797 		break;
1798 	default:
1799 		return -EINVAL;
1800 	}
1801 
1802 	page_num = link_tbl->npages;
1803 	entry = link_tbl->table.buf;
1804 	memset(req_a, 0, sizeof(*req_a));
1805 	memset(req_b, 0, sizeof(*req_b));
1806 
1807 	for (i = 0; i < 2; i++) {
1808 		hns_roce_cmq_setup_basic_desc(&desc[i], opcode, false);
1809 
1810 		if (i == 0)
1811 			desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1812 		else
1813 			desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1814 
1815 		if (i == 0) {
1816 			req_a->base_addr_l =
1817 				cpu_to_le32(link_tbl->table.map & 0xffffffff);
1818 			req_a->base_addr_h =
1819 				cpu_to_le32(link_tbl->table.map >> 32);
1820 			roce_set_field(req_a->depth_pgsz_init_en,
1821 				       CFG_LLM_QUE_DEPTH_M,
1822 				       CFG_LLM_QUE_DEPTH_S,
1823 				       link_tbl->npages);
1824 			roce_set_field(req_a->depth_pgsz_init_en,
1825 				       CFG_LLM_QUE_PGSZ_M,
1826 				       CFG_LLM_QUE_PGSZ_S,
1827 				       link_tbl->pg_sz);
1828 			req_a->head_ba_l = cpu_to_le32(entry[0].blk_ba0);
1829 			req_a->head_ba_h_nxtptr =
1830 				cpu_to_le32(entry[0].blk_ba1_nxt_ptr);
1831 			roce_set_field(req_a->head_ptr,
1832 				       CFG_LLM_HEAD_PTR_M,
1833 				       CFG_LLM_HEAD_PTR_S, 0);
1834 		} else {
1835 			req_b->tail_ba_l =
1836 				cpu_to_le32(entry[page_num - 1].blk_ba0);
1837 			roce_set_field(req_b->tail_ba_h,
1838 				       CFG_LLM_TAIL_BA_H_M,
1839 				       CFG_LLM_TAIL_BA_H_S,
1840 				       entry[page_num - 1].blk_ba1_nxt_ptr &
1841 				       HNS_ROCE_LINK_TABLE_BA1_M);
1842 			roce_set_field(req_b->tail_ptr,
1843 				       CFG_LLM_TAIL_PTR_M,
1844 				       CFG_LLM_TAIL_PTR_S,
1845 				       (entry[page_num - 2].blk_ba1_nxt_ptr &
1846 				       HNS_ROCE_LINK_TABLE_NXT_PTR_M) >>
1847 				       HNS_ROCE_LINK_TABLE_NXT_PTR_S);
1848 		}
1849 	}
1850 	roce_set_field(req_a->depth_pgsz_init_en,
1851 		       CFG_LLM_INIT_EN_M, CFG_LLM_INIT_EN_S, 1);
1852 
1853 	return hns_roce_cmq_send(hr_dev, desc, 2);
1854 }
1855 
1856 static int hns_roce_init_link_table(struct hns_roce_dev *hr_dev,
1857 				    enum hns_roce_link_table_type type)
1858 {
1859 	struct hns_roce_v2_priv *priv = hr_dev->priv;
1860 	struct hns_roce_link_table *link_tbl;
1861 	struct hns_roce_link_table_entry *entry;
1862 	struct device *dev = hr_dev->dev;
1863 	u32 buf_chk_sz;
1864 	dma_addr_t t;
1865 	int func_num = 1;
1866 	int pg_num_a;
1867 	int pg_num_b;
1868 	int pg_num;
1869 	int size;
1870 	int i;
1871 
1872 	switch (type) {
1873 	case TSQ_LINK_TABLE:
1874 		link_tbl = &priv->tsq;
1875 		buf_chk_sz = 1 << (hr_dev->caps.tsq_buf_pg_sz + PAGE_SHIFT);
1876 		pg_num_a = hr_dev->caps.num_qps * 8 / buf_chk_sz;
1877 		pg_num_b = hr_dev->caps.sl_num * 4 + 2;
1878 		break;
1879 	case TPQ_LINK_TABLE:
1880 		link_tbl = &priv->tpq;
1881 		buf_chk_sz = 1 << (hr_dev->caps.tpq_buf_pg_sz +	PAGE_SHIFT);
1882 		pg_num_a = hr_dev->caps.num_cqs * 4 / buf_chk_sz;
1883 		pg_num_b = 2 * 4 * func_num + 2;
1884 		break;
1885 	default:
1886 		return -EINVAL;
1887 	}
1888 
1889 	pg_num = max(pg_num_a, pg_num_b);
1890 	size = pg_num * sizeof(struct hns_roce_link_table_entry);
1891 
1892 	link_tbl->table.buf = dma_alloc_coherent(dev, size,
1893 						 &link_tbl->table.map,
1894 						 GFP_KERNEL);
1895 	if (!link_tbl->table.buf)
1896 		goto out;
1897 
1898 	link_tbl->pg_list = kcalloc(pg_num, sizeof(*link_tbl->pg_list),
1899 				    GFP_KERNEL);
1900 	if (!link_tbl->pg_list)
1901 		goto err_kcalloc_failed;
1902 
1903 	entry = link_tbl->table.buf;
1904 	for (i = 0; i < pg_num; ++i) {
1905 		link_tbl->pg_list[i].buf = dma_alloc_coherent(dev, buf_chk_sz,
1906 							      &t, GFP_KERNEL);
1907 		if (!link_tbl->pg_list[i].buf)
1908 			goto err_alloc_buf_failed;
1909 
1910 		link_tbl->pg_list[i].map = t;
1911 
1912 		entry[i].blk_ba0 = (u32)(t >> 12);
1913 		entry[i].blk_ba1_nxt_ptr = (u32)(t >> 44);
1914 
1915 		if (i < (pg_num - 1))
1916 			entry[i].blk_ba1_nxt_ptr |=
1917 				(i + 1) << HNS_ROCE_LINK_TABLE_NXT_PTR_S;
1918 
1919 	}
1920 	link_tbl->npages = pg_num;
1921 	link_tbl->pg_sz = buf_chk_sz;
1922 
1923 	return hns_roce_config_link_table(hr_dev, type);
1924 
1925 err_alloc_buf_failed:
1926 	for (i -= 1; i >= 0; i--)
1927 		dma_free_coherent(dev, buf_chk_sz,
1928 				  link_tbl->pg_list[i].buf,
1929 				  link_tbl->pg_list[i].map);
1930 	kfree(link_tbl->pg_list);
1931 
1932 err_kcalloc_failed:
1933 	dma_free_coherent(dev, size, link_tbl->table.buf,
1934 			  link_tbl->table.map);
1935 
1936 out:
1937 	return -ENOMEM;
1938 }
1939 
1940 static void hns_roce_free_link_table(struct hns_roce_dev *hr_dev,
1941 				     struct hns_roce_link_table *link_tbl)
1942 {
1943 	struct device *dev = hr_dev->dev;
1944 	int size;
1945 	int i;
1946 
1947 	size = link_tbl->npages * sizeof(struct hns_roce_link_table_entry);
1948 
1949 	for (i = 0; i < link_tbl->npages; ++i)
1950 		if (link_tbl->pg_list[i].buf)
1951 			dma_free_coherent(dev, link_tbl->pg_sz,
1952 					  link_tbl->pg_list[i].buf,
1953 					  link_tbl->pg_list[i].map);
1954 	kfree(link_tbl->pg_list);
1955 
1956 	dma_free_coherent(dev, size, link_tbl->table.buf,
1957 			  link_tbl->table.map);
1958 }
1959 
1960 static int hns_roce_v2_init(struct hns_roce_dev *hr_dev)
1961 {
1962 	struct hns_roce_v2_priv *priv = hr_dev->priv;
1963 	int qpc_count, cqc_count;
1964 	int ret, i;
1965 
1966 	/* TSQ includes SQ doorbell and ack doorbell */
1967 	ret = hns_roce_init_link_table(hr_dev, TSQ_LINK_TABLE);
1968 	if (ret) {
1969 		dev_err(hr_dev->dev, "TSQ init failed, ret = %d.\n", ret);
1970 		return ret;
1971 	}
1972 
1973 	ret = hns_roce_init_link_table(hr_dev, TPQ_LINK_TABLE);
1974 	if (ret) {
1975 		dev_err(hr_dev->dev, "TPQ init failed, ret = %d.\n", ret);
1976 		goto err_tpq_init_failed;
1977 	}
1978 
1979 	/* Alloc memory for QPC Timer buffer space chunk */
1980 	for (qpc_count = 0; qpc_count < hr_dev->caps.qpc_timer_bt_num;
1981 	     qpc_count++) {
1982 		ret = hns_roce_table_get(hr_dev, &hr_dev->qpc_timer_table,
1983 					 qpc_count);
1984 		if (ret) {
1985 			dev_err(hr_dev->dev, "QPC Timer get failed\n");
1986 			goto err_qpc_timer_failed;
1987 		}
1988 	}
1989 
1990 	/* Alloc memory for CQC Timer buffer space chunk */
1991 	for (cqc_count = 0; cqc_count < hr_dev->caps.cqc_timer_bt_num;
1992 	     cqc_count++) {
1993 		ret = hns_roce_table_get(hr_dev, &hr_dev->cqc_timer_table,
1994 					 cqc_count);
1995 		if (ret) {
1996 			dev_err(hr_dev->dev, "CQC Timer get failed\n");
1997 			goto err_cqc_timer_failed;
1998 		}
1999 	}
2000 
2001 	return 0;
2002 
2003 err_cqc_timer_failed:
2004 	for (i = 0; i < cqc_count; i++)
2005 		hns_roce_table_put(hr_dev, &hr_dev->cqc_timer_table, i);
2006 
2007 err_qpc_timer_failed:
2008 	for (i = 0; i < qpc_count; i++)
2009 		hns_roce_table_put(hr_dev, &hr_dev->qpc_timer_table, i);
2010 
2011 	hns_roce_free_link_table(hr_dev, &priv->tpq);
2012 
2013 err_tpq_init_failed:
2014 	hns_roce_free_link_table(hr_dev, &priv->tsq);
2015 
2016 	return ret;
2017 }
2018 
2019 static void hns_roce_v2_exit(struct hns_roce_dev *hr_dev)
2020 {
2021 	struct hns_roce_v2_priv *priv = hr_dev->priv;
2022 
2023 	if (hr_dev->pci_dev->revision == 0x21)
2024 		hns_roce_function_clear(hr_dev);
2025 
2026 	hns_roce_free_link_table(hr_dev, &priv->tpq);
2027 	hns_roce_free_link_table(hr_dev, &priv->tsq);
2028 }
2029 
2030 static int hns_roce_query_mbox_status(struct hns_roce_dev *hr_dev)
2031 {
2032 	struct hns_roce_cmq_desc desc;
2033 	struct hns_roce_mbox_status *mb_st =
2034 				       (struct hns_roce_mbox_status *)desc.data;
2035 	enum hns_roce_cmd_return_status status;
2036 
2037 	hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_MB_ST, true);
2038 
2039 	status = hns_roce_cmq_send(hr_dev, &desc, 1);
2040 	if (status)
2041 		return status;
2042 
2043 	return le32_to_cpu(mb_st->mb_status_hw_run);
2044 }
2045 
2046 static int hns_roce_v2_cmd_pending(struct hns_roce_dev *hr_dev)
2047 {
2048 	u32 status = hns_roce_query_mbox_status(hr_dev);
2049 
2050 	return status >> HNS_ROCE_HW_RUN_BIT_SHIFT;
2051 }
2052 
2053 static int hns_roce_v2_cmd_complete(struct hns_roce_dev *hr_dev)
2054 {
2055 	u32 status = hns_roce_query_mbox_status(hr_dev);
2056 
2057 	return status & HNS_ROCE_HW_MB_STATUS_MASK;
2058 }
2059 
2060 static int hns_roce_mbox_post(struct hns_roce_dev *hr_dev, u64 in_param,
2061 			      u64 out_param, u32 in_modifier, u8 op_modifier,
2062 			      u16 op, u16 token, int event)
2063 {
2064 	struct hns_roce_cmq_desc desc;
2065 	struct hns_roce_post_mbox *mb = (struct hns_roce_post_mbox *)desc.data;
2066 
2067 	hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_POST_MB, false);
2068 
2069 	mb->in_param_l = cpu_to_le32(in_param);
2070 	mb->in_param_h = cpu_to_le32(in_param >> 32);
2071 	mb->out_param_l = cpu_to_le32(out_param);
2072 	mb->out_param_h = cpu_to_le32(out_param >> 32);
2073 	mb->cmd_tag = cpu_to_le32(in_modifier << 8 | op);
2074 	mb->token_event_en = cpu_to_le32(event << 16 | token);
2075 
2076 	return hns_roce_cmq_send(hr_dev, &desc, 1);
2077 }
2078 
2079 static int hns_roce_v2_post_mbox(struct hns_roce_dev *hr_dev, u64 in_param,
2080 				 u64 out_param, u32 in_modifier, u8 op_modifier,
2081 				 u16 op, u16 token, int event)
2082 {
2083 	struct device *dev = hr_dev->dev;
2084 	unsigned long end;
2085 	int ret;
2086 
2087 	end = msecs_to_jiffies(HNS_ROCE_V2_GO_BIT_TIMEOUT_MSECS) + jiffies;
2088 	while (hns_roce_v2_cmd_pending(hr_dev)) {
2089 		if (time_after(jiffies, end)) {
2090 			dev_dbg(dev, "jiffies=%d end=%d\n", (int)jiffies,
2091 				(int)end);
2092 			return -EAGAIN;
2093 		}
2094 		cond_resched();
2095 	}
2096 
2097 	ret = hns_roce_mbox_post(hr_dev, in_param, out_param, in_modifier,
2098 				 op_modifier, op, token, event);
2099 	if (ret)
2100 		dev_err(dev, "Post mailbox fail(%d)\n", ret);
2101 
2102 	return ret;
2103 }
2104 
2105 static int hns_roce_v2_chk_mbox(struct hns_roce_dev *hr_dev,
2106 				unsigned long timeout)
2107 {
2108 	struct device *dev = hr_dev->dev;
2109 	unsigned long end;
2110 	u32 status;
2111 
2112 	end = msecs_to_jiffies(timeout) + jiffies;
2113 	while (hns_roce_v2_cmd_pending(hr_dev) && time_before(jiffies, end))
2114 		cond_resched();
2115 
2116 	if (hns_roce_v2_cmd_pending(hr_dev)) {
2117 		dev_err(dev, "[cmd_poll]hw run cmd TIMEDOUT!\n");
2118 		return -ETIMEDOUT;
2119 	}
2120 
2121 	status = hns_roce_v2_cmd_complete(hr_dev);
2122 	if (status != 0x1) {
2123 		if (status == CMD_RST_PRC_EBUSY)
2124 			return status;
2125 
2126 		dev_err(dev, "mailbox status 0x%x!\n", status);
2127 		return -EBUSY;
2128 	}
2129 
2130 	return 0;
2131 }
2132 
2133 static int hns_roce_config_sgid_table(struct hns_roce_dev *hr_dev,
2134 				      int gid_index, const union ib_gid *gid,
2135 				      enum hns_roce_sgid_type sgid_type)
2136 {
2137 	struct hns_roce_cmq_desc desc;
2138 	struct hns_roce_cfg_sgid_tb *sgid_tb =
2139 				    (struct hns_roce_cfg_sgid_tb *)desc.data;
2140 	u32 *p;
2141 
2142 	hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_SGID_TB, false);
2143 
2144 	roce_set_field(sgid_tb->table_idx_rsv,
2145 		       CFG_SGID_TB_TABLE_IDX_M,
2146 		       CFG_SGID_TB_TABLE_IDX_S, gid_index);
2147 	roce_set_field(sgid_tb->vf_sgid_type_rsv,
2148 		       CFG_SGID_TB_VF_SGID_TYPE_M,
2149 		       CFG_SGID_TB_VF_SGID_TYPE_S, sgid_type);
2150 
2151 	p = (u32 *)&gid->raw[0];
2152 	sgid_tb->vf_sgid_l = cpu_to_le32(*p);
2153 
2154 	p = (u32 *)&gid->raw[4];
2155 	sgid_tb->vf_sgid_ml = cpu_to_le32(*p);
2156 
2157 	p = (u32 *)&gid->raw[8];
2158 	sgid_tb->vf_sgid_mh = cpu_to_le32(*p);
2159 
2160 	p = (u32 *)&gid->raw[0xc];
2161 	sgid_tb->vf_sgid_h = cpu_to_le32(*p);
2162 
2163 	return hns_roce_cmq_send(hr_dev, &desc, 1);
2164 }
2165 
2166 static int hns_roce_v2_set_gid(struct hns_roce_dev *hr_dev, u8 port,
2167 			       int gid_index, const union ib_gid *gid,
2168 			       const struct ib_gid_attr *attr)
2169 {
2170 	enum hns_roce_sgid_type sgid_type = GID_TYPE_FLAG_ROCE_V1;
2171 	int ret;
2172 
2173 	if (!gid || !attr)
2174 		return -EINVAL;
2175 
2176 	if (attr->gid_type == IB_GID_TYPE_ROCE)
2177 		sgid_type = GID_TYPE_FLAG_ROCE_V1;
2178 
2179 	if (attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) {
2180 		if (ipv6_addr_v4mapped((void *)gid))
2181 			sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV4;
2182 		else
2183 			sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV6;
2184 	}
2185 
2186 	ret = hns_roce_config_sgid_table(hr_dev, gid_index, gid, sgid_type);
2187 	if (ret)
2188 		dev_err(hr_dev->dev, "Configure sgid table failed(%d)!\n", ret);
2189 
2190 	return ret;
2191 }
2192 
2193 static int hns_roce_v2_set_mac(struct hns_roce_dev *hr_dev, u8 phy_port,
2194 			       u8 *addr)
2195 {
2196 	struct hns_roce_cmq_desc desc;
2197 	struct hns_roce_cfg_smac_tb *smac_tb =
2198 				    (struct hns_roce_cfg_smac_tb *)desc.data;
2199 	u16 reg_smac_h;
2200 	u32 reg_smac_l;
2201 
2202 	hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_SMAC_TB, false);
2203 
2204 	reg_smac_l = *(u32 *)(&addr[0]);
2205 	reg_smac_h = *(u16 *)(&addr[4]);
2206 
2207 	memset(smac_tb, 0, sizeof(*smac_tb));
2208 	roce_set_field(smac_tb->tb_idx_rsv,
2209 		       CFG_SMAC_TB_IDX_M,
2210 		       CFG_SMAC_TB_IDX_S, phy_port);
2211 	roce_set_field(smac_tb->vf_smac_h_rsv,
2212 		       CFG_SMAC_TB_VF_SMAC_H_M,
2213 		       CFG_SMAC_TB_VF_SMAC_H_S, reg_smac_h);
2214 	smac_tb->vf_smac_l = cpu_to_le32(reg_smac_l);
2215 
2216 	return hns_roce_cmq_send(hr_dev, &desc, 1);
2217 }
2218 
2219 static int set_mtpt_pbl(struct hns_roce_v2_mpt_entry *mpt_entry,
2220 			struct hns_roce_mr *mr)
2221 {
2222 	struct sg_dma_page_iter sg_iter;
2223 	u64 page_addr;
2224 	u64 *pages;
2225 	int i;
2226 
2227 	mpt_entry->pbl_size = cpu_to_le32(mr->pbl_size);
2228 	mpt_entry->pbl_ba_l = cpu_to_le32(lower_32_bits(mr->pbl_ba >> 3));
2229 	roce_set_field(mpt_entry->byte_48_mode_ba,
2230 		       V2_MPT_BYTE_48_PBL_BA_H_M, V2_MPT_BYTE_48_PBL_BA_H_S,
2231 		       upper_32_bits(mr->pbl_ba >> 3));
2232 
2233 	pages = (u64 *)__get_free_page(GFP_KERNEL);
2234 	if (!pages)
2235 		return -ENOMEM;
2236 
2237 	i = 0;
2238 	for_each_sg_dma_page(mr->umem->sg_head.sgl, &sg_iter, mr->umem->nmap, 0) {
2239 		page_addr = sg_page_iter_dma_address(&sg_iter);
2240 		pages[i] = page_addr >> 6;
2241 
2242 		/* Record the first 2 entry directly to MTPT table */
2243 		if (i >= HNS_ROCE_V2_MAX_INNER_MTPT_NUM - 1)
2244 			goto found;
2245 		i++;
2246 	}
2247 found:
2248 	mpt_entry->pa0_l = cpu_to_le32(lower_32_bits(pages[0]));
2249 	roce_set_field(mpt_entry->byte_56_pa0_h, V2_MPT_BYTE_56_PA0_H_M,
2250 		       V2_MPT_BYTE_56_PA0_H_S, upper_32_bits(pages[0]));
2251 
2252 	mpt_entry->pa1_l = cpu_to_le32(lower_32_bits(pages[1]));
2253 	roce_set_field(mpt_entry->byte_64_buf_pa1, V2_MPT_BYTE_64_PA1_H_M,
2254 		       V2_MPT_BYTE_64_PA1_H_S, upper_32_bits(pages[1]));
2255 	roce_set_field(mpt_entry->byte_64_buf_pa1,
2256 		       V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M,
2257 		       V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S,
2258 		       mr->pbl_buf_pg_sz + PG_SHIFT_OFFSET);
2259 
2260 	free_page((unsigned long)pages);
2261 
2262 	return 0;
2263 }
2264 
2265 static int hns_roce_v2_write_mtpt(void *mb_buf, struct hns_roce_mr *mr,
2266 				  unsigned long mtpt_idx)
2267 {
2268 	struct hns_roce_v2_mpt_entry *mpt_entry;
2269 	int ret;
2270 
2271 	mpt_entry = mb_buf;
2272 	memset(mpt_entry, 0, sizeof(*mpt_entry));
2273 
2274 	roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M,
2275 		       V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_VALID);
2276 	roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PBL_HOP_NUM_M,
2277 		       V2_MPT_BYTE_4_PBL_HOP_NUM_S, mr->pbl_hop_num ==
2278 		       HNS_ROCE_HOP_NUM_0 ? 0 : mr->pbl_hop_num);
2279 	roce_set_field(mpt_entry->byte_4_pd_hop_st,
2280 		       V2_MPT_BYTE_4_PBL_BA_PG_SZ_M,
2281 		       V2_MPT_BYTE_4_PBL_BA_PG_SZ_S,
2282 		       mr->pbl_ba_pg_sz + PG_SHIFT_OFFSET);
2283 	roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
2284 		       V2_MPT_BYTE_4_PD_S, mr->pd);
2285 
2286 	roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RA_EN_S, 0);
2287 	roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 0);
2288 	roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 1);
2289 	roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_BIND_EN_S,
2290 		     (mr->access & IB_ACCESS_MW_BIND ? 1 : 0));
2291 	roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_ATOMIC_EN_S,
2292 		     mr->access & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0);
2293 	roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RR_EN_S,
2294 		     (mr->access & IB_ACCESS_REMOTE_READ ? 1 : 0));
2295 	roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RW_EN_S,
2296 		     (mr->access & IB_ACCESS_REMOTE_WRITE ? 1 : 0));
2297 	roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_LW_EN_S,
2298 		     (mr->access & IB_ACCESS_LOCAL_WRITE ? 1 : 0));
2299 
2300 	roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_PA_S,
2301 		     mr->type == MR_TYPE_MR ? 0 : 1);
2302 	roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_INNER_PA_VLD_S,
2303 		     1);
2304 
2305 	mpt_entry->len_l = cpu_to_le32(lower_32_bits(mr->size));
2306 	mpt_entry->len_h = cpu_to_le32(upper_32_bits(mr->size));
2307 	mpt_entry->lkey = cpu_to_le32(mr->key);
2308 	mpt_entry->va_l = cpu_to_le32(lower_32_bits(mr->iova));
2309 	mpt_entry->va_h = cpu_to_le32(upper_32_bits(mr->iova));
2310 
2311 	if (mr->type == MR_TYPE_DMA)
2312 		return 0;
2313 
2314 	ret = set_mtpt_pbl(mpt_entry, mr);
2315 
2316 	return ret;
2317 }
2318 
2319 static int hns_roce_v2_rereg_write_mtpt(struct hns_roce_dev *hr_dev,
2320 					struct hns_roce_mr *mr, int flags,
2321 					u32 pdn, int mr_access_flags, u64 iova,
2322 					u64 size, void *mb_buf)
2323 {
2324 	struct hns_roce_v2_mpt_entry *mpt_entry = mb_buf;
2325 	int ret = 0;
2326 
2327 	roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M,
2328 		       V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_VALID);
2329 
2330 	if (flags & IB_MR_REREG_PD) {
2331 		roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
2332 			       V2_MPT_BYTE_4_PD_S, pdn);
2333 		mr->pd = pdn;
2334 	}
2335 
2336 	if (flags & IB_MR_REREG_ACCESS) {
2337 		roce_set_bit(mpt_entry->byte_8_mw_cnt_en,
2338 			     V2_MPT_BYTE_8_BIND_EN_S,
2339 			     (mr_access_flags & IB_ACCESS_MW_BIND ? 1 : 0));
2340 		roce_set_bit(mpt_entry->byte_8_mw_cnt_en,
2341 			     V2_MPT_BYTE_8_ATOMIC_EN_S,
2342 			     mr_access_flags & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0);
2343 		roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RR_EN_S,
2344 			     mr_access_flags & IB_ACCESS_REMOTE_READ ? 1 : 0);
2345 		roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RW_EN_S,
2346 			     mr_access_flags & IB_ACCESS_REMOTE_WRITE ? 1 : 0);
2347 		roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_LW_EN_S,
2348 			     mr_access_flags & IB_ACCESS_LOCAL_WRITE ? 1 : 0);
2349 	}
2350 
2351 	if (flags & IB_MR_REREG_TRANS) {
2352 		mpt_entry->va_l = cpu_to_le32(lower_32_bits(iova));
2353 		mpt_entry->va_h = cpu_to_le32(upper_32_bits(iova));
2354 		mpt_entry->len_l = cpu_to_le32(lower_32_bits(size));
2355 		mpt_entry->len_h = cpu_to_le32(upper_32_bits(size));
2356 
2357 		mr->iova = iova;
2358 		mr->size = size;
2359 
2360 		ret = set_mtpt_pbl(mpt_entry, mr);
2361 	}
2362 
2363 	return ret;
2364 }
2365 
2366 static int hns_roce_v2_frmr_write_mtpt(void *mb_buf, struct hns_roce_mr *mr)
2367 {
2368 	struct hns_roce_v2_mpt_entry *mpt_entry;
2369 
2370 	mpt_entry = mb_buf;
2371 	memset(mpt_entry, 0, sizeof(*mpt_entry));
2372 
2373 	roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M,
2374 		       V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_FREE);
2375 	roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PBL_HOP_NUM_M,
2376 		       V2_MPT_BYTE_4_PBL_HOP_NUM_S, 1);
2377 	roce_set_field(mpt_entry->byte_4_pd_hop_st,
2378 		       V2_MPT_BYTE_4_PBL_BA_PG_SZ_M,
2379 		       V2_MPT_BYTE_4_PBL_BA_PG_SZ_S,
2380 		       mr->pbl_ba_pg_sz + PG_SHIFT_OFFSET);
2381 	roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
2382 		       V2_MPT_BYTE_4_PD_S, mr->pd);
2383 
2384 	roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RA_EN_S, 1);
2385 	roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 1);
2386 	roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 1);
2387 
2388 	roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_FRE_S, 1);
2389 	roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_PA_S, 0);
2390 	roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_MR_MW_S, 0);
2391 	roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_BPD_S, 1);
2392 
2393 	mpt_entry->pbl_size = cpu_to_le32(mr->pbl_size);
2394 
2395 	mpt_entry->pbl_ba_l = cpu_to_le32(lower_32_bits(mr->pbl_ba >> 3));
2396 	roce_set_field(mpt_entry->byte_48_mode_ba, V2_MPT_BYTE_48_PBL_BA_H_M,
2397 		       V2_MPT_BYTE_48_PBL_BA_H_S,
2398 		       upper_32_bits(mr->pbl_ba >> 3));
2399 
2400 	roce_set_field(mpt_entry->byte_64_buf_pa1,
2401 		       V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M,
2402 		       V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S,
2403 		       mr->pbl_buf_pg_sz + PG_SHIFT_OFFSET);
2404 
2405 	return 0;
2406 }
2407 
2408 static int hns_roce_v2_mw_write_mtpt(void *mb_buf, struct hns_roce_mw *mw)
2409 {
2410 	struct hns_roce_v2_mpt_entry *mpt_entry;
2411 
2412 	mpt_entry = mb_buf;
2413 	memset(mpt_entry, 0, sizeof(*mpt_entry));
2414 
2415 	roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M,
2416 		       V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_FREE);
2417 	roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
2418 		       V2_MPT_BYTE_4_PD_S, mw->pdn);
2419 	roce_set_field(mpt_entry->byte_4_pd_hop_st,
2420 		       V2_MPT_BYTE_4_PBL_HOP_NUM_M,
2421 		       V2_MPT_BYTE_4_PBL_HOP_NUM_S,
2422 		       mw->pbl_hop_num == HNS_ROCE_HOP_NUM_0 ?
2423 		       0 : mw->pbl_hop_num);
2424 	roce_set_field(mpt_entry->byte_4_pd_hop_st,
2425 		       V2_MPT_BYTE_4_PBL_BA_PG_SZ_M,
2426 		       V2_MPT_BYTE_4_PBL_BA_PG_SZ_S,
2427 		       mw->pbl_ba_pg_sz + PG_SHIFT_OFFSET);
2428 
2429 	roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 1);
2430 	roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 1);
2431 
2432 	roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_PA_S, 0);
2433 	roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_MR_MW_S, 1);
2434 	roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_BPD_S, 1);
2435 	roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_BQP_S,
2436 		     mw->ibmw.type == IB_MW_TYPE_1 ? 0 : 1);
2437 
2438 	roce_set_field(mpt_entry->byte_64_buf_pa1,
2439 		       V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M,
2440 		       V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S,
2441 		       mw->pbl_buf_pg_sz + PG_SHIFT_OFFSET);
2442 
2443 	mpt_entry->lkey = cpu_to_le32(mw->rkey);
2444 
2445 	return 0;
2446 }
2447 
2448 static void *get_cqe_v2(struct hns_roce_cq *hr_cq, int n)
2449 {
2450 	return hns_roce_buf_offset(&hr_cq->buf, n * HNS_ROCE_V2_CQE_ENTRY_SIZE);
2451 }
2452 
2453 static void *get_sw_cqe_v2(struct hns_roce_cq *hr_cq, int n)
2454 {
2455 	struct hns_roce_v2_cqe *cqe = get_cqe_v2(hr_cq, n & hr_cq->ib_cq.cqe);
2456 
2457 	/* Get cqe when Owner bit is Conversely with the MSB of cons_idx */
2458 	return (roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_OWNER_S) ^
2459 		!!(n & hr_cq->cq_depth)) ? cqe : NULL;
2460 }
2461 
2462 static struct hns_roce_v2_cqe *next_cqe_sw_v2(struct hns_roce_cq *hr_cq)
2463 {
2464 	return get_sw_cqe_v2(hr_cq, hr_cq->cons_index);
2465 }
2466 
2467 static void *get_srq_wqe(struct hns_roce_srq *srq, int n)
2468 {
2469 	return hns_roce_buf_offset(&srq->buf, n << srq->wqe_shift);
2470 }
2471 
2472 static void hns_roce_free_srq_wqe(struct hns_roce_srq *srq, int wqe_index)
2473 {
2474 	/* always called with interrupts disabled. */
2475 	spin_lock(&srq->lock);
2476 
2477 	bitmap_clear(srq->idx_que.bitmap, wqe_index, 1);
2478 	srq->tail++;
2479 
2480 	spin_unlock(&srq->lock);
2481 }
2482 
2483 static void hns_roce_v2_cq_set_ci(struct hns_roce_cq *hr_cq, u32 cons_index)
2484 {
2485 	*hr_cq->set_ci_db = cons_index & 0xffffff;
2486 }
2487 
2488 static void __hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
2489 				   struct hns_roce_srq *srq)
2490 {
2491 	struct hns_roce_v2_cqe *cqe, *dest;
2492 	u32 prod_index;
2493 	int nfreed = 0;
2494 	int wqe_index;
2495 	u8 owner_bit;
2496 
2497 	for (prod_index = hr_cq->cons_index; get_sw_cqe_v2(hr_cq, prod_index);
2498 	     ++prod_index) {
2499 		if (prod_index > hr_cq->cons_index + hr_cq->ib_cq.cqe)
2500 			break;
2501 	}
2502 
2503 	/*
2504 	 * Now backwards through the CQ, removing CQ entries
2505 	 * that match our QP by overwriting them with next entries.
2506 	 */
2507 	while ((int) --prod_index - (int) hr_cq->cons_index >= 0) {
2508 		cqe = get_cqe_v2(hr_cq, prod_index & hr_cq->ib_cq.cqe);
2509 		if ((roce_get_field(cqe->byte_16, V2_CQE_BYTE_16_LCL_QPN_M,
2510 				    V2_CQE_BYTE_16_LCL_QPN_S) &
2511 				    HNS_ROCE_V2_CQE_QPN_MASK) == qpn) {
2512 			if (srq &&
2513 			    roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_S_R_S)) {
2514 				wqe_index = roce_get_field(cqe->byte_4,
2515 						     V2_CQE_BYTE_4_WQE_INDX_M,
2516 						     V2_CQE_BYTE_4_WQE_INDX_S);
2517 				hns_roce_free_srq_wqe(srq, wqe_index);
2518 			}
2519 			++nfreed;
2520 		} else if (nfreed) {
2521 			dest = get_cqe_v2(hr_cq, (prod_index + nfreed) &
2522 					  hr_cq->ib_cq.cqe);
2523 			owner_bit = roce_get_bit(dest->byte_4,
2524 						 V2_CQE_BYTE_4_OWNER_S);
2525 			memcpy(dest, cqe, sizeof(*cqe));
2526 			roce_set_bit(dest->byte_4, V2_CQE_BYTE_4_OWNER_S,
2527 				     owner_bit);
2528 		}
2529 	}
2530 
2531 	if (nfreed) {
2532 		hr_cq->cons_index += nfreed;
2533 		/*
2534 		 * Make sure update of buffer contents is done before
2535 		 * updating consumer index.
2536 		 */
2537 		wmb();
2538 		hns_roce_v2_cq_set_ci(hr_cq, hr_cq->cons_index);
2539 	}
2540 }
2541 
2542 static void hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
2543 				 struct hns_roce_srq *srq)
2544 {
2545 	spin_lock_irq(&hr_cq->lock);
2546 	__hns_roce_v2_cq_clean(hr_cq, qpn, srq);
2547 	spin_unlock_irq(&hr_cq->lock);
2548 }
2549 
2550 static void hns_roce_v2_write_cqc(struct hns_roce_dev *hr_dev,
2551 				  struct hns_roce_cq *hr_cq, void *mb_buf,
2552 				  u64 *mtts, dma_addr_t dma_handle)
2553 {
2554 	struct hns_roce_v2_cq_context *cq_context;
2555 
2556 	cq_context = mb_buf;
2557 	memset(cq_context, 0, sizeof(*cq_context));
2558 
2559 	roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_CQ_ST_M,
2560 		       V2_CQC_BYTE_4_CQ_ST_S, V2_CQ_STATE_VALID);
2561 	roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_ARM_ST_M,
2562 		       V2_CQC_BYTE_4_ARM_ST_S, REG_NXT_CEQE);
2563 	roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_SHIFT_M,
2564 		       V2_CQC_BYTE_4_SHIFT_S,
2565 		       ilog2(hr_cq->cq_depth));
2566 	roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_CEQN_M,
2567 		       V2_CQC_BYTE_4_CEQN_S, hr_cq->vector);
2568 
2569 	roce_set_field(cq_context->byte_8_cqn, V2_CQC_BYTE_8_CQN_M,
2570 		       V2_CQC_BYTE_8_CQN_S, hr_cq->cqn);
2571 
2572 	cq_context->cqe_cur_blk_addr = cpu_to_le32(mtts[0] >> PAGE_ADDR_SHIFT);
2573 
2574 	roce_set_field(cq_context->byte_16_hop_addr,
2575 		       V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_M,
2576 		       V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_S,
2577 		       mtts[0] >> (32 + PAGE_ADDR_SHIFT));
2578 	roce_set_field(cq_context->byte_16_hop_addr,
2579 		       V2_CQC_BYTE_16_CQE_HOP_NUM_M,
2580 		       V2_CQC_BYTE_16_CQE_HOP_NUM_S, hr_dev->caps.cqe_hop_num ==
2581 		       HNS_ROCE_HOP_NUM_0 ? 0 : hr_dev->caps.cqe_hop_num);
2582 
2583 	cq_context->cqe_nxt_blk_addr = cpu_to_le32(mtts[1] >> PAGE_ADDR_SHIFT);
2584 	roce_set_field(cq_context->byte_24_pgsz_addr,
2585 		       V2_CQC_BYTE_24_CQE_NXT_BLK_ADDR_M,
2586 		       V2_CQC_BYTE_24_CQE_NXT_BLK_ADDR_S,
2587 		       mtts[1] >> (32 + PAGE_ADDR_SHIFT));
2588 	roce_set_field(cq_context->byte_24_pgsz_addr,
2589 		       V2_CQC_BYTE_24_CQE_BA_PG_SZ_M,
2590 		       V2_CQC_BYTE_24_CQE_BA_PG_SZ_S,
2591 		       hr_dev->caps.cqe_ba_pg_sz + PG_SHIFT_OFFSET);
2592 	roce_set_field(cq_context->byte_24_pgsz_addr,
2593 		       V2_CQC_BYTE_24_CQE_BUF_PG_SZ_M,
2594 		       V2_CQC_BYTE_24_CQE_BUF_PG_SZ_S,
2595 		       hr_dev->caps.cqe_buf_pg_sz + PG_SHIFT_OFFSET);
2596 
2597 	cq_context->cqe_ba = cpu_to_le32(dma_handle >> 3);
2598 
2599 	roce_set_field(cq_context->byte_40_cqe_ba, V2_CQC_BYTE_40_CQE_BA_M,
2600 		       V2_CQC_BYTE_40_CQE_BA_S, (dma_handle >> (32 + 3)));
2601 
2602 	if (hr_cq->db_en)
2603 		roce_set_bit(cq_context->byte_44_db_record,
2604 			     V2_CQC_BYTE_44_DB_RECORD_EN_S, 1);
2605 
2606 	roce_set_field(cq_context->byte_44_db_record,
2607 		       V2_CQC_BYTE_44_DB_RECORD_ADDR_M,
2608 		       V2_CQC_BYTE_44_DB_RECORD_ADDR_S,
2609 		       ((u32)hr_cq->db.dma) >> 1);
2610 	cq_context->db_record_addr = cpu_to_le32(hr_cq->db.dma >> 32);
2611 
2612 	roce_set_field(cq_context->byte_56_cqe_period_maxcnt,
2613 		       V2_CQC_BYTE_56_CQ_MAX_CNT_M,
2614 		       V2_CQC_BYTE_56_CQ_MAX_CNT_S,
2615 		       HNS_ROCE_V2_CQ_DEFAULT_BURST_NUM);
2616 	roce_set_field(cq_context->byte_56_cqe_period_maxcnt,
2617 		       V2_CQC_BYTE_56_CQ_PERIOD_M,
2618 		       V2_CQC_BYTE_56_CQ_PERIOD_S,
2619 		       HNS_ROCE_V2_CQ_DEFAULT_INTERVAL);
2620 }
2621 
2622 static int hns_roce_v2_req_notify_cq(struct ib_cq *ibcq,
2623 				     enum ib_cq_notify_flags flags)
2624 {
2625 	struct hns_roce_dev *hr_dev = to_hr_dev(ibcq->device);
2626 	struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
2627 	u32 notification_flag;
2628 	__le32 doorbell[2];
2629 
2630 	doorbell[0] = 0;
2631 	doorbell[1] = 0;
2632 
2633 	notification_flag = (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ?
2634 			     V2_CQ_DB_REQ_NOT : V2_CQ_DB_REQ_NOT_SOL;
2635 	/*
2636 	 * flags = 0; Notification Flag = 1, next
2637 	 * flags = 1; Notification Flag = 0, solocited
2638 	 */
2639 	roce_set_field(doorbell[0], V2_CQ_DB_BYTE_4_TAG_M, V2_DB_BYTE_4_TAG_S,
2640 		       hr_cq->cqn);
2641 	roce_set_field(doorbell[0], V2_CQ_DB_BYTE_4_CMD_M, V2_DB_BYTE_4_CMD_S,
2642 		       HNS_ROCE_V2_CQ_DB_NTR);
2643 	roce_set_field(doorbell[1], V2_CQ_DB_PARAMETER_CONS_IDX_M,
2644 		       V2_CQ_DB_PARAMETER_CONS_IDX_S,
2645 		       hr_cq->cons_index & ((hr_cq->cq_depth << 1) - 1));
2646 	roce_set_field(doorbell[1], V2_CQ_DB_PARAMETER_CMD_SN_M,
2647 		       V2_CQ_DB_PARAMETER_CMD_SN_S, hr_cq->arm_sn & 0x3);
2648 	roce_set_bit(doorbell[1], V2_CQ_DB_PARAMETER_NOTIFY_S,
2649 		     notification_flag);
2650 
2651 	hns_roce_write64(hr_dev, doorbell, hr_cq->cq_db_l);
2652 
2653 	return 0;
2654 }
2655 
2656 static int hns_roce_handle_recv_inl_wqe(struct hns_roce_v2_cqe *cqe,
2657 						    struct hns_roce_qp **cur_qp,
2658 						    struct ib_wc *wc)
2659 {
2660 	struct hns_roce_rinl_sge *sge_list;
2661 	u32 wr_num, wr_cnt, sge_num;
2662 	u32 sge_cnt, data_len, size;
2663 	void *wqe_buf;
2664 
2665 	wr_num = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_WQE_INDX_M,
2666 				V2_CQE_BYTE_4_WQE_INDX_S) & 0xffff;
2667 	wr_cnt = wr_num & ((*cur_qp)->rq.wqe_cnt - 1);
2668 
2669 	sge_list = (*cur_qp)->rq_inl_buf.wqe_list[wr_cnt].sg_list;
2670 	sge_num = (*cur_qp)->rq_inl_buf.wqe_list[wr_cnt].sge_cnt;
2671 	wqe_buf = get_recv_wqe(*cur_qp, wr_cnt);
2672 	data_len = wc->byte_len;
2673 
2674 	for (sge_cnt = 0; (sge_cnt < sge_num) && (data_len); sge_cnt++) {
2675 		size = min(sge_list[sge_cnt].len, data_len);
2676 		memcpy((void *)sge_list[sge_cnt].addr, wqe_buf, size);
2677 
2678 		data_len -= size;
2679 		wqe_buf += size;
2680 	}
2681 
2682 	if (data_len) {
2683 		wc->status = IB_WC_LOC_LEN_ERR;
2684 		return -EAGAIN;
2685 	}
2686 
2687 	return 0;
2688 }
2689 
2690 static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq,
2691 				struct hns_roce_qp **cur_qp, struct ib_wc *wc)
2692 {
2693 	struct hns_roce_srq *srq = NULL;
2694 	struct hns_roce_dev *hr_dev;
2695 	struct hns_roce_v2_cqe *cqe;
2696 	struct hns_roce_qp *hr_qp;
2697 	struct hns_roce_wq *wq;
2698 	struct ib_qp_attr attr;
2699 	int attr_mask;
2700 	int is_send;
2701 	u16 wqe_ctr;
2702 	u32 opcode;
2703 	u32 status;
2704 	int qpn;
2705 	int ret;
2706 
2707 	/* Find cqe according to consumer index */
2708 	cqe = next_cqe_sw_v2(hr_cq);
2709 	if (!cqe)
2710 		return -EAGAIN;
2711 
2712 	++hr_cq->cons_index;
2713 	/* Memory barrier */
2714 	rmb();
2715 
2716 	/* 0->SQ, 1->RQ */
2717 	is_send = !roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_S_R_S);
2718 
2719 	qpn = roce_get_field(cqe->byte_16, V2_CQE_BYTE_16_LCL_QPN_M,
2720 				V2_CQE_BYTE_16_LCL_QPN_S);
2721 
2722 	if (!*cur_qp || (qpn & HNS_ROCE_V2_CQE_QPN_MASK) != (*cur_qp)->qpn) {
2723 		hr_dev = to_hr_dev(hr_cq->ib_cq.device);
2724 		hr_qp = __hns_roce_qp_lookup(hr_dev, qpn);
2725 		if (unlikely(!hr_qp)) {
2726 			dev_err(hr_dev->dev, "CQ %06lx with entry for unknown QPN %06x\n",
2727 				hr_cq->cqn, (qpn & HNS_ROCE_V2_CQE_QPN_MASK));
2728 			return -EINVAL;
2729 		}
2730 		*cur_qp = hr_qp;
2731 	}
2732 
2733 	wc->qp = &(*cur_qp)->ibqp;
2734 	wc->vendor_err = 0;
2735 
2736 	if (is_send) {
2737 		wq = &(*cur_qp)->sq;
2738 		if ((*cur_qp)->sq_signal_bits) {
2739 			/*
2740 			 * If sg_signal_bit is 1,
2741 			 * firstly tail pointer updated to wqe
2742 			 * which current cqe correspond to
2743 			 */
2744 			wqe_ctr = (u16)roce_get_field(cqe->byte_4,
2745 						      V2_CQE_BYTE_4_WQE_INDX_M,
2746 						      V2_CQE_BYTE_4_WQE_INDX_S);
2747 			wq->tail += (wqe_ctr - (u16)wq->tail) &
2748 				    (wq->wqe_cnt - 1);
2749 		}
2750 
2751 		wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
2752 		++wq->tail;
2753 	} else if ((*cur_qp)->ibqp.srq) {
2754 		srq = to_hr_srq((*cur_qp)->ibqp.srq);
2755 		wqe_ctr = (u16)roce_get_field(cqe->byte_4,
2756 					      V2_CQE_BYTE_4_WQE_INDX_M,
2757 					      V2_CQE_BYTE_4_WQE_INDX_S);
2758 		wc->wr_id = srq->wrid[wqe_ctr];
2759 		hns_roce_free_srq_wqe(srq, wqe_ctr);
2760 	} else {
2761 		/* Update tail pointer, record wr_id */
2762 		wq = &(*cur_qp)->rq;
2763 		wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
2764 		++wq->tail;
2765 	}
2766 
2767 	status = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_STATUS_M,
2768 				V2_CQE_BYTE_4_STATUS_S);
2769 	switch (status & HNS_ROCE_V2_CQE_STATUS_MASK) {
2770 	case HNS_ROCE_CQE_V2_SUCCESS:
2771 		wc->status = IB_WC_SUCCESS;
2772 		break;
2773 	case HNS_ROCE_CQE_V2_LOCAL_LENGTH_ERR:
2774 		wc->status = IB_WC_LOC_LEN_ERR;
2775 		break;
2776 	case HNS_ROCE_CQE_V2_LOCAL_QP_OP_ERR:
2777 		wc->status = IB_WC_LOC_QP_OP_ERR;
2778 		break;
2779 	case HNS_ROCE_CQE_V2_LOCAL_PROT_ERR:
2780 		wc->status = IB_WC_LOC_PROT_ERR;
2781 		break;
2782 	case HNS_ROCE_CQE_V2_WR_FLUSH_ERR:
2783 		wc->status = IB_WC_WR_FLUSH_ERR;
2784 		break;
2785 	case HNS_ROCE_CQE_V2_MW_BIND_ERR:
2786 		wc->status = IB_WC_MW_BIND_ERR;
2787 		break;
2788 	case HNS_ROCE_CQE_V2_BAD_RESP_ERR:
2789 		wc->status = IB_WC_BAD_RESP_ERR;
2790 		break;
2791 	case HNS_ROCE_CQE_V2_LOCAL_ACCESS_ERR:
2792 		wc->status = IB_WC_LOC_ACCESS_ERR;
2793 		break;
2794 	case HNS_ROCE_CQE_V2_REMOTE_INVAL_REQ_ERR:
2795 		wc->status = IB_WC_REM_INV_REQ_ERR;
2796 		break;
2797 	case HNS_ROCE_CQE_V2_REMOTE_ACCESS_ERR:
2798 		wc->status = IB_WC_REM_ACCESS_ERR;
2799 		break;
2800 	case HNS_ROCE_CQE_V2_REMOTE_OP_ERR:
2801 		wc->status = IB_WC_REM_OP_ERR;
2802 		break;
2803 	case HNS_ROCE_CQE_V2_TRANSPORT_RETRY_EXC_ERR:
2804 		wc->status = IB_WC_RETRY_EXC_ERR;
2805 		break;
2806 	case HNS_ROCE_CQE_V2_RNR_RETRY_EXC_ERR:
2807 		wc->status = IB_WC_RNR_RETRY_EXC_ERR;
2808 		break;
2809 	case HNS_ROCE_CQE_V2_REMOTE_ABORT_ERR:
2810 		wc->status = IB_WC_REM_ABORT_ERR;
2811 		break;
2812 	default:
2813 		wc->status = IB_WC_GENERAL_ERR;
2814 		break;
2815 	}
2816 
2817 	/* flush cqe if wc status is error, excluding flush error */
2818 	if ((wc->status != IB_WC_SUCCESS) &&
2819 	    (wc->status != IB_WC_WR_FLUSH_ERR)) {
2820 		attr_mask = IB_QP_STATE;
2821 		attr.qp_state = IB_QPS_ERR;
2822 		return hns_roce_v2_modify_qp(&(*cur_qp)->ibqp,
2823 					     &attr, attr_mask,
2824 					     (*cur_qp)->state, IB_QPS_ERR);
2825 	}
2826 
2827 	if (wc->status == IB_WC_WR_FLUSH_ERR)
2828 		return 0;
2829 
2830 	if (is_send) {
2831 		wc->wc_flags = 0;
2832 		/* SQ corresponding to CQE */
2833 		switch (roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_OPCODE_M,
2834 				       V2_CQE_BYTE_4_OPCODE_S) & 0x1f) {
2835 		case HNS_ROCE_SQ_OPCODE_SEND:
2836 			wc->opcode = IB_WC_SEND;
2837 			break;
2838 		case HNS_ROCE_SQ_OPCODE_SEND_WITH_INV:
2839 			wc->opcode = IB_WC_SEND;
2840 			break;
2841 		case HNS_ROCE_SQ_OPCODE_SEND_WITH_IMM:
2842 			wc->opcode = IB_WC_SEND;
2843 			wc->wc_flags |= IB_WC_WITH_IMM;
2844 			break;
2845 		case HNS_ROCE_SQ_OPCODE_RDMA_READ:
2846 			wc->opcode = IB_WC_RDMA_READ;
2847 			wc->byte_len = le32_to_cpu(cqe->byte_cnt);
2848 			break;
2849 		case HNS_ROCE_SQ_OPCODE_RDMA_WRITE:
2850 			wc->opcode = IB_WC_RDMA_WRITE;
2851 			break;
2852 		case HNS_ROCE_SQ_OPCODE_RDMA_WRITE_WITH_IMM:
2853 			wc->opcode = IB_WC_RDMA_WRITE;
2854 			wc->wc_flags |= IB_WC_WITH_IMM;
2855 			break;
2856 		case HNS_ROCE_SQ_OPCODE_LOCAL_INV:
2857 			wc->opcode = IB_WC_LOCAL_INV;
2858 			wc->wc_flags |= IB_WC_WITH_INVALIDATE;
2859 			break;
2860 		case HNS_ROCE_SQ_OPCODE_ATOMIC_COMP_AND_SWAP:
2861 			wc->opcode = IB_WC_COMP_SWAP;
2862 			wc->byte_len  = 8;
2863 			break;
2864 		case HNS_ROCE_SQ_OPCODE_ATOMIC_FETCH_AND_ADD:
2865 			wc->opcode = IB_WC_FETCH_ADD;
2866 			wc->byte_len  = 8;
2867 			break;
2868 		case HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_COMP_AND_SWAP:
2869 			wc->opcode = IB_WC_MASKED_COMP_SWAP;
2870 			wc->byte_len  = 8;
2871 			break;
2872 		case HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_FETCH_AND_ADD:
2873 			wc->opcode = IB_WC_MASKED_FETCH_ADD;
2874 			wc->byte_len  = 8;
2875 			break;
2876 		case HNS_ROCE_SQ_OPCODE_FAST_REG_WR:
2877 			wc->opcode = IB_WC_REG_MR;
2878 			break;
2879 		case HNS_ROCE_SQ_OPCODE_BIND_MW:
2880 			wc->opcode = IB_WC_REG_MR;
2881 			break;
2882 		default:
2883 			wc->status = IB_WC_GENERAL_ERR;
2884 			break;
2885 		}
2886 	} else {
2887 		/* RQ correspond to CQE */
2888 		wc->byte_len = le32_to_cpu(cqe->byte_cnt);
2889 
2890 		opcode = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_OPCODE_M,
2891 					V2_CQE_BYTE_4_OPCODE_S);
2892 		switch (opcode & 0x1f) {
2893 		case HNS_ROCE_V2_OPCODE_RDMA_WRITE_IMM:
2894 			wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
2895 			wc->wc_flags = IB_WC_WITH_IMM;
2896 			wc->ex.imm_data =
2897 				cpu_to_be32(le32_to_cpu(cqe->immtdata));
2898 			break;
2899 		case HNS_ROCE_V2_OPCODE_SEND:
2900 			wc->opcode = IB_WC_RECV;
2901 			wc->wc_flags = 0;
2902 			break;
2903 		case HNS_ROCE_V2_OPCODE_SEND_WITH_IMM:
2904 			wc->opcode = IB_WC_RECV;
2905 			wc->wc_flags = IB_WC_WITH_IMM;
2906 			wc->ex.imm_data =
2907 				cpu_to_be32(le32_to_cpu(cqe->immtdata));
2908 			break;
2909 		case HNS_ROCE_V2_OPCODE_SEND_WITH_INV:
2910 			wc->opcode = IB_WC_RECV;
2911 			wc->wc_flags = IB_WC_WITH_INVALIDATE;
2912 			wc->ex.invalidate_rkey = le32_to_cpu(cqe->rkey);
2913 			break;
2914 		default:
2915 			wc->status = IB_WC_GENERAL_ERR;
2916 			break;
2917 		}
2918 
2919 		if ((wc->qp->qp_type == IB_QPT_RC ||
2920 		     wc->qp->qp_type == IB_QPT_UC) &&
2921 		    (opcode == HNS_ROCE_V2_OPCODE_SEND ||
2922 		    opcode == HNS_ROCE_V2_OPCODE_SEND_WITH_IMM ||
2923 		    opcode == HNS_ROCE_V2_OPCODE_SEND_WITH_INV) &&
2924 		    (roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_RQ_INLINE_S))) {
2925 			ret = hns_roce_handle_recv_inl_wqe(cqe, cur_qp, wc);
2926 			if (ret)
2927 				return -EAGAIN;
2928 		}
2929 
2930 		wc->sl = (u8)roce_get_field(cqe->byte_32, V2_CQE_BYTE_32_SL_M,
2931 					    V2_CQE_BYTE_32_SL_S);
2932 		wc->src_qp = (u8)roce_get_field(cqe->byte_32,
2933 						V2_CQE_BYTE_32_RMT_QPN_M,
2934 						V2_CQE_BYTE_32_RMT_QPN_S);
2935 		wc->slid = 0;
2936 		wc->wc_flags |= (roce_get_bit(cqe->byte_32,
2937 					      V2_CQE_BYTE_32_GRH_S) ?
2938 					      IB_WC_GRH : 0);
2939 		wc->port_num = roce_get_field(cqe->byte_32,
2940 				V2_CQE_BYTE_32_PORTN_M, V2_CQE_BYTE_32_PORTN_S);
2941 		wc->pkey_index = 0;
2942 		memcpy(wc->smac, cqe->smac, 4);
2943 		wc->smac[4] = roce_get_field(cqe->byte_28,
2944 					     V2_CQE_BYTE_28_SMAC_4_M,
2945 					     V2_CQE_BYTE_28_SMAC_4_S);
2946 		wc->smac[5] = roce_get_field(cqe->byte_28,
2947 					     V2_CQE_BYTE_28_SMAC_5_M,
2948 					     V2_CQE_BYTE_28_SMAC_5_S);
2949 		wc->wc_flags |= IB_WC_WITH_SMAC;
2950 		if (roce_get_bit(cqe->byte_28, V2_CQE_BYTE_28_VID_VLD_S)) {
2951 			wc->vlan_id = (u16)roce_get_field(cqe->byte_28,
2952 							  V2_CQE_BYTE_28_VID_M,
2953 							  V2_CQE_BYTE_28_VID_S);
2954 			wc->wc_flags |= IB_WC_WITH_VLAN;
2955 		} else {
2956 			wc->vlan_id = 0xffff;
2957 		}
2958 
2959 		wc->network_hdr_type = roce_get_field(cqe->byte_28,
2960 						    V2_CQE_BYTE_28_PORT_TYPE_M,
2961 						    V2_CQE_BYTE_28_PORT_TYPE_S);
2962 	}
2963 
2964 	return 0;
2965 }
2966 
2967 static int hns_roce_v2_poll_cq(struct ib_cq *ibcq, int num_entries,
2968 			       struct ib_wc *wc)
2969 {
2970 	struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
2971 	struct hns_roce_qp *cur_qp = NULL;
2972 	unsigned long flags;
2973 	int npolled;
2974 
2975 	spin_lock_irqsave(&hr_cq->lock, flags);
2976 
2977 	for (npolled = 0; npolled < num_entries; ++npolled) {
2978 		if (hns_roce_v2_poll_one(hr_cq, &cur_qp, wc + npolled))
2979 			break;
2980 	}
2981 
2982 	if (npolled) {
2983 		/* Memory barrier */
2984 		wmb();
2985 		hns_roce_v2_cq_set_ci(hr_cq, hr_cq->cons_index);
2986 	}
2987 
2988 	spin_unlock_irqrestore(&hr_cq->lock, flags);
2989 
2990 	return npolled;
2991 }
2992 
2993 static int get_op_for_set_hem(struct hns_roce_dev *hr_dev, u32 type,
2994 			      int step_idx)
2995 {
2996 	int op;
2997 
2998 	if (type == HEM_TYPE_SCCC && step_idx)
2999 		return -EINVAL;
3000 
3001 	switch (type) {
3002 	case HEM_TYPE_QPC:
3003 		op = HNS_ROCE_CMD_WRITE_QPC_BT0;
3004 		break;
3005 	case HEM_TYPE_MTPT:
3006 		op = HNS_ROCE_CMD_WRITE_MPT_BT0;
3007 		break;
3008 	case HEM_TYPE_CQC:
3009 		op = HNS_ROCE_CMD_WRITE_CQC_BT0;
3010 		break;
3011 	case HEM_TYPE_SRQC:
3012 		op = HNS_ROCE_CMD_WRITE_SRQC_BT0;
3013 		break;
3014 	case HEM_TYPE_SCCC:
3015 		op = HNS_ROCE_CMD_WRITE_SCCC_BT0;
3016 		break;
3017 	case HEM_TYPE_QPC_TIMER:
3018 		op = HNS_ROCE_CMD_WRITE_QPC_TIMER_BT0;
3019 		break;
3020 	case HEM_TYPE_CQC_TIMER:
3021 		op = HNS_ROCE_CMD_WRITE_CQC_TIMER_BT0;
3022 		break;
3023 	default:
3024 		dev_warn(hr_dev->dev,
3025 			 "Table %d not to be written by mailbox!\n", type);
3026 		return -EINVAL;
3027 	}
3028 
3029 	return op + step_idx;
3030 }
3031 
3032 static int hns_roce_v2_set_hem(struct hns_roce_dev *hr_dev,
3033 			       struct hns_roce_hem_table *table, int obj,
3034 			       int step_idx)
3035 {
3036 	struct hns_roce_cmd_mailbox *mailbox;
3037 	struct hns_roce_hem_iter iter;
3038 	struct hns_roce_hem_mhop mhop;
3039 	struct hns_roce_hem *hem;
3040 	unsigned long mhop_obj = obj;
3041 	int i, j, k;
3042 	int ret = 0;
3043 	u64 hem_idx = 0;
3044 	u64 l1_idx = 0;
3045 	u64 bt_ba = 0;
3046 	u32 chunk_ba_num;
3047 	u32 hop_num;
3048 	int op;
3049 
3050 	if (!hns_roce_check_whether_mhop(hr_dev, table->type))
3051 		return 0;
3052 
3053 	hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop);
3054 	i = mhop.l0_idx;
3055 	j = mhop.l1_idx;
3056 	k = mhop.l2_idx;
3057 	hop_num = mhop.hop_num;
3058 	chunk_ba_num = mhop.bt_chunk_size / 8;
3059 
3060 	if (hop_num == 2) {
3061 		hem_idx = i * chunk_ba_num * chunk_ba_num + j * chunk_ba_num +
3062 			  k;
3063 		l1_idx = i * chunk_ba_num + j;
3064 	} else if (hop_num == 1) {
3065 		hem_idx = i * chunk_ba_num + j;
3066 	} else if (hop_num == HNS_ROCE_HOP_NUM_0) {
3067 		hem_idx = i;
3068 	}
3069 
3070 	op = get_op_for_set_hem(hr_dev, table->type, step_idx);
3071 	if (op == -EINVAL)
3072 		return 0;
3073 
3074 	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
3075 	if (IS_ERR(mailbox))
3076 		return PTR_ERR(mailbox);
3077 
3078 	if (table->type == HEM_TYPE_SCCC)
3079 		obj = mhop.l0_idx;
3080 
3081 	if (check_whether_last_step(hop_num, step_idx)) {
3082 		hem = table->hem[hem_idx];
3083 		for (hns_roce_hem_first(hem, &iter);
3084 		     !hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) {
3085 			bt_ba = hns_roce_hem_addr(&iter);
3086 
3087 			/* configure the ba, tag, and op */
3088 			ret = hns_roce_cmd_mbox(hr_dev, bt_ba, mailbox->dma,
3089 						obj, 0, op,
3090 						HNS_ROCE_CMD_TIMEOUT_MSECS);
3091 		}
3092 	} else {
3093 		if (step_idx == 0)
3094 			bt_ba = table->bt_l0_dma_addr[i];
3095 		else if (step_idx == 1 && hop_num == 2)
3096 			bt_ba = table->bt_l1_dma_addr[l1_idx];
3097 
3098 		/* configure the ba, tag, and op */
3099 		ret = hns_roce_cmd_mbox(hr_dev, bt_ba, mailbox->dma, obj,
3100 					0, op, HNS_ROCE_CMD_TIMEOUT_MSECS);
3101 	}
3102 
3103 	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
3104 	return ret;
3105 }
3106 
3107 static int hns_roce_v2_clear_hem(struct hns_roce_dev *hr_dev,
3108 				 struct hns_roce_hem_table *table, int obj,
3109 				 int step_idx)
3110 {
3111 	struct device *dev = hr_dev->dev;
3112 	struct hns_roce_cmd_mailbox *mailbox;
3113 	int ret;
3114 	u16 op = 0xff;
3115 
3116 	if (!hns_roce_check_whether_mhop(hr_dev, table->type))
3117 		return 0;
3118 
3119 	switch (table->type) {
3120 	case HEM_TYPE_QPC:
3121 		op = HNS_ROCE_CMD_DESTROY_QPC_BT0;
3122 		break;
3123 	case HEM_TYPE_MTPT:
3124 		op = HNS_ROCE_CMD_DESTROY_MPT_BT0;
3125 		break;
3126 	case HEM_TYPE_CQC:
3127 		op = HNS_ROCE_CMD_DESTROY_CQC_BT0;
3128 		break;
3129 	case HEM_TYPE_SCCC:
3130 	case HEM_TYPE_QPC_TIMER:
3131 	case HEM_TYPE_CQC_TIMER:
3132 		break;
3133 	case HEM_TYPE_SRQC:
3134 		op = HNS_ROCE_CMD_DESTROY_SRQC_BT0;
3135 		break;
3136 	default:
3137 		dev_warn(dev, "Table %d not to be destroyed by mailbox!\n",
3138 			 table->type);
3139 		return 0;
3140 	}
3141 
3142 	if (table->type == HEM_TYPE_SCCC ||
3143 	    table->type == HEM_TYPE_QPC_TIMER ||
3144 	    table->type == HEM_TYPE_CQC_TIMER)
3145 		return 0;
3146 
3147 	op += step_idx;
3148 
3149 	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
3150 	if (IS_ERR(mailbox))
3151 		return PTR_ERR(mailbox);
3152 
3153 	/* configure the tag and op */
3154 	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, obj, 0, op,
3155 				HNS_ROCE_CMD_TIMEOUT_MSECS);
3156 
3157 	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
3158 	return ret;
3159 }
3160 
3161 static int hns_roce_v2_qp_modify(struct hns_roce_dev *hr_dev,
3162 				 enum ib_qp_state cur_state,
3163 				 enum ib_qp_state new_state,
3164 				 struct hns_roce_v2_qp_context *context,
3165 				 struct hns_roce_qp *hr_qp)
3166 {
3167 	struct hns_roce_cmd_mailbox *mailbox;
3168 	int ret;
3169 
3170 	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
3171 	if (IS_ERR(mailbox))
3172 		return PTR_ERR(mailbox);
3173 
3174 	memcpy(mailbox->buf, context, sizeof(*context) * 2);
3175 
3176 	ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_qp->qpn, 0,
3177 				HNS_ROCE_CMD_MODIFY_QPC,
3178 				HNS_ROCE_CMD_TIMEOUT_MSECS);
3179 
3180 	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
3181 
3182 	return ret;
3183 }
3184 
3185 static void set_access_flags(struct hns_roce_qp *hr_qp,
3186 			     struct hns_roce_v2_qp_context *context,
3187 			     struct hns_roce_v2_qp_context *qpc_mask,
3188 			     const struct ib_qp_attr *attr, int attr_mask)
3189 {
3190 	u8 dest_rd_atomic;
3191 	u32 access_flags;
3192 
3193 	dest_rd_atomic = (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) ?
3194 			 attr->max_dest_rd_atomic : hr_qp->resp_depth;
3195 
3196 	access_flags = (attr_mask & IB_QP_ACCESS_FLAGS) ?
3197 		       attr->qp_access_flags : hr_qp->atomic_rd_en;
3198 
3199 	if (!dest_rd_atomic)
3200 		access_flags &= IB_ACCESS_REMOTE_WRITE;
3201 
3202 	roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
3203 		     !!(access_flags & IB_ACCESS_REMOTE_READ));
3204 	roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S, 0);
3205 
3206 	roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
3207 		     !!(access_flags & IB_ACCESS_REMOTE_WRITE));
3208 	roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S, 0);
3209 
3210 	roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
3211 		     !!(access_flags & IB_ACCESS_REMOTE_ATOMIC));
3212 	roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S, 0);
3213 }
3214 
3215 static void set_qpc_wqe_cnt(struct hns_roce_qp *hr_qp,
3216 			    struct hns_roce_v2_qp_context *context,
3217 			    struct hns_roce_v2_qp_context *qpc_mask)
3218 {
3219 	if (hr_qp->ibqp.qp_type == IB_QPT_GSI)
3220 		roce_set_field(context->byte_4_sqpn_tst,
3221 			       V2_QPC_BYTE_4_SGE_SHIFT_M,
3222 			       V2_QPC_BYTE_4_SGE_SHIFT_S,
3223 			       ilog2((unsigned int)hr_qp->sge.sge_cnt));
3224 	else
3225 		roce_set_field(context->byte_4_sqpn_tst,
3226 			       V2_QPC_BYTE_4_SGE_SHIFT_M,
3227 			       V2_QPC_BYTE_4_SGE_SHIFT_S,
3228 			       hr_qp->sq.max_gs >
3229 			       HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE ?
3230 			       ilog2((unsigned int)hr_qp->sge.sge_cnt) : 0);
3231 
3232 	roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SGE_SHIFT_M,
3233 		       V2_QPC_BYTE_4_SGE_SHIFT_S, 0);
3234 
3235 	roce_set_field(context->byte_20_smac_sgid_idx,
3236 		       V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S,
3237 		       ilog2((unsigned int)hr_qp->sq.wqe_cnt));
3238 	roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
3239 		       V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S, 0);
3240 
3241 	roce_set_field(context->byte_20_smac_sgid_idx,
3242 		       V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S,
3243 		       (hr_qp->ibqp.qp_type == IB_QPT_XRC_INI ||
3244 		       hr_qp->ibqp.qp_type == IB_QPT_XRC_TGT ||
3245 		       hr_qp->ibqp.srq) ? 0 :
3246 		       ilog2((unsigned int)hr_qp->rq.wqe_cnt));
3247 
3248 	roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
3249 		       V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S, 0);
3250 }
3251 
3252 static void modify_qp_reset_to_init(struct ib_qp *ibqp,
3253 				    const struct ib_qp_attr *attr,
3254 				    int attr_mask,
3255 				    struct hns_roce_v2_qp_context *context,
3256 				    struct hns_roce_v2_qp_context *qpc_mask)
3257 {
3258 	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
3259 	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
3260 
3261 	/*
3262 	 * In v2 engine, software pass context and context mask to hardware
3263 	 * when modifying qp. If software need modify some fields in context,
3264 	 * we should set all bits of the relevant fields in context mask to
3265 	 * 0 at the same time, else set them to 0x1.
3266 	 */
3267 	roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,
3268 		       V2_QPC_BYTE_4_TST_S, to_hr_qp_type(hr_qp->ibqp.qp_type));
3269 	roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,
3270 		       V2_QPC_BYTE_4_TST_S, 0);
3271 
3272 	roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
3273 		       V2_QPC_BYTE_4_SQPN_S, hr_qp->qpn);
3274 	roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
3275 		       V2_QPC_BYTE_4_SQPN_S, 0);
3276 
3277 	roce_set_field(context->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,
3278 		       V2_QPC_BYTE_16_PD_S, to_hr_pd(ibqp->pd)->pdn);
3279 	roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,
3280 		       V2_QPC_BYTE_16_PD_S, 0);
3281 
3282 	roce_set_field(context->byte_20_smac_sgid_idx, V2_QPC_BYTE_20_RQWS_M,
3283 		       V2_QPC_BYTE_20_RQWS_S, ilog2(hr_qp->rq.max_gs));
3284 	roce_set_field(qpc_mask->byte_20_smac_sgid_idx, V2_QPC_BYTE_20_RQWS_M,
3285 		       V2_QPC_BYTE_20_RQWS_S, 0);
3286 
3287 	set_qpc_wqe_cnt(hr_qp, context, qpc_mask);
3288 
3289 	/* No VLAN need to set 0xFFF */
3290 	roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_ID_M,
3291 		       V2_QPC_BYTE_24_VLAN_ID_S, 0xfff);
3292 	roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_ID_M,
3293 		       V2_QPC_BYTE_24_VLAN_ID_S, 0);
3294 
3295 	/*
3296 	 * Set some fields in context to zero, Because the default values
3297 	 * of all fields in context are zero, we need not set them to 0 again.
3298 	 * but we should set the relevant fields of context mask to 0.
3299 	 */
3300 	roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_SQ_TX_ERR_S, 0);
3301 	roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_SQ_RX_ERR_S, 0);
3302 	roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_RQ_TX_ERR_S, 0);
3303 	roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_RQ_RX_ERR_S, 0);
3304 
3305 	roce_set_field(qpc_mask->byte_60_qpst_tempid, V2_QPC_BYTE_60_TEMPID_M,
3306 		       V2_QPC_BYTE_60_TEMPID_S, 0);
3307 
3308 	roce_set_field(qpc_mask->byte_60_qpst_tempid,
3309 		       V2_QPC_BYTE_60_SCC_TOKEN_M, V2_QPC_BYTE_60_SCC_TOKEN_S,
3310 		       0);
3311 	roce_set_bit(qpc_mask->byte_60_qpst_tempid,
3312 		     V2_QPC_BYTE_60_SQ_DB_DOING_S, 0);
3313 	roce_set_bit(qpc_mask->byte_60_qpst_tempid,
3314 		     V2_QPC_BYTE_60_RQ_DB_DOING_S, 0);
3315 	roce_set_bit(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_CNP_TX_FLAG_S, 0);
3316 	roce_set_bit(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_CE_FLAG_S, 0);
3317 
3318 	if (hr_qp->rdb_en) {
3319 		roce_set_bit(context->byte_68_rq_db,
3320 			     V2_QPC_BYTE_68_RQ_RECORD_EN_S, 1);
3321 		roce_set_bit(qpc_mask->byte_68_rq_db,
3322 			     V2_QPC_BYTE_68_RQ_RECORD_EN_S, 0);
3323 	}
3324 
3325 	roce_set_field(context->byte_68_rq_db,
3326 		       V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_M,
3327 		       V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_S,
3328 		       ((u32)hr_qp->rdb.dma) >> 1);
3329 	roce_set_field(qpc_mask->byte_68_rq_db,
3330 		       V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_M,
3331 		       V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_S, 0);
3332 	context->rq_db_record_addr = cpu_to_le32(hr_qp->rdb.dma >> 32);
3333 	qpc_mask->rq_db_record_addr = 0;
3334 
3335 	roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RQIE_S,
3336 		    (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) ? 1 : 0);
3337 	roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RQIE_S, 0);
3338 
3339 	roce_set_field(context->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M,
3340 		       V2_QPC_BYTE_80_RX_CQN_S, to_hr_cq(ibqp->recv_cq)->cqn);
3341 	roce_set_field(qpc_mask->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M,
3342 		       V2_QPC_BYTE_80_RX_CQN_S, 0);
3343 	if (ibqp->srq) {
3344 		roce_set_field(context->byte_76_srqn_op_en,
3345 			       V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S,
3346 			       to_hr_srq(ibqp->srq)->srqn);
3347 		roce_set_field(qpc_mask->byte_76_srqn_op_en,
3348 			       V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S, 0);
3349 		roce_set_bit(context->byte_76_srqn_op_en,
3350 			     V2_QPC_BYTE_76_SRQ_EN_S, 1);
3351 		roce_set_bit(qpc_mask->byte_76_srqn_op_en,
3352 			     V2_QPC_BYTE_76_SRQ_EN_S, 0);
3353 	}
3354 
3355 	roce_set_field(qpc_mask->byte_84_rq_ci_pi,
3356 		       V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
3357 		       V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0);
3358 	roce_set_field(qpc_mask->byte_84_rq_ci_pi,
3359 		       V2_QPC_BYTE_84_RQ_CONSUMER_IDX_M,
3360 		       V2_QPC_BYTE_84_RQ_CONSUMER_IDX_S, 0);
3361 
3362 	roce_set_field(qpc_mask->byte_92_srq_info, V2_QPC_BYTE_92_SRQ_INFO_M,
3363 		       V2_QPC_BYTE_92_SRQ_INFO_S, 0);
3364 
3365 	roce_set_field(qpc_mask->byte_96_rx_reqmsn, V2_QPC_BYTE_96_RX_REQ_MSN_M,
3366 		       V2_QPC_BYTE_96_RX_REQ_MSN_S, 0);
3367 
3368 	roce_set_field(qpc_mask->byte_104_rq_sge,
3369 		       V2_QPC_BYTE_104_RQ_CUR_WQE_SGE_NUM_M,
3370 		       V2_QPC_BYTE_104_RQ_CUR_WQE_SGE_NUM_S, 0);
3371 
3372 	roce_set_bit(qpc_mask->byte_108_rx_reqepsn,
3373 		     V2_QPC_BYTE_108_RX_REQ_PSN_ERR_S, 0);
3374 	roce_set_field(qpc_mask->byte_108_rx_reqepsn,
3375 		       V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_M,
3376 		       V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_S, 0);
3377 	roce_set_bit(qpc_mask->byte_108_rx_reqepsn,
3378 		     V2_QPC_BYTE_108_RX_REQ_RNR_S, 0);
3379 
3380 	qpc_mask->rq_rnr_timer = 0;
3381 	qpc_mask->rx_msg_len = 0;
3382 	qpc_mask->rx_rkey_pkt_info = 0;
3383 	qpc_mask->rx_va = 0;
3384 
3385 	roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_HEAD_MAX_M,
3386 		       V2_QPC_BYTE_132_TRRL_HEAD_MAX_S, 0);
3387 	roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_TAIL_MAX_M,
3388 		       V2_QPC_BYTE_132_TRRL_TAIL_MAX_S, 0);
3389 
3390 	roce_set_bit(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RQ_RTY_WAIT_DO_S,
3391 		     0);
3392 	roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RAQ_TRRL_HEAD_M,
3393 		       V2_QPC_BYTE_140_RAQ_TRRL_HEAD_S, 0);
3394 	roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RAQ_TRRL_TAIL_M,
3395 		       V2_QPC_BYTE_140_RAQ_TRRL_TAIL_S, 0);
3396 
3397 	roce_set_field(qpc_mask->byte_144_raq,
3398 		       V2_QPC_BYTE_144_RAQ_RTY_INI_PSN_M,
3399 		       V2_QPC_BYTE_144_RAQ_RTY_INI_PSN_S, 0);
3400 	roce_set_field(qpc_mask->byte_144_raq, V2_QPC_BYTE_144_RAQ_CREDIT_M,
3401 		       V2_QPC_BYTE_144_RAQ_CREDIT_S, 0);
3402 	roce_set_bit(qpc_mask->byte_144_raq, V2_QPC_BYTE_144_RESP_RTY_FLG_S, 0);
3403 
3404 	roce_set_field(qpc_mask->byte_148_raq, V2_QPC_BYTE_148_RQ_MSN_M,
3405 		       V2_QPC_BYTE_148_RQ_MSN_S, 0);
3406 	roce_set_field(qpc_mask->byte_148_raq, V2_QPC_BYTE_148_RAQ_SYNDROME_M,
3407 		       V2_QPC_BYTE_148_RAQ_SYNDROME_S, 0);
3408 
3409 	roce_set_field(qpc_mask->byte_152_raq, V2_QPC_BYTE_152_RAQ_PSN_M,
3410 		       V2_QPC_BYTE_152_RAQ_PSN_S, 0);
3411 	roce_set_field(qpc_mask->byte_152_raq,
3412 		       V2_QPC_BYTE_152_RAQ_TRRL_RTY_HEAD_M,
3413 		       V2_QPC_BYTE_152_RAQ_TRRL_RTY_HEAD_S, 0);
3414 
3415 	roce_set_field(qpc_mask->byte_156_raq, V2_QPC_BYTE_156_RAQ_USE_PKTN_M,
3416 		       V2_QPC_BYTE_156_RAQ_USE_PKTN_S, 0);
3417 
3418 	roce_set_field(qpc_mask->byte_160_sq_ci_pi,
3419 		       V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M,
3420 		       V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S, 0);
3421 	roce_set_field(qpc_mask->byte_160_sq_ci_pi,
3422 		       V2_QPC_BYTE_160_SQ_CONSUMER_IDX_M,
3423 		       V2_QPC_BYTE_160_SQ_CONSUMER_IDX_S, 0);
3424 
3425 	roce_set_bit(qpc_mask->byte_168_irrl_idx,
3426 		     V2_QPC_BYTE_168_POLL_DB_WAIT_DO_S, 0);
3427 	roce_set_bit(qpc_mask->byte_168_irrl_idx,
3428 		     V2_QPC_BYTE_168_SCC_TOKEN_FORBID_SQ_DEQ_S, 0);
3429 	roce_set_bit(qpc_mask->byte_168_irrl_idx,
3430 		     V2_QPC_BYTE_168_WAIT_ACK_TIMEOUT_S, 0);
3431 	roce_set_bit(qpc_mask->byte_168_irrl_idx,
3432 		     V2_QPC_BYTE_168_MSG_RTY_LP_FLG_S, 0);
3433 	roce_set_bit(qpc_mask->byte_168_irrl_idx,
3434 		     V2_QPC_BYTE_168_SQ_INVLD_FLG_S, 0);
3435 	roce_set_field(qpc_mask->byte_168_irrl_idx,
3436 		       V2_QPC_BYTE_168_IRRL_IDX_LSB_M,
3437 		       V2_QPC_BYTE_168_IRRL_IDX_LSB_S, 0);
3438 
3439 	roce_set_field(context->byte_172_sq_psn, V2_QPC_BYTE_172_ACK_REQ_FREQ_M,
3440 		       V2_QPC_BYTE_172_ACK_REQ_FREQ_S, 4);
3441 	roce_set_field(qpc_mask->byte_172_sq_psn,
3442 		       V2_QPC_BYTE_172_ACK_REQ_FREQ_M,
3443 		       V2_QPC_BYTE_172_ACK_REQ_FREQ_S, 0);
3444 
3445 	roce_set_bit(qpc_mask->byte_172_sq_psn, V2_QPC_BYTE_172_MSG_RNR_FLG_S,
3446 		     0);
3447 
3448 	roce_set_bit(context->byte_172_sq_psn, V2_QPC_BYTE_172_FRE_S, 1);
3449 	roce_set_bit(qpc_mask->byte_172_sq_psn, V2_QPC_BYTE_172_FRE_S, 0);
3450 
3451 	roce_set_field(qpc_mask->byte_176_msg_pktn,
3452 		       V2_QPC_BYTE_176_MSG_USE_PKTN_M,
3453 		       V2_QPC_BYTE_176_MSG_USE_PKTN_S, 0);
3454 	roce_set_field(qpc_mask->byte_176_msg_pktn,
3455 		       V2_QPC_BYTE_176_IRRL_HEAD_PRE_M,
3456 		       V2_QPC_BYTE_176_IRRL_HEAD_PRE_S, 0);
3457 
3458 	roce_set_field(qpc_mask->byte_184_irrl_idx,
3459 		       V2_QPC_BYTE_184_IRRL_IDX_MSB_M,
3460 		       V2_QPC_BYTE_184_IRRL_IDX_MSB_S, 0);
3461 
3462 	qpc_mask->cur_sge_offset = 0;
3463 
3464 	roce_set_field(qpc_mask->byte_192_ext_sge,
3465 		       V2_QPC_BYTE_192_CUR_SGE_IDX_M,
3466 		       V2_QPC_BYTE_192_CUR_SGE_IDX_S, 0);
3467 	roce_set_field(qpc_mask->byte_192_ext_sge,
3468 		       V2_QPC_BYTE_192_EXT_SGE_NUM_LEFT_M,
3469 		       V2_QPC_BYTE_192_EXT_SGE_NUM_LEFT_S, 0);
3470 
3471 	roce_set_field(qpc_mask->byte_196_sq_psn, V2_QPC_BYTE_196_IRRL_HEAD_M,
3472 		       V2_QPC_BYTE_196_IRRL_HEAD_S, 0);
3473 
3474 	roce_set_field(qpc_mask->byte_200_sq_max, V2_QPC_BYTE_200_SQ_MAX_IDX_M,
3475 		       V2_QPC_BYTE_200_SQ_MAX_IDX_S, 0);
3476 	roce_set_field(qpc_mask->byte_200_sq_max,
3477 		       V2_QPC_BYTE_200_LCL_OPERATED_CNT_M,
3478 		       V2_QPC_BYTE_200_LCL_OPERATED_CNT_S, 0);
3479 
3480 	roce_set_bit(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_PKT_RNR_FLG_S, 0);
3481 	roce_set_bit(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_PKT_RTY_FLG_S, 0);
3482 
3483 	roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_CHECK_FLG_M,
3484 		       V2_QPC_BYTE_212_CHECK_FLG_S, 0);
3485 
3486 	qpc_mask->sq_timer = 0;
3487 
3488 	roce_set_field(qpc_mask->byte_220_retry_psn_msn,
3489 		       V2_QPC_BYTE_220_RETRY_MSG_MSN_M,
3490 		       V2_QPC_BYTE_220_RETRY_MSG_MSN_S, 0);
3491 	roce_set_field(qpc_mask->byte_232_irrl_sge,
3492 		       V2_QPC_BYTE_232_IRRL_SGE_IDX_M,
3493 		       V2_QPC_BYTE_232_IRRL_SGE_IDX_S, 0);
3494 
3495 	roce_set_bit(qpc_mask->byte_232_irrl_sge, V2_QPC_BYTE_232_SO_LP_VLD_S,
3496 		     0);
3497 	roce_set_bit(qpc_mask->byte_232_irrl_sge,
3498 		     V2_QPC_BYTE_232_FENCE_LP_VLD_S, 0);
3499 	roce_set_bit(qpc_mask->byte_232_irrl_sge, V2_QPC_BYTE_232_IRRL_LP_VLD_S,
3500 		     0);
3501 
3502 	qpc_mask->irrl_cur_sge_offset = 0;
3503 
3504 	roce_set_field(qpc_mask->byte_240_irrl_tail,
3505 		       V2_QPC_BYTE_240_IRRL_TAIL_REAL_M,
3506 		       V2_QPC_BYTE_240_IRRL_TAIL_REAL_S, 0);
3507 	roce_set_field(qpc_mask->byte_240_irrl_tail,
3508 		       V2_QPC_BYTE_240_IRRL_TAIL_RD_M,
3509 		       V2_QPC_BYTE_240_IRRL_TAIL_RD_S, 0);
3510 	roce_set_field(qpc_mask->byte_240_irrl_tail,
3511 		       V2_QPC_BYTE_240_RX_ACK_MSN_M,
3512 		       V2_QPC_BYTE_240_RX_ACK_MSN_S, 0);
3513 
3514 	roce_set_field(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_IRRL_PSN_M,
3515 		       V2_QPC_BYTE_248_IRRL_PSN_S, 0);
3516 	roce_set_bit(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_ACK_PSN_ERR_S,
3517 		     0);
3518 	roce_set_field(qpc_mask->byte_248_ack_psn,
3519 		       V2_QPC_BYTE_248_ACK_LAST_OPTYPE_M,
3520 		       V2_QPC_BYTE_248_ACK_LAST_OPTYPE_S, 0);
3521 	roce_set_bit(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_IRRL_PSN_VLD_S,
3522 		     0);
3523 	roce_set_bit(qpc_mask->byte_248_ack_psn,
3524 		     V2_QPC_BYTE_248_RNR_RETRY_FLAG_S, 0);
3525 	roce_set_bit(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_CQ_ERR_IND_S,
3526 		     0);
3527 
3528 	hr_qp->access_flags = attr->qp_access_flags;
3529 	roce_set_field(context->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,
3530 		       V2_QPC_BYTE_252_TX_CQN_S, to_hr_cq(ibqp->send_cq)->cqn);
3531 	roce_set_field(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,
3532 		       V2_QPC_BYTE_252_TX_CQN_S, 0);
3533 
3534 	roce_set_field(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_ERR_TYPE_M,
3535 		       V2_QPC_BYTE_252_ERR_TYPE_S, 0);
3536 
3537 	roce_set_field(qpc_mask->byte_256_sqflush_rqcqe,
3538 		       V2_QPC_BYTE_256_RQ_CQE_IDX_M,
3539 		       V2_QPC_BYTE_256_RQ_CQE_IDX_S, 0);
3540 	roce_set_field(qpc_mask->byte_256_sqflush_rqcqe,
3541 		       V2_QPC_BYTE_256_SQ_FLUSH_IDX_M,
3542 		       V2_QPC_BYTE_256_SQ_FLUSH_IDX_S, 0);
3543 }
3544 
3545 static void modify_qp_init_to_init(struct ib_qp *ibqp,
3546 				   const struct ib_qp_attr *attr, int attr_mask,
3547 				   struct hns_roce_v2_qp_context *context,
3548 				   struct hns_roce_v2_qp_context *qpc_mask)
3549 {
3550 	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
3551 
3552 	/*
3553 	 * In v2 engine, software pass context and context mask to hardware
3554 	 * when modifying qp. If software need modify some fields in context,
3555 	 * we should set all bits of the relevant fields in context mask to
3556 	 * 0 at the same time, else set them to 0x1.
3557 	 */
3558 	roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,
3559 		       V2_QPC_BYTE_4_TST_S, to_hr_qp_type(hr_qp->ibqp.qp_type));
3560 	roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,
3561 		       V2_QPC_BYTE_4_TST_S, 0);
3562 
3563 	if (attr_mask & IB_QP_ACCESS_FLAGS) {
3564 		roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
3565 			     !!(attr->qp_access_flags & IB_ACCESS_REMOTE_READ));
3566 		roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
3567 			     0);
3568 
3569 		roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
3570 			     !!(attr->qp_access_flags &
3571 			     IB_ACCESS_REMOTE_WRITE));
3572 		roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
3573 			     0);
3574 
3575 		roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
3576 			     !!(attr->qp_access_flags &
3577 			     IB_ACCESS_REMOTE_ATOMIC));
3578 		roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
3579 			     0);
3580 	} else {
3581 		roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
3582 			     !!(hr_qp->access_flags & IB_ACCESS_REMOTE_READ));
3583 		roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
3584 			     0);
3585 
3586 		roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
3587 			     !!(hr_qp->access_flags & IB_ACCESS_REMOTE_WRITE));
3588 		roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
3589 			     0);
3590 
3591 		roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
3592 			     !!(hr_qp->access_flags & IB_ACCESS_REMOTE_ATOMIC));
3593 		roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
3594 			     0);
3595 	}
3596 
3597 	roce_set_field(context->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,
3598 		       V2_QPC_BYTE_16_PD_S, to_hr_pd(ibqp->pd)->pdn);
3599 	roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,
3600 		       V2_QPC_BYTE_16_PD_S, 0);
3601 
3602 	roce_set_field(context->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M,
3603 		       V2_QPC_BYTE_80_RX_CQN_S, to_hr_cq(ibqp->recv_cq)->cqn);
3604 	roce_set_field(qpc_mask->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M,
3605 		       V2_QPC_BYTE_80_RX_CQN_S, 0);
3606 
3607 	roce_set_field(context->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,
3608 		       V2_QPC_BYTE_252_TX_CQN_S, to_hr_cq(ibqp->send_cq)->cqn);
3609 	roce_set_field(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,
3610 		       V2_QPC_BYTE_252_TX_CQN_S, 0);
3611 
3612 	if (ibqp->srq) {
3613 		roce_set_bit(context->byte_76_srqn_op_en,
3614 			     V2_QPC_BYTE_76_SRQ_EN_S, 1);
3615 		roce_set_bit(qpc_mask->byte_76_srqn_op_en,
3616 			     V2_QPC_BYTE_76_SRQ_EN_S, 0);
3617 		roce_set_field(context->byte_76_srqn_op_en,
3618 			       V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S,
3619 			       to_hr_srq(ibqp->srq)->srqn);
3620 		roce_set_field(qpc_mask->byte_76_srqn_op_en,
3621 			       V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S, 0);
3622 	}
3623 
3624 	roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
3625 		       V2_QPC_BYTE_4_SQPN_S, hr_qp->qpn);
3626 	roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
3627 		       V2_QPC_BYTE_4_SQPN_S, 0);
3628 
3629 	if (attr_mask & IB_QP_DEST_QPN) {
3630 		roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_DQPN_M,
3631 			       V2_QPC_BYTE_56_DQPN_S, hr_qp->qpn);
3632 		roce_set_field(qpc_mask->byte_56_dqpn_err,
3633 			       V2_QPC_BYTE_56_DQPN_M, V2_QPC_BYTE_56_DQPN_S, 0);
3634 	}
3635 }
3636 
3637 static bool check_wqe_rq_mtt_count(struct hns_roce_dev *hr_dev,
3638 				   struct hns_roce_qp *hr_qp, int mtt_cnt,
3639 				   u32 page_size)
3640 {
3641 	struct device *dev = hr_dev->dev;
3642 
3643 	if (hr_qp->rq.wqe_cnt < 1)
3644 		return true;
3645 
3646 	if (mtt_cnt < 1) {
3647 		dev_err(dev, "qp(0x%lx) rqwqe buf ba find failed\n",
3648 			hr_qp->qpn);
3649 		return false;
3650 	}
3651 
3652 	if (mtt_cnt < MTT_MIN_COUNT &&
3653 		(hr_qp->rq.offset + page_size) < hr_qp->buff_size) {
3654 		dev_err(dev, "qp(0x%lx) next rqwqe buf ba find failed\n",
3655 			hr_qp->qpn);
3656 		return false;
3657 	}
3658 
3659 	return true;
3660 }
3661 
3662 static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
3663 				 const struct ib_qp_attr *attr, int attr_mask,
3664 				 struct hns_roce_v2_qp_context *context,
3665 				 struct hns_roce_v2_qp_context *qpc_mask)
3666 {
3667 	const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
3668 	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
3669 	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
3670 	struct device *dev = hr_dev->dev;
3671 	u64 mtts[MTT_MIN_COUNT] = { 0 };
3672 	dma_addr_t dma_handle_3;
3673 	dma_addr_t dma_handle_2;
3674 	u64 wqe_sge_ba;
3675 	u32 page_size;
3676 	u8 port_num;
3677 	u64 *mtts_3;
3678 	u64 *mtts_2;
3679 	int count;
3680 	u8 *dmac;
3681 	u8 *smac;
3682 	int port;
3683 
3684 	/* Search qp buf's mtts */
3685 	page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT);
3686 	count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr,
3687 				  hr_qp->rq.offset / page_size, mtts,
3688 				  MTT_MIN_COUNT, &wqe_sge_ba);
3689 	if (!ibqp->srq)
3690 		if (!check_wqe_rq_mtt_count(hr_dev, hr_qp, count, page_size))
3691 			return -EINVAL;
3692 
3693 	/* Search IRRL's mtts */
3694 	mtts_2 = hns_roce_table_find(hr_dev, &hr_dev->qp_table.irrl_table,
3695 				     hr_qp->qpn, &dma_handle_2);
3696 	if (!mtts_2) {
3697 		dev_err(dev, "qp irrl_table find failed\n");
3698 		return -EINVAL;
3699 	}
3700 
3701 	/* Search TRRL's mtts */
3702 	mtts_3 = hns_roce_table_find(hr_dev, &hr_dev->qp_table.trrl_table,
3703 				     hr_qp->qpn, &dma_handle_3);
3704 	if (!mtts_3) {
3705 		dev_err(dev, "qp trrl_table find failed\n");
3706 		return -EINVAL;
3707 	}
3708 
3709 	if (attr_mask & IB_QP_ALT_PATH) {
3710 		dev_err(dev, "INIT2RTR attr_mask (0x%x) error\n", attr_mask);
3711 		return -EINVAL;
3712 	}
3713 
3714 	dmac = (u8 *)attr->ah_attr.roce.dmac;
3715 	context->wqe_sge_ba = cpu_to_le32(wqe_sge_ba >> 3);
3716 	qpc_mask->wqe_sge_ba = 0;
3717 
3718 	/*
3719 	 * In v2 engine, software pass context and context mask to hardware
3720 	 * when modifying qp. If software need modify some fields in context,
3721 	 * we should set all bits of the relevant fields in context mask to
3722 	 * 0 at the same time, else set them to 0x1.
3723 	 */
3724 	roce_set_field(context->byte_12_sq_hop, V2_QPC_BYTE_12_WQE_SGE_BA_M,
3725 		       V2_QPC_BYTE_12_WQE_SGE_BA_S, wqe_sge_ba >> (32 + 3));
3726 	roce_set_field(qpc_mask->byte_12_sq_hop, V2_QPC_BYTE_12_WQE_SGE_BA_M,
3727 		       V2_QPC_BYTE_12_WQE_SGE_BA_S, 0);
3728 
3729 	roce_set_field(context->byte_12_sq_hop, V2_QPC_BYTE_12_SQ_HOP_NUM_M,
3730 		       V2_QPC_BYTE_12_SQ_HOP_NUM_S,
3731 		       hr_dev->caps.wqe_sq_hop_num == HNS_ROCE_HOP_NUM_0 ?
3732 		       0 : hr_dev->caps.wqe_sq_hop_num);
3733 	roce_set_field(qpc_mask->byte_12_sq_hop, V2_QPC_BYTE_12_SQ_HOP_NUM_M,
3734 		       V2_QPC_BYTE_12_SQ_HOP_NUM_S, 0);
3735 
3736 	roce_set_field(context->byte_20_smac_sgid_idx,
3737 		       V2_QPC_BYTE_20_SGE_HOP_NUM_M,
3738 		       V2_QPC_BYTE_20_SGE_HOP_NUM_S,
3739 		       ((ibqp->qp_type == IB_QPT_GSI) ||
3740 		       hr_qp->sq.max_gs > HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) ?
3741 		       hr_dev->caps.wqe_sge_hop_num : 0);
3742 	roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
3743 		       V2_QPC_BYTE_20_SGE_HOP_NUM_M,
3744 		       V2_QPC_BYTE_20_SGE_HOP_NUM_S, 0);
3745 
3746 	roce_set_field(context->byte_20_smac_sgid_idx,
3747 		       V2_QPC_BYTE_20_RQ_HOP_NUM_M,
3748 		       V2_QPC_BYTE_20_RQ_HOP_NUM_S,
3749 		       hr_dev->caps.wqe_rq_hop_num == HNS_ROCE_HOP_NUM_0 ?
3750 		       0 : hr_dev->caps.wqe_rq_hop_num);
3751 	roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
3752 		       V2_QPC_BYTE_20_RQ_HOP_NUM_M,
3753 		       V2_QPC_BYTE_20_RQ_HOP_NUM_S, 0);
3754 
3755 	roce_set_field(context->byte_16_buf_ba_pg_sz,
3756 		       V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M,
3757 		       V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S,
3758 		       hr_qp->wqe_bt_pg_shift + PG_SHIFT_OFFSET);
3759 	roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz,
3760 		       V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M,
3761 		       V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S, 0);
3762 
3763 	roce_set_field(context->byte_16_buf_ba_pg_sz,
3764 		       V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M,
3765 		       V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S,
3766 		       hr_dev->caps.mtt_buf_pg_sz + PG_SHIFT_OFFSET);
3767 	roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz,
3768 		       V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M,
3769 		       V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S, 0);
3770 
3771 	context->rq_cur_blk_addr = cpu_to_le32(mtts[0] >> PAGE_ADDR_SHIFT);
3772 	qpc_mask->rq_cur_blk_addr = 0;
3773 
3774 	roce_set_field(context->byte_92_srq_info,
3775 		       V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M,
3776 		       V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S,
3777 		       mtts[0] >> (32 + PAGE_ADDR_SHIFT));
3778 	roce_set_field(qpc_mask->byte_92_srq_info,
3779 		       V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M,
3780 		       V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S, 0);
3781 
3782 	context->rq_nxt_blk_addr = cpu_to_le32(mtts[1] >> PAGE_ADDR_SHIFT);
3783 	qpc_mask->rq_nxt_blk_addr = 0;
3784 
3785 	roce_set_field(context->byte_104_rq_sge,
3786 		       V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M,
3787 		       V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S,
3788 		       mtts[1] >> (32 + PAGE_ADDR_SHIFT));
3789 	roce_set_field(qpc_mask->byte_104_rq_sge,
3790 		       V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M,
3791 		       V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S, 0);
3792 
3793 	roce_set_field(context->byte_132_trrl, V2_QPC_BYTE_132_TRRL_BA_M,
3794 		       V2_QPC_BYTE_132_TRRL_BA_S, dma_handle_3 >> 4);
3795 	roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_BA_M,
3796 		       V2_QPC_BYTE_132_TRRL_BA_S, 0);
3797 	context->trrl_ba = cpu_to_le32(dma_handle_3 >> (16 + 4));
3798 	qpc_mask->trrl_ba = 0;
3799 	roce_set_field(context->byte_140_raq, V2_QPC_BYTE_140_TRRL_BA_M,
3800 		       V2_QPC_BYTE_140_TRRL_BA_S,
3801 		       (u32)(dma_handle_3 >> (32 + 16 + 4)));
3802 	roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_TRRL_BA_M,
3803 		       V2_QPC_BYTE_140_TRRL_BA_S, 0);
3804 
3805 	context->irrl_ba = cpu_to_le32(dma_handle_2 >> 6);
3806 	qpc_mask->irrl_ba = 0;
3807 	roce_set_field(context->byte_208_irrl, V2_QPC_BYTE_208_IRRL_BA_M,
3808 		       V2_QPC_BYTE_208_IRRL_BA_S,
3809 		       dma_handle_2 >> (32 + 6));
3810 	roce_set_field(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_IRRL_BA_M,
3811 		       V2_QPC_BYTE_208_IRRL_BA_S, 0);
3812 
3813 	roce_set_bit(context->byte_208_irrl, V2_QPC_BYTE_208_RMT_E2E_S, 1);
3814 	roce_set_bit(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_RMT_E2E_S, 0);
3815 
3816 	roce_set_bit(context->byte_252_err_txcqn, V2_QPC_BYTE_252_SIG_TYPE_S,
3817 		     hr_qp->sq_signal_bits);
3818 	roce_set_bit(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_SIG_TYPE_S,
3819 		     0);
3820 
3821 	port = (attr_mask & IB_QP_PORT) ? (attr->port_num - 1) : hr_qp->port;
3822 
3823 	smac = (u8 *)hr_dev->dev_addr[port];
3824 	/* when dmac equals smac or loop_idc is 1, it should loopback */
3825 	if (ether_addr_equal_unaligned(dmac, smac) ||
3826 	    hr_dev->loop_idc == 0x1) {
3827 		roce_set_bit(context->byte_28_at_fl, V2_QPC_BYTE_28_LBI_S, 1);
3828 		roce_set_bit(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_LBI_S, 0);
3829 	}
3830 
3831 	if (attr_mask & IB_QP_DEST_QPN) {
3832 		roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_DQPN_M,
3833 			       V2_QPC_BYTE_56_DQPN_S, attr->dest_qp_num);
3834 		roce_set_field(qpc_mask->byte_56_dqpn_err,
3835 			       V2_QPC_BYTE_56_DQPN_M, V2_QPC_BYTE_56_DQPN_S, 0);
3836 	}
3837 
3838 	/* Configure GID index */
3839 	port_num = rdma_ah_get_port_num(&attr->ah_attr);
3840 	roce_set_field(context->byte_20_smac_sgid_idx,
3841 		       V2_QPC_BYTE_20_SGID_IDX_M,
3842 		       V2_QPC_BYTE_20_SGID_IDX_S,
3843 		       hns_get_gid_index(hr_dev, port_num - 1,
3844 					 grh->sgid_index));
3845 	roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
3846 		       V2_QPC_BYTE_20_SGID_IDX_M,
3847 		       V2_QPC_BYTE_20_SGID_IDX_S, 0);
3848 	memcpy(&(context->dmac), dmac, sizeof(u32));
3849 	roce_set_field(context->byte_52_udpspn_dmac, V2_QPC_BYTE_52_DMAC_M,
3850 		       V2_QPC_BYTE_52_DMAC_S, *((u16 *)(&dmac[4])));
3851 	qpc_mask->dmac = 0;
3852 	roce_set_field(qpc_mask->byte_52_udpspn_dmac, V2_QPC_BYTE_52_DMAC_M,
3853 		       V2_QPC_BYTE_52_DMAC_S, 0);
3854 
3855 	/* mtu*(2^LP_PKTN_INI) should not bigger than 1 message length 64kb */
3856 	roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_LP_PKTN_INI_M,
3857 		       V2_QPC_BYTE_56_LP_PKTN_INI_S, 4);
3858 	roce_set_field(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_LP_PKTN_INI_M,
3859 		       V2_QPC_BYTE_56_LP_PKTN_INI_S, 0);
3860 
3861 	if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_UD)
3862 		roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M,
3863 			       V2_QPC_BYTE_24_MTU_S, IB_MTU_4096);
3864 	else if (attr_mask & IB_QP_PATH_MTU)
3865 		roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M,
3866 			       V2_QPC_BYTE_24_MTU_S, attr->path_mtu);
3867 
3868 	roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M,
3869 		       V2_QPC_BYTE_24_MTU_S, 0);
3870 
3871 	roce_set_field(context->byte_84_rq_ci_pi,
3872 		       V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
3873 		       V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, hr_qp->rq.head);
3874 	roce_set_field(qpc_mask->byte_84_rq_ci_pi,
3875 		       V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
3876 		       V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0);
3877 
3878 	roce_set_field(qpc_mask->byte_84_rq_ci_pi,
3879 		       V2_QPC_BYTE_84_RQ_CONSUMER_IDX_M,
3880 		       V2_QPC_BYTE_84_RQ_CONSUMER_IDX_S, 0);
3881 	roce_set_bit(qpc_mask->byte_108_rx_reqepsn,
3882 		     V2_QPC_BYTE_108_RX_REQ_PSN_ERR_S, 0);
3883 	roce_set_field(qpc_mask->byte_96_rx_reqmsn, V2_QPC_BYTE_96_RX_REQ_MSN_M,
3884 		       V2_QPC_BYTE_96_RX_REQ_MSN_S, 0);
3885 	roce_set_field(qpc_mask->byte_108_rx_reqepsn,
3886 		       V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_M,
3887 		       V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_S, 0);
3888 
3889 	context->rq_rnr_timer = 0;
3890 	qpc_mask->rq_rnr_timer = 0;
3891 
3892 	roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_HEAD_MAX_M,
3893 		       V2_QPC_BYTE_132_TRRL_HEAD_MAX_S, 0);
3894 	roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_TAIL_MAX_M,
3895 		       V2_QPC_BYTE_132_TRRL_TAIL_MAX_S, 0);
3896 
3897 	/* rocee send 2^lp_sgen_ini segs every time */
3898 	roce_set_field(context->byte_168_irrl_idx,
3899 		       V2_QPC_BYTE_168_LP_SGEN_INI_M,
3900 		       V2_QPC_BYTE_168_LP_SGEN_INI_S, 3);
3901 	roce_set_field(qpc_mask->byte_168_irrl_idx,
3902 		       V2_QPC_BYTE_168_LP_SGEN_INI_M,
3903 		       V2_QPC_BYTE_168_LP_SGEN_INI_S, 0);
3904 
3905 	return 0;
3906 }
3907 
3908 static int modify_qp_rtr_to_rts(struct ib_qp *ibqp,
3909 				const struct ib_qp_attr *attr, int attr_mask,
3910 				struct hns_roce_v2_qp_context *context,
3911 				struct hns_roce_v2_qp_context *qpc_mask)
3912 {
3913 	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
3914 	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
3915 	struct device *dev = hr_dev->dev;
3916 	u64 sge_cur_blk = 0;
3917 	u64 sq_cur_blk = 0;
3918 	u32 page_size;
3919 	int count;
3920 
3921 	/* Search qp buf's mtts */
3922 	count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, 0, &sq_cur_blk, 1, NULL);
3923 	if (count < 1) {
3924 		dev_err(dev, "qp(0x%lx) buf pa find failed\n", hr_qp->qpn);
3925 		return -EINVAL;
3926 	}
3927 
3928 	if (hr_qp->sge.offset) {
3929 		page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT);
3930 		count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr,
3931 					  hr_qp->sge.offset / page_size,
3932 					  &sge_cur_blk, 1, NULL);
3933 		if (count < 1) {
3934 			dev_err(dev, "qp(0x%lx) sge pa find failed\n",
3935 				hr_qp->qpn);
3936 			return -EINVAL;
3937 		}
3938 	}
3939 
3940 	/* Not support alternate path and path migration */
3941 	if ((attr_mask & IB_QP_ALT_PATH) ||
3942 	    (attr_mask & IB_QP_PATH_MIG_STATE)) {
3943 		dev_err(dev, "RTR2RTS attr_mask (0x%x)error\n", attr_mask);
3944 		return -EINVAL;
3945 	}
3946 
3947 	/*
3948 	 * In v2 engine, software pass context and context mask to hardware
3949 	 * when modifying qp. If software need modify some fields in context,
3950 	 * we should set all bits of the relevant fields in context mask to
3951 	 * 0 at the same time, else set them to 0x1.
3952 	 */
3953 	context->sq_cur_blk_addr = cpu_to_le32(sq_cur_blk >> PAGE_ADDR_SHIFT);
3954 	roce_set_field(context->byte_168_irrl_idx,
3955 		       V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M,
3956 		       V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S,
3957 		       sq_cur_blk >> (32 + PAGE_ADDR_SHIFT));
3958 	qpc_mask->sq_cur_blk_addr = 0;
3959 	roce_set_field(qpc_mask->byte_168_irrl_idx,
3960 		       V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M,
3961 		       V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S, 0);
3962 
3963 	context->sq_cur_sge_blk_addr = ((ibqp->qp_type == IB_QPT_GSI) ||
3964 		       hr_qp->sq.max_gs > HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) ?
3965 		       cpu_to_le32(sge_cur_blk >>
3966 		       PAGE_ADDR_SHIFT) : 0;
3967 	roce_set_field(context->byte_184_irrl_idx,
3968 		       V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M,
3969 		       V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S,
3970 		       ((ibqp->qp_type == IB_QPT_GSI) || hr_qp->sq.max_gs >
3971 		       HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) ?
3972 		       (sge_cur_blk >>
3973 		       (32 + PAGE_ADDR_SHIFT)) : 0);
3974 	qpc_mask->sq_cur_sge_blk_addr = 0;
3975 	roce_set_field(qpc_mask->byte_184_irrl_idx,
3976 		       V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M,
3977 		       V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S, 0);
3978 
3979 	context->rx_sq_cur_blk_addr =
3980 		cpu_to_le32(sq_cur_blk >> PAGE_ADDR_SHIFT);
3981 	roce_set_field(context->byte_232_irrl_sge,
3982 		       V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M,
3983 		       V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_S,
3984 		       sq_cur_blk >> (32 + PAGE_ADDR_SHIFT));
3985 	qpc_mask->rx_sq_cur_blk_addr = 0;
3986 	roce_set_field(qpc_mask->byte_232_irrl_sge,
3987 		       V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M,
3988 		       V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_S, 0);
3989 
3990 	/*
3991 	 * Set some fields in context to zero, Because the default values
3992 	 * of all fields in context are zero, we need not set them to 0 again.
3993 	 * but we should set the relevant fields of context mask to 0.
3994 	 */
3995 	roce_set_field(qpc_mask->byte_232_irrl_sge,
3996 		       V2_QPC_BYTE_232_IRRL_SGE_IDX_M,
3997 		       V2_QPC_BYTE_232_IRRL_SGE_IDX_S, 0);
3998 
3999 	roce_set_field(qpc_mask->byte_240_irrl_tail,
4000 		       V2_QPC_BYTE_240_RX_ACK_MSN_M,
4001 		       V2_QPC_BYTE_240_RX_ACK_MSN_S, 0);
4002 
4003 	roce_set_field(qpc_mask->byte_248_ack_psn,
4004 		       V2_QPC_BYTE_248_ACK_LAST_OPTYPE_M,
4005 		       V2_QPC_BYTE_248_ACK_LAST_OPTYPE_S, 0);
4006 	roce_set_bit(qpc_mask->byte_248_ack_psn,
4007 		     V2_QPC_BYTE_248_IRRL_PSN_VLD_S, 0);
4008 	roce_set_field(qpc_mask->byte_248_ack_psn,
4009 		       V2_QPC_BYTE_248_IRRL_PSN_M,
4010 		       V2_QPC_BYTE_248_IRRL_PSN_S, 0);
4011 
4012 	roce_set_field(qpc_mask->byte_240_irrl_tail,
4013 		       V2_QPC_BYTE_240_IRRL_TAIL_REAL_M,
4014 		       V2_QPC_BYTE_240_IRRL_TAIL_REAL_S, 0);
4015 
4016 	roce_set_field(qpc_mask->byte_220_retry_psn_msn,
4017 		       V2_QPC_BYTE_220_RETRY_MSG_MSN_M,
4018 		       V2_QPC_BYTE_220_RETRY_MSG_MSN_S, 0);
4019 
4020 	roce_set_bit(qpc_mask->byte_248_ack_psn,
4021 		     V2_QPC_BYTE_248_RNR_RETRY_FLAG_S, 0);
4022 
4023 	roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_CHECK_FLG_M,
4024 		       V2_QPC_BYTE_212_CHECK_FLG_S, 0);
4025 
4026 	roce_set_field(context->byte_212_lsn, V2_QPC_BYTE_212_LSN_M,
4027 		       V2_QPC_BYTE_212_LSN_S, 0x100);
4028 	roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_LSN_M,
4029 		       V2_QPC_BYTE_212_LSN_S, 0);
4030 
4031 	roce_set_field(qpc_mask->byte_196_sq_psn, V2_QPC_BYTE_196_IRRL_HEAD_M,
4032 		       V2_QPC_BYTE_196_IRRL_HEAD_S, 0);
4033 
4034 	return 0;
4035 }
4036 
4037 static inline bool hns_roce_v2_check_qp_stat(enum ib_qp_state cur_state,
4038 					     enum ib_qp_state new_state)
4039 {
4040 
4041 	if ((cur_state != IB_QPS_RESET &&
4042 	    (new_state == IB_QPS_ERR || new_state == IB_QPS_RESET)) ||
4043 	    ((cur_state == IB_QPS_RTS || cur_state == IB_QPS_SQD) &&
4044 	    (new_state == IB_QPS_RTS || new_state == IB_QPS_SQD)) ||
4045 	    (cur_state == IB_QPS_SQE && new_state == IB_QPS_RTS))
4046 		return true;
4047 
4048 	return false;
4049 
4050 }
4051 
4052 static int hns_roce_v2_set_path(struct ib_qp *ibqp,
4053 				const struct ib_qp_attr *attr,
4054 				int attr_mask,
4055 				struct hns_roce_v2_qp_context *context,
4056 				struct hns_roce_v2_qp_context *qpc_mask)
4057 {
4058 	const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
4059 	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4060 	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4061 	const struct ib_gid_attr *gid_attr = NULL;
4062 	int is_roce_protocol;
4063 	u16 vlan_id = 0xffff;
4064 	bool is_udp = false;
4065 	u8 ib_port;
4066 	u8 hr_port;
4067 	int ret;
4068 
4069 	ib_port = (attr_mask & IB_QP_PORT) ? attr->port_num : hr_qp->port + 1;
4070 	hr_port = ib_port - 1;
4071 	is_roce_protocol = rdma_cap_eth_ah(&hr_dev->ib_dev, ib_port) &&
4072 			   rdma_ah_get_ah_flags(&attr->ah_attr) & IB_AH_GRH;
4073 
4074 	if (is_roce_protocol) {
4075 		gid_attr = attr->ah_attr.grh.sgid_attr;
4076 		ret = rdma_read_gid_l2_fields(gid_attr, &vlan_id, NULL);
4077 		if (ret)
4078 			return ret;
4079 
4080 		if (gid_attr)
4081 			is_udp = (gid_attr->gid_type ==
4082 				 IB_GID_TYPE_ROCE_UDP_ENCAP);
4083 	}
4084 
4085 	if (vlan_id < VLAN_N_VID) {
4086 		roce_set_bit(context->byte_76_srqn_op_en,
4087 			     V2_QPC_BYTE_76_RQ_VLAN_EN_S, 1);
4088 		roce_set_bit(qpc_mask->byte_76_srqn_op_en,
4089 			     V2_QPC_BYTE_76_RQ_VLAN_EN_S, 0);
4090 		roce_set_bit(context->byte_168_irrl_idx,
4091 			     V2_QPC_BYTE_168_SQ_VLAN_EN_S, 1);
4092 		roce_set_bit(qpc_mask->byte_168_irrl_idx,
4093 			     V2_QPC_BYTE_168_SQ_VLAN_EN_S, 0);
4094 	}
4095 
4096 	roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_ID_M,
4097 		       V2_QPC_BYTE_24_VLAN_ID_S, vlan_id);
4098 	roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_ID_M,
4099 		       V2_QPC_BYTE_24_VLAN_ID_S, 0);
4100 
4101 	if (grh->sgid_index >= hr_dev->caps.gid_table_len[hr_port]) {
4102 		dev_err(hr_dev->dev, "sgid_index(%u) too large. max is %d\n",
4103 			grh->sgid_index, hr_dev->caps.gid_table_len[hr_port]);
4104 		return -EINVAL;
4105 	}
4106 
4107 	if (attr->ah_attr.type != RDMA_AH_ATTR_TYPE_ROCE) {
4108 		dev_err(hr_dev->dev, "ah attr is not RDMA roce type\n");
4109 		return -EINVAL;
4110 	}
4111 
4112 	roce_set_field(context->byte_52_udpspn_dmac, V2_QPC_BYTE_52_UDPSPN_M,
4113 		       V2_QPC_BYTE_52_UDPSPN_S,
4114 		       is_udp ? 0x12b7 : 0);
4115 
4116 	roce_set_field(qpc_mask->byte_52_udpspn_dmac, V2_QPC_BYTE_52_UDPSPN_M,
4117 		       V2_QPC_BYTE_52_UDPSPN_S, 0);
4118 
4119 	roce_set_field(context->byte_20_smac_sgid_idx,
4120 		       V2_QPC_BYTE_20_SGID_IDX_M, V2_QPC_BYTE_20_SGID_IDX_S,
4121 		       grh->sgid_index);
4122 
4123 	roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
4124 		       V2_QPC_BYTE_20_SGID_IDX_M, V2_QPC_BYTE_20_SGID_IDX_S, 0);
4125 
4126 	roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_HOP_LIMIT_M,
4127 		       V2_QPC_BYTE_24_HOP_LIMIT_S, grh->hop_limit);
4128 	roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_HOP_LIMIT_M,
4129 		       V2_QPC_BYTE_24_HOP_LIMIT_S, 0);
4130 
4131 	if (hr_dev->pci_dev->revision == 0x21 && is_udp)
4132 		roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M,
4133 			       V2_QPC_BYTE_24_TC_S, grh->traffic_class >> 2);
4134 	else
4135 		roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M,
4136 			       V2_QPC_BYTE_24_TC_S, grh->traffic_class);
4137 	roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M,
4138 		       V2_QPC_BYTE_24_TC_S, 0);
4139 	roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_FL_M,
4140 		       V2_QPC_BYTE_28_FL_S, grh->flow_label);
4141 	roce_set_field(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_FL_M,
4142 		       V2_QPC_BYTE_28_FL_S, 0);
4143 	memcpy(context->dgid, grh->dgid.raw, sizeof(grh->dgid.raw));
4144 	memset(qpc_mask->dgid, 0, sizeof(grh->dgid.raw));
4145 	roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_SL_M,
4146 		       V2_QPC_BYTE_28_SL_S, rdma_ah_get_sl(&attr->ah_attr));
4147 	roce_set_field(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_SL_M,
4148 		       V2_QPC_BYTE_28_SL_S, 0);
4149 	hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr);
4150 
4151 	return 0;
4152 }
4153 
4154 static int hns_roce_v2_set_abs_fields(struct ib_qp *ibqp,
4155 				      const struct ib_qp_attr *attr,
4156 				      int attr_mask,
4157 				      enum ib_qp_state cur_state,
4158 				      enum ib_qp_state new_state,
4159 				      struct hns_roce_v2_qp_context *context,
4160 				      struct hns_roce_v2_qp_context *qpc_mask)
4161 {
4162 	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4163 	int ret = 0;
4164 
4165 	if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
4166 		memset(qpc_mask, 0, sizeof(*qpc_mask));
4167 		modify_qp_reset_to_init(ibqp, attr, attr_mask, context,
4168 					qpc_mask);
4169 	} else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) {
4170 		modify_qp_init_to_init(ibqp, attr, attr_mask, context,
4171 				       qpc_mask);
4172 	} else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
4173 		ret = modify_qp_init_to_rtr(ibqp, attr, attr_mask, context,
4174 					    qpc_mask);
4175 		if (ret)
4176 			goto out;
4177 	} else if (cur_state == IB_QPS_RTR && new_state == IB_QPS_RTS) {
4178 		ret = modify_qp_rtr_to_rts(ibqp, attr, attr_mask, context,
4179 					   qpc_mask);
4180 		if (ret)
4181 			goto out;
4182 	} else if (hns_roce_v2_check_qp_stat(cur_state, new_state)) {
4183 		/* Nothing */
4184 		;
4185 	} else {
4186 		dev_err(hr_dev->dev, "Illegal state for QP!\n");
4187 		ret = -EINVAL;
4188 		goto out;
4189 	}
4190 
4191 out:
4192 	return ret;
4193 }
4194 
4195 static int hns_roce_v2_set_opt_fields(struct ib_qp *ibqp,
4196 				      const struct ib_qp_attr *attr,
4197 				      int attr_mask,
4198 				      struct hns_roce_v2_qp_context *context,
4199 				      struct hns_roce_v2_qp_context *qpc_mask)
4200 {
4201 	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4202 	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4203 	int ret = 0;
4204 
4205 	if (attr_mask & IB_QP_AV) {
4206 		ret = hns_roce_v2_set_path(ibqp, attr, attr_mask, context,
4207 					   qpc_mask);
4208 		if (ret)
4209 			return ret;
4210 	}
4211 
4212 	if (attr_mask & IB_QP_TIMEOUT) {
4213 		if (attr->timeout < 31) {
4214 			roce_set_field(context->byte_28_at_fl,
4215 				       V2_QPC_BYTE_28_AT_M, V2_QPC_BYTE_28_AT_S,
4216 				       attr->timeout);
4217 			roce_set_field(qpc_mask->byte_28_at_fl,
4218 				       V2_QPC_BYTE_28_AT_M, V2_QPC_BYTE_28_AT_S,
4219 				       0);
4220 		} else {
4221 			dev_warn(hr_dev->dev,
4222 				 "Local ACK timeout shall be 0 to 30.\n");
4223 		}
4224 	}
4225 
4226 	if (attr_mask & IB_QP_RETRY_CNT) {
4227 		roce_set_field(context->byte_212_lsn,
4228 			       V2_QPC_BYTE_212_RETRY_NUM_INIT_M,
4229 			       V2_QPC_BYTE_212_RETRY_NUM_INIT_S,
4230 			       attr->retry_cnt);
4231 		roce_set_field(qpc_mask->byte_212_lsn,
4232 			       V2_QPC_BYTE_212_RETRY_NUM_INIT_M,
4233 			       V2_QPC_BYTE_212_RETRY_NUM_INIT_S, 0);
4234 
4235 		roce_set_field(context->byte_212_lsn,
4236 			       V2_QPC_BYTE_212_RETRY_CNT_M,
4237 			       V2_QPC_BYTE_212_RETRY_CNT_S,
4238 			       attr->retry_cnt);
4239 		roce_set_field(qpc_mask->byte_212_lsn,
4240 			       V2_QPC_BYTE_212_RETRY_CNT_M,
4241 			       V2_QPC_BYTE_212_RETRY_CNT_S, 0);
4242 	}
4243 
4244 	if (attr_mask & IB_QP_RNR_RETRY) {
4245 		roce_set_field(context->byte_244_rnr_rxack,
4246 			       V2_QPC_BYTE_244_RNR_NUM_INIT_M,
4247 			       V2_QPC_BYTE_244_RNR_NUM_INIT_S, attr->rnr_retry);
4248 		roce_set_field(qpc_mask->byte_244_rnr_rxack,
4249 			       V2_QPC_BYTE_244_RNR_NUM_INIT_M,
4250 			       V2_QPC_BYTE_244_RNR_NUM_INIT_S, 0);
4251 
4252 		roce_set_field(context->byte_244_rnr_rxack,
4253 			       V2_QPC_BYTE_244_RNR_CNT_M,
4254 			       V2_QPC_BYTE_244_RNR_CNT_S, attr->rnr_retry);
4255 		roce_set_field(qpc_mask->byte_244_rnr_rxack,
4256 			       V2_QPC_BYTE_244_RNR_CNT_M,
4257 			       V2_QPC_BYTE_244_RNR_CNT_S, 0);
4258 	}
4259 
4260 	/* RC&UC&UD required attr */
4261 	if (attr_mask & IB_QP_SQ_PSN) {
4262 		roce_set_field(context->byte_172_sq_psn,
4263 			       V2_QPC_BYTE_172_SQ_CUR_PSN_M,
4264 			       V2_QPC_BYTE_172_SQ_CUR_PSN_S, attr->sq_psn);
4265 		roce_set_field(qpc_mask->byte_172_sq_psn,
4266 			       V2_QPC_BYTE_172_SQ_CUR_PSN_M,
4267 			       V2_QPC_BYTE_172_SQ_CUR_PSN_S, 0);
4268 
4269 		roce_set_field(context->byte_196_sq_psn,
4270 			       V2_QPC_BYTE_196_SQ_MAX_PSN_M,
4271 			       V2_QPC_BYTE_196_SQ_MAX_PSN_S, attr->sq_psn);
4272 		roce_set_field(qpc_mask->byte_196_sq_psn,
4273 			       V2_QPC_BYTE_196_SQ_MAX_PSN_M,
4274 			       V2_QPC_BYTE_196_SQ_MAX_PSN_S, 0);
4275 
4276 		roce_set_field(context->byte_220_retry_psn_msn,
4277 			       V2_QPC_BYTE_220_RETRY_MSG_PSN_M,
4278 			       V2_QPC_BYTE_220_RETRY_MSG_PSN_S, attr->sq_psn);
4279 		roce_set_field(qpc_mask->byte_220_retry_psn_msn,
4280 			       V2_QPC_BYTE_220_RETRY_MSG_PSN_M,
4281 			       V2_QPC_BYTE_220_RETRY_MSG_PSN_S, 0);
4282 
4283 		roce_set_field(context->byte_224_retry_msg,
4284 			       V2_QPC_BYTE_224_RETRY_MSG_PSN_M,
4285 			       V2_QPC_BYTE_224_RETRY_MSG_PSN_S,
4286 			       attr->sq_psn >> V2_QPC_BYTE_220_RETRY_MSG_PSN_S);
4287 		roce_set_field(qpc_mask->byte_224_retry_msg,
4288 			       V2_QPC_BYTE_224_RETRY_MSG_PSN_M,
4289 			       V2_QPC_BYTE_224_RETRY_MSG_PSN_S, 0);
4290 
4291 		roce_set_field(context->byte_224_retry_msg,
4292 			       V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_M,
4293 			       V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_S,
4294 			       attr->sq_psn);
4295 		roce_set_field(qpc_mask->byte_224_retry_msg,
4296 			       V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_M,
4297 			       V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_S, 0);
4298 
4299 		roce_set_field(context->byte_244_rnr_rxack,
4300 			       V2_QPC_BYTE_244_RX_ACK_EPSN_M,
4301 			       V2_QPC_BYTE_244_RX_ACK_EPSN_S, attr->sq_psn);
4302 		roce_set_field(qpc_mask->byte_244_rnr_rxack,
4303 			       V2_QPC_BYTE_244_RX_ACK_EPSN_M,
4304 			       V2_QPC_BYTE_244_RX_ACK_EPSN_S, 0);
4305 	}
4306 
4307 	if ((attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) &&
4308 	     attr->max_dest_rd_atomic) {
4309 		roce_set_field(context->byte_140_raq, V2_QPC_BYTE_140_RR_MAX_M,
4310 			       V2_QPC_BYTE_140_RR_MAX_S,
4311 			       fls(attr->max_dest_rd_atomic - 1));
4312 		roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RR_MAX_M,
4313 			       V2_QPC_BYTE_140_RR_MAX_S, 0);
4314 	}
4315 
4316 	if ((attr_mask & IB_QP_MAX_QP_RD_ATOMIC) && attr->max_rd_atomic) {
4317 		roce_set_field(context->byte_208_irrl, V2_QPC_BYTE_208_SR_MAX_M,
4318 			       V2_QPC_BYTE_208_SR_MAX_S,
4319 			       fls(attr->max_rd_atomic - 1));
4320 		roce_set_field(qpc_mask->byte_208_irrl,
4321 			       V2_QPC_BYTE_208_SR_MAX_M,
4322 			       V2_QPC_BYTE_208_SR_MAX_S, 0);
4323 	}
4324 
4325 	if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC))
4326 		set_access_flags(hr_qp, context, qpc_mask, attr, attr_mask);
4327 
4328 	if (attr_mask & IB_QP_MIN_RNR_TIMER) {
4329 		roce_set_field(context->byte_80_rnr_rx_cqn,
4330 			       V2_QPC_BYTE_80_MIN_RNR_TIME_M,
4331 			       V2_QPC_BYTE_80_MIN_RNR_TIME_S,
4332 			       attr->min_rnr_timer);
4333 		roce_set_field(qpc_mask->byte_80_rnr_rx_cqn,
4334 			       V2_QPC_BYTE_80_MIN_RNR_TIME_M,
4335 			       V2_QPC_BYTE_80_MIN_RNR_TIME_S, 0);
4336 	}
4337 
4338 	/* RC&UC required attr */
4339 	if (attr_mask & IB_QP_RQ_PSN) {
4340 		roce_set_field(context->byte_108_rx_reqepsn,
4341 			       V2_QPC_BYTE_108_RX_REQ_EPSN_M,
4342 			       V2_QPC_BYTE_108_RX_REQ_EPSN_S, attr->rq_psn);
4343 		roce_set_field(qpc_mask->byte_108_rx_reqepsn,
4344 			       V2_QPC_BYTE_108_RX_REQ_EPSN_M,
4345 			       V2_QPC_BYTE_108_RX_REQ_EPSN_S, 0);
4346 
4347 		roce_set_field(context->byte_152_raq, V2_QPC_BYTE_152_RAQ_PSN_M,
4348 			       V2_QPC_BYTE_152_RAQ_PSN_S, attr->rq_psn - 1);
4349 		roce_set_field(qpc_mask->byte_152_raq,
4350 			       V2_QPC_BYTE_152_RAQ_PSN_M,
4351 			       V2_QPC_BYTE_152_RAQ_PSN_S, 0);
4352 	}
4353 
4354 	if (attr_mask & IB_QP_QKEY) {
4355 		context->qkey_xrcd = cpu_to_le32(attr->qkey);
4356 		qpc_mask->qkey_xrcd = 0;
4357 		hr_qp->qkey = attr->qkey;
4358 	}
4359 
4360 	return ret;
4361 }
4362 
4363 static void hns_roce_v2_record_opt_fields(struct ib_qp *ibqp,
4364 					  const struct ib_qp_attr *attr,
4365 					  int attr_mask)
4366 {
4367 	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4368 	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4369 
4370 	if (attr_mask & IB_QP_ACCESS_FLAGS)
4371 		hr_qp->atomic_rd_en = attr->qp_access_flags;
4372 
4373 	if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
4374 		hr_qp->resp_depth = attr->max_dest_rd_atomic;
4375 	if (attr_mask & IB_QP_PORT) {
4376 		hr_qp->port = attr->port_num - 1;
4377 		hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port];
4378 	}
4379 }
4380 
4381 static int hns_roce_v2_modify_qp(struct ib_qp *ibqp,
4382 				 const struct ib_qp_attr *attr,
4383 				 int attr_mask, enum ib_qp_state cur_state,
4384 				 enum ib_qp_state new_state)
4385 {
4386 	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4387 	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4388 	struct hns_roce_v2_qp_context ctx[2];
4389 	struct hns_roce_v2_qp_context *context = ctx;
4390 	struct hns_roce_v2_qp_context *qpc_mask = ctx + 1;
4391 	struct device *dev = hr_dev->dev;
4392 	int ret;
4393 
4394 	/*
4395 	 * In v2 engine, software pass context and context mask to hardware
4396 	 * when modifying qp. If software need modify some fields in context,
4397 	 * we should set all bits of the relevant fields in context mask to
4398 	 * 0 at the same time, else set them to 0x1.
4399 	 */
4400 	memset(context, 0, sizeof(*context));
4401 	memset(qpc_mask, 0xff, sizeof(*qpc_mask));
4402 	ret = hns_roce_v2_set_abs_fields(ibqp, attr, attr_mask, cur_state,
4403 					 new_state, context, qpc_mask);
4404 	if (ret)
4405 		goto out;
4406 
4407 	/* When QP state is err, SQ and RQ WQE should be flushed */
4408 	if (new_state == IB_QPS_ERR) {
4409 		roce_set_field(context->byte_160_sq_ci_pi,
4410 			       V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M,
4411 			       V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S,
4412 			       hr_qp->sq.head);
4413 		roce_set_field(qpc_mask->byte_160_sq_ci_pi,
4414 			       V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M,
4415 			       V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S, 0);
4416 
4417 		if (!ibqp->srq) {
4418 			roce_set_field(context->byte_84_rq_ci_pi,
4419 			       V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
4420 			       V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S,
4421 			       hr_qp->rq.head);
4422 			roce_set_field(qpc_mask->byte_84_rq_ci_pi,
4423 			       V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
4424 			       V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0);
4425 		}
4426 	}
4427 
4428 	/* Configure the optional fields */
4429 	ret = hns_roce_v2_set_opt_fields(ibqp, attr, attr_mask, context,
4430 					 qpc_mask);
4431 	if (ret)
4432 		goto out;
4433 
4434 	roce_set_bit(context->byte_108_rx_reqepsn, V2_QPC_BYTE_108_INV_CREDIT_S,
4435 		     ibqp->srq ? 1 : 0);
4436 	roce_set_bit(qpc_mask->byte_108_rx_reqepsn,
4437 		     V2_QPC_BYTE_108_INV_CREDIT_S, 0);
4438 
4439 	/* Every status migrate must change state */
4440 	roce_set_field(context->byte_60_qpst_tempid, V2_QPC_BYTE_60_QP_ST_M,
4441 		       V2_QPC_BYTE_60_QP_ST_S, new_state);
4442 	roce_set_field(qpc_mask->byte_60_qpst_tempid, V2_QPC_BYTE_60_QP_ST_M,
4443 		       V2_QPC_BYTE_60_QP_ST_S, 0);
4444 
4445 	/* SW pass context to HW */
4446 	ret = hns_roce_v2_qp_modify(hr_dev, cur_state, new_state, ctx, hr_qp);
4447 	if (ret) {
4448 		dev_err(dev, "hns_roce_qp_modify failed(%d)\n", ret);
4449 		goto out;
4450 	}
4451 
4452 	hr_qp->state = new_state;
4453 
4454 	hns_roce_v2_record_opt_fields(ibqp, attr, attr_mask);
4455 
4456 	if (new_state == IB_QPS_RESET && !ibqp->uobject) {
4457 		hns_roce_v2_cq_clean(to_hr_cq(ibqp->recv_cq), hr_qp->qpn,
4458 				     ibqp->srq ? to_hr_srq(ibqp->srq) : NULL);
4459 		if (ibqp->send_cq != ibqp->recv_cq)
4460 			hns_roce_v2_cq_clean(to_hr_cq(ibqp->send_cq),
4461 					     hr_qp->qpn, NULL);
4462 
4463 		hr_qp->rq.head = 0;
4464 		hr_qp->rq.tail = 0;
4465 		hr_qp->sq.head = 0;
4466 		hr_qp->sq.tail = 0;
4467 		hr_qp->sq_next_wqe = 0;
4468 		hr_qp->next_sge = 0;
4469 		if (hr_qp->rq.wqe_cnt)
4470 			*hr_qp->rdb.db_record = 0;
4471 	}
4472 
4473 out:
4474 	return ret;
4475 }
4476 
4477 static inline enum ib_qp_state to_ib_qp_st(enum hns_roce_v2_qp_state state)
4478 {
4479 	switch (state) {
4480 	case HNS_ROCE_QP_ST_RST:	return IB_QPS_RESET;
4481 	case HNS_ROCE_QP_ST_INIT:	return IB_QPS_INIT;
4482 	case HNS_ROCE_QP_ST_RTR:	return IB_QPS_RTR;
4483 	case HNS_ROCE_QP_ST_RTS:	return IB_QPS_RTS;
4484 	case HNS_ROCE_QP_ST_SQ_DRAINING:
4485 	case HNS_ROCE_QP_ST_SQD:	return IB_QPS_SQD;
4486 	case HNS_ROCE_QP_ST_SQER:	return IB_QPS_SQE;
4487 	case HNS_ROCE_QP_ST_ERR:	return IB_QPS_ERR;
4488 	default:			return -1;
4489 	}
4490 }
4491 
4492 static int hns_roce_v2_query_qpc(struct hns_roce_dev *hr_dev,
4493 				 struct hns_roce_qp *hr_qp,
4494 				 struct hns_roce_v2_qp_context *hr_context)
4495 {
4496 	struct hns_roce_cmd_mailbox *mailbox;
4497 	int ret;
4498 
4499 	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
4500 	if (IS_ERR(mailbox))
4501 		return PTR_ERR(mailbox);
4502 
4503 	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, hr_qp->qpn, 0,
4504 				HNS_ROCE_CMD_QUERY_QPC,
4505 				HNS_ROCE_CMD_TIMEOUT_MSECS);
4506 	if (ret) {
4507 		dev_err(hr_dev->dev, "QUERY QP cmd process error\n");
4508 		goto out;
4509 	}
4510 
4511 	memcpy(hr_context, mailbox->buf, sizeof(*hr_context));
4512 
4513 out:
4514 	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
4515 	return ret;
4516 }
4517 
4518 static int hns_roce_v2_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
4519 				int qp_attr_mask,
4520 				struct ib_qp_init_attr *qp_init_attr)
4521 {
4522 	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4523 	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4524 	struct hns_roce_v2_qp_context context = {};
4525 	struct device *dev = hr_dev->dev;
4526 	int tmp_qp_state;
4527 	int state;
4528 	int ret;
4529 
4530 	memset(qp_attr, 0, sizeof(*qp_attr));
4531 	memset(qp_init_attr, 0, sizeof(*qp_init_attr));
4532 
4533 	mutex_lock(&hr_qp->mutex);
4534 
4535 	if (hr_qp->state == IB_QPS_RESET) {
4536 		qp_attr->qp_state = IB_QPS_RESET;
4537 		ret = 0;
4538 		goto done;
4539 	}
4540 
4541 	ret = hns_roce_v2_query_qpc(hr_dev, hr_qp, &context);
4542 	if (ret) {
4543 		dev_err(dev, "query qpc error\n");
4544 		ret = -EINVAL;
4545 		goto out;
4546 	}
4547 
4548 	state = roce_get_field(context.byte_60_qpst_tempid,
4549 			       V2_QPC_BYTE_60_QP_ST_M, V2_QPC_BYTE_60_QP_ST_S);
4550 	tmp_qp_state = to_ib_qp_st((enum hns_roce_v2_qp_state)state);
4551 	if (tmp_qp_state == -1) {
4552 		dev_err(dev, "Illegal ib_qp_state\n");
4553 		ret = -EINVAL;
4554 		goto out;
4555 	}
4556 	hr_qp->state = (u8)tmp_qp_state;
4557 	qp_attr->qp_state = (enum ib_qp_state)hr_qp->state;
4558 	qp_attr->path_mtu = (enum ib_mtu)roce_get_field(context.byte_24_mtu_tc,
4559 							V2_QPC_BYTE_24_MTU_M,
4560 							V2_QPC_BYTE_24_MTU_S);
4561 	qp_attr->path_mig_state = IB_MIG_ARMED;
4562 	qp_attr->ah_attr.type   = RDMA_AH_ATTR_TYPE_ROCE;
4563 	if (hr_qp->ibqp.qp_type == IB_QPT_UD)
4564 		qp_attr->qkey = V2_QKEY_VAL;
4565 
4566 	qp_attr->rq_psn = roce_get_field(context.byte_108_rx_reqepsn,
4567 					 V2_QPC_BYTE_108_RX_REQ_EPSN_M,
4568 					 V2_QPC_BYTE_108_RX_REQ_EPSN_S);
4569 	qp_attr->sq_psn = (u32)roce_get_field(context.byte_172_sq_psn,
4570 					      V2_QPC_BYTE_172_SQ_CUR_PSN_M,
4571 					      V2_QPC_BYTE_172_SQ_CUR_PSN_S);
4572 	qp_attr->dest_qp_num = (u8)roce_get_field(context.byte_56_dqpn_err,
4573 						  V2_QPC_BYTE_56_DQPN_M,
4574 						  V2_QPC_BYTE_56_DQPN_S);
4575 	qp_attr->qp_access_flags = ((roce_get_bit(context.byte_76_srqn_op_en,
4576 				    V2_QPC_BYTE_76_RRE_S)) << V2_QP_RRE_S) |
4577 				    ((roce_get_bit(context.byte_76_srqn_op_en,
4578 				    V2_QPC_BYTE_76_RWE_S)) << V2_QP_RWE_S) |
4579 				    ((roce_get_bit(context.byte_76_srqn_op_en,
4580 				    V2_QPC_BYTE_76_ATE_S)) << V2_QP_ATE_S);
4581 
4582 	if (hr_qp->ibqp.qp_type == IB_QPT_RC ||
4583 	    hr_qp->ibqp.qp_type == IB_QPT_UC) {
4584 		struct ib_global_route *grh =
4585 				rdma_ah_retrieve_grh(&qp_attr->ah_attr);
4586 
4587 		rdma_ah_set_sl(&qp_attr->ah_attr,
4588 			       roce_get_field(context.byte_28_at_fl,
4589 					      V2_QPC_BYTE_28_SL_M,
4590 					      V2_QPC_BYTE_28_SL_S));
4591 		grh->flow_label = roce_get_field(context.byte_28_at_fl,
4592 						 V2_QPC_BYTE_28_FL_M,
4593 						 V2_QPC_BYTE_28_FL_S);
4594 		grh->sgid_index = roce_get_field(context.byte_20_smac_sgid_idx,
4595 						 V2_QPC_BYTE_20_SGID_IDX_M,
4596 						 V2_QPC_BYTE_20_SGID_IDX_S);
4597 		grh->hop_limit = roce_get_field(context.byte_24_mtu_tc,
4598 						V2_QPC_BYTE_24_HOP_LIMIT_M,
4599 						V2_QPC_BYTE_24_HOP_LIMIT_S);
4600 		grh->traffic_class = roce_get_field(context.byte_24_mtu_tc,
4601 						    V2_QPC_BYTE_24_TC_M,
4602 						    V2_QPC_BYTE_24_TC_S);
4603 
4604 		memcpy(grh->dgid.raw, context.dgid, sizeof(grh->dgid.raw));
4605 	}
4606 
4607 	qp_attr->port_num = hr_qp->port + 1;
4608 	qp_attr->sq_draining = 0;
4609 	qp_attr->max_rd_atomic = 1 << roce_get_field(context.byte_208_irrl,
4610 						     V2_QPC_BYTE_208_SR_MAX_M,
4611 						     V2_QPC_BYTE_208_SR_MAX_S);
4612 	qp_attr->max_dest_rd_atomic = 1 << roce_get_field(context.byte_140_raq,
4613 						     V2_QPC_BYTE_140_RR_MAX_M,
4614 						     V2_QPC_BYTE_140_RR_MAX_S);
4615 	qp_attr->min_rnr_timer = (u8)roce_get_field(context.byte_80_rnr_rx_cqn,
4616 						 V2_QPC_BYTE_80_MIN_RNR_TIME_M,
4617 						 V2_QPC_BYTE_80_MIN_RNR_TIME_S);
4618 	qp_attr->timeout = (u8)roce_get_field(context.byte_28_at_fl,
4619 					      V2_QPC_BYTE_28_AT_M,
4620 					      V2_QPC_BYTE_28_AT_S);
4621 	qp_attr->retry_cnt = roce_get_field(context.byte_212_lsn,
4622 					    V2_QPC_BYTE_212_RETRY_CNT_M,
4623 					    V2_QPC_BYTE_212_RETRY_CNT_S);
4624 	qp_attr->rnr_retry = le32_to_cpu(context.rq_rnr_timer);
4625 
4626 done:
4627 	qp_attr->cur_qp_state = qp_attr->qp_state;
4628 	qp_attr->cap.max_recv_wr = hr_qp->rq.wqe_cnt;
4629 	qp_attr->cap.max_recv_sge = hr_qp->rq.max_gs;
4630 
4631 	if (!ibqp->uobject) {
4632 		qp_attr->cap.max_send_wr = hr_qp->sq.wqe_cnt;
4633 		qp_attr->cap.max_send_sge = hr_qp->sq.max_gs;
4634 	} else {
4635 		qp_attr->cap.max_send_wr = 0;
4636 		qp_attr->cap.max_send_sge = 0;
4637 	}
4638 
4639 	qp_init_attr->cap = qp_attr->cap;
4640 
4641 out:
4642 	mutex_unlock(&hr_qp->mutex);
4643 	return ret;
4644 }
4645 
4646 static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev,
4647 					 struct hns_roce_qp *hr_qp,
4648 					 struct ib_udata *udata)
4649 {
4650 	struct hns_roce_cq *send_cq, *recv_cq;
4651 	struct ib_device *ibdev = &hr_dev->ib_dev;
4652 	int ret = 0;
4653 
4654 	if (hr_qp->ibqp.qp_type == IB_QPT_RC && hr_qp->state != IB_QPS_RESET) {
4655 		/* Modify qp to reset before destroying qp */
4656 		ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, NULL, 0,
4657 					    hr_qp->state, IB_QPS_RESET);
4658 		if (ret)
4659 			ibdev_err(ibdev, "modify QP to Reset failed.\n");
4660 	}
4661 
4662 	send_cq = to_hr_cq(hr_qp->ibqp.send_cq);
4663 	recv_cq = to_hr_cq(hr_qp->ibqp.recv_cq);
4664 
4665 	hns_roce_lock_cqs(send_cq, recv_cq);
4666 
4667 	if (!udata) {
4668 		__hns_roce_v2_cq_clean(recv_cq, hr_qp->qpn, hr_qp->ibqp.srq ?
4669 				       to_hr_srq(hr_qp->ibqp.srq) : NULL);
4670 		if (send_cq != recv_cq)
4671 			__hns_roce_v2_cq_clean(send_cq, hr_qp->qpn, NULL);
4672 	}
4673 
4674 	hns_roce_qp_remove(hr_dev, hr_qp);
4675 
4676 	hns_roce_unlock_cqs(send_cq, recv_cq);
4677 
4678 	hns_roce_qp_free(hr_dev, hr_qp);
4679 
4680 	/* Not special_QP, free their QPN */
4681 	if ((hr_qp->ibqp.qp_type == IB_QPT_RC) ||
4682 	    (hr_qp->ibqp.qp_type == IB_QPT_UC) ||
4683 	    (hr_qp->ibqp.qp_type == IB_QPT_UD))
4684 		hns_roce_release_range_qp(hr_dev, hr_qp->qpn, 1);
4685 
4686 	hns_roce_mtr_cleanup(hr_dev, &hr_qp->mtr);
4687 
4688 	if (udata) {
4689 		struct hns_roce_ucontext *context =
4690 			rdma_udata_to_drv_context(
4691 				udata,
4692 				struct hns_roce_ucontext,
4693 				ibucontext);
4694 
4695 		if (hr_qp->sq.wqe_cnt && (hr_qp->sdb_en == 1))
4696 			hns_roce_db_unmap_user(context, &hr_qp->sdb);
4697 
4698 		if (hr_qp->rq.wqe_cnt && (hr_qp->rdb_en == 1))
4699 			hns_roce_db_unmap_user(context, &hr_qp->rdb);
4700 	} else {
4701 		kfree(hr_qp->sq.wrid);
4702 		kfree(hr_qp->rq.wrid);
4703 		hns_roce_buf_free(hr_dev, hr_qp->buff_size, &hr_qp->hr_buf);
4704 		if (hr_qp->rq.wqe_cnt)
4705 			hns_roce_free_db(hr_dev, &hr_qp->rdb);
4706 	}
4707 	ib_umem_release(hr_qp->umem);
4708 
4709 	if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) &&
4710 	     hr_qp->rq.wqe_cnt) {
4711 		kfree(hr_qp->rq_inl_buf.wqe_list[0].sg_list);
4712 		kfree(hr_qp->rq_inl_buf.wqe_list);
4713 	}
4714 
4715 	return ret;
4716 }
4717 
4718 static int hns_roce_v2_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
4719 {
4720 	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4721 	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4722 	int ret;
4723 
4724 	ret = hns_roce_v2_destroy_qp_common(hr_dev, hr_qp, udata);
4725 	if (ret)
4726 		ibdev_err(&hr_dev->ib_dev, "Destroy qp 0x%06lx failed(%d)\n",
4727 			  hr_qp->qpn, ret);
4728 
4729 	kfree(hr_qp);
4730 
4731 	return 0;
4732 }
4733 
4734 static int hns_roce_v2_qp_flow_control_init(struct hns_roce_dev *hr_dev,
4735 						struct hns_roce_qp *hr_qp)
4736 {
4737 	struct hns_roce_sccc_clr_done *resp;
4738 	struct hns_roce_sccc_clr *clr;
4739 	struct hns_roce_cmq_desc desc;
4740 	int ret, i;
4741 
4742 	mutex_lock(&hr_dev->qp_table.scc_mutex);
4743 
4744 	/* set scc ctx clear done flag */
4745 	hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_RESET_SCCC, false);
4746 	ret =  hns_roce_cmq_send(hr_dev, &desc, 1);
4747 	if (ret) {
4748 		dev_err(hr_dev->dev, "Reset SCC ctx  failed(%d)\n", ret);
4749 		goto out;
4750 	}
4751 
4752 	/* clear scc context */
4753 	hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CLR_SCCC, false);
4754 	clr = (struct hns_roce_sccc_clr *)desc.data;
4755 	clr->qpn = cpu_to_le32(hr_qp->qpn);
4756 	ret =  hns_roce_cmq_send(hr_dev, &desc, 1);
4757 	if (ret) {
4758 		dev_err(hr_dev->dev, "Clear SCC ctx failed(%d)\n", ret);
4759 		goto out;
4760 	}
4761 
4762 	/* query scc context clear is done or not */
4763 	resp = (struct hns_roce_sccc_clr_done *)desc.data;
4764 	for (i = 0; i <= HNS_ROCE_CMQ_SCC_CLR_DONE_CNT; i++) {
4765 		hns_roce_cmq_setup_basic_desc(&desc,
4766 					      HNS_ROCE_OPC_QUERY_SCCC, true);
4767 		ret = hns_roce_cmq_send(hr_dev, &desc, 1);
4768 		if (ret) {
4769 			dev_err(hr_dev->dev, "Query clr cmq failed(%d)\n", ret);
4770 			goto out;
4771 		}
4772 
4773 		if (resp->clr_done)
4774 			goto out;
4775 
4776 		msleep(20);
4777 	}
4778 
4779 	dev_err(hr_dev->dev, "Query SCC clr done flag overtime.\n");
4780 	ret = -ETIMEDOUT;
4781 
4782 out:
4783 	mutex_unlock(&hr_dev->qp_table.scc_mutex);
4784 	return ret;
4785 }
4786 
4787 static int hns_roce_v2_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)
4788 {
4789 	struct hns_roce_dev *hr_dev = to_hr_dev(cq->device);
4790 	struct hns_roce_v2_cq_context *cq_context;
4791 	struct hns_roce_cq *hr_cq = to_hr_cq(cq);
4792 	struct hns_roce_v2_cq_context *cqc_mask;
4793 	struct hns_roce_cmd_mailbox *mailbox;
4794 	int ret;
4795 
4796 	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
4797 	if (IS_ERR(mailbox))
4798 		return PTR_ERR(mailbox);
4799 
4800 	cq_context = mailbox->buf;
4801 	cqc_mask = (struct hns_roce_v2_cq_context *)mailbox->buf + 1;
4802 
4803 	memset(cqc_mask, 0xff, sizeof(*cqc_mask));
4804 
4805 	roce_set_field(cq_context->byte_56_cqe_period_maxcnt,
4806 		       V2_CQC_BYTE_56_CQ_MAX_CNT_M, V2_CQC_BYTE_56_CQ_MAX_CNT_S,
4807 		       cq_count);
4808 	roce_set_field(cqc_mask->byte_56_cqe_period_maxcnt,
4809 		       V2_CQC_BYTE_56_CQ_MAX_CNT_M, V2_CQC_BYTE_56_CQ_MAX_CNT_S,
4810 		       0);
4811 	roce_set_field(cq_context->byte_56_cqe_period_maxcnt,
4812 		       V2_CQC_BYTE_56_CQ_PERIOD_M, V2_CQC_BYTE_56_CQ_PERIOD_S,
4813 		       cq_period);
4814 	roce_set_field(cqc_mask->byte_56_cqe_period_maxcnt,
4815 		       V2_CQC_BYTE_56_CQ_PERIOD_M, V2_CQC_BYTE_56_CQ_PERIOD_S,
4816 		       0);
4817 
4818 	ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_cq->cqn, 1,
4819 				HNS_ROCE_CMD_MODIFY_CQC,
4820 				HNS_ROCE_CMD_TIMEOUT_MSECS);
4821 	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
4822 	if (ret)
4823 		dev_err(hr_dev->dev, "MODIFY CQ Failed to cmd mailbox.\n");
4824 
4825 	return ret;
4826 }
4827 
4828 static void hns_roce_set_qps_to_err(struct hns_roce_dev *hr_dev, u32 qpn)
4829 {
4830 	struct hns_roce_qp *hr_qp;
4831 	struct ib_qp_attr attr;
4832 	int attr_mask;
4833 	int ret;
4834 
4835 	hr_qp = __hns_roce_qp_lookup(hr_dev, qpn);
4836 	if (!hr_qp) {
4837 		dev_warn(hr_dev->dev, "no hr_qp can be found!\n");
4838 		return;
4839 	}
4840 
4841 	if (hr_qp->ibqp.uobject) {
4842 		if (hr_qp->sdb_en == 1) {
4843 			hr_qp->sq.head = *(int *)(hr_qp->sdb.virt_addr);
4844 			if (hr_qp->rdb_en == 1)
4845 				hr_qp->rq.head = *(int *)(hr_qp->rdb.virt_addr);
4846 		} else {
4847 			dev_warn(hr_dev->dev, "flush cqe is unsupported in userspace!\n");
4848 			return;
4849 		}
4850 	}
4851 
4852 	attr_mask = IB_QP_STATE;
4853 	attr.qp_state = IB_QPS_ERR;
4854 	ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, &attr, attr_mask,
4855 				    hr_qp->state, IB_QPS_ERR);
4856 	if (ret)
4857 		dev_err(hr_dev->dev, "failed to modify qp %d to err state.\n",
4858 			qpn);
4859 }
4860 
4861 static void hns_roce_irq_work_handle(struct work_struct *work)
4862 {
4863 	struct hns_roce_work *irq_work =
4864 				container_of(work, struct hns_roce_work, work);
4865 	struct device *dev = irq_work->hr_dev->dev;
4866 	u32 qpn = irq_work->qpn;
4867 	u32 cqn = irq_work->cqn;
4868 
4869 	switch (irq_work->event_type) {
4870 	case HNS_ROCE_EVENT_TYPE_PATH_MIG:
4871 		dev_info(dev, "Path migrated succeeded.\n");
4872 		break;
4873 	case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED:
4874 		dev_warn(dev, "Path migration failed.\n");
4875 		break;
4876 	case HNS_ROCE_EVENT_TYPE_COMM_EST:
4877 		break;
4878 	case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:
4879 		dev_warn(dev, "Send queue drained.\n");
4880 		break;
4881 	case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
4882 		dev_err(dev, "Local work queue 0x%x catas error, sub_type:%d\n",
4883 			qpn, irq_work->sub_type);
4884 		hns_roce_set_qps_to_err(irq_work->hr_dev, qpn);
4885 		break;
4886 	case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
4887 		dev_err(dev, "Invalid request local work queue 0x%x error.\n",
4888 			qpn);
4889 		hns_roce_set_qps_to_err(irq_work->hr_dev, qpn);
4890 		break;
4891 	case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
4892 		dev_err(dev, "Local access violation work queue 0x%x error, sub_type:%d\n",
4893 			qpn, irq_work->sub_type);
4894 		hns_roce_set_qps_to_err(irq_work->hr_dev, qpn);
4895 		break;
4896 	case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH:
4897 		dev_warn(dev, "SRQ limit reach.\n");
4898 		break;
4899 	case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:
4900 		dev_warn(dev, "SRQ last wqe reach.\n");
4901 		break;
4902 	case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:
4903 		dev_err(dev, "SRQ catas error.\n");
4904 		break;
4905 	case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:
4906 		dev_err(dev, "CQ 0x%x access err.\n", cqn);
4907 		break;
4908 	case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:
4909 		dev_warn(dev, "CQ 0x%x overflow\n", cqn);
4910 		break;
4911 	case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW:
4912 		dev_warn(dev, "DB overflow.\n");
4913 		break;
4914 	case HNS_ROCE_EVENT_TYPE_FLR:
4915 		dev_warn(dev, "Function level reset.\n");
4916 		break;
4917 	default:
4918 		break;
4919 	}
4920 
4921 	kfree(irq_work);
4922 }
4923 
4924 static void hns_roce_v2_init_irq_work(struct hns_roce_dev *hr_dev,
4925 				      struct hns_roce_eq *eq,
4926 				      u32 qpn, u32 cqn)
4927 {
4928 	struct hns_roce_work *irq_work;
4929 
4930 	irq_work = kzalloc(sizeof(struct hns_roce_work), GFP_ATOMIC);
4931 	if (!irq_work)
4932 		return;
4933 
4934 	INIT_WORK(&(irq_work->work), hns_roce_irq_work_handle);
4935 	irq_work->hr_dev = hr_dev;
4936 	irq_work->qpn = qpn;
4937 	irq_work->cqn = cqn;
4938 	irq_work->event_type = eq->event_type;
4939 	irq_work->sub_type = eq->sub_type;
4940 	queue_work(hr_dev->irq_workq, &(irq_work->work));
4941 }
4942 
4943 static void set_eq_cons_index_v2(struct hns_roce_eq *eq)
4944 {
4945 	struct hns_roce_dev *hr_dev = eq->hr_dev;
4946 	__le32 doorbell[2] = {};
4947 
4948 	if (eq->type_flag == HNS_ROCE_AEQ) {
4949 		roce_set_field(doorbell[0], HNS_ROCE_V2_EQ_DB_CMD_M,
4950 			       HNS_ROCE_V2_EQ_DB_CMD_S,
4951 			       eq->arm_st == HNS_ROCE_V2_EQ_ALWAYS_ARMED ?
4952 			       HNS_ROCE_EQ_DB_CMD_AEQ :
4953 			       HNS_ROCE_EQ_DB_CMD_AEQ_ARMED);
4954 	} else {
4955 		roce_set_field(doorbell[0], HNS_ROCE_V2_EQ_DB_TAG_M,
4956 			       HNS_ROCE_V2_EQ_DB_TAG_S, eq->eqn);
4957 
4958 		roce_set_field(doorbell[0], HNS_ROCE_V2_EQ_DB_CMD_M,
4959 			       HNS_ROCE_V2_EQ_DB_CMD_S,
4960 			       eq->arm_st == HNS_ROCE_V2_EQ_ALWAYS_ARMED ?
4961 			       HNS_ROCE_EQ_DB_CMD_CEQ :
4962 			       HNS_ROCE_EQ_DB_CMD_CEQ_ARMED);
4963 	}
4964 
4965 	roce_set_field(doorbell[1], HNS_ROCE_V2_EQ_DB_PARA_M,
4966 		       HNS_ROCE_V2_EQ_DB_PARA_S,
4967 		       (eq->cons_index & HNS_ROCE_V2_CONS_IDX_M));
4968 
4969 	hns_roce_write64(hr_dev, doorbell, eq->doorbell);
4970 }
4971 
4972 static struct hns_roce_aeqe *get_aeqe_v2(struct hns_roce_eq *eq, u32 entry)
4973 {
4974 	u32 buf_chk_sz;
4975 	unsigned long off;
4976 
4977 	buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
4978 	off = (entry & (eq->entries - 1)) * HNS_ROCE_AEQ_ENTRY_SIZE;
4979 
4980 	return (struct hns_roce_aeqe *)((char *)(eq->buf_list->buf) +
4981 		off % buf_chk_sz);
4982 }
4983 
4984 static struct hns_roce_aeqe *mhop_get_aeqe(struct hns_roce_eq *eq, u32 entry)
4985 {
4986 	u32 buf_chk_sz;
4987 	unsigned long off;
4988 
4989 	buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
4990 
4991 	off = (entry & (eq->entries - 1)) * HNS_ROCE_AEQ_ENTRY_SIZE;
4992 
4993 	if (eq->hop_num == HNS_ROCE_HOP_NUM_0)
4994 		return (struct hns_roce_aeqe *)((u8 *)(eq->bt_l0) +
4995 			off % buf_chk_sz);
4996 	else
4997 		return (struct hns_roce_aeqe *)((u8 *)
4998 			(eq->buf[off / buf_chk_sz]) + off % buf_chk_sz);
4999 }
5000 
5001 static struct hns_roce_aeqe *next_aeqe_sw_v2(struct hns_roce_eq *eq)
5002 {
5003 	struct hns_roce_aeqe *aeqe;
5004 
5005 	if (!eq->hop_num)
5006 		aeqe = get_aeqe_v2(eq, eq->cons_index);
5007 	else
5008 		aeqe = mhop_get_aeqe(eq, eq->cons_index);
5009 
5010 	return (roce_get_bit(aeqe->asyn, HNS_ROCE_V2_AEQ_AEQE_OWNER_S) ^
5011 		!!(eq->cons_index & eq->entries)) ? aeqe : NULL;
5012 }
5013 
5014 static int hns_roce_v2_aeq_int(struct hns_roce_dev *hr_dev,
5015 			       struct hns_roce_eq *eq)
5016 {
5017 	struct device *dev = hr_dev->dev;
5018 	struct hns_roce_aeqe *aeqe = next_aeqe_sw_v2(eq);
5019 	int aeqe_found = 0;
5020 	int event_type;
5021 	int sub_type;
5022 	u32 srqn;
5023 	u32 qpn;
5024 	u32 cqn;
5025 
5026 	while (aeqe) {
5027 		/* Make sure we read AEQ entry after we have checked the
5028 		 * ownership bit
5029 		 */
5030 		dma_rmb();
5031 
5032 		event_type = roce_get_field(aeqe->asyn,
5033 					    HNS_ROCE_V2_AEQE_EVENT_TYPE_M,
5034 					    HNS_ROCE_V2_AEQE_EVENT_TYPE_S);
5035 		sub_type = roce_get_field(aeqe->asyn,
5036 					  HNS_ROCE_V2_AEQE_SUB_TYPE_M,
5037 					  HNS_ROCE_V2_AEQE_SUB_TYPE_S);
5038 		qpn = roce_get_field(aeqe->event.qp_event.qp,
5039 				     HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M,
5040 				     HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S);
5041 		cqn = roce_get_field(aeqe->event.cq_event.cq,
5042 				     HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M,
5043 				     HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S);
5044 		srqn = roce_get_field(aeqe->event.srq_event.srq,
5045 				     HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M,
5046 				     HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S);
5047 
5048 		switch (event_type) {
5049 		case HNS_ROCE_EVENT_TYPE_PATH_MIG:
5050 		case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED:
5051 		case HNS_ROCE_EVENT_TYPE_COMM_EST:
5052 		case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:
5053 		case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
5054 		case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:
5055 		case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
5056 		case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
5057 			hns_roce_qp_event(hr_dev, qpn, event_type);
5058 			break;
5059 		case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH:
5060 		case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:
5061 			hns_roce_srq_event(hr_dev, srqn, event_type);
5062 			break;
5063 		case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:
5064 		case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:
5065 			hns_roce_cq_event(hr_dev, cqn, event_type);
5066 			break;
5067 		case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW:
5068 			break;
5069 		case HNS_ROCE_EVENT_TYPE_MB:
5070 			hns_roce_cmd_event(hr_dev,
5071 					le16_to_cpu(aeqe->event.cmd.token),
5072 					aeqe->event.cmd.status,
5073 					le64_to_cpu(aeqe->event.cmd.out_param));
5074 			break;
5075 		case HNS_ROCE_EVENT_TYPE_CEQ_OVERFLOW:
5076 			break;
5077 		case HNS_ROCE_EVENT_TYPE_FLR:
5078 			break;
5079 		default:
5080 			dev_err(dev, "Unhandled event %d on EQ %d at idx %u.\n",
5081 				event_type, eq->eqn, eq->cons_index);
5082 			break;
5083 		}
5084 
5085 		eq->event_type = event_type;
5086 		eq->sub_type = sub_type;
5087 		++eq->cons_index;
5088 		aeqe_found = 1;
5089 
5090 		if (eq->cons_index > (2 * eq->entries - 1))
5091 			eq->cons_index = 0;
5092 
5093 		hns_roce_v2_init_irq_work(hr_dev, eq, qpn, cqn);
5094 
5095 		aeqe = next_aeqe_sw_v2(eq);
5096 	}
5097 
5098 	set_eq_cons_index_v2(eq);
5099 	return aeqe_found;
5100 }
5101 
5102 static struct hns_roce_ceqe *get_ceqe_v2(struct hns_roce_eq *eq, u32 entry)
5103 {
5104 	u32 buf_chk_sz;
5105 	unsigned long off;
5106 
5107 	buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
5108 	off = (entry & (eq->entries - 1)) * HNS_ROCE_CEQ_ENTRY_SIZE;
5109 
5110 	return (struct hns_roce_ceqe *)((char *)(eq->buf_list->buf) +
5111 		off % buf_chk_sz);
5112 }
5113 
5114 static struct hns_roce_ceqe *mhop_get_ceqe(struct hns_roce_eq *eq, u32 entry)
5115 {
5116 	u32 buf_chk_sz;
5117 	unsigned long off;
5118 
5119 	buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
5120 
5121 	off = (entry & (eq->entries - 1)) * HNS_ROCE_CEQ_ENTRY_SIZE;
5122 
5123 	if (eq->hop_num == HNS_ROCE_HOP_NUM_0)
5124 		return (struct hns_roce_ceqe *)((u8 *)(eq->bt_l0) +
5125 			off % buf_chk_sz);
5126 	else
5127 		return (struct hns_roce_ceqe *)((u8 *)(eq->buf[off /
5128 			buf_chk_sz]) + off % buf_chk_sz);
5129 }
5130 
5131 static struct hns_roce_ceqe *next_ceqe_sw_v2(struct hns_roce_eq *eq)
5132 {
5133 	struct hns_roce_ceqe *ceqe;
5134 
5135 	if (!eq->hop_num)
5136 		ceqe = get_ceqe_v2(eq, eq->cons_index);
5137 	else
5138 		ceqe = mhop_get_ceqe(eq, eq->cons_index);
5139 
5140 	return (!!(roce_get_bit(ceqe->comp, HNS_ROCE_V2_CEQ_CEQE_OWNER_S))) ^
5141 		(!!(eq->cons_index & eq->entries)) ? ceqe : NULL;
5142 }
5143 
5144 static int hns_roce_v2_ceq_int(struct hns_roce_dev *hr_dev,
5145 			       struct hns_roce_eq *eq)
5146 {
5147 	struct device *dev = hr_dev->dev;
5148 	struct hns_roce_ceqe *ceqe = next_ceqe_sw_v2(eq);
5149 	int ceqe_found = 0;
5150 	u32 cqn;
5151 
5152 	while (ceqe) {
5153 		/* Make sure we read CEQ entry after we have checked the
5154 		 * ownership bit
5155 		 */
5156 		dma_rmb();
5157 
5158 		cqn = roce_get_field(ceqe->comp,
5159 				     HNS_ROCE_V2_CEQE_COMP_CQN_M,
5160 				     HNS_ROCE_V2_CEQE_COMP_CQN_S);
5161 
5162 		hns_roce_cq_completion(hr_dev, cqn);
5163 
5164 		++eq->cons_index;
5165 		ceqe_found = 1;
5166 
5167 		if (eq->cons_index > (EQ_DEPTH_COEFF * eq->entries - 1)) {
5168 			dev_warn(dev, "cons_index overflow, set back to 0.\n");
5169 			eq->cons_index = 0;
5170 		}
5171 
5172 		ceqe = next_ceqe_sw_v2(eq);
5173 	}
5174 
5175 	set_eq_cons_index_v2(eq);
5176 
5177 	return ceqe_found;
5178 }
5179 
5180 static irqreturn_t hns_roce_v2_msix_interrupt_eq(int irq, void *eq_ptr)
5181 {
5182 	struct hns_roce_eq *eq = eq_ptr;
5183 	struct hns_roce_dev *hr_dev = eq->hr_dev;
5184 	int int_work = 0;
5185 
5186 	if (eq->type_flag == HNS_ROCE_CEQ)
5187 		/* Completion event interrupt */
5188 		int_work = hns_roce_v2_ceq_int(hr_dev, eq);
5189 	else
5190 		/* Asychronous event interrupt */
5191 		int_work = hns_roce_v2_aeq_int(hr_dev, eq);
5192 
5193 	return IRQ_RETVAL(int_work);
5194 }
5195 
5196 static irqreturn_t hns_roce_v2_msix_interrupt_abn(int irq, void *dev_id)
5197 {
5198 	struct hns_roce_dev *hr_dev = dev_id;
5199 	struct device *dev = hr_dev->dev;
5200 	int int_work = 0;
5201 	u32 int_st;
5202 	u32 int_en;
5203 
5204 	/* Abnormal interrupt */
5205 	int_st = roce_read(hr_dev, ROCEE_VF_ABN_INT_ST_REG);
5206 	int_en = roce_read(hr_dev, ROCEE_VF_ABN_INT_EN_REG);
5207 
5208 	if (int_st & BIT(HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S)) {
5209 		struct pci_dev *pdev = hr_dev->pci_dev;
5210 		struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
5211 		const struct hnae3_ae_ops *ops = ae_dev->ops;
5212 
5213 		dev_err(dev, "AEQ overflow!\n");
5214 
5215 		int_st |= 1 << HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S;
5216 		roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st);
5217 
5218 		/* Set reset level for reset_event() */
5219 		if (ops->set_default_reset_request)
5220 			ops->set_default_reset_request(ae_dev,
5221 						       HNAE3_FUNC_RESET);
5222 		if (ops->reset_event)
5223 			ops->reset_event(pdev, NULL);
5224 
5225 		int_en |= 1 << HNS_ROCE_V2_VF_ABN_INT_EN_S;
5226 		roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en);
5227 
5228 		int_work = 1;
5229 	} else if (int_st & BIT(HNS_ROCE_V2_VF_INT_ST_BUS_ERR_S)) {
5230 		dev_err(dev, "BUS ERR!\n");
5231 
5232 		int_st |= 1 << HNS_ROCE_V2_VF_INT_ST_BUS_ERR_S;
5233 		roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st);
5234 
5235 		int_en |= 1 << HNS_ROCE_V2_VF_ABN_INT_EN_S;
5236 		roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en);
5237 
5238 		int_work = 1;
5239 	} else if (int_st & BIT(HNS_ROCE_V2_VF_INT_ST_OTHER_ERR_S)) {
5240 		dev_err(dev, "OTHER ERR!\n");
5241 
5242 		int_st |= 1 << HNS_ROCE_V2_VF_INT_ST_OTHER_ERR_S;
5243 		roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st);
5244 
5245 		int_en |= 1 << HNS_ROCE_V2_VF_ABN_INT_EN_S;
5246 		roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en);
5247 
5248 		int_work = 1;
5249 	} else
5250 		dev_err(dev, "There is no abnormal irq found!\n");
5251 
5252 	return IRQ_RETVAL(int_work);
5253 }
5254 
5255 static void hns_roce_v2_int_mask_enable(struct hns_roce_dev *hr_dev,
5256 					int eq_num, int enable_flag)
5257 {
5258 	int i;
5259 
5260 	if (enable_flag == EQ_ENABLE) {
5261 		for (i = 0; i < eq_num; i++)
5262 			roce_write(hr_dev, ROCEE_VF_EVENT_INT_EN_REG +
5263 				   i * EQ_REG_OFFSET,
5264 				   HNS_ROCE_V2_VF_EVENT_INT_EN_M);
5265 
5266 		roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG,
5267 			   HNS_ROCE_V2_VF_ABN_INT_EN_M);
5268 		roce_write(hr_dev, ROCEE_VF_ABN_INT_CFG_REG,
5269 			   HNS_ROCE_V2_VF_ABN_INT_CFG_M);
5270 	} else {
5271 		for (i = 0; i < eq_num; i++)
5272 			roce_write(hr_dev, ROCEE_VF_EVENT_INT_EN_REG +
5273 				   i * EQ_REG_OFFSET,
5274 				   HNS_ROCE_V2_VF_EVENT_INT_EN_M & 0x0);
5275 
5276 		roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG,
5277 			   HNS_ROCE_V2_VF_ABN_INT_EN_M & 0x0);
5278 		roce_write(hr_dev, ROCEE_VF_ABN_INT_CFG_REG,
5279 			   HNS_ROCE_V2_VF_ABN_INT_CFG_M & 0x0);
5280 	}
5281 }
5282 
5283 static void hns_roce_v2_destroy_eqc(struct hns_roce_dev *hr_dev, int eqn)
5284 {
5285 	struct device *dev = hr_dev->dev;
5286 	int ret;
5287 
5288 	if (eqn < hr_dev->caps.num_comp_vectors)
5289 		ret = hns_roce_cmd_mbox(hr_dev, 0, 0, eqn & HNS_ROCE_V2_EQN_M,
5290 					0, HNS_ROCE_CMD_DESTROY_CEQC,
5291 					HNS_ROCE_CMD_TIMEOUT_MSECS);
5292 	else
5293 		ret = hns_roce_cmd_mbox(hr_dev, 0, 0, eqn & HNS_ROCE_V2_EQN_M,
5294 					0, HNS_ROCE_CMD_DESTROY_AEQC,
5295 					HNS_ROCE_CMD_TIMEOUT_MSECS);
5296 	if (ret)
5297 		dev_err(dev, "[mailbox cmd] destroy eqc(%d) failed.\n", eqn);
5298 }
5299 
5300 static void hns_roce_mhop_free_eq(struct hns_roce_dev *hr_dev,
5301 				  struct hns_roce_eq *eq)
5302 {
5303 	struct device *dev = hr_dev->dev;
5304 	u64 idx;
5305 	u64 size;
5306 	u32 buf_chk_sz;
5307 	u32 bt_chk_sz;
5308 	u32 mhop_num;
5309 	int eqe_alloc;
5310 	int i = 0;
5311 	int j = 0;
5312 
5313 	mhop_num = hr_dev->caps.eqe_hop_num;
5314 	buf_chk_sz = 1 << (hr_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT);
5315 	bt_chk_sz = 1 << (hr_dev->caps.eqe_ba_pg_sz + PAGE_SHIFT);
5316 
5317 	if (mhop_num == HNS_ROCE_HOP_NUM_0) {
5318 		dma_free_coherent(dev, (unsigned int)(eq->entries *
5319 				  eq->eqe_size), eq->bt_l0, eq->l0_dma);
5320 		return;
5321 	}
5322 
5323 	dma_free_coherent(dev, bt_chk_sz, eq->bt_l0, eq->l0_dma);
5324 	if (mhop_num == 1) {
5325 		for (i = 0; i < eq->l0_last_num; i++) {
5326 			if (i == eq->l0_last_num - 1) {
5327 				eqe_alloc = i * (buf_chk_sz / eq->eqe_size);
5328 				size = (eq->entries - eqe_alloc) * eq->eqe_size;
5329 				dma_free_coherent(dev, size, eq->buf[i],
5330 						  eq->buf_dma[i]);
5331 				break;
5332 			}
5333 			dma_free_coherent(dev, buf_chk_sz, eq->buf[i],
5334 					  eq->buf_dma[i]);
5335 		}
5336 	} else if (mhop_num == 2) {
5337 		for (i = 0; i < eq->l0_last_num; i++) {
5338 			dma_free_coherent(dev, bt_chk_sz, eq->bt_l1[i],
5339 					  eq->l1_dma[i]);
5340 
5341 			for (j = 0; j < bt_chk_sz / BA_BYTE_LEN; j++) {
5342 				idx = i * (bt_chk_sz / BA_BYTE_LEN) + j;
5343 				if ((i == eq->l0_last_num - 1)
5344 				     && j == eq->l1_last_num - 1) {
5345 					eqe_alloc = (buf_chk_sz / eq->eqe_size)
5346 						    * idx;
5347 					size = (eq->entries - eqe_alloc)
5348 						* eq->eqe_size;
5349 					dma_free_coherent(dev, size,
5350 							  eq->buf[idx],
5351 							  eq->buf_dma[idx]);
5352 					break;
5353 				}
5354 				dma_free_coherent(dev, buf_chk_sz, eq->buf[idx],
5355 						  eq->buf_dma[idx]);
5356 			}
5357 		}
5358 	}
5359 	kfree(eq->buf_dma);
5360 	kfree(eq->buf);
5361 	kfree(eq->l1_dma);
5362 	kfree(eq->bt_l1);
5363 	eq->buf_dma = NULL;
5364 	eq->buf = NULL;
5365 	eq->l1_dma = NULL;
5366 	eq->bt_l1 = NULL;
5367 }
5368 
5369 static void hns_roce_v2_free_eq(struct hns_roce_dev *hr_dev,
5370 				struct hns_roce_eq *eq)
5371 {
5372 	u32 buf_chk_sz;
5373 
5374 	buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
5375 
5376 	if (hr_dev->caps.eqe_hop_num) {
5377 		hns_roce_mhop_free_eq(hr_dev, eq);
5378 		return;
5379 	}
5380 
5381 	dma_free_coherent(hr_dev->dev, buf_chk_sz, eq->buf_list->buf,
5382 			  eq->buf_list->map);
5383 	kfree(eq->buf_list);
5384 }
5385 
5386 static void hns_roce_config_eqc(struct hns_roce_dev *hr_dev,
5387 				struct hns_roce_eq *eq,
5388 				void *mb_buf)
5389 {
5390 	struct hns_roce_eq_context *eqc;
5391 
5392 	eqc = mb_buf;
5393 	memset(eqc, 0, sizeof(struct hns_roce_eq_context));
5394 
5395 	/* init eqc */
5396 	eq->doorbell = hr_dev->reg_base + ROCEE_VF_EQ_DB_CFG0_REG;
5397 	eq->hop_num = hr_dev->caps.eqe_hop_num;
5398 	eq->cons_index = 0;
5399 	eq->over_ignore = HNS_ROCE_V2_EQ_OVER_IGNORE_0;
5400 	eq->coalesce = HNS_ROCE_V2_EQ_COALESCE_0;
5401 	eq->arm_st = HNS_ROCE_V2_EQ_ALWAYS_ARMED;
5402 	eq->eqe_ba_pg_sz = hr_dev->caps.eqe_ba_pg_sz;
5403 	eq->eqe_buf_pg_sz = hr_dev->caps.eqe_buf_pg_sz;
5404 	eq->shift = ilog2((unsigned int)eq->entries);
5405 
5406 	if (!eq->hop_num)
5407 		eq->eqe_ba = eq->buf_list->map;
5408 	else
5409 		eq->eqe_ba = eq->l0_dma;
5410 
5411 	/* set eqc state */
5412 	roce_set_field(eqc->byte_4,
5413 		       HNS_ROCE_EQC_EQ_ST_M,
5414 		       HNS_ROCE_EQC_EQ_ST_S,
5415 		       HNS_ROCE_V2_EQ_STATE_VALID);
5416 
5417 	/* set eqe hop num */
5418 	roce_set_field(eqc->byte_4,
5419 		       HNS_ROCE_EQC_HOP_NUM_M,
5420 		       HNS_ROCE_EQC_HOP_NUM_S, eq->hop_num);
5421 
5422 	/* set eqc over_ignore */
5423 	roce_set_field(eqc->byte_4,
5424 		       HNS_ROCE_EQC_OVER_IGNORE_M,
5425 		       HNS_ROCE_EQC_OVER_IGNORE_S, eq->over_ignore);
5426 
5427 	/* set eqc coalesce */
5428 	roce_set_field(eqc->byte_4,
5429 		       HNS_ROCE_EQC_COALESCE_M,
5430 		       HNS_ROCE_EQC_COALESCE_S, eq->coalesce);
5431 
5432 	/* set eqc arm_state */
5433 	roce_set_field(eqc->byte_4,
5434 		       HNS_ROCE_EQC_ARM_ST_M,
5435 		       HNS_ROCE_EQC_ARM_ST_S, eq->arm_st);
5436 
5437 	/* set eqn */
5438 	roce_set_field(eqc->byte_4,
5439 		       HNS_ROCE_EQC_EQN_M,
5440 		       HNS_ROCE_EQC_EQN_S, eq->eqn);
5441 
5442 	/* set eqe_cnt */
5443 	roce_set_field(eqc->byte_4,
5444 		       HNS_ROCE_EQC_EQE_CNT_M,
5445 		       HNS_ROCE_EQC_EQE_CNT_S,
5446 		       HNS_ROCE_EQ_INIT_EQE_CNT);
5447 
5448 	/* set eqe_ba_pg_sz */
5449 	roce_set_field(eqc->byte_8,
5450 		       HNS_ROCE_EQC_BA_PG_SZ_M,
5451 		       HNS_ROCE_EQC_BA_PG_SZ_S,
5452 		       eq->eqe_ba_pg_sz + PG_SHIFT_OFFSET);
5453 
5454 	/* set eqe_buf_pg_sz */
5455 	roce_set_field(eqc->byte_8,
5456 		       HNS_ROCE_EQC_BUF_PG_SZ_M,
5457 		       HNS_ROCE_EQC_BUF_PG_SZ_S,
5458 		       eq->eqe_buf_pg_sz + PG_SHIFT_OFFSET);
5459 
5460 	/* set eq_producer_idx */
5461 	roce_set_field(eqc->byte_8,
5462 		       HNS_ROCE_EQC_PROD_INDX_M,
5463 		       HNS_ROCE_EQC_PROD_INDX_S,
5464 		       HNS_ROCE_EQ_INIT_PROD_IDX);
5465 
5466 	/* set eq_max_cnt */
5467 	roce_set_field(eqc->byte_12,
5468 		       HNS_ROCE_EQC_MAX_CNT_M,
5469 		       HNS_ROCE_EQC_MAX_CNT_S, eq->eq_max_cnt);
5470 
5471 	/* set eq_period */
5472 	roce_set_field(eqc->byte_12,
5473 		       HNS_ROCE_EQC_PERIOD_M,
5474 		       HNS_ROCE_EQC_PERIOD_S, eq->eq_period);
5475 
5476 	/* set eqe_report_timer */
5477 	roce_set_field(eqc->eqe_report_timer,
5478 		       HNS_ROCE_EQC_REPORT_TIMER_M,
5479 		       HNS_ROCE_EQC_REPORT_TIMER_S,
5480 		       HNS_ROCE_EQ_INIT_REPORT_TIMER);
5481 
5482 	/* set eqe_ba [34:3] */
5483 	roce_set_field(eqc->eqe_ba0,
5484 		       HNS_ROCE_EQC_EQE_BA_L_M,
5485 		       HNS_ROCE_EQC_EQE_BA_L_S, eq->eqe_ba >> 3);
5486 
5487 	/* set eqe_ba [64:35] */
5488 	roce_set_field(eqc->eqe_ba1,
5489 		       HNS_ROCE_EQC_EQE_BA_H_M,
5490 		       HNS_ROCE_EQC_EQE_BA_H_S, eq->eqe_ba >> 35);
5491 
5492 	/* set eq shift */
5493 	roce_set_field(eqc->byte_28,
5494 		       HNS_ROCE_EQC_SHIFT_M,
5495 		       HNS_ROCE_EQC_SHIFT_S, eq->shift);
5496 
5497 	/* set eq MSI_IDX */
5498 	roce_set_field(eqc->byte_28,
5499 		       HNS_ROCE_EQC_MSI_INDX_M,
5500 		       HNS_ROCE_EQC_MSI_INDX_S,
5501 		       HNS_ROCE_EQ_INIT_MSI_IDX);
5502 
5503 	/* set cur_eqe_ba [27:12] */
5504 	roce_set_field(eqc->byte_28,
5505 		       HNS_ROCE_EQC_CUR_EQE_BA_L_M,
5506 		       HNS_ROCE_EQC_CUR_EQE_BA_L_S, eq->cur_eqe_ba >> 12);
5507 
5508 	/* set cur_eqe_ba [59:28] */
5509 	roce_set_field(eqc->byte_32,
5510 		       HNS_ROCE_EQC_CUR_EQE_BA_M_M,
5511 		       HNS_ROCE_EQC_CUR_EQE_BA_M_S, eq->cur_eqe_ba >> 28);
5512 
5513 	/* set cur_eqe_ba [63:60] */
5514 	roce_set_field(eqc->byte_36,
5515 		       HNS_ROCE_EQC_CUR_EQE_BA_H_M,
5516 		       HNS_ROCE_EQC_CUR_EQE_BA_H_S, eq->cur_eqe_ba >> 60);
5517 
5518 	/* set eq consumer idx */
5519 	roce_set_field(eqc->byte_36,
5520 		       HNS_ROCE_EQC_CONS_INDX_M,
5521 		       HNS_ROCE_EQC_CONS_INDX_S,
5522 		       HNS_ROCE_EQ_INIT_CONS_IDX);
5523 
5524 	/* set nex_eqe_ba[43:12] */
5525 	roce_set_field(eqc->nxt_eqe_ba0,
5526 		       HNS_ROCE_EQC_NXT_EQE_BA_L_M,
5527 		       HNS_ROCE_EQC_NXT_EQE_BA_L_S, eq->nxt_eqe_ba >> 12);
5528 
5529 	/* set nex_eqe_ba[63:44] */
5530 	roce_set_field(eqc->nxt_eqe_ba1,
5531 		       HNS_ROCE_EQC_NXT_EQE_BA_H_M,
5532 		       HNS_ROCE_EQC_NXT_EQE_BA_H_S, eq->nxt_eqe_ba >> 44);
5533 }
5534 
5535 static int hns_roce_mhop_alloc_eq(struct hns_roce_dev *hr_dev,
5536 				  struct hns_roce_eq *eq)
5537 {
5538 	struct device *dev = hr_dev->dev;
5539 	int eq_alloc_done = 0;
5540 	int eq_buf_cnt = 0;
5541 	int eqe_alloc;
5542 	u32 buf_chk_sz;
5543 	u32 bt_chk_sz;
5544 	u32 mhop_num;
5545 	u64 size;
5546 	u64 idx;
5547 	int ba_num;
5548 	int bt_num;
5549 	int record_i;
5550 	int record_j;
5551 	int i = 0;
5552 	int j = 0;
5553 
5554 	mhop_num = hr_dev->caps.eqe_hop_num;
5555 	buf_chk_sz = 1 << (hr_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT);
5556 	bt_chk_sz = 1 << (hr_dev->caps.eqe_ba_pg_sz + PAGE_SHIFT);
5557 
5558 	ba_num = DIV_ROUND_UP(PAGE_ALIGN(eq->entries * eq->eqe_size),
5559 			      buf_chk_sz);
5560 	bt_num = DIV_ROUND_UP(ba_num, bt_chk_sz / BA_BYTE_LEN);
5561 
5562 	if (mhop_num == HNS_ROCE_HOP_NUM_0) {
5563 		if (eq->entries > buf_chk_sz / eq->eqe_size) {
5564 			dev_err(dev, "eq entries %d is larger than buf_pg_sz!",
5565 				eq->entries);
5566 			return -EINVAL;
5567 		}
5568 		eq->bt_l0 = dma_alloc_coherent(dev, eq->entries * eq->eqe_size,
5569 					       &(eq->l0_dma), GFP_KERNEL);
5570 		if (!eq->bt_l0)
5571 			return -ENOMEM;
5572 
5573 		eq->cur_eqe_ba = eq->l0_dma;
5574 		eq->nxt_eqe_ba = 0;
5575 
5576 		return 0;
5577 	}
5578 
5579 	eq->buf_dma = kcalloc(ba_num, sizeof(*eq->buf_dma), GFP_KERNEL);
5580 	if (!eq->buf_dma)
5581 		return -ENOMEM;
5582 	eq->buf = kcalloc(ba_num, sizeof(*eq->buf), GFP_KERNEL);
5583 	if (!eq->buf)
5584 		goto err_kcalloc_buf;
5585 
5586 	if (mhop_num == 2) {
5587 		eq->l1_dma = kcalloc(bt_num, sizeof(*eq->l1_dma), GFP_KERNEL);
5588 		if (!eq->l1_dma)
5589 			goto err_kcalloc_l1_dma;
5590 
5591 		eq->bt_l1 = kcalloc(bt_num, sizeof(*eq->bt_l1), GFP_KERNEL);
5592 		if (!eq->bt_l1)
5593 			goto err_kcalloc_bt_l1;
5594 	}
5595 
5596 	/* alloc L0 BT */
5597 	eq->bt_l0 = dma_alloc_coherent(dev, bt_chk_sz, &eq->l0_dma, GFP_KERNEL);
5598 	if (!eq->bt_l0)
5599 		goto err_dma_alloc_l0;
5600 
5601 	if (mhop_num == 1) {
5602 		if (ba_num > (bt_chk_sz / BA_BYTE_LEN))
5603 			dev_err(dev, "ba_num %d is too large for 1 hop\n",
5604 				ba_num);
5605 
5606 		/* alloc buf */
5607 		for (i = 0; i < bt_chk_sz / BA_BYTE_LEN; i++) {
5608 			if (eq_buf_cnt + 1 < ba_num) {
5609 				size = buf_chk_sz;
5610 			} else {
5611 				eqe_alloc = i * (buf_chk_sz / eq->eqe_size);
5612 				size = (eq->entries - eqe_alloc) * eq->eqe_size;
5613 			}
5614 			eq->buf[i] = dma_alloc_coherent(dev, size,
5615 							&(eq->buf_dma[i]),
5616 							GFP_KERNEL);
5617 			if (!eq->buf[i])
5618 				goto err_dma_alloc_buf;
5619 
5620 			*(eq->bt_l0 + i) = eq->buf_dma[i];
5621 
5622 			eq_buf_cnt++;
5623 			if (eq_buf_cnt >= ba_num)
5624 				break;
5625 		}
5626 		eq->cur_eqe_ba = eq->buf_dma[0];
5627 		if (ba_num > 1)
5628 			eq->nxt_eqe_ba = eq->buf_dma[1];
5629 
5630 	} else if (mhop_num == 2) {
5631 		/* alloc L1 BT and buf */
5632 		for (i = 0; i < bt_chk_sz / BA_BYTE_LEN; i++) {
5633 			eq->bt_l1[i] = dma_alloc_coherent(dev, bt_chk_sz,
5634 							  &(eq->l1_dma[i]),
5635 							  GFP_KERNEL);
5636 			if (!eq->bt_l1[i])
5637 				goto err_dma_alloc_l1;
5638 			*(eq->bt_l0 + i) = eq->l1_dma[i];
5639 
5640 			for (j = 0; j < bt_chk_sz / BA_BYTE_LEN; j++) {
5641 				idx = i * bt_chk_sz / BA_BYTE_LEN + j;
5642 				if (eq_buf_cnt + 1 < ba_num) {
5643 					size = buf_chk_sz;
5644 				} else {
5645 					eqe_alloc = (buf_chk_sz / eq->eqe_size)
5646 						    * idx;
5647 					size = (eq->entries - eqe_alloc)
5648 						* eq->eqe_size;
5649 				}
5650 				eq->buf[idx] = dma_alloc_coherent(dev, size,
5651 								  &(eq->buf_dma[idx]),
5652 								  GFP_KERNEL);
5653 				if (!eq->buf[idx])
5654 					goto err_dma_alloc_buf;
5655 
5656 				*(eq->bt_l1[i] + j) = eq->buf_dma[idx];
5657 
5658 				eq_buf_cnt++;
5659 				if (eq_buf_cnt >= ba_num) {
5660 					eq_alloc_done = 1;
5661 					break;
5662 				}
5663 			}
5664 
5665 			if (eq_alloc_done)
5666 				break;
5667 		}
5668 		eq->cur_eqe_ba = eq->buf_dma[0];
5669 		if (ba_num > 1)
5670 			eq->nxt_eqe_ba = eq->buf_dma[1];
5671 	}
5672 
5673 	eq->l0_last_num = i + 1;
5674 	if (mhop_num == 2)
5675 		eq->l1_last_num = j + 1;
5676 
5677 	return 0;
5678 
5679 err_dma_alloc_l1:
5680 	dma_free_coherent(dev, bt_chk_sz, eq->bt_l0, eq->l0_dma);
5681 	eq->bt_l0 = NULL;
5682 	eq->l0_dma = 0;
5683 	for (i -= 1; i >= 0; i--) {
5684 		dma_free_coherent(dev, bt_chk_sz, eq->bt_l1[i],
5685 				  eq->l1_dma[i]);
5686 
5687 		for (j = 0; j < bt_chk_sz / BA_BYTE_LEN; j++) {
5688 			idx = i * bt_chk_sz / BA_BYTE_LEN + j;
5689 			dma_free_coherent(dev, buf_chk_sz, eq->buf[idx],
5690 					  eq->buf_dma[idx]);
5691 		}
5692 	}
5693 	goto err_dma_alloc_l0;
5694 
5695 err_dma_alloc_buf:
5696 	dma_free_coherent(dev, bt_chk_sz, eq->bt_l0, eq->l0_dma);
5697 	eq->bt_l0 = NULL;
5698 	eq->l0_dma = 0;
5699 
5700 	if (mhop_num == 1)
5701 		for (i -= 1; i >= 0; i--)
5702 			dma_free_coherent(dev, buf_chk_sz, eq->buf[i],
5703 					  eq->buf_dma[i]);
5704 	else if (mhop_num == 2) {
5705 		record_i = i;
5706 		record_j = j;
5707 		for (; i >= 0; i--) {
5708 			dma_free_coherent(dev, bt_chk_sz, eq->bt_l1[i],
5709 					  eq->l1_dma[i]);
5710 
5711 			for (j = 0; j < bt_chk_sz / BA_BYTE_LEN; j++) {
5712 				if (i == record_i && j >= record_j)
5713 					break;
5714 
5715 				idx = i * bt_chk_sz / BA_BYTE_LEN + j;
5716 				dma_free_coherent(dev, buf_chk_sz,
5717 						  eq->buf[idx],
5718 						  eq->buf_dma[idx]);
5719 			}
5720 		}
5721 	}
5722 
5723 err_dma_alloc_l0:
5724 	kfree(eq->bt_l1);
5725 	eq->bt_l1 = NULL;
5726 
5727 err_kcalloc_bt_l1:
5728 	kfree(eq->l1_dma);
5729 	eq->l1_dma = NULL;
5730 
5731 err_kcalloc_l1_dma:
5732 	kfree(eq->buf);
5733 	eq->buf = NULL;
5734 
5735 err_kcalloc_buf:
5736 	kfree(eq->buf_dma);
5737 	eq->buf_dma = NULL;
5738 
5739 	return -ENOMEM;
5740 }
5741 
5742 static int hns_roce_v2_create_eq(struct hns_roce_dev *hr_dev,
5743 				 struct hns_roce_eq *eq,
5744 				 unsigned int eq_cmd)
5745 {
5746 	struct device *dev = hr_dev->dev;
5747 	struct hns_roce_cmd_mailbox *mailbox;
5748 	u32 buf_chk_sz = 0;
5749 	int ret;
5750 
5751 	/* Allocate mailbox memory */
5752 	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
5753 	if (IS_ERR(mailbox))
5754 		return PTR_ERR(mailbox);
5755 
5756 	if (!hr_dev->caps.eqe_hop_num) {
5757 		buf_chk_sz = 1 << (hr_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT);
5758 
5759 		eq->buf_list = kzalloc(sizeof(struct hns_roce_buf_list),
5760 				       GFP_KERNEL);
5761 		if (!eq->buf_list) {
5762 			ret = -ENOMEM;
5763 			goto free_cmd_mbox;
5764 		}
5765 
5766 		eq->buf_list->buf = dma_alloc_coherent(dev, buf_chk_sz,
5767 						       &(eq->buf_list->map),
5768 						       GFP_KERNEL);
5769 		if (!eq->buf_list->buf) {
5770 			ret = -ENOMEM;
5771 			goto err_alloc_buf;
5772 		}
5773 
5774 	} else {
5775 		ret = hns_roce_mhop_alloc_eq(hr_dev, eq);
5776 		if (ret) {
5777 			ret = -ENOMEM;
5778 			goto free_cmd_mbox;
5779 		}
5780 	}
5781 
5782 	hns_roce_config_eqc(hr_dev, eq, mailbox->buf);
5783 
5784 	ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, eq->eqn, 0,
5785 				eq_cmd, HNS_ROCE_CMD_TIMEOUT_MSECS);
5786 	if (ret) {
5787 		dev_err(dev, "[mailbox cmd] create eqc failed.\n");
5788 		goto err_cmd_mbox;
5789 	}
5790 
5791 	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
5792 
5793 	return 0;
5794 
5795 err_cmd_mbox:
5796 	if (!hr_dev->caps.eqe_hop_num)
5797 		dma_free_coherent(dev, buf_chk_sz, eq->buf_list->buf,
5798 				  eq->buf_list->map);
5799 	else {
5800 		hns_roce_mhop_free_eq(hr_dev, eq);
5801 		goto free_cmd_mbox;
5802 	}
5803 
5804 err_alloc_buf:
5805 	kfree(eq->buf_list);
5806 
5807 free_cmd_mbox:
5808 	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
5809 
5810 	return ret;
5811 }
5812 
5813 static int __hns_roce_request_irq(struct hns_roce_dev *hr_dev, int irq_num,
5814 				  int comp_num, int aeq_num, int other_num)
5815 {
5816 	struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;
5817 	int i, j;
5818 	int ret;
5819 
5820 	for (i = 0; i < irq_num; i++) {
5821 		hr_dev->irq_names[i] = kzalloc(HNS_ROCE_INT_NAME_LEN,
5822 					       GFP_KERNEL);
5823 		if (!hr_dev->irq_names[i]) {
5824 			ret = -ENOMEM;
5825 			goto err_kzalloc_failed;
5826 		}
5827 	}
5828 
5829 	/* irq contains: abnormal + AEQ + CEQ */
5830 	for (j = 0; j < other_num; j++)
5831 		snprintf((char *)hr_dev->irq_names[j],
5832 			 HNS_ROCE_INT_NAME_LEN, "hns-abn-%d", j);
5833 
5834 	for (j = other_num; j < (other_num + aeq_num); j++)
5835 		snprintf((char *)hr_dev->irq_names[j],
5836 			 HNS_ROCE_INT_NAME_LEN, "hns-aeq-%d",
5837 			 j - other_num);
5838 
5839 	for (j = (other_num + aeq_num); j < irq_num; j++)
5840 		snprintf((char *)hr_dev->irq_names[j],
5841 			 HNS_ROCE_INT_NAME_LEN, "hns-ceq-%d",
5842 			 j - other_num - aeq_num);
5843 
5844 	for (j = 0; j < irq_num; j++) {
5845 		if (j < other_num)
5846 			ret = request_irq(hr_dev->irq[j],
5847 					  hns_roce_v2_msix_interrupt_abn,
5848 					  0, hr_dev->irq_names[j], hr_dev);
5849 
5850 		else if (j < (other_num + comp_num))
5851 			ret = request_irq(eq_table->eq[j - other_num].irq,
5852 					  hns_roce_v2_msix_interrupt_eq,
5853 					  0, hr_dev->irq_names[j + aeq_num],
5854 					  &eq_table->eq[j - other_num]);
5855 		else
5856 			ret = request_irq(eq_table->eq[j - other_num].irq,
5857 					  hns_roce_v2_msix_interrupt_eq,
5858 					  0, hr_dev->irq_names[j - comp_num],
5859 					  &eq_table->eq[j - other_num]);
5860 		if (ret) {
5861 			dev_err(hr_dev->dev, "Request irq error!\n");
5862 			goto err_request_failed;
5863 		}
5864 	}
5865 
5866 	return 0;
5867 
5868 err_request_failed:
5869 	for (j -= 1; j >= 0; j--)
5870 		if (j < other_num)
5871 			free_irq(hr_dev->irq[j], hr_dev);
5872 		else
5873 			free_irq(eq_table->eq[j - other_num].irq,
5874 				 &eq_table->eq[j - other_num]);
5875 
5876 err_kzalloc_failed:
5877 	for (i -= 1; i >= 0; i--)
5878 		kfree(hr_dev->irq_names[i]);
5879 
5880 	return ret;
5881 }
5882 
5883 static void __hns_roce_free_irq(struct hns_roce_dev *hr_dev)
5884 {
5885 	int irq_num;
5886 	int eq_num;
5887 	int i;
5888 
5889 	eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors;
5890 	irq_num = eq_num + hr_dev->caps.num_other_vectors;
5891 
5892 	for (i = 0; i < hr_dev->caps.num_other_vectors; i++)
5893 		free_irq(hr_dev->irq[i], hr_dev);
5894 
5895 	for (i = 0; i < eq_num; i++)
5896 		free_irq(hr_dev->eq_table.eq[i].irq, &hr_dev->eq_table.eq[i]);
5897 
5898 	for (i = 0; i < irq_num; i++)
5899 		kfree(hr_dev->irq_names[i]);
5900 }
5901 
5902 static int hns_roce_v2_init_eq_table(struct hns_roce_dev *hr_dev)
5903 {
5904 	struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;
5905 	struct device *dev = hr_dev->dev;
5906 	struct hns_roce_eq *eq;
5907 	unsigned int eq_cmd;
5908 	int irq_num;
5909 	int eq_num;
5910 	int other_num;
5911 	int comp_num;
5912 	int aeq_num;
5913 	int i;
5914 	int ret;
5915 
5916 	other_num = hr_dev->caps.num_other_vectors;
5917 	comp_num = hr_dev->caps.num_comp_vectors;
5918 	aeq_num = hr_dev->caps.num_aeq_vectors;
5919 
5920 	eq_num = comp_num + aeq_num;
5921 	irq_num = eq_num + other_num;
5922 
5923 	eq_table->eq = kcalloc(eq_num, sizeof(*eq_table->eq), GFP_KERNEL);
5924 	if (!eq_table->eq)
5925 		return -ENOMEM;
5926 
5927 	/* create eq */
5928 	for (i = 0; i < eq_num; i++) {
5929 		eq = &eq_table->eq[i];
5930 		eq->hr_dev = hr_dev;
5931 		eq->eqn = i;
5932 		if (i < comp_num) {
5933 			/* CEQ */
5934 			eq_cmd = HNS_ROCE_CMD_CREATE_CEQC;
5935 			eq->type_flag = HNS_ROCE_CEQ;
5936 			eq->entries = hr_dev->caps.ceqe_depth;
5937 			eq->eqe_size = HNS_ROCE_CEQ_ENTRY_SIZE;
5938 			eq->irq = hr_dev->irq[i + other_num + aeq_num];
5939 			eq->eq_max_cnt = HNS_ROCE_CEQ_DEFAULT_BURST_NUM;
5940 			eq->eq_period = HNS_ROCE_CEQ_DEFAULT_INTERVAL;
5941 		} else {
5942 			/* AEQ */
5943 			eq_cmd = HNS_ROCE_CMD_CREATE_AEQC;
5944 			eq->type_flag = HNS_ROCE_AEQ;
5945 			eq->entries = hr_dev->caps.aeqe_depth;
5946 			eq->eqe_size = HNS_ROCE_AEQ_ENTRY_SIZE;
5947 			eq->irq = hr_dev->irq[i - comp_num + other_num];
5948 			eq->eq_max_cnt = HNS_ROCE_AEQ_DEFAULT_BURST_NUM;
5949 			eq->eq_period = HNS_ROCE_AEQ_DEFAULT_INTERVAL;
5950 		}
5951 
5952 		ret = hns_roce_v2_create_eq(hr_dev, eq, eq_cmd);
5953 		if (ret) {
5954 			dev_err(dev, "eq create failed.\n");
5955 			goto err_create_eq_fail;
5956 		}
5957 	}
5958 
5959 	/* enable irq */
5960 	hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_ENABLE);
5961 
5962 	ret = __hns_roce_request_irq(hr_dev, irq_num, comp_num,
5963 				     aeq_num, other_num);
5964 	if (ret) {
5965 		dev_err(dev, "Request irq failed.\n");
5966 		goto err_request_irq_fail;
5967 	}
5968 
5969 	hr_dev->irq_workq =
5970 		create_singlethread_workqueue("hns_roce_irq_workqueue");
5971 	if (!hr_dev->irq_workq) {
5972 		dev_err(dev, "Create irq workqueue failed!\n");
5973 		ret = -ENOMEM;
5974 		goto err_create_wq_fail;
5975 	}
5976 
5977 	return 0;
5978 
5979 err_create_wq_fail:
5980 	__hns_roce_free_irq(hr_dev);
5981 
5982 err_request_irq_fail:
5983 	hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_DISABLE);
5984 
5985 err_create_eq_fail:
5986 	for (i -= 1; i >= 0; i--)
5987 		hns_roce_v2_free_eq(hr_dev, &eq_table->eq[i]);
5988 	kfree(eq_table->eq);
5989 
5990 	return ret;
5991 }
5992 
5993 static void hns_roce_v2_cleanup_eq_table(struct hns_roce_dev *hr_dev)
5994 {
5995 	struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;
5996 	int eq_num;
5997 	int i;
5998 
5999 	eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors;
6000 
6001 	/* Disable irq */
6002 	hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_DISABLE);
6003 
6004 	__hns_roce_free_irq(hr_dev);
6005 
6006 	for (i = 0; i < eq_num; i++) {
6007 		hns_roce_v2_destroy_eqc(hr_dev, i);
6008 
6009 		hns_roce_v2_free_eq(hr_dev, &eq_table->eq[i]);
6010 	}
6011 
6012 	kfree(eq_table->eq);
6013 
6014 	flush_workqueue(hr_dev->irq_workq);
6015 	destroy_workqueue(hr_dev->irq_workq);
6016 }
6017 
6018 static void hns_roce_v2_write_srqc(struct hns_roce_dev *hr_dev,
6019 				   struct hns_roce_srq *srq, u32 pdn, u16 xrcd,
6020 				   u32 cqn, void *mb_buf, u64 *mtts_wqe,
6021 				   u64 *mtts_idx, dma_addr_t dma_handle_wqe,
6022 				   dma_addr_t dma_handle_idx)
6023 {
6024 	struct hns_roce_srq_context *srq_context;
6025 
6026 	srq_context = mb_buf;
6027 	memset(srq_context, 0, sizeof(*srq_context));
6028 
6029 	roce_set_field(srq_context->byte_4_srqn_srqst, SRQC_BYTE_4_SRQ_ST_M,
6030 		       SRQC_BYTE_4_SRQ_ST_S, 1);
6031 
6032 	roce_set_field(srq_context->byte_4_srqn_srqst,
6033 		       SRQC_BYTE_4_SRQ_WQE_HOP_NUM_M,
6034 		       SRQC_BYTE_4_SRQ_WQE_HOP_NUM_S,
6035 		       (hr_dev->caps.srqwqe_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 :
6036 		       hr_dev->caps.srqwqe_hop_num));
6037 	roce_set_field(srq_context->byte_4_srqn_srqst,
6038 		       SRQC_BYTE_4_SRQ_SHIFT_M, SRQC_BYTE_4_SRQ_SHIFT_S,
6039 		       ilog2(srq->wqe_cnt));
6040 
6041 	roce_set_field(srq_context->byte_4_srqn_srqst, SRQC_BYTE_4_SRQN_M,
6042 		       SRQC_BYTE_4_SRQN_S, srq->srqn);
6043 
6044 	roce_set_field(srq_context->byte_8_limit_wl, SRQC_BYTE_8_SRQ_LIMIT_WL_M,
6045 		       SRQC_BYTE_8_SRQ_LIMIT_WL_S, 0);
6046 
6047 	roce_set_field(srq_context->byte_12_xrcd, SRQC_BYTE_12_SRQ_XRCD_M,
6048 		       SRQC_BYTE_12_SRQ_XRCD_S, xrcd);
6049 
6050 	srq_context->wqe_bt_ba = cpu_to_le32((u32)(dma_handle_wqe >> 3));
6051 
6052 	roce_set_field(srq_context->byte_24_wqe_bt_ba,
6053 		       SRQC_BYTE_24_SRQ_WQE_BT_BA_M,
6054 		       SRQC_BYTE_24_SRQ_WQE_BT_BA_S,
6055 		       dma_handle_wqe >> 35);
6056 
6057 	roce_set_field(srq_context->byte_28_rqws_pd, SRQC_BYTE_28_PD_M,
6058 		       SRQC_BYTE_28_PD_S, pdn);
6059 	roce_set_field(srq_context->byte_28_rqws_pd, SRQC_BYTE_28_RQWS_M,
6060 		       SRQC_BYTE_28_RQWS_S, srq->max_gs <= 0 ? 0 :
6061 		       fls(srq->max_gs - 1));
6062 
6063 	srq_context->idx_bt_ba = cpu_to_le32(dma_handle_idx >> 3);
6064 	roce_set_field(srq_context->rsv_idx_bt_ba,
6065 		       SRQC_BYTE_36_SRQ_IDX_BT_BA_M,
6066 		       SRQC_BYTE_36_SRQ_IDX_BT_BA_S,
6067 		       dma_handle_idx >> 35);
6068 
6069 	srq_context->idx_cur_blk_addr =
6070 		cpu_to_le32(mtts_idx[0] >> PAGE_ADDR_SHIFT);
6071 	roce_set_field(srq_context->byte_44_idxbufpgsz_addr,
6072 		       SRQC_BYTE_44_SRQ_IDX_CUR_BLK_ADDR_M,
6073 		       SRQC_BYTE_44_SRQ_IDX_CUR_BLK_ADDR_S,
6074 		       mtts_idx[0] >> (32 + PAGE_ADDR_SHIFT));
6075 	roce_set_field(srq_context->byte_44_idxbufpgsz_addr,
6076 		       SRQC_BYTE_44_SRQ_IDX_HOP_NUM_M,
6077 		       SRQC_BYTE_44_SRQ_IDX_HOP_NUM_S,
6078 		       hr_dev->caps.idx_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 :
6079 		       hr_dev->caps.idx_hop_num);
6080 
6081 	roce_set_field(srq_context->byte_44_idxbufpgsz_addr,
6082 		       SRQC_BYTE_44_SRQ_IDX_BA_PG_SZ_M,
6083 		       SRQC_BYTE_44_SRQ_IDX_BA_PG_SZ_S,
6084 		       hr_dev->caps.idx_ba_pg_sz + PG_SHIFT_OFFSET);
6085 	roce_set_field(srq_context->byte_44_idxbufpgsz_addr,
6086 		       SRQC_BYTE_44_SRQ_IDX_BUF_PG_SZ_M,
6087 		       SRQC_BYTE_44_SRQ_IDX_BUF_PG_SZ_S,
6088 		       hr_dev->caps.idx_buf_pg_sz + PG_SHIFT_OFFSET);
6089 
6090 	srq_context->idx_nxt_blk_addr =
6091 		cpu_to_le32(mtts_idx[1] >> PAGE_ADDR_SHIFT);
6092 	roce_set_field(srq_context->rsv_idxnxtblkaddr,
6093 		       SRQC_BYTE_52_SRQ_IDX_NXT_BLK_ADDR_M,
6094 		       SRQC_BYTE_52_SRQ_IDX_NXT_BLK_ADDR_S,
6095 		       mtts_idx[1] >> (32 + PAGE_ADDR_SHIFT));
6096 	roce_set_field(srq_context->byte_56_xrc_cqn,
6097 		       SRQC_BYTE_56_SRQ_XRC_CQN_M, SRQC_BYTE_56_SRQ_XRC_CQN_S,
6098 		       cqn);
6099 	roce_set_field(srq_context->byte_56_xrc_cqn,
6100 		       SRQC_BYTE_56_SRQ_WQE_BA_PG_SZ_M,
6101 		       SRQC_BYTE_56_SRQ_WQE_BA_PG_SZ_S,
6102 		       hr_dev->caps.srqwqe_ba_pg_sz + PG_SHIFT_OFFSET);
6103 	roce_set_field(srq_context->byte_56_xrc_cqn,
6104 		       SRQC_BYTE_56_SRQ_WQE_BUF_PG_SZ_M,
6105 		       SRQC_BYTE_56_SRQ_WQE_BUF_PG_SZ_S,
6106 		       hr_dev->caps.srqwqe_buf_pg_sz + PG_SHIFT_OFFSET);
6107 
6108 	roce_set_bit(srq_context->db_record_addr_record_en,
6109 		     SRQC_BYTE_60_SRQ_RECORD_EN_S, 0);
6110 }
6111 
6112 static int hns_roce_v2_modify_srq(struct ib_srq *ibsrq,
6113 				  struct ib_srq_attr *srq_attr,
6114 				  enum ib_srq_attr_mask srq_attr_mask,
6115 				  struct ib_udata *udata)
6116 {
6117 	struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
6118 	struct hns_roce_srq *srq = to_hr_srq(ibsrq);
6119 	struct hns_roce_srq_context *srq_context;
6120 	struct hns_roce_srq_context *srqc_mask;
6121 	struct hns_roce_cmd_mailbox *mailbox;
6122 	int ret;
6123 
6124 	if (srq_attr_mask & IB_SRQ_LIMIT) {
6125 		if (srq_attr->srq_limit >= srq->wqe_cnt)
6126 			return -EINVAL;
6127 
6128 		mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
6129 		if (IS_ERR(mailbox))
6130 			return PTR_ERR(mailbox);
6131 
6132 		srq_context = mailbox->buf;
6133 		srqc_mask = (struct hns_roce_srq_context *)mailbox->buf + 1;
6134 
6135 		memset(srqc_mask, 0xff, sizeof(*srqc_mask));
6136 
6137 		roce_set_field(srq_context->byte_8_limit_wl,
6138 			       SRQC_BYTE_8_SRQ_LIMIT_WL_M,
6139 			       SRQC_BYTE_8_SRQ_LIMIT_WL_S, srq_attr->srq_limit);
6140 		roce_set_field(srqc_mask->byte_8_limit_wl,
6141 			       SRQC_BYTE_8_SRQ_LIMIT_WL_M,
6142 			       SRQC_BYTE_8_SRQ_LIMIT_WL_S, 0);
6143 
6144 		ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, srq->srqn, 0,
6145 					HNS_ROCE_CMD_MODIFY_SRQC,
6146 					HNS_ROCE_CMD_TIMEOUT_MSECS);
6147 		hns_roce_free_cmd_mailbox(hr_dev, mailbox);
6148 		if (ret) {
6149 			dev_err(hr_dev->dev,
6150 				"MODIFY SRQ Failed to cmd mailbox.\n");
6151 			return ret;
6152 		}
6153 	}
6154 
6155 	return 0;
6156 }
6157 
6158 static int hns_roce_v2_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr)
6159 {
6160 	struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
6161 	struct hns_roce_srq *srq = to_hr_srq(ibsrq);
6162 	struct hns_roce_srq_context *srq_context;
6163 	struct hns_roce_cmd_mailbox *mailbox;
6164 	int limit_wl;
6165 	int ret;
6166 
6167 	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
6168 	if (IS_ERR(mailbox))
6169 		return PTR_ERR(mailbox);
6170 
6171 	srq_context = mailbox->buf;
6172 	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, srq->srqn, 0,
6173 				HNS_ROCE_CMD_QUERY_SRQC,
6174 				HNS_ROCE_CMD_TIMEOUT_MSECS);
6175 	if (ret) {
6176 		dev_err(hr_dev->dev, "QUERY SRQ cmd process error\n");
6177 		goto out;
6178 	}
6179 
6180 	limit_wl = roce_get_field(srq_context->byte_8_limit_wl,
6181 				  SRQC_BYTE_8_SRQ_LIMIT_WL_M,
6182 				  SRQC_BYTE_8_SRQ_LIMIT_WL_S);
6183 
6184 	attr->srq_limit = limit_wl;
6185 	attr->max_wr    = srq->wqe_cnt - 1;
6186 	attr->max_sge   = srq->max_gs;
6187 
6188 	memcpy(srq_context, mailbox->buf, sizeof(*srq_context));
6189 
6190 out:
6191 	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
6192 	return ret;
6193 }
6194 
6195 static int find_empty_entry(struct hns_roce_idx_que *idx_que,
6196 			    unsigned long size)
6197 {
6198 	int wqe_idx;
6199 
6200 	if (unlikely(bitmap_full(idx_que->bitmap, size)))
6201 		return -ENOSPC;
6202 
6203 	wqe_idx = find_first_zero_bit(idx_que->bitmap, size);
6204 
6205 	bitmap_set(idx_que->bitmap, wqe_idx, 1);
6206 
6207 	return wqe_idx;
6208 }
6209 
6210 static void fill_idx_queue(struct hns_roce_idx_que *idx_que,
6211 			   int cur_idx, int wqe_idx)
6212 {
6213 	unsigned int *addr;
6214 
6215 	addr = (unsigned int *)hns_roce_buf_offset(&idx_que->idx_buf,
6216 						   cur_idx * idx_que->entry_sz);
6217 	*addr = wqe_idx;
6218 }
6219 
6220 static int hns_roce_v2_post_srq_recv(struct ib_srq *ibsrq,
6221 				     const struct ib_recv_wr *wr,
6222 				     const struct ib_recv_wr **bad_wr)
6223 {
6224 	struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
6225 	struct hns_roce_srq *srq = to_hr_srq(ibsrq);
6226 	struct hns_roce_v2_wqe_data_seg *dseg;
6227 	struct hns_roce_v2_db srq_db;
6228 	unsigned long flags;
6229 	int ret = 0;
6230 	int wqe_idx;
6231 	void *wqe;
6232 	int nreq;
6233 	int ind;
6234 	int i;
6235 
6236 	spin_lock_irqsave(&srq->lock, flags);
6237 
6238 	ind = srq->head & (srq->wqe_cnt - 1);
6239 
6240 	for (nreq = 0; wr; ++nreq, wr = wr->next) {
6241 		if (unlikely(wr->num_sge > srq->max_gs)) {
6242 			ret = -EINVAL;
6243 			*bad_wr = wr;
6244 			break;
6245 		}
6246 
6247 		if (unlikely(srq->head == srq->tail)) {
6248 			ret = -ENOMEM;
6249 			*bad_wr = wr;
6250 			break;
6251 		}
6252 
6253 		wqe_idx = find_empty_entry(&srq->idx_que, srq->wqe_cnt);
6254 		if (wqe_idx < 0) {
6255 			ret = -ENOMEM;
6256 			*bad_wr = wr;
6257 			break;
6258 		}
6259 
6260 		fill_idx_queue(&srq->idx_que, ind, wqe_idx);
6261 		wqe = get_srq_wqe(srq, wqe_idx);
6262 		dseg = (struct hns_roce_v2_wqe_data_seg *)wqe;
6263 
6264 		for (i = 0; i < wr->num_sge; ++i) {
6265 			dseg[i].len = cpu_to_le32(wr->sg_list[i].length);
6266 			dseg[i].lkey = cpu_to_le32(wr->sg_list[i].lkey);
6267 			dseg[i].addr = cpu_to_le64(wr->sg_list[i].addr);
6268 		}
6269 
6270 		if (i < srq->max_gs) {
6271 			dseg[i].len = 0;
6272 			dseg[i].lkey = cpu_to_le32(0x100);
6273 			dseg[i].addr = 0;
6274 		}
6275 
6276 		srq->wrid[wqe_idx] = wr->wr_id;
6277 		ind = (ind + 1) & (srq->wqe_cnt - 1);
6278 	}
6279 
6280 	if (likely(nreq)) {
6281 		srq->head += nreq;
6282 
6283 		/*
6284 		 * Make sure that descriptors are written before
6285 		 * doorbell record.
6286 		 */
6287 		wmb();
6288 
6289 		srq_db.byte_4 =
6290 			cpu_to_le32(HNS_ROCE_V2_SRQ_DB << V2_DB_BYTE_4_CMD_S |
6291 				    (srq->srqn & V2_DB_BYTE_4_TAG_M));
6292 		srq_db.parameter = cpu_to_le32(srq->head);
6293 
6294 		hns_roce_write64(hr_dev, (__le32 *)&srq_db, srq->db_reg_l);
6295 
6296 	}
6297 
6298 	spin_unlock_irqrestore(&srq->lock, flags);
6299 
6300 	return ret;
6301 }
6302 
6303 static const struct hns_roce_dfx_hw hns_roce_dfx_hw_v2 = {
6304 	.query_cqc_info = hns_roce_v2_query_cqc_info,
6305 };
6306 
6307 static const struct ib_device_ops hns_roce_v2_dev_ops = {
6308 	.destroy_qp = hns_roce_v2_destroy_qp,
6309 	.modify_cq = hns_roce_v2_modify_cq,
6310 	.poll_cq = hns_roce_v2_poll_cq,
6311 	.post_recv = hns_roce_v2_post_recv,
6312 	.post_send = hns_roce_v2_post_send,
6313 	.query_qp = hns_roce_v2_query_qp,
6314 	.req_notify_cq = hns_roce_v2_req_notify_cq,
6315 };
6316 
6317 static const struct ib_device_ops hns_roce_v2_dev_srq_ops = {
6318 	.modify_srq = hns_roce_v2_modify_srq,
6319 	.post_srq_recv = hns_roce_v2_post_srq_recv,
6320 	.query_srq = hns_roce_v2_query_srq,
6321 };
6322 
6323 static const struct hns_roce_hw hns_roce_hw_v2 = {
6324 	.cmq_init = hns_roce_v2_cmq_init,
6325 	.cmq_exit = hns_roce_v2_cmq_exit,
6326 	.hw_profile = hns_roce_v2_profile,
6327 	.hw_init = hns_roce_v2_init,
6328 	.hw_exit = hns_roce_v2_exit,
6329 	.post_mbox = hns_roce_v2_post_mbox,
6330 	.chk_mbox = hns_roce_v2_chk_mbox,
6331 	.rst_prc_mbox = hns_roce_v2_rst_process_cmd,
6332 	.set_gid = hns_roce_v2_set_gid,
6333 	.set_mac = hns_roce_v2_set_mac,
6334 	.write_mtpt = hns_roce_v2_write_mtpt,
6335 	.rereg_write_mtpt = hns_roce_v2_rereg_write_mtpt,
6336 	.frmr_write_mtpt = hns_roce_v2_frmr_write_mtpt,
6337 	.mw_write_mtpt = hns_roce_v2_mw_write_mtpt,
6338 	.write_cqc = hns_roce_v2_write_cqc,
6339 	.set_hem = hns_roce_v2_set_hem,
6340 	.clear_hem = hns_roce_v2_clear_hem,
6341 	.modify_qp = hns_roce_v2_modify_qp,
6342 	.query_qp = hns_roce_v2_query_qp,
6343 	.destroy_qp = hns_roce_v2_destroy_qp,
6344 	.qp_flow_control_init = hns_roce_v2_qp_flow_control_init,
6345 	.modify_cq = hns_roce_v2_modify_cq,
6346 	.post_send = hns_roce_v2_post_send,
6347 	.post_recv = hns_roce_v2_post_recv,
6348 	.req_notify_cq = hns_roce_v2_req_notify_cq,
6349 	.poll_cq = hns_roce_v2_poll_cq,
6350 	.init_eq = hns_roce_v2_init_eq_table,
6351 	.cleanup_eq = hns_roce_v2_cleanup_eq_table,
6352 	.write_srqc = hns_roce_v2_write_srqc,
6353 	.modify_srq = hns_roce_v2_modify_srq,
6354 	.query_srq = hns_roce_v2_query_srq,
6355 	.post_srq_recv = hns_roce_v2_post_srq_recv,
6356 	.hns_roce_dev_ops = &hns_roce_v2_dev_ops,
6357 	.hns_roce_dev_srq_ops = &hns_roce_v2_dev_srq_ops,
6358 };
6359 
6360 static const struct pci_device_id hns_roce_hw_v2_pci_tbl[] = {
6361 	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA), 0},
6362 	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC), 0},
6363 	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA), 0},
6364 	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC), 0},
6365 	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC), 0},
6366 	/* required last entry */
6367 	{0, }
6368 };
6369 
6370 MODULE_DEVICE_TABLE(pci, hns_roce_hw_v2_pci_tbl);
6371 
6372 static void hns_roce_hw_v2_get_cfg(struct hns_roce_dev *hr_dev,
6373 				  struct hnae3_handle *handle)
6374 {
6375 	struct hns_roce_v2_priv *priv = hr_dev->priv;
6376 	int i;
6377 
6378 	hr_dev->pci_dev = handle->pdev;
6379 	hr_dev->dev = &handle->pdev->dev;
6380 	hr_dev->hw = &hns_roce_hw_v2;
6381 	hr_dev->dfx = &hns_roce_dfx_hw_v2;
6382 	hr_dev->sdb_offset = ROCEE_DB_SQ_L_0_REG;
6383 	hr_dev->odb_offset = hr_dev->sdb_offset;
6384 
6385 	/* Get info from NIC driver. */
6386 	hr_dev->reg_base = handle->rinfo.roce_io_base;
6387 	hr_dev->caps.num_ports = 1;
6388 	hr_dev->iboe.netdevs[0] = handle->rinfo.netdev;
6389 	hr_dev->iboe.phy_port[0] = 0;
6390 
6391 	addrconf_addr_eui48((u8 *)&hr_dev->ib_dev.node_guid,
6392 			    hr_dev->iboe.netdevs[0]->dev_addr);
6393 
6394 	for (i = 0; i < HNS_ROCE_V2_MAX_IRQ_NUM; i++)
6395 		hr_dev->irq[i] = pci_irq_vector(handle->pdev,
6396 						i + handle->rinfo.base_vector);
6397 
6398 	/* cmd issue mode: 0 is poll, 1 is event */
6399 	hr_dev->cmd_mod = 1;
6400 	hr_dev->loop_idc = 0;
6401 
6402 	hr_dev->reset_cnt = handle->ae_algo->ops->ae_dev_reset_cnt(handle);
6403 	priv->handle = handle;
6404 }
6405 
6406 static int __hns_roce_hw_v2_init_instance(struct hnae3_handle *handle)
6407 {
6408 	struct hns_roce_dev *hr_dev;
6409 	int ret;
6410 
6411 	hr_dev = ib_alloc_device(hns_roce_dev, ib_dev);
6412 	if (!hr_dev)
6413 		return -ENOMEM;
6414 
6415 	hr_dev->priv = kzalloc(sizeof(struct hns_roce_v2_priv), GFP_KERNEL);
6416 	if (!hr_dev->priv) {
6417 		ret = -ENOMEM;
6418 		goto error_failed_kzalloc;
6419 	}
6420 
6421 	hns_roce_hw_v2_get_cfg(hr_dev, handle);
6422 
6423 	ret = hns_roce_init(hr_dev);
6424 	if (ret) {
6425 		dev_err(hr_dev->dev, "RoCE Engine init failed!\n");
6426 		goto error_failed_get_cfg;
6427 	}
6428 
6429 	handle->priv = hr_dev;
6430 
6431 	return 0;
6432 
6433 error_failed_get_cfg:
6434 	kfree(hr_dev->priv);
6435 
6436 error_failed_kzalloc:
6437 	ib_dealloc_device(&hr_dev->ib_dev);
6438 
6439 	return ret;
6440 }
6441 
6442 static void __hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle,
6443 					   bool reset)
6444 {
6445 	struct hns_roce_dev *hr_dev = (struct hns_roce_dev *)handle->priv;
6446 
6447 	if (!hr_dev)
6448 		return;
6449 
6450 	handle->priv = NULL;
6451 	hns_roce_exit(hr_dev);
6452 	kfree(hr_dev->priv);
6453 	ib_dealloc_device(&hr_dev->ib_dev);
6454 }
6455 
6456 static int hns_roce_hw_v2_init_instance(struct hnae3_handle *handle)
6457 {
6458 	const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
6459 	const struct pci_device_id *id;
6460 	struct device *dev = &handle->pdev->dev;
6461 	int ret;
6462 
6463 	handle->rinfo.instance_state = HNS_ROCE_STATE_INIT;
6464 
6465 	if (ops->ae_dev_resetting(handle) || ops->get_hw_reset_stat(handle)) {
6466 		handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT;
6467 		goto reset_chk_err;
6468 	}
6469 
6470 	id = pci_match_id(hns_roce_hw_v2_pci_tbl, handle->pdev);
6471 	if (!id)
6472 		return 0;
6473 
6474 	ret = __hns_roce_hw_v2_init_instance(handle);
6475 	if (ret) {
6476 		handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT;
6477 		dev_err(dev, "RoCE instance init failed! ret = %d\n", ret);
6478 		if (ops->ae_dev_resetting(handle) ||
6479 		    ops->get_hw_reset_stat(handle))
6480 			goto reset_chk_err;
6481 		else
6482 			return ret;
6483 	}
6484 
6485 	handle->rinfo.instance_state = HNS_ROCE_STATE_INITED;
6486 
6487 
6488 	return 0;
6489 
6490 reset_chk_err:
6491 	dev_err(dev, "Device is busy in resetting state.\n"
6492 		     "please retry later.\n");
6493 
6494 	return -EBUSY;
6495 }
6496 
6497 static void hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle,
6498 					   bool reset)
6499 {
6500 	if (handle->rinfo.instance_state != HNS_ROCE_STATE_INITED)
6501 		return;
6502 
6503 	handle->rinfo.instance_state = HNS_ROCE_STATE_UNINIT;
6504 
6505 	__hns_roce_hw_v2_uninit_instance(handle, reset);
6506 
6507 	handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT;
6508 }
6509 static int hns_roce_hw_v2_reset_notify_down(struct hnae3_handle *handle)
6510 {
6511 	struct hns_roce_dev *hr_dev;
6512 	struct ib_event event;
6513 
6514 	if (handle->rinfo.instance_state != HNS_ROCE_STATE_INITED) {
6515 		set_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state);
6516 		return 0;
6517 	}
6518 
6519 	handle->rinfo.reset_state = HNS_ROCE_STATE_RST_DOWN;
6520 	clear_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state);
6521 
6522 	hr_dev = (struct hns_roce_dev *)handle->priv;
6523 	if (!hr_dev)
6524 		return 0;
6525 
6526 	hr_dev->is_reset = true;
6527 	hr_dev->active = false;
6528 	hr_dev->dis_db = true;
6529 
6530 	event.event = IB_EVENT_DEVICE_FATAL;
6531 	event.device = &hr_dev->ib_dev;
6532 	event.element.port_num = 1;
6533 	ib_dispatch_event(&event);
6534 
6535 	return 0;
6536 }
6537 
6538 static int hns_roce_hw_v2_reset_notify_init(struct hnae3_handle *handle)
6539 {
6540 	struct device *dev = &handle->pdev->dev;
6541 	int ret;
6542 
6543 	if (test_and_clear_bit(HNS_ROCE_RST_DIRECT_RETURN,
6544 			       &handle->rinfo.state)) {
6545 		handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INITED;
6546 		return 0;
6547 	}
6548 
6549 	handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INIT;
6550 
6551 	dev_info(&handle->pdev->dev, "In reset process RoCE client reinit.\n");
6552 	ret = __hns_roce_hw_v2_init_instance(handle);
6553 	if (ret) {
6554 		/* when reset notify type is HNAE3_INIT_CLIENT In reset notify
6555 		 * callback function, RoCE Engine reinitialize. If RoCE reinit
6556 		 * failed, we should inform NIC driver.
6557 		 */
6558 		handle->priv = NULL;
6559 		dev_err(dev, "In reset process RoCE reinit failed %d.\n", ret);
6560 	} else {
6561 		handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INITED;
6562 		dev_info(dev, "Reset done, RoCE client reinit finished.\n");
6563 	}
6564 
6565 	return ret;
6566 }
6567 
6568 static int hns_roce_hw_v2_reset_notify_uninit(struct hnae3_handle *handle)
6569 {
6570 	if (test_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state))
6571 		return 0;
6572 
6573 	handle->rinfo.reset_state = HNS_ROCE_STATE_RST_UNINIT;
6574 	dev_info(&handle->pdev->dev, "In reset process RoCE client uninit.\n");
6575 	msleep(HNS_ROCE_V2_HW_RST_UNINT_DELAY);
6576 	__hns_roce_hw_v2_uninit_instance(handle, false);
6577 
6578 	return 0;
6579 }
6580 
6581 static int hns_roce_hw_v2_reset_notify(struct hnae3_handle *handle,
6582 				       enum hnae3_reset_notify_type type)
6583 {
6584 	int ret = 0;
6585 
6586 	switch (type) {
6587 	case HNAE3_DOWN_CLIENT:
6588 		ret = hns_roce_hw_v2_reset_notify_down(handle);
6589 		break;
6590 	case HNAE3_INIT_CLIENT:
6591 		ret = hns_roce_hw_v2_reset_notify_init(handle);
6592 		break;
6593 	case HNAE3_UNINIT_CLIENT:
6594 		ret = hns_roce_hw_v2_reset_notify_uninit(handle);
6595 		break;
6596 	default:
6597 		break;
6598 	}
6599 
6600 	return ret;
6601 }
6602 
6603 static const struct hnae3_client_ops hns_roce_hw_v2_ops = {
6604 	.init_instance = hns_roce_hw_v2_init_instance,
6605 	.uninit_instance = hns_roce_hw_v2_uninit_instance,
6606 	.reset_notify = hns_roce_hw_v2_reset_notify,
6607 };
6608 
6609 static struct hnae3_client hns_roce_hw_v2_client = {
6610 	.name = "hns_roce_hw_v2",
6611 	.type = HNAE3_CLIENT_ROCE,
6612 	.ops = &hns_roce_hw_v2_ops,
6613 };
6614 
6615 static int __init hns_roce_hw_v2_init(void)
6616 {
6617 	return hnae3_register_client(&hns_roce_hw_v2_client);
6618 }
6619 
6620 static void __exit hns_roce_hw_v2_exit(void)
6621 {
6622 	hnae3_unregister_client(&hns_roce_hw_v2_client);
6623 }
6624 
6625 module_init(hns_roce_hw_v2_init);
6626 module_exit(hns_roce_hw_v2_exit);
6627 
6628 MODULE_LICENSE("Dual BSD/GPL");
6629 MODULE_AUTHOR("Wei Hu <xavier.huwei@huawei.com>");
6630 MODULE_AUTHOR("Lijun Ou <oulijun@huawei.com>");
6631 MODULE_AUTHOR("Shaobo Xu <xushaobo2@huawei.com>");
6632 MODULE_DESCRIPTION("Hisilicon Hip08 Family RoCE Driver");
6633