1 /*
2  * Copyright (c) 2016 Hisilicon Limited.
3  * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
4  *
5  * This software is available to you under a choice of one of two
6  * licenses.  You may choose to be licensed under the terms of the GNU
7  * General Public License (GPL) Version 2, available from the file
8  * COPYING in the main directory of this source tree, or the
9  * OpenIB.org BSD license below:
10  *
11  *     Redistribution and use in source and binary forms, with or
12  *     without modification, are permitted provided that the following
13  *     conditions are met:
14  *
15  *      - Redistributions of source code must retain the above
16  *        copyright notice, this list of conditions and the following
17  *        disclaimer.
18  *
19  *      - Redistributions in binary form must reproduce the above
20  *        copyright notice, this list of conditions and the following
21  *        disclaimer in the documentation and/or other materials
22  *        provided with the distribution.
23  *
24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31  * SOFTWARE.
32  */
33 
34 #include <linux/pci.h>
35 #include <rdma/ib_addr.h>
36 #include <rdma/ib_umem.h>
37 #include <rdma/uverbs_ioctl.h>
38 #include "hns_roce_common.h"
39 #include "hns_roce_device.h"
40 #include "hns_roce_hem.h"
41 
42 static void flush_work_handle(struct work_struct *work)
43 {
44 	struct hns_roce_work *flush_work = container_of(work,
45 					struct hns_roce_work, work);
46 	struct hns_roce_qp *hr_qp = container_of(flush_work,
47 					struct hns_roce_qp, flush_work);
48 	struct device *dev = flush_work->hr_dev->dev;
49 	struct ib_qp_attr attr;
50 	int attr_mask;
51 	int ret;
52 
53 	attr_mask = IB_QP_STATE;
54 	attr.qp_state = IB_QPS_ERR;
55 
56 	if (test_and_clear_bit(HNS_ROCE_FLUSH_FLAG, &hr_qp->flush_flag)) {
57 		ret = hns_roce_modify_qp(&hr_qp->ibqp, &attr, attr_mask, NULL);
58 		if (ret)
59 			dev_err(dev, "modify QP to error state failed(%d) during CQE flush\n",
60 				ret);
61 	}
62 
63 	/*
64 	 * make sure we signal QP destroy leg that flush QP was completed
65 	 * so that it can safely proceed ahead now and destroy QP
66 	 */
67 	if (refcount_dec_and_test(&hr_qp->refcount))
68 		complete(&hr_qp->free);
69 }
70 
71 void init_flush_work(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
72 {
73 	struct hns_roce_work *flush_work = &hr_qp->flush_work;
74 
75 	flush_work->hr_dev = hr_dev;
76 	INIT_WORK(&flush_work->work, flush_work_handle);
77 	refcount_inc(&hr_qp->refcount);
78 	queue_work(hr_dev->irq_workq, &flush_work->work);
79 }
80 
81 void flush_cqe(struct hns_roce_dev *dev, struct hns_roce_qp *qp)
82 {
83 	/*
84 	 * Hip08 hardware cannot flush the WQEs in SQ/RQ if the QP state
85 	 * gets into errored mode. Hence, as a workaround to this
86 	 * hardware limitation, driver needs to assist in flushing. But
87 	 * the flushing operation uses mailbox to convey the QP state to
88 	 * the hardware and which can sleep due to the mutex protection
89 	 * around the mailbox calls. Hence, use the deferred flush for
90 	 * now.
91 	 */
92 	if (!test_and_set_bit(HNS_ROCE_FLUSH_FLAG, &qp->flush_flag))
93 		init_flush_work(dev, qp);
94 }
95 
96 void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type)
97 {
98 	struct device *dev = hr_dev->dev;
99 	struct hns_roce_qp *qp;
100 
101 	xa_lock(&hr_dev->qp_table_xa);
102 	qp = __hns_roce_qp_lookup(hr_dev, qpn);
103 	if (qp)
104 		refcount_inc(&qp->refcount);
105 	xa_unlock(&hr_dev->qp_table_xa);
106 
107 	if (!qp) {
108 		dev_warn(dev, "async event for bogus QP %08x\n", qpn);
109 		return;
110 	}
111 
112 	if (event_type == HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR ||
113 	    event_type == HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR ||
114 	    event_type == HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR ||
115 	    event_type == HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION ||
116 	    event_type == HNS_ROCE_EVENT_TYPE_INVALID_XRCETH) {
117 		qp->state = IB_QPS_ERR;
118 
119 		flush_cqe(hr_dev, qp);
120 	}
121 
122 	qp->event(qp, (enum hns_roce_event)event_type);
123 
124 	if (refcount_dec_and_test(&qp->refcount))
125 		complete(&qp->free);
126 }
127 
128 static void hns_roce_ib_qp_event(struct hns_roce_qp *hr_qp,
129 				 enum hns_roce_event type)
130 {
131 	struct ib_qp *ibqp = &hr_qp->ibqp;
132 	struct ib_event event;
133 
134 	if (ibqp->event_handler) {
135 		event.device = ibqp->device;
136 		event.element.qp = ibqp;
137 		switch (type) {
138 		case HNS_ROCE_EVENT_TYPE_PATH_MIG:
139 			event.event = IB_EVENT_PATH_MIG;
140 			break;
141 		case HNS_ROCE_EVENT_TYPE_COMM_EST:
142 			event.event = IB_EVENT_COMM_EST;
143 			break;
144 		case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:
145 			event.event = IB_EVENT_SQ_DRAINED;
146 			break;
147 		case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:
148 			event.event = IB_EVENT_QP_LAST_WQE_REACHED;
149 			break;
150 		case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
151 			event.event = IB_EVENT_QP_FATAL;
152 			break;
153 		case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED:
154 			event.event = IB_EVENT_PATH_MIG_ERR;
155 			break;
156 		case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
157 			event.event = IB_EVENT_QP_REQ_ERR;
158 			break;
159 		case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
160 		case HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION:
161 		case HNS_ROCE_EVENT_TYPE_INVALID_XRCETH:
162 			event.event = IB_EVENT_QP_ACCESS_ERR;
163 			break;
164 		default:
165 			dev_dbg(ibqp->device->dev.parent, "roce_ib: Unexpected event type %d on QP %06lx\n",
166 				type, hr_qp->qpn);
167 			return;
168 		}
169 		ibqp->event_handler(&event, ibqp->qp_context);
170 	}
171 }
172 
173 static u8 get_least_load_bankid_for_qp(struct hns_roce_bank *bank)
174 {
175 	u32 least_load = bank[0].inuse;
176 	u8 bankid = 0;
177 	u32 bankcnt;
178 	u8 i;
179 
180 	for (i = 1; i < HNS_ROCE_QP_BANK_NUM; i++) {
181 		bankcnt = bank[i].inuse;
182 		if (bankcnt < least_load) {
183 			least_load = bankcnt;
184 			bankid = i;
185 		}
186 	}
187 
188 	return bankid;
189 }
190 
191 static int alloc_qpn_with_bankid(struct hns_roce_bank *bank, u8 bankid,
192 				 unsigned long *qpn)
193 {
194 	int id;
195 
196 	id = ida_alloc_range(&bank->ida, bank->next, bank->max, GFP_KERNEL);
197 	if (id < 0) {
198 		id = ida_alloc_range(&bank->ida, bank->min, bank->max,
199 				     GFP_KERNEL);
200 		if (id < 0)
201 			return id;
202 	}
203 
204 	/* the QPN should keep increasing until the max value is reached. */
205 	bank->next = (id + 1) > bank->max ? bank->min : id + 1;
206 
207 	/* the lower 3 bits is bankid */
208 	*qpn = (id << 3) | bankid;
209 
210 	return 0;
211 }
212 static int alloc_qpn(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
213 {
214 	struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
215 	unsigned long num = 0;
216 	u8 bankid;
217 	int ret;
218 
219 	if (hr_qp->ibqp.qp_type == IB_QPT_GSI) {
220 		num = 1;
221 	} else {
222 		mutex_lock(&qp_table->bank_mutex);
223 		bankid = get_least_load_bankid_for_qp(qp_table->bank);
224 
225 		ret = alloc_qpn_with_bankid(&qp_table->bank[bankid], bankid,
226 					    &num);
227 		if (ret) {
228 			ibdev_err(&hr_dev->ib_dev,
229 				  "failed to alloc QPN, ret = %d\n", ret);
230 			mutex_unlock(&qp_table->bank_mutex);
231 			return ret;
232 		}
233 
234 		qp_table->bank[bankid].inuse++;
235 		mutex_unlock(&qp_table->bank_mutex);
236 	}
237 
238 	hr_qp->qpn = num;
239 
240 	return 0;
241 }
242 
243 static void add_qp_to_list(struct hns_roce_dev *hr_dev,
244 			   struct hns_roce_qp *hr_qp,
245 			   struct ib_cq *send_cq, struct ib_cq *recv_cq)
246 {
247 	struct hns_roce_cq *hr_send_cq, *hr_recv_cq;
248 	unsigned long flags;
249 
250 	hr_send_cq = send_cq ? to_hr_cq(send_cq) : NULL;
251 	hr_recv_cq = recv_cq ? to_hr_cq(recv_cq) : NULL;
252 
253 	spin_lock_irqsave(&hr_dev->qp_list_lock, flags);
254 	hns_roce_lock_cqs(hr_send_cq, hr_recv_cq);
255 
256 	list_add_tail(&hr_qp->node, &hr_dev->qp_list);
257 	if (hr_send_cq)
258 		list_add_tail(&hr_qp->sq_node, &hr_send_cq->sq_list);
259 	if (hr_recv_cq)
260 		list_add_tail(&hr_qp->rq_node, &hr_recv_cq->rq_list);
261 
262 	hns_roce_unlock_cqs(hr_send_cq, hr_recv_cq);
263 	spin_unlock_irqrestore(&hr_dev->qp_list_lock, flags);
264 }
265 
266 static int hns_roce_qp_store(struct hns_roce_dev *hr_dev,
267 			     struct hns_roce_qp *hr_qp,
268 			     struct ib_qp_init_attr *init_attr)
269 {
270 	struct xarray *xa = &hr_dev->qp_table_xa;
271 	int ret;
272 
273 	if (!hr_qp->qpn)
274 		return -EINVAL;
275 
276 	ret = xa_err(xa_store_irq(xa, hr_qp->qpn, hr_qp, GFP_KERNEL));
277 	if (ret)
278 		dev_err(hr_dev->dev, "failed to xa store for QPC\n");
279 	else
280 		/* add QP to device's QP list for softwc */
281 		add_qp_to_list(hr_dev, hr_qp, init_attr->send_cq,
282 			       init_attr->recv_cq);
283 
284 	return ret;
285 }
286 
287 static int alloc_qpc(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
288 {
289 	struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
290 	struct device *dev = hr_dev->dev;
291 	int ret;
292 
293 	if (!hr_qp->qpn)
294 		return -EINVAL;
295 
296 	/* Alloc memory for QPC */
297 	ret = hns_roce_table_get(hr_dev, &qp_table->qp_table, hr_qp->qpn);
298 	if (ret) {
299 		dev_err(dev, "failed to get QPC table\n");
300 		goto err_out;
301 	}
302 
303 	/* Alloc memory for IRRL */
304 	ret = hns_roce_table_get(hr_dev, &qp_table->irrl_table, hr_qp->qpn);
305 	if (ret) {
306 		dev_err(dev, "failed to get IRRL table\n");
307 		goto err_put_qp;
308 	}
309 
310 	if (hr_dev->caps.trrl_entry_sz) {
311 		/* Alloc memory for TRRL */
312 		ret = hns_roce_table_get(hr_dev, &qp_table->trrl_table,
313 					 hr_qp->qpn);
314 		if (ret) {
315 			dev_err(dev, "failed to get TRRL table\n");
316 			goto err_put_irrl;
317 		}
318 	}
319 
320 	if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) {
321 		/* Alloc memory for SCC CTX */
322 		ret = hns_roce_table_get(hr_dev, &qp_table->sccc_table,
323 					 hr_qp->qpn);
324 		if (ret) {
325 			dev_err(dev, "failed to get SCC CTX table\n");
326 			goto err_put_trrl;
327 		}
328 	}
329 
330 	return 0;
331 
332 err_put_trrl:
333 	if (hr_dev->caps.trrl_entry_sz)
334 		hns_roce_table_put(hr_dev, &qp_table->trrl_table, hr_qp->qpn);
335 
336 err_put_irrl:
337 	hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn);
338 
339 err_put_qp:
340 	hns_roce_table_put(hr_dev, &qp_table->qp_table, hr_qp->qpn);
341 
342 err_out:
343 	return ret;
344 }
345 
346 static void qp_user_mmap_entry_remove(struct hns_roce_qp *hr_qp)
347 {
348 	rdma_user_mmap_entry_remove(&hr_qp->dwqe_mmap_entry->rdma_entry);
349 }
350 
351 void hns_roce_qp_remove(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
352 {
353 	struct xarray *xa = &hr_dev->qp_table_xa;
354 	unsigned long flags;
355 
356 	list_del(&hr_qp->node);
357 
358 	if (hr_qp->ibqp.qp_type != IB_QPT_XRC_TGT)
359 		list_del(&hr_qp->sq_node);
360 
361 	if (hr_qp->ibqp.qp_type != IB_QPT_XRC_INI &&
362 	    hr_qp->ibqp.qp_type != IB_QPT_XRC_TGT)
363 		list_del(&hr_qp->rq_node);
364 
365 	xa_lock_irqsave(xa, flags);
366 	__xa_erase(xa, hr_qp->qpn);
367 	xa_unlock_irqrestore(xa, flags);
368 }
369 
370 static void free_qpc(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
371 {
372 	struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
373 
374 	if (hr_dev->caps.trrl_entry_sz)
375 		hns_roce_table_put(hr_dev, &qp_table->trrl_table, hr_qp->qpn);
376 	hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn);
377 }
378 
379 static inline u8 get_qp_bankid(unsigned long qpn)
380 {
381 	/* The lower 3 bits of QPN are used to hash to different banks */
382 	return (u8)(qpn & GENMASK(2, 0));
383 }
384 
385 static void free_qpn(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
386 {
387 	u8 bankid;
388 
389 	if (hr_qp->ibqp.qp_type == IB_QPT_GSI)
390 		return;
391 
392 	if (hr_qp->qpn < hr_dev->caps.reserved_qps)
393 		return;
394 
395 	bankid = get_qp_bankid(hr_qp->qpn);
396 
397 	ida_free(&hr_dev->qp_table.bank[bankid].ida, hr_qp->qpn >> 3);
398 
399 	mutex_lock(&hr_dev->qp_table.bank_mutex);
400 	hr_dev->qp_table.bank[bankid].inuse--;
401 	mutex_unlock(&hr_dev->qp_table.bank_mutex);
402 }
403 
404 static u32 proc_rq_sge(struct hns_roce_dev *dev, struct hns_roce_qp *hr_qp,
405 		       bool user)
406 {
407 	u32 max_sge = dev->caps.max_rq_sg;
408 
409 	if (dev->pci_dev->revision >= PCI_REVISION_ID_HIP09)
410 		return max_sge;
411 
412 	/* Reserve SGEs only for HIP08 in kernel; The userspace driver will
413 	 * calculate number of max_sge with reserved SGEs when allocating wqe
414 	 * buf, so there is no need to do this again in kernel. But the number
415 	 * may exceed the capacity of SGEs recorded in the firmware, so the
416 	 * kernel driver should just adapt the value accordingly.
417 	 */
418 	if (user)
419 		max_sge = roundup_pow_of_two(max_sge + 1);
420 	else
421 		hr_qp->rq.rsv_sge = 1;
422 
423 	return max_sge;
424 }
425 
426 static int set_rq_size(struct hns_roce_dev *hr_dev, struct ib_qp_cap *cap,
427 		       struct hns_roce_qp *hr_qp, int has_rq, bool user)
428 {
429 	u32 max_sge = proc_rq_sge(hr_dev, hr_qp, user);
430 	u32 cnt;
431 
432 	/* If srq exist, set zero for relative number of rq */
433 	if (!has_rq) {
434 		hr_qp->rq.wqe_cnt = 0;
435 		hr_qp->rq.max_gs = 0;
436 		hr_qp->rq_inl_buf.wqe_cnt = 0;
437 		cap->max_recv_wr = 0;
438 		cap->max_recv_sge = 0;
439 
440 		return 0;
441 	}
442 
443 	/* Check the validity of QP support capacity */
444 	if (!cap->max_recv_wr || cap->max_recv_wr > hr_dev->caps.max_wqes ||
445 	    cap->max_recv_sge > max_sge) {
446 		ibdev_err(&hr_dev->ib_dev,
447 			  "RQ config error, depth = %u, sge = %u\n",
448 			  cap->max_recv_wr, cap->max_recv_sge);
449 		return -EINVAL;
450 	}
451 
452 	cnt = roundup_pow_of_two(max(cap->max_recv_wr, hr_dev->caps.min_wqes));
453 	if (cnt > hr_dev->caps.max_wqes) {
454 		ibdev_err(&hr_dev->ib_dev, "rq depth %u too large\n",
455 			  cap->max_recv_wr);
456 		return -EINVAL;
457 	}
458 
459 	hr_qp->rq.max_gs = roundup_pow_of_two(max(1U, cap->max_recv_sge) +
460 					      hr_qp->rq.rsv_sge);
461 
462 	hr_qp->rq.wqe_shift = ilog2(hr_dev->caps.max_rq_desc_sz *
463 				    hr_qp->rq.max_gs);
464 
465 	hr_qp->rq.wqe_cnt = cnt;
466 	if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE &&
467 	    hr_qp->ibqp.qp_type != IB_QPT_UD &&
468 	    hr_qp->ibqp.qp_type != IB_QPT_GSI)
469 		hr_qp->rq_inl_buf.wqe_cnt = cnt;
470 	else
471 		hr_qp->rq_inl_buf.wqe_cnt = 0;
472 
473 	cap->max_recv_wr = cnt;
474 	cap->max_recv_sge = hr_qp->rq.max_gs - hr_qp->rq.rsv_sge;
475 
476 	return 0;
477 }
478 
479 static u32 get_wqe_ext_sge_cnt(struct hns_roce_qp *qp)
480 {
481 	/* GSI/UD QP only has extended sge */
482 	if (qp->ibqp.qp_type == IB_QPT_GSI || qp->ibqp.qp_type == IB_QPT_UD)
483 		return qp->sq.max_gs;
484 
485 	if (qp->sq.max_gs > HNS_ROCE_SGE_IN_WQE)
486 		return qp->sq.max_gs - HNS_ROCE_SGE_IN_WQE;
487 
488 	return 0;
489 }
490 
491 static void set_ext_sge_param(struct hns_roce_dev *hr_dev, u32 sq_wqe_cnt,
492 			      struct hns_roce_qp *hr_qp, struct ib_qp_cap *cap)
493 {
494 	u32 total_sge_cnt;
495 	u32 wqe_sge_cnt;
496 
497 	hr_qp->sge.sge_shift = HNS_ROCE_SGE_SHIFT;
498 
499 	hr_qp->sq.max_gs = max(1U, cap->max_send_sge);
500 
501 	wqe_sge_cnt = get_wqe_ext_sge_cnt(hr_qp);
502 
503 	/* If the number of extended sge is not zero, they MUST use the
504 	 * space of HNS_HW_PAGE_SIZE at least.
505 	 */
506 	if (wqe_sge_cnt) {
507 		total_sge_cnt = roundup_pow_of_two(sq_wqe_cnt * wqe_sge_cnt);
508 		hr_qp->sge.sge_cnt = max(total_sge_cnt,
509 				(u32)HNS_HW_PAGE_SIZE / HNS_ROCE_SGE_SIZE);
510 	}
511 }
512 
513 static int check_sq_size_with_integrity(struct hns_roce_dev *hr_dev,
514 					struct ib_qp_cap *cap,
515 					struct hns_roce_ib_create_qp *ucmd)
516 {
517 	u32 roundup_sq_stride = roundup_pow_of_two(hr_dev->caps.max_sq_desc_sz);
518 	u8 max_sq_stride = ilog2(roundup_sq_stride);
519 
520 	/* Sanity check SQ size before proceeding */
521 	if (ucmd->log_sq_stride > max_sq_stride ||
522 	    ucmd->log_sq_stride < HNS_ROCE_IB_MIN_SQ_STRIDE) {
523 		ibdev_err(&hr_dev->ib_dev, "failed to check SQ stride size.\n");
524 		return -EINVAL;
525 	}
526 
527 	if (cap->max_send_sge > hr_dev->caps.max_sq_sg) {
528 		ibdev_err(&hr_dev->ib_dev, "failed to check SQ SGE size %u.\n",
529 			  cap->max_send_sge);
530 		return -EINVAL;
531 	}
532 
533 	return 0;
534 }
535 
536 static int set_user_sq_size(struct hns_roce_dev *hr_dev,
537 			    struct ib_qp_cap *cap, struct hns_roce_qp *hr_qp,
538 			    struct hns_roce_ib_create_qp *ucmd)
539 {
540 	struct ib_device *ibdev = &hr_dev->ib_dev;
541 	u32 cnt = 0;
542 	int ret;
543 
544 	if (check_shl_overflow(1, ucmd->log_sq_bb_count, &cnt) ||
545 	    cnt > hr_dev->caps.max_wqes)
546 		return -EINVAL;
547 
548 	ret = check_sq_size_with_integrity(hr_dev, cap, ucmd);
549 	if (ret) {
550 		ibdev_err(ibdev, "failed to check user SQ size, ret = %d.\n",
551 			  ret);
552 		return ret;
553 	}
554 
555 	set_ext_sge_param(hr_dev, cnt, hr_qp, cap);
556 
557 	hr_qp->sq.wqe_shift = ucmd->log_sq_stride;
558 	hr_qp->sq.wqe_cnt = cnt;
559 
560 	return 0;
561 }
562 
563 static int set_wqe_buf_attr(struct hns_roce_dev *hr_dev,
564 			    struct hns_roce_qp *hr_qp,
565 			    struct hns_roce_buf_attr *buf_attr)
566 {
567 	int buf_size;
568 	int idx = 0;
569 
570 	hr_qp->buff_size = 0;
571 
572 	/* SQ WQE */
573 	hr_qp->sq.offset = 0;
574 	buf_size = to_hr_hem_entries_size(hr_qp->sq.wqe_cnt,
575 					  hr_qp->sq.wqe_shift);
576 	if (buf_size > 0 && idx < ARRAY_SIZE(buf_attr->region)) {
577 		buf_attr->region[idx].size = buf_size;
578 		buf_attr->region[idx].hopnum = hr_dev->caps.wqe_sq_hop_num;
579 		idx++;
580 		hr_qp->buff_size += buf_size;
581 	}
582 
583 	/* extend SGE WQE in SQ */
584 	hr_qp->sge.offset = hr_qp->buff_size;
585 	buf_size = to_hr_hem_entries_size(hr_qp->sge.sge_cnt,
586 					  hr_qp->sge.sge_shift);
587 	if (buf_size > 0 && idx < ARRAY_SIZE(buf_attr->region)) {
588 		buf_attr->region[idx].size = buf_size;
589 		buf_attr->region[idx].hopnum = hr_dev->caps.wqe_sge_hop_num;
590 		idx++;
591 		hr_qp->buff_size += buf_size;
592 	}
593 
594 	/* RQ WQE */
595 	hr_qp->rq.offset = hr_qp->buff_size;
596 	buf_size = to_hr_hem_entries_size(hr_qp->rq.wqe_cnt,
597 					  hr_qp->rq.wqe_shift);
598 	if (buf_size > 0 && idx < ARRAY_SIZE(buf_attr->region)) {
599 		buf_attr->region[idx].size = buf_size;
600 		buf_attr->region[idx].hopnum = hr_dev->caps.wqe_rq_hop_num;
601 		idx++;
602 		hr_qp->buff_size += buf_size;
603 	}
604 
605 	if (hr_qp->buff_size < 1)
606 		return -EINVAL;
607 
608 	buf_attr->page_shift = HNS_HW_PAGE_SHIFT + hr_dev->caps.mtt_buf_pg_sz;
609 	buf_attr->region_count = idx;
610 
611 	return 0;
612 }
613 
614 static int set_kernel_sq_size(struct hns_roce_dev *hr_dev,
615 			      struct ib_qp_cap *cap, struct hns_roce_qp *hr_qp)
616 {
617 	struct ib_device *ibdev = &hr_dev->ib_dev;
618 	u32 cnt;
619 
620 	if (!cap->max_send_wr || cap->max_send_wr > hr_dev->caps.max_wqes ||
621 	    cap->max_send_sge > hr_dev->caps.max_sq_sg) {
622 		ibdev_err(ibdev, "failed to check SQ WR or SGE num.\n");
623 		return -EINVAL;
624 	}
625 
626 	cnt = roundup_pow_of_two(max(cap->max_send_wr, hr_dev->caps.min_wqes));
627 	if (cnt > hr_dev->caps.max_wqes) {
628 		ibdev_err(ibdev, "failed to check WQE num, WQE num = %u.\n",
629 			  cnt);
630 		return -EINVAL;
631 	}
632 
633 	hr_qp->sq.wqe_shift = ilog2(hr_dev->caps.max_sq_desc_sz);
634 	hr_qp->sq.wqe_cnt = cnt;
635 
636 	set_ext_sge_param(hr_dev, cnt, hr_qp, cap);
637 
638 	/* sync the parameters of kernel QP to user's configuration */
639 	cap->max_send_wr = cnt;
640 	cap->max_send_sge = hr_qp->sq.max_gs;
641 
642 	return 0;
643 }
644 
645 static int hns_roce_qp_has_sq(struct ib_qp_init_attr *attr)
646 {
647 	if (attr->qp_type == IB_QPT_XRC_TGT || !attr->cap.max_send_wr)
648 		return 0;
649 
650 	return 1;
651 }
652 
653 static int hns_roce_qp_has_rq(struct ib_qp_init_attr *attr)
654 {
655 	if (attr->qp_type == IB_QPT_XRC_INI ||
656 	    attr->qp_type == IB_QPT_XRC_TGT || attr->srq ||
657 	    !attr->cap.max_recv_wr)
658 		return 0;
659 
660 	return 1;
661 }
662 
663 static int alloc_rq_inline_buf(struct hns_roce_qp *hr_qp,
664 			       struct ib_qp_init_attr *init_attr)
665 {
666 	u32 max_recv_sge = init_attr->cap.max_recv_sge;
667 	u32 wqe_cnt = hr_qp->rq_inl_buf.wqe_cnt;
668 	struct hns_roce_rinl_wqe *wqe_list;
669 	int i;
670 
671 	/* allocate recv inline buf */
672 	wqe_list = kcalloc(wqe_cnt, sizeof(struct hns_roce_rinl_wqe),
673 			   GFP_KERNEL);
674 	if (!wqe_list)
675 		goto err;
676 
677 	/* Allocate a continuous buffer for all inline sge we need */
678 	wqe_list[0].sg_list = kcalloc(wqe_cnt, (max_recv_sge *
679 				      sizeof(struct hns_roce_rinl_sge)),
680 				      GFP_KERNEL);
681 	if (!wqe_list[0].sg_list)
682 		goto err_wqe_list;
683 
684 	/* Assign buffers of sg_list to each inline wqe */
685 	for (i = 1; i < wqe_cnt; i++)
686 		wqe_list[i].sg_list = &wqe_list[0].sg_list[i * max_recv_sge];
687 
688 	hr_qp->rq_inl_buf.wqe_list = wqe_list;
689 
690 	return 0;
691 
692 err_wqe_list:
693 	kfree(wqe_list);
694 
695 err:
696 	return -ENOMEM;
697 }
698 
699 static void free_rq_inline_buf(struct hns_roce_qp *hr_qp)
700 {
701 	if (hr_qp->rq_inl_buf.wqe_list)
702 		kfree(hr_qp->rq_inl_buf.wqe_list[0].sg_list);
703 	kfree(hr_qp->rq_inl_buf.wqe_list);
704 }
705 
706 static int alloc_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
707 			struct ib_qp_init_attr *init_attr,
708 			struct ib_udata *udata, unsigned long addr)
709 {
710 	struct ib_device *ibdev = &hr_dev->ib_dev;
711 	struct hns_roce_buf_attr buf_attr = {};
712 	int ret;
713 
714 	if (!udata && hr_qp->rq_inl_buf.wqe_cnt) {
715 		ret = alloc_rq_inline_buf(hr_qp, init_attr);
716 		if (ret) {
717 			ibdev_err(ibdev,
718 				  "failed to alloc inline buf, ret = %d.\n",
719 				  ret);
720 			return ret;
721 		}
722 	} else {
723 		hr_qp->rq_inl_buf.wqe_list = NULL;
724 	}
725 
726 	ret = set_wqe_buf_attr(hr_dev, hr_qp, &buf_attr);
727 	if (ret) {
728 		ibdev_err(ibdev, "failed to split WQE buf, ret = %d.\n", ret);
729 		goto err_inline;
730 	}
731 	ret = hns_roce_mtr_create(hr_dev, &hr_qp->mtr, &buf_attr,
732 				  PAGE_SHIFT + hr_dev->caps.mtt_ba_pg_sz,
733 				  udata, addr);
734 	if (ret) {
735 		ibdev_err(ibdev, "failed to create WQE mtr, ret = %d.\n", ret);
736 		goto err_inline;
737 	}
738 
739 	if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_DIRECT_WQE)
740 		hr_qp->en_flags |= HNS_ROCE_QP_CAP_DIRECT_WQE;
741 
742 	return 0;
743 
744 err_inline:
745 	free_rq_inline_buf(hr_qp);
746 
747 	return ret;
748 }
749 
750 static void free_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
751 {
752 	hns_roce_mtr_destroy(hr_dev, &hr_qp->mtr);
753 	free_rq_inline_buf(hr_qp);
754 }
755 
756 static inline bool user_qp_has_sdb(struct hns_roce_dev *hr_dev,
757 				   struct ib_qp_init_attr *init_attr,
758 				   struct ib_udata *udata,
759 				   struct hns_roce_ib_create_qp_resp *resp,
760 				   struct hns_roce_ib_create_qp *ucmd)
761 {
762 	return ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) &&
763 		udata->outlen >= offsetofend(typeof(*resp), cap_flags) &&
764 		hns_roce_qp_has_sq(init_attr) &&
765 		udata->inlen >= offsetofend(typeof(*ucmd), sdb_addr));
766 }
767 
768 static inline bool user_qp_has_rdb(struct hns_roce_dev *hr_dev,
769 				   struct ib_qp_init_attr *init_attr,
770 				   struct ib_udata *udata,
771 				   struct hns_roce_ib_create_qp_resp *resp)
772 {
773 	return ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) &&
774 		udata->outlen >= offsetofend(typeof(*resp), cap_flags) &&
775 		hns_roce_qp_has_rq(init_attr));
776 }
777 
778 static inline bool kernel_qp_has_rdb(struct hns_roce_dev *hr_dev,
779 				     struct ib_qp_init_attr *init_attr)
780 {
781 	return ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) &&
782 		hns_roce_qp_has_rq(init_attr));
783 }
784 
785 static int qp_mmap_entry(struct hns_roce_qp *hr_qp,
786 			 struct hns_roce_dev *hr_dev,
787 			 struct ib_udata *udata,
788 			 struct hns_roce_ib_create_qp_resp *resp)
789 {
790 	struct hns_roce_ucontext *uctx =
791 		rdma_udata_to_drv_context(udata,
792 			struct hns_roce_ucontext, ibucontext);
793 	struct rdma_user_mmap_entry *rdma_entry;
794 	u64 address;
795 
796 	address = hr_dev->dwqe_page + hr_qp->qpn * HNS_ROCE_DWQE_SIZE;
797 
798 	hr_qp->dwqe_mmap_entry =
799 		hns_roce_user_mmap_entry_insert(&uctx->ibucontext, address,
800 						HNS_ROCE_DWQE_SIZE,
801 						HNS_ROCE_MMAP_TYPE_DWQE);
802 
803 	if (!hr_qp->dwqe_mmap_entry) {
804 		ibdev_err(&hr_dev->ib_dev, "failed to get dwqe mmap entry.\n");
805 		return -ENOMEM;
806 	}
807 
808 	rdma_entry = &hr_qp->dwqe_mmap_entry->rdma_entry;
809 	resp->dwqe_mmap_key = rdma_user_mmap_get_offset(rdma_entry);
810 
811 	return 0;
812 }
813 
814 static int alloc_user_qp_db(struct hns_roce_dev *hr_dev,
815 			    struct hns_roce_qp *hr_qp,
816 			    struct ib_qp_init_attr *init_attr,
817 			    struct ib_udata *udata,
818 			    struct hns_roce_ib_create_qp *ucmd,
819 			    struct hns_roce_ib_create_qp_resp *resp)
820 {
821 	struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context(udata,
822 		struct hns_roce_ucontext, ibucontext);
823 	struct ib_device *ibdev = &hr_dev->ib_dev;
824 	int ret;
825 
826 	if (user_qp_has_sdb(hr_dev, init_attr, udata, resp, ucmd)) {
827 		ret = hns_roce_db_map_user(uctx, ucmd->sdb_addr, &hr_qp->sdb);
828 		if (ret) {
829 			ibdev_err(ibdev,
830 				  "failed to map user SQ doorbell, ret = %d.\n",
831 				  ret);
832 			goto err_out;
833 		}
834 		hr_qp->en_flags |= HNS_ROCE_QP_CAP_SQ_RECORD_DB;
835 	}
836 
837 	if (user_qp_has_rdb(hr_dev, init_attr, udata, resp)) {
838 		ret = hns_roce_db_map_user(uctx, ucmd->db_addr, &hr_qp->rdb);
839 		if (ret) {
840 			ibdev_err(ibdev,
841 				  "failed to map user RQ doorbell, ret = %d.\n",
842 				  ret);
843 			goto err_sdb;
844 		}
845 		hr_qp->en_flags |= HNS_ROCE_QP_CAP_RQ_RECORD_DB;
846 	}
847 
848 	return 0;
849 
850 err_sdb:
851 	if (hr_qp->en_flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB)
852 		hns_roce_db_unmap_user(uctx, &hr_qp->sdb);
853 err_out:
854 	return ret;
855 }
856 
857 static int alloc_kernel_qp_db(struct hns_roce_dev *hr_dev,
858 			      struct hns_roce_qp *hr_qp,
859 			      struct ib_qp_init_attr *init_attr)
860 {
861 	struct ib_device *ibdev = &hr_dev->ib_dev;
862 	int ret;
863 
864 	if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09)
865 		hr_qp->sq.db_reg = hr_dev->mem_base +
866 				   HNS_ROCE_DWQE_SIZE * hr_qp->qpn;
867 	else
868 		hr_qp->sq.db_reg = hr_dev->reg_base + hr_dev->sdb_offset +
869 				   DB_REG_OFFSET * hr_dev->priv_uar.index;
870 
871 	hr_qp->rq.db_reg = hr_dev->reg_base + hr_dev->odb_offset +
872 			   DB_REG_OFFSET * hr_dev->priv_uar.index;
873 
874 	if (kernel_qp_has_rdb(hr_dev, init_attr)) {
875 		ret = hns_roce_alloc_db(hr_dev, &hr_qp->rdb, 0);
876 		if (ret) {
877 			ibdev_err(ibdev,
878 				  "failed to alloc kernel RQ doorbell, ret = %d.\n",
879 				  ret);
880 			return ret;
881 		}
882 		*hr_qp->rdb.db_record = 0;
883 		hr_qp->en_flags |= HNS_ROCE_QP_CAP_RQ_RECORD_DB;
884 	}
885 
886 	return 0;
887 }
888 
889 static int alloc_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
890 		       struct ib_qp_init_attr *init_attr,
891 		       struct ib_udata *udata,
892 		       struct hns_roce_ib_create_qp *ucmd,
893 		       struct hns_roce_ib_create_qp_resp *resp)
894 {
895 	int ret;
896 
897 	if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SDI_MODE)
898 		hr_qp->en_flags |= HNS_ROCE_QP_CAP_OWNER_DB;
899 
900 	if (udata) {
901 		if (hr_qp->en_flags & HNS_ROCE_QP_CAP_DIRECT_WQE) {
902 			ret = qp_mmap_entry(hr_qp, hr_dev, udata, resp);
903 			if (ret)
904 				return ret;
905 		}
906 
907 		ret = alloc_user_qp_db(hr_dev, hr_qp, init_attr, udata, ucmd,
908 				       resp);
909 		if (ret)
910 			goto err_remove_qp;
911 	} else {
912 		ret = alloc_kernel_qp_db(hr_dev, hr_qp, init_attr);
913 		if (ret)
914 			return ret;
915 	}
916 
917 	return 0;
918 
919 err_remove_qp:
920 	if (hr_qp->en_flags & HNS_ROCE_QP_CAP_DIRECT_WQE)
921 		qp_user_mmap_entry_remove(hr_qp);
922 
923 	return ret;
924 }
925 
926 static void free_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
927 		       struct ib_udata *udata)
928 {
929 	struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context(
930 		udata, struct hns_roce_ucontext, ibucontext);
931 
932 	if (udata) {
933 		if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB)
934 			hns_roce_db_unmap_user(uctx, &hr_qp->rdb);
935 		if (hr_qp->en_flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB)
936 			hns_roce_db_unmap_user(uctx, &hr_qp->sdb);
937 		if (hr_qp->en_flags & HNS_ROCE_QP_CAP_DIRECT_WQE)
938 			qp_user_mmap_entry_remove(hr_qp);
939 	} else {
940 		if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB)
941 			hns_roce_free_db(hr_dev, &hr_qp->rdb);
942 	}
943 }
944 
945 static int alloc_kernel_wrid(struct hns_roce_dev *hr_dev,
946 			     struct hns_roce_qp *hr_qp)
947 {
948 	struct ib_device *ibdev = &hr_dev->ib_dev;
949 	u64 *sq_wrid = NULL;
950 	u64 *rq_wrid = NULL;
951 	int ret;
952 
953 	sq_wrid = kcalloc(hr_qp->sq.wqe_cnt, sizeof(u64), GFP_KERNEL);
954 	if (ZERO_OR_NULL_PTR(sq_wrid)) {
955 		ibdev_err(ibdev, "failed to alloc SQ wrid.\n");
956 		return -ENOMEM;
957 	}
958 
959 	if (hr_qp->rq.wqe_cnt) {
960 		rq_wrid = kcalloc(hr_qp->rq.wqe_cnt, sizeof(u64), GFP_KERNEL);
961 		if (ZERO_OR_NULL_PTR(rq_wrid)) {
962 			ibdev_err(ibdev, "failed to alloc RQ wrid.\n");
963 			ret = -ENOMEM;
964 			goto err_sq;
965 		}
966 	}
967 
968 	hr_qp->sq.wrid = sq_wrid;
969 	hr_qp->rq.wrid = rq_wrid;
970 	return 0;
971 err_sq:
972 	kfree(sq_wrid);
973 
974 	return ret;
975 }
976 
977 static void free_kernel_wrid(struct hns_roce_qp *hr_qp)
978 {
979 	kfree(hr_qp->rq.wrid);
980 	kfree(hr_qp->sq.wrid);
981 }
982 
983 static int set_qp_param(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
984 			struct ib_qp_init_attr *init_attr,
985 			struct ib_udata *udata,
986 			struct hns_roce_ib_create_qp *ucmd)
987 {
988 	struct ib_device *ibdev = &hr_dev->ib_dev;
989 	int ret;
990 
991 	if (init_attr->cap.max_inline_data > hr_dev->caps.max_sq_inline)
992 		init_attr->cap.max_inline_data = hr_dev->caps.max_sq_inline;
993 
994 	hr_qp->max_inline_data = init_attr->cap.max_inline_data;
995 
996 	if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)
997 		hr_qp->sq_signal_bits = IB_SIGNAL_ALL_WR;
998 	else
999 		hr_qp->sq_signal_bits = IB_SIGNAL_REQ_WR;
1000 
1001 	ret = set_rq_size(hr_dev, &init_attr->cap, hr_qp,
1002 			  hns_roce_qp_has_rq(init_attr), !!udata);
1003 	if (ret) {
1004 		ibdev_err(ibdev, "failed to set user RQ size, ret = %d.\n",
1005 			  ret);
1006 		return ret;
1007 	}
1008 
1009 	if (udata) {
1010 		ret = ib_copy_from_udata(ucmd, udata,
1011 					 min(udata->inlen, sizeof(*ucmd)));
1012 		if (ret) {
1013 			ibdev_err(ibdev,
1014 				  "failed to copy QP ucmd, ret = %d\n", ret);
1015 			return ret;
1016 		}
1017 
1018 		ret = set_user_sq_size(hr_dev, &init_attr->cap, hr_qp, ucmd);
1019 		if (ret)
1020 			ibdev_err(ibdev,
1021 				  "failed to set user SQ size, ret = %d.\n",
1022 				  ret);
1023 	} else {
1024 		ret = set_kernel_sq_size(hr_dev, &init_attr->cap, hr_qp);
1025 		if (ret)
1026 			ibdev_err(ibdev,
1027 				  "failed to set kernel SQ size, ret = %d.\n",
1028 				  ret);
1029 	}
1030 
1031 	return ret;
1032 }
1033 
1034 static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev,
1035 				     struct ib_pd *ib_pd,
1036 				     struct ib_qp_init_attr *init_attr,
1037 				     struct ib_udata *udata,
1038 				     struct hns_roce_qp *hr_qp)
1039 {
1040 	struct hns_roce_ib_create_qp_resp resp = {};
1041 	struct ib_device *ibdev = &hr_dev->ib_dev;
1042 	struct hns_roce_ib_create_qp ucmd;
1043 	int ret;
1044 
1045 	mutex_init(&hr_qp->mutex);
1046 	spin_lock_init(&hr_qp->sq.lock);
1047 	spin_lock_init(&hr_qp->rq.lock);
1048 
1049 	hr_qp->state = IB_QPS_RESET;
1050 	hr_qp->flush_flag = 0;
1051 
1052 	if (init_attr->create_flags)
1053 		return -EOPNOTSUPP;
1054 
1055 	ret = set_qp_param(hr_dev, hr_qp, init_attr, udata, &ucmd);
1056 	if (ret) {
1057 		ibdev_err(ibdev, "failed to set QP param, ret = %d.\n", ret);
1058 		return ret;
1059 	}
1060 
1061 	if (!udata) {
1062 		ret = alloc_kernel_wrid(hr_dev, hr_qp);
1063 		if (ret) {
1064 			ibdev_err(ibdev, "failed to alloc wrid, ret = %d.\n",
1065 				  ret);
1066 			return ret;
1067 		}
1068 	}
1069 
1070 	ret = alloc_qp_buf(hr_dev, hr_qp, init_attr, udata, ucmd.buf_addr);
1071 	if (ret) {
1072 		ibdev_err(ibdev, "failed to alloc QP buffer, ret = %d.\n", ret);
1073 		goto err_buf;
1074 	}
1075 
1076 	ret = alloc_qpn(hr_dev, hr_qp);
1077 	if (ret) {
1078 		ibdev_err(ibdev, "failed to alloc QPN, ret = %d.\n", ret);
1079 		goto err_qpn;
1080 	}
1081 
1082 	ret = alloc_qp_db(hr_dev, hr_qp, init_attr, udata, &ucmd, &resp);
1083 	if (ret) {
1084 		ibdev_err(ibdev, "failed to alloc QP doorbell, ret = %d.\n",
1085 			  ret);
1086 		goto err_db;
1087 	}
1088 
1089 	ret = alloc_qpc(hr_dev, hr_qp);
1090 	if (ret) {
1091 		ibdev_err(ibdev, "failed to alloc QP context, ret = %d.\n",
1092 			  ret);
1093 		goto err_qpc;
1094 	}
1095 
1096 	ret = hns_roce_qp_store(hr_dev, hr_qp, init_attr);
1097 	if (ret) {
1098 		ibdev_err(ibdev, "failed to store QP, ret = %d.\n", ret);
1099 		goto err_store;
1100 	}
1101 
1102 	if (udata) {
1103 		resp.cap_flags = hr_qp->en_flags;
1104 		ret = ib_copy_to_udata(udata, &resp,
1105 				       min(udata->outlen, sizeof(resp)));
1106 		if (ret) {
1107 			ibdev_err(ibdev, "copy qp resp failed!\n");
1108 			goto err_store;
1109 		}
1110 	}
1111 
1112 	if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) {
1113 		ret = hr_dev->hw->qp_flow_control_init(hr_dev, hr_qp);
1114 		if (ret)
1115 			goto err_flow_ctrl;
1116 	}
1117 
1118 	hr_qp->ibqp.qp_num = hr_qp->qpn;
1119 	hr_qp->event = hns_roce_ib_qp_event;
1120 	refcount_set(&hr_qp->refcount, 1);
1121 	init_completion(&hr_qp->free);
1122 
1123 	return 0;
1124 
1125 err_flow_ctrl:
1126 	hns_roce_qp_remove(hr_dev, hr_qp);
1127 err_store:
1128 	free_qpc(hr_dev, hr_qp);
1129 err_qpc:
1130 	free_qp_db(hr_dev, hr_qp, udata);
1131 err_db:
1132 	free_qpn(hr_dev, hr_qp);
1133 err_qpn:
1134 	free_qp_buf(hr_dev, hr_qp);
1135 err_buf:
1136 	free_kernel_wrid(hr_qp);
1137 	return ret;
1138 }
1139 
1140 void hns_roce_qp_destroy(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
1141 			 struct ib_udata *udata)
1142 {
1143 	if (refcount_dec_and_test(&hr_qp->refcount))
1144 		complete(&hr_qp->free);
1145 	wait_for_completion(&hr_qp->free);
1146 
1147 	free_qpc(hr_dev, hr_qp);
1148 	free_qpn(hr_dev, hr_qp);
1149 	free_qp_buf(hr_dev, hr_qp);
1150 	free_kernel_wrid(hr_qp);
1151 	free_qp_db(hr_dev, hr_qp, udata);
1152 }
1153 
1154 static int check_qp_type(struct hns_roce_dev *hr_dev, enum ib_qp_type type,
1155 			 bool is_user)
1156 {
1157 	switch (type) {
1158 	case IB_QPT_XRC_INI:
1159 	case IB_QPT_XRC_TGT:
1160 		if (!(hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_XRC))
1161 			goto out;
1162 		break;
1163 	case IB_QPT_UD:
1164 		if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08 &&
1165 		    is_user)
1166 			goto out;
1167 		break;
1168 	case IB_QPT_RC:
1169 	case IB_QPT_GSI:
1170 		break;
1171 	default:
1172 		goto out;
1173 	}
1174 
1175 	return 0;
1176 
1177 out:
1178 	ibdev_err(&hr_dev->ib_dev, "not support QP type %d\n", type);
1179 
1180 	return -EOPNOTSUPP;
1181 }
1182 
1183 int hns_roce_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *init_attr,
1184 		       struct ib_udata *udata)
1185 {
1186 	struct ib_device *ibdev = qp->device;
1187 	struct hns_roce_dev *hr_dev = to_hr_dev(ibdev);
1188 	struct hns_roce_qp *hr_qp = to_hr_qp(qp);
1189 	struct ib_pd *pd = qp->pd;
1190 	int ret;
1191 
1192 	ret = check_qp_type(hr_dev, init_attr->qp_type, !!udata);
1193 	if (ret)
1194 		return ret;
1195 
1196 	if (init_attr->qp_type == IB_QPT_XRC_TGT)
1197 		hr_qp->xrcdn = to_hr_xrcd(init_attr->xrcd)->xrcdn;
1198 
1199 	if (init_attr->qp_type == IB_QPT_GSI) {
1200 		hr_qp->port = init_attr->port_num - 1;
1201 		hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port];
1202 	}
1203 
1204 	ret = hns_roce_create_qp_common(hr_dev, pd, init_attr, udata, hr_qp);
1205 	if (ret)
1206 		ibdev_err(ibdev, "create QP type 0x%x failed(%d)\n",
1207 			  init_attr->qp_type, ret);
1208 
1209 	return ret;
1210 }
1211 
1212 int to_hr_qp_type(int qp_type)
1213 {
1214 	switch (qp_type) {
1215 	case IB_QPT_RC:
1216 		return SERV_TYPE_RC;
1217 	case IB_QPT_UD:
1218 	case IB_QPT_GSI:
1219 		return SERV_TYPE_UD;
1220 	case IB_QPT_XRC_INI:
1221 	case IB_QPT_XRC_TGT:
1222 		return SERV_TYPE_XRC;
1223 	default:
1224 		return -1;
1225 	}
1226 }
1227 
1228 static int check_mtu_validate(struct hns_roce_dev *hr_dev,
1229 			      struct hns_roce_qp *hr_qp,
1230 			      struct ib_qp_attr *attr, int attr_mask)
1231 {
1232 	enum ib_mtu active_mtu;
1233 	int p;
1234 
1235 	p = attr_mask & IB_QP_PORT ? (attr->port_num - 1) : hr_qp->port;
1236 	active_mtu = iboe_get_mtu(hr_dev->iboe.netdevs[p]->mtu);
1237 
1238 	if ((hr_dev->caps.max_mtu >= IB_MTU_2048 &&
1239 	    attr->path_mtu > hr_dev->caps.max_mtu) ||
1240 	    attr->path_mtu < IB_MTU_256 || attr->path_mtu > active_mtu) {
1241 		ibdev_err(&hr_dev->ib_dev,
1242 			"attr path_mtu(%d)invalid while modify qp",
1243 			attr->path_mtu);
1244 		return -EINVAL;
1245 	}
1246 
1247 	return 0;
1248 }
1249 
1250 static int hns_roce_check_qp_attr(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1251 				  int attr_mask)
1252 {
1253 	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
1254 	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
1255 	int p;
1256 
1257 	if ((attr_mask & IB_QP_PORT) &&
1258 	    (attr->port_num == 0 || attr->port_num > hr_dev->caps.num_ports)) {
1259 		ibdev_err(&hr_dev->ib_dev, "invalid attr, port_num = %u.\n",
1260 			  attr->port_num);
1261 		return -EINVAL;
1262 	}
1263 
1264 	if (attr_mask & IB_QP_PKEY_INDEX) {
1265 		p = attr_mask & IB_QP_PORT ? (attr->port_num - 1) : hr_qp->port;
1266 		if (attr->pkey_index >= hr_dev->caps.pkey_table_len[p]) {
1267 			ibdev_err(&hr_dev->ib_dev,
1268 				  "invalid attr, pkey_index = %u.\n",
1269 				  attr->pkey_index);
1270 			return -EINVAL;
1271 		}
1272 	}
1273 
1274 	if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC &&
1275 	    attr->max_rd_atomic > hr_dev->caps.max_qp_init_rdma) {
1276 		ibdev_err(&hr_dev->ib_dev,
1277 			  "invalid attr, max_rd_atomic = %u.\n",
1278 			  attr->max_rd_atomic);
1279 		return -EINVAL;
1280 	}
1281 
1282 	if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC &&
1283 	    attr->max_dest_rd_atomic > hr_dev->caps.max_qp_dest_rdma) {
1284 		ibdev_err(&hr_dev->ib_dev,
1285 			  "invalid attr, max_dest_rd_atomic = %u.\n",
1286 			  attr->max_dest_rd_atomic);
1287 		return -EINVAL;
1288 	}
1289 
1290 	if (attr_mask & IB_QP_PATH_MTU)
1291 		return check_mtu_validate(hr_dev, hr_qp, attr, attr_mask);
1292 
1293 	return 0;
1294 }
1295 
1296 int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1297 		       int attr_mask, struct ib_udata *udata)
1298 {
1299 	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
1300 	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
1301 	enum ib_qp_state cur_state, new_state;
1302 	int ret = -EINVAL;
1303 
1304 	mutex_lock(&hr_qp->mutex);
1305 
1306 	if (attr_mask & IB_QP_CUR_STATE && attr->cur_qp_state != hr_qp->state)
1307 		goto out;
1308 
1309 	cur_state = hr_qp->state;
1310 	new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state;
1311 
1312 	if (ibqp->uobject &&
1313 	    (attr_mask & IB_QP_STATE) && new_state == IB_QPS_ERR) {
1314 		if (hr_qp->en_flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB) {
1315 			hr_qp->sq.head = *(int *)(hr_qp->sdb.virt_addr);
1316 
1317 			if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB)
1318 				hr_qp->rq.head = *(int *)(hr_qp->rdb.virt_addr);
1319 		} else {
1320 			ibdev_warn(&hr_dev->ib_dev,
1321 				  "flush cqe is not supported in userspace!\n");
1322 			goto out;
1323 		}
1324 	}
1325 
1326 	if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type,
1327 				attr_mask)) {
1328 		ibdev_err(&hr_dev->ib_dev, "ib_modify_qp_is_ok failed\n");
1329 		goto out;
1330 	}
1331 
1332 	ret = hns_roce_check_qp_attr(ibqp, attr, attr_mask);
1333 	if (ret)
1334 		goto out;
1335 
1336 	if (cur_state == new_state && cur_state == IB_QPS_RESET)
1337 		goto out;
1338 
1339 	ret = hr_dev->hw->modify_qp(ibqp, attr, attr_mask, cur_state,
1340 				    new_state);
1341 
1342 out:
1343 	mutex_unlock(&hr_qp->mutex);
1344 
1345 	return ret;
1346 }
1347 
1348 void hns_roce_lock_cqs(struct hns_roce_cq *send_cq, struct hns_roce_cq *recv_cq)
1349 		       __acquires(&send_cq->lock) __acquires(&recv_cq->lock)
1350 {
1351 	if (unlikely(send_cq == NULL && recv_cq == NULL)) {
1352 		__acquire(&send_cq->lock);
1353 		__acquire(&recv_cq->lock);
1354 	} else if (unlikely(send_cq != NULL && recv_cq == NULL)) {
1355 		spin_lock_irq(&send_cq->lock);
1356 		__acquire(&recv_cq->lock);
1357 	} else if (unlikely(send_cq == NULL && recv_cq != NULL)) {
1358 		spin_lock_irq(&recv_cq->lock);
1359 		__acquire(&send_cq->lock);
1360 	} else if (send_cq == recv_cq) {
1361 		spin_lock_irq(&send_cq->lock);
1362 		__acquire(&recv_cq->lock);
1363 	} else if (send_cq->cqn < recv_cq->cqn) {
1364 		spin_lock_irq(&send_cq->lock);
1365 		spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING);
1366 	} else {
1367 		spin_lock_irq(&recv_cq->lock);
1368 		spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING);
1369 	}
1370 }
1371 
1372 void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq,
1373 			 struct hns_roce_cq *recv_cq) __releases(&send_cq->lock)
1374 			 __releases(&recv_cq->lock)
1375 {
1376 	if (unlikely(send_cq == NULL && recv_cq == NULL)) {
1377 		__release(&recv_cq->lock);
1378 		__release(&send_cq->lock);
1379 	} else if (unlikely(send_cq != NULL && recv_cq == NULL)) {
1380 		__release(&recv_cq->lock);
1381 		spin_unlock(&send_cq->lock);
1382 	} else if (unlikely(send_cq == NULL && recv_cq != NULL)) {
1383 		__release(&send_cq->lock);
1384 		spin_unlock(&recv_cq->lock);
1385 	} else if (send_cq == recv_cq) {
1386 		__release(&recv_cq->lock);
1387 		spin_unlock_irq(&send_cq->lock);
1388 	} else if (send_cq->cqn < recv_cq->cqn) {
1389 		spin_unlock(&recv_cq->lock);
1390 		spin_unlock_irq(&send_cq->lock);
1391 	} else {
1392 		spin_unlock(&send_cq->lock);
1393 		spin_unlock_irq(&recv_cq->lock);
1394 	}
1395 }
1396 
1397 static inline void *get_wqe(struct hns_roce_qp *hr_qp, u32 offset)
1398 {
1399 	return hns_roce_buf_offset(hr_qp->mtr.kmem, offset);
1400 }
1401 
1402 void *hns_roce_get_recv_wqe(struct hns_roce_qp *hr_qp, unsigned int n)
1403 {
1404 	return get_wqe(hr_qp, hr_qp->rq.offset + (n << hr_qp->rq.wqe_shift));
1405 }
1406 
1407 void *hns_roce_get_send_wqe(struct hns_roce_qp *hr_qp, unsigned int n)
1408 {
1409 	return get_wqe(hr_qp, hr_qp->sq.offset + (n << hr_qp->sq.wqe_shift));
1410 }
1411 
1412 void *hns_roce_get_extend_sge(struct hns_roce_qp *hr_qp, unsigned int n)
1413 {
1414 	return get_wqe(hr_qp, hr_qp->sge.offset + (n << hr_qp->sge.sge_shift));
1415 }
1416 
1417 bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, u32 nreq,
1418 			  struct ib_cq *ib_cq)
1419 {
1420 	struct hns_roce_cq *hr_cq;
1421 	u32 cur;
1422 
1423 	cur = hr_wq->head - hr_wq->tail;
1424 	if (likely(cur + nreq < hr_wq->wqe_cnt))
1425 		return false;
1426 
1427 	hr_cq = to_hr_cq(ib_cq);
1428 	spin_lock(&hr_cq->lock);
1429 	cur = hr_wq->head - hr_wq->tail;
1430 	spin_unlock(&hr_cq->lock);
1431 
1432 	return cur + nreq >= hr_wq->wqe_cnt;
1433 }
1434 
1435 int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev)
1436 {
1437 	struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
1438 	unsigned int reserved_from_bot;
1439 	unsigned int i;
1440 
1441 	qp_table->idx_table.spare_idx = kcalloc(hr_dev->caps.num_qps,
1442 					sizeof(u32), GFP_KERNEL);
1443 	if (!qp_table->idx_table.spare_idx)
1444 		return -ENOMEM;
1445 
1446 	mutex_init(&qp_table->scc_mutex);
1447 	mutex_init(&qp_table->bank_mutex);
1448 	xa_init(&hr_dev->qp_table_xa);
1449 
1450 	reserved_from_bot = hr_dev->caps.reserved_qps;
1451 
1452 	for (i = 0; i < reserved_from_bot; i++) {
1453 		hr_dev->qp_table.bank[get_qp_bankid(i)].inuse++;
1454 		hr_dev->qp_table.bank[get_qp_bankid(i)].min++;
1455 	}
1456 
1457 	for (i = 0; i < HNS_ROCE_QP_BANK_NUM; i++) {
1458 		ida_init(&hr_dev->qp_table.bank[i].ida);
1459 		hr_dev->qp_table.bank[i].max = hr_dev->caps.num_qps /
1460 					       HNS_ROCE_QP_BANK_NUM - 1;
1461 		hr_dev->qp_table.bank[i].next = hr_dev->qp_table.bank[i].min;
1462 	}
1463 
1464 	return 0;
1465 }
1466 
1467 void hns_roce_cleanup_qp_table(struct hns_roce_dev *hr_dev)
1468 {
1469 	int i;
1470 
1471 	for (i = 0; i < HNS_ROCE_QP_BANK_NUM; i++)
1472 		ida_destroy(&hr_dev->qp_table.bank[i].ida);
1473 	kfree(hr_dev->qp_table.idx_table.spare_idx);
1474 }
1475