1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /*
3  * Copyright (c) 2018 Hisilicon Limited.
4  */
5 
6 #include <rdma/ib_umem.h>
7 #include <rdma/hns-abi.h>
8 #include "hns_roce_device.h"
9 #include "hns_roce_cmd.h"
10 #include "hns_roce_hem.h"
11 
12 void hns_roce_srq_event(struct hns_roce_dev *hr_dev, u32 srqn, int event_type)
13 {
14 	struct hns_roce_srq_table *srq_table = &hr_dev->srq_table;
15 	struct hns_roce_srq *srq;
16 
17 	xa_lock(&srq_table->xa);
18 	srq = xa_load(&srq_table->xa, srqn & (hr_dev->caps.num_srqs - 1));
19 	if (srq)
20 		atomic_inc(&srq->refcount);
21 	xa_unlock(&srq_table->xa);
22 
23 	if (!srq) {
24 		dev_warn(hr_dev->dev, "Async event for bogus SRQ %08x\n", srqn);
25 		return;
26 	}
27 
28 	srq->event(srq, event_type);
29 
30 	if (atomic_dec_and_test(&srq->refcount))
31 		complete(&srq->free);
32 }
33 EXPORT_SYMBOL_GPL(hns_roce_srq_event);
34 
35 static void hns_roce_ib_srq_event(struct hns_roce_srq *srq,
36 				  enum hns_roce_event event_type)
37 {
38 	struct hns_roce_dev *hr_dev = to_hr_dev(srq->ibsrq.device);
39 	struct ib_srq *ibsrq = &srq->ibsrq;
40 	struct ib_event event;
41 
42 	if (ibsrq->event_handler) {
43 		event.device      = ibsrq->device;
44 		event.element.srq = ibsrq;
45 		switch (event_type) {
46 		case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH:
47 			event.event = IB_EVENT_SRQ_LIMIT_REACHED;
48 			break;
49 		case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:
50 			event.event = IB_EVENT_SRQ_ERR;
51 			break;
52 		default:
53 			dev_err(hr_dev->dev,
54 			   "hns_roce:Unexpected event type 0x%x on SRQ %06lx\n",
55 			   event_type, srq->srqn);
56 			return;
57 		}
58 
59 		ibsrq->event_handler(&event, ibsrq->srq_context);
60 	}
61 }
62 
63 static int hns_roce_sw2hw_srq(struct hns_roce_dev *dev,
64 			      struct hns_roce_cmd_mailbox *mailbox,
65 			      unsigned long srq_num)
66 {
67 	return hns_roce_cmd_mbox(dev, mailbox->dma, 0, srq_num, 0,
68 				 HNS_ROCE_CMD_SW2HW_SRQ,
69 				 HNS_ROCE_CMD_TIMEOUT_MSECS);
70 }
71 
72 static int hns_roce_hw2sw_srq(struct hns_roce_dev *dev,
73 			     struct hns_roce_cmd_mailbox *mailbox,
74 			     unsigned long srq_num)
75 {
76 	return hns_roce_cmd_mbox(dev, 0, mailbox ? mailbox->dma : 0, srq_num,
77 				 mailbox ? 0 : 1, HNS_ROCE_CMD_HW2SW_SRQ,
78 				 HNS_ROCE_CMD_TIMEOUT_MSECS);
79 }
80 
81 int hns_roce_srq_alloc(struct hns_roce_dev *hr_dev, u32 pdn, u32 cqn, u16 xrcd,
82 		       struct hns_roce_mtt *hr_mtt, u64 db_rec_addr,
83 		       struct hns_roce_srq *srq)
84 {
85 	struct hns_roce_srq_table *srq_table = &hr_dev->srq_table;
86 	struct hns_roce_cmd_mailbox *mailbox;
87 	dma_addr_t dma_handle_wqe;
88 	dma_addr_t dma_handle_idx;
89 	u64 *mtts_wqe;
90 	u64 *mtts_idx;
91 	int ret;
92 
93 	/* Get the physical address of srq buf */
94 	mtts_wqe = hns_roce_table_find(hr_dev,
95 				       &hr_dev->mr_table.mtt_srqwqe_table,
96 				       srq->mtt.first_seg,
97 				       &dma_handle_wqe);
98 	if (!mtts_wqe) {
99 		dev_err(hr_dev->dev,
100 			"SRQ alloc.Failed to find srq buf addr.\n");
101 		return -EINVAL;
102 	}
103 
104 	/* Get physical address of idx que buf */
105 	mtts_idx = hns_roce_table_find(hr_dev, &hr_dev->mr_table.mtt_idx_table,
106 				       srq->idx_que.mtt.first_seg,
107 				       &dma_handle_idx);
108 	if (!mtts_idx) {
109 		dev_err(hr_dev->dev,
110 			"SRQ alloc.Failed to find idx que buf addr.\n");
111 		return -EINVAL;
112 	}
113 
114 	ret = hns_roce_bitmap_alloc(&srq_table->bitmap, &srq->srqn);
115 	if (ret == -1) {
116 		dev_err(hr_dev->dev, "SRQ alloc.Failed to alloc index.\n");
117 		return -ENOMEM;
118 	}
119 
120 	ret = hns_roce_table_get(hr_dev, &srq_table->table, srq->srqn);
121 	if (ret)
122 		goto err_out;
123 
124 	ret = xa_err(xa_store(&srq_table->xa, srq->srqn, srq, GFP_KERNEL));
125 	if (ret)
126 		goto err_put;
127 
128 	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
129 	if (IS_ERR(mailbox)) {
130 		ret = PTR_ERR(mailbox);
131 		goto err_xa;
132 	}
133 
134 	hr_dev->hw->write_srqc(hr_dev, srq, pdn, xrcd, cqn, mailbox->buf,
135 			       mtts_wqe, mtts_idx, dma_handle_wqe,
136 			       dma_handle_idx);
137 
138 	ret = hns_roce_sw2hw_srq(hr_dev, mailbox, srq->srqn);
139 	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
140 	if (ret)
141 		goto err_xa;
142 
143 	atomic_set(&srq->refcount, 1);
144 	init_completion(&srq->free);
145 	return ret;
146 
147 err_xa:
148 	xa_erase(&srq_table->xa, srq->srqn);
149 
150 err_put:
151 	hns_roce_table_put(hr_dev, &srq_table->table, srq->srqn);
152 
153 err_out:
154 	hns_roce_bitmap_free(&srq_table->bitmap, srq->srqn, BITMAP_NO_RR);
155 	return ret;
156 }
157 
158 void hns_roce_srq_free(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq)
159 {
160 	struct hns_roce_srq_table *srq_table = &hr_dev->srq_table;
161 	int ret;
162 
163 	ret = hns_roce_hw2sw_srq(hr_dev, NULL, srq->srqn);
164 	if (ret)
165 		dev_err(hr_dev->dev, "HW2SW_SRQ failed (%d) for CQN %06lx\n",
166 			ret, srq->srqn);
167 
168 	xa_erase(&srq_table->xa, srq->srqn);
169 
170 	if (atomic_dec_and_test(&srq->refcount))
171 		complete(&srq->free);
172 	wait_for_completion(&srq->free);
173 
174 	hns_roce_table_put(hr_dev, &srq_table->table, srq->srqn);
175 	hns_roce_bitmap_free(&srq_table->bitmap, srq->srqn, BITMAP_NO_RR);
176 }
177 
178 static int hns_roce_create_idx_que(struct ib_pd *pd, struct hns_roce_srq *srq,
179 				   u32 page_shift)
180 {
181 	struct hns_roce_dev *hr_dev = to_hr_dev(pd->device);
182 	struct hns_roce_idx_que *idx_que = &srq->idx_que;
183 	u32 bitmap_num;
184 	int i;
185 
186 	bitmap_num = HNS_ROCE_ALOGN_UP(srq->max, 8 * sizeof(u64));
187 
188 	idx_que->bitmap = kcalloc(1, bitmap_num / 8, GFP_KERNEL);
189 	if (!idx_que->bitmap)
190 		return -ENOMEM;
191 
192 	bitmap_num = bitmap_num / (8 * sizeof(u64));
193 
194 	idx_que->buf_size = srq->idx_que.buf_size;
195 
196 	if (hns_roce_buf_alloc(hr_dev, idx_que->buf_size, (1 << page_shift) * 2,
197 			       &idx_que->idx_buf, page_shift)) {
198 		kfree(idx_que->bitmap);
199 		return -ENOMEM;
200 	}
201 
202 	for (i = 0; i < bitmap_num; i++)
203 		idx_que->bitmap[i] = ~(0UL);
204 
205 	return 0;
206 }
207 
208 struct ib_srq *hns_roce_create_srq(struct ib_pd *pd,
209 				   struct ib_srq_init_attr *srq_init_attr,
210 				   struct ib_udata *udata)
211 {
212 	struct hns_roce_dev *hr_dev = to_hr_dev(pd->device);
213 	struct hns_roce_srq *srq;
214 	int srq_desc_size;
215 	int srq_buf_size;
216 	u32 page_shift;
217 	int ret = 0;
218 	u32 npages;
219 	u32 cqn;
220 
221 	/* Check the actual SRQ wqe and SRQ sge num */
222 	if (srq_init_attr->attr.max_wr >= hr_dev->caps.max_srq_wrs ||
223 	    srq_init_attr->attr.max_sge > hr_dev->caps.max_srq_sges)
224 		return ERR_PTR(-EINVAL);
225 
226 	srq = kzalloc(sizeof(*srq), GFP_KERNEL);
227 	if (!srq)
228 		return ERR_PTR(-ENOMEM);
229 
230 	mutex_init(&srq->mutex);
231 	spin_lock_init(&srq->lock);
232 
233 	srq->max = roundup_pow_of_two(srq_init_attr->attr.max_wr + 1);
234 	srq->max_gs = srq_init_attr->attr.max_sge;
235 
236 	srq_desc_size = max(16, 16 * srq->max_gs);
237 
238 	srq->wqe_shift = ilog2(srq_desc_size);
239 
240 	srq_buf_size = srq->max * srq_desc_size;
241 
242 	srq->idx_que.entry_sz = HNS_ROCE_IDX_QUE_ENTRY_SZ;
243 	srq->idx_que.buf_size = srq->max * srq->idx_que.entry_sz;
244 	srq->mtt.mtt_type = MTT_TYPE_SRQWQE;
245 	srq->idx_que.mtt.mtt_type = MTT_TYPE_IDX;
246 
247 	if (udata) {
248 		struct hns_roce_ib_create_srq  ucmd;
249 
250 		if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd))) {
251 			ret = -EFAULT;
252 			goto err_srq;
253 		}
254 
255 		srq->umem = ib_umem_get(pd->uobject->context, ucmd.buf_addr,
256 					srq_buf_size, 0, 0);
257 		if (IS_ERR(srq->umem)) {
258 			ret = PTR_ERR(srq->umem);
259 			goto err_srq;
260 		}
261 
262 		if (hr_dev->caps.srqwqe_buf_pg_sz) {
263 			npages = (ib_umem_page_count(srq->umem) +
264 				  (1 << hr_dev->caps.srqwqe_buf_pg_sz) - 1) /
265 				  (1 << hr_dev->caps.srqwqe_buf_pg_sz);
266 			page_shift = PAGE_SHIFT + hr_dev->caps.srqwqe_buf_pg_sz;
267 			ret = hns_roce_mtt_init(hr_dev, npages,
268 						page_shift,
269 						&srq->mtt);
270 		} else
271 			ret = hns_roce_mtt_init(hr_dev,
272 						ib_umem_page_count(srq->umem),
273 						srq->umem->page_shift,
274 						&srq->mtt);
275 		if (ret)
276 			goto err_buf;
277 
278 		ret = hns_roce_ib_umem_write_mtt(hr_dev, &srq->mtt, srq->umem);
279 		if (ret)
280 			goto err_srq_mtt;
281 
282 		/* config index queue BA */
283 		srq->idx_que.umem = ib_umem_get(pd->uobject->context,
284 						ucmd.que_addr,
285 						srq->idx_que.buf_size, 0, 0);
286 		if (IS_ERR(srq->idx_que.umem)) {
287 			dev_err(hr_dev->dev,
288 				"ib_umem_get error for index queue\n");
289 			ret = PTR_ERR(srq->idx_que.umem);
290 			goto err_srq_mtt;
291 		}
292 
293 		if (hr_dev->caps.idx_buf_pg_sz) {
294 			npages = (ib_umem_page_count(srq->idx_que.umem) +
295 				  (1 << hr_dev->caps.idx_buf_pg_sz) - 1) /
296 				  (1 << hr_dev->caps.idx_buf_pg_sz);
297 			page_shift = PAGE_SHIFT + hr_dev->caps.idx_buf_pg_sz;
298 			ret = hns_roce_mtt_init(hr_dev, npages,
299 						page_shift, &srq->idx_que.mtt);
300 		} else {
301 			ret = hns_roce_mtt_init(hr_dev,
302 				       ib_umem_page_count(srq->idx_que.umem),
303 				       srq->idx_que.umem->page_shift,
304 				       &srq->idx_que.mtt);
305 		}
306 
307 		if (ret) {
308 			dev_err(hr_dev->dev,
309 				"hns_roce_mtt_init error for idx que\n");
310 			goto err_idx_mtt;
311 		}
312 
313 		ret = hns_roce_ib_umem_write_mtt(hr_dev, &srq->idx_que.mtt,
314 						 srq->idx_que.umem);
315 		if (ret) {
316 			dev_err(hr_dev->dev,
317 			      "hns_roce_ib_umem_write_mtt error for idx que\n");
318 			goto err_idx_buf;
319 		}
320 	} else {
321 		page_shift = PAGE_SHIFT + hr_dev->caps.srqwqe_buf_pg_sz;
322 		if (hns_roce_buf_alloc(hr_dev, srq_buf_size,
323 				      (1 << page_shift) * 2,
324 				      &srq->buf, page_shift)) {
325 			ret = -ENOMEM;
326 			goto err_srq;
327 		}
328 
329 		srq->head = 0;
330 		srq->tail = srq->max - 1;
331 
332 		ret = hns_roce_mtt_init(hr_dev, srq->buf.npages,
333 					srq->buf.page_shift, &srq->mtt);
334 		if (ret)
335 			goto err_buf;
336 
337 		ret = hns_roce_buf_write_mtt(hr_dev, &srq->mtt, &srq->buf);
338 		if (ret)
339 			goto err_srq_mtt;
340 
341 		page_shift = PAGE_SHIFT + hr_dev->caps.idx_buf_pg_sz;
342 		ret = hns_roce_create_idx_que(pd, srq, page_shift);
343 		if (ret) {
344 			dev_err(hr_dev->dev, "Create idx queue fail(%d)!\n",
345 				ret);
346 			goto err_srq_mtt;
347 		}
348 
349 		/* Init mtt table for idx_que */
350 		ret = hns_roce_mtt_init(hr_dev, srq->idx_que.idx_buf.npages,
351 					srq->idx_que.idx_buf.page_shift,
352 					&srq->idx_que.mtt);
353 		if (ret)
354 			goto err_create_idx;
355 
356 		/* Write buffer address into the mtt table */
357 		ret = hns_roce_buf_write_mtt(hr_dev, &srq->idx_que.mtt,
358 					     &srq->idx_que.idx_buf);
359 		if (ret)
360 			goto err_idx_buf;
361 
362 		srq->wrid = kvmalloc_array(srq->max, sizeof(u64), GFP_KERNEL);
363 		if (!srq->wrid) {
364 			ret = -ENOMEM;
365 			goto err_idx_buf;
366 		}
367 	}
368 
369 	cqn = ib_srq_has_cq(srq_init_attr->srq_type) ?
370 	      to_hr_cq(srq_init_attr->ext.cq)->cqn : 0;
371 
372 	srq->db_reg_l = hr_dev->reg_base + SRQ_DB_REG;
373 
374 	ret = hns_roce_srq_alloc(hr_dev, to_hr_pd(pd)->pdn, cqn, 0,
375 				 &srq->mtt, 0, srq);
376 	if (ret)
377 		goto err_wrid;
378 
379 	srq->event = hns_roce_ib_srq_event;
380 	srq->ibsrq.ext.xrc.srq_num = srq->srqn;
381 
382 	if (udata) {
383 		if (ib_copy_to_udata(udata, &srq->srqn, sizeof(__u32))) {
384 			ret = -EFAULT;
385 			goto err_wrid;
386 		}
387 	}
388 
389 	return &srq->ibsrq;
390 
391 err_wrid:
392 	kvfree(srq->wrid);
393 
394 err_idx_buf:
395 	hns_roce_mtt_cleanup(hr_dev, &srq->idx_que.mtt);
396 
397 err_idx_mtt:
398 	if (udata)
399 		ib_umem_release(srq->idx_que.umem);
400 
401 err_create_idx:
402 	hns_roce_buf_free(hr_dev, srq->idx_que.buf_size,
403 			  &srq->idx_que.idx_buf);
404 	kfree(srq->idx_que.bitmap);
405 
406 err_srq_mtt:
407 	hns_roce_mtt_cleanup(hr_dev, &srq->mtt);
408 
409 err_buf:
410 	if (udata)
411 		ib_umem_release(srq->umem);
412 	else
413 		hns_roce_buf_free(hr_dev, srq_buf_size, &srq->buf);
414 
415 err_srq:
416 	kfree(srq);
417 	return ERR_PTR(ret);
418 }
419 
420 int hns_roce_destroy_srq(struct ib_srq *ibsrq)
421 {
422 	struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
423 	struct hns_roce_srq *srq = to_hr_srq(ibsrq);
424 
425 	hns_roce_srq_free(hr_dev, srq);
426 	hns_roce_mtt_cleanup(hr_dev, &srq->mtt);
427 
428 	if (ibsrq->uobject) {
429 		hns_roce_mtt_cleanup(hr_dev, &srq->idx_que.mtt);
430 		ib_umem_release(srq->idx_que.umem);
431 		ib_umem_release(srq->umem);
432 	} else {
433 		kvfree(srq->wrid);
434 		hns_roce_buf_free(hr_dev, srq->max << srq->wqe_shift,
435 				  &srq->buf);
436 	}
437 
438 	kfree(srq);
439 
440 	return 0;
441 }
442 
443 int hns_roce_init_srq_table(struct hns_roce_dev *hr_dev)
444 {
445 	struct hns_roce_srq_table *srq_table = &hr_dev->srq_table;
446 
447 	xa_init(&srq_table->xa);
448 
449 	return hns_roce_bitmap_init(&srq_table->bitmap, hr_dev->caps.num_srqs,
450 				    hr_dev->caps.num_srqs - 1,
451 				    hr_dev->caps.reserved_srqs, 0);
452 }
453 
454 void hns_roce_cleanup_srq_table(struct hns_roce_dev *hr_dev)
455 {
456 	hns_roce_bitmap_cleanup(&hr_dev->srq_table.bitmap);
457 }
458