xref: /openbmc/linux/drivers/infiniband/hw/mana/qp.c (revision b755c25f)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (c) 2022, Microsoft Corporation. All rights reserved.
4  */
5 
6 #include "mana_ib.h"
7 
8 static int mana_ib_cfg_vport_steering(struct mana_ib_dev *dev,
9 				      struct net_device *ndev,
10 				      mana_handle_t default_rxobj,
11 				      mana_handle_t ind_table[],
12 				      u32 log_ind_tbl_size, u32 rx_hash_key_len,
13 				      u8 *rx_hash_key)
14 {
15 	struct mana_port_context *mpc = netdev_priv(ndev);
16 	struct mana_cfg_rx_steer_req_v2 *req;
17 	struct mana_cfg_rx_steer_resp resp = {};
18 	mana_handle_t *req_indir_tab;
19 	struct gdma_context *gc;
20 	struct gdma_dev *mdev;
21 	u32 req_buf_size;
22 	int i, err;
23 
24 	mdev = dev->gdma_dev;
25 	gc = mdev->gdma_context;
26 
27 	req_buf_size =
28 		sizeof(*req) + sizeof(mana_handle_t) * MANA_INDIRECT_TABLE_SIZE;
29 	req = kzalloc(req_buf_size, GFP_KERNEL);
30 	if (!req)
31 		return -ENOMEM;
32 
33 	mana_gd_init_req_hdr(&req->hdr, MANA_CONFIG_VPORT_RX, req_buf_size,
34 			     sizeof(resp));
35 
36 	req->hdr.req.msg_version = GDMA_MESSAGE_V2;
37 
38 	req->vport = mpc->port_handle;
39 	req->rx_enable = 1;
40 	req->update_default_rxobj = 1;
41 	req->default_rxobj = default_rxobj;
42 	req->hdr.dev_id = mdev->dev_id;
43 
44 	/* If there are more than 1 entries in indirection table, enable RSS */
45 	if (log_ind_tbl_size)
46 		req->rss_enable = true;
47 
48 	req->num_indir_entries = MANA_INDIRECT_TABLE_SIZE;
49 	req->indir_tab_offset = sizeof(*req);
50 	req->update_indir_tab = true;
51 	req->cqe_coalescing_enable = 1;
52 
53 	req_indir_tab = (mana_handle_t *)(req + 1);
54 	/* The ind table passed to the hardware must have
55 	 * MANA_INDIRECT_TABLE_SIZE entries. Adjust the verb
56 	 * ind_table to MANA_INDIRECT_TABLE_SIZE if required
57 	 */
58 	ibdev_dbg(&dev->ib_dev, "ind table size %u\n", 1 << log_ind_tbl_size);
59 	for (i = 0; i < MANA_INDIRECT_TABLE_SIZE; i++) {
60 		req_indir_tab[i] = ind_table[i % (1 << log_ind_tbl_size)];
61 		ibdev_dbg(&dev->ib_dev, "index %u handle 0x%llx\n", i,
62 			  req_indir_tab[i]);
63 	}
64 
65 	req->update_hashkey = true;
66 	if (rx_hash_key_len)
67 		memcpy(req->hashkey, rx_hash_key, rx_hash_key_len);
68 	else
69 		netdev_rss_key_fill(req->hashkey, MANA_HASH_KEY_SIZE);
70 
71 	ibdev_dbg(&dev->ib_dev, "vport handle %llu default_rxobj 0x%llx\n",
72 		  req->vport, default_rxobj);
73 
74 	err = mana_gd_send_request(gc, req_buf_size, req, sizeof(resp), &resp);
75 	if (err) {
76 		netdev_err(ndev, "Failed to configure vPort RX: %d\n", err);
77 		goto out;
78 	}
79 
80 	if (resp.hdr.status) {
81 		netdev_err(ndev, "vPort RX configuration failed: 0x%x\n",
82 			   resp.hdr.status);
83 		err = -EPROTO;
84 		goto out;
85 	}
86 
87 	netdev_info(ndev, "Configured steering vPort %llu log_entries %u\n",
88 		    mpc->port_handle, log_ind_tbl_size);
89 
90 out:
91 	kfree(req);
92 	return err;
93 }
94 
95 static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd,
96 				 struct ib_qp_init_attr *attr,
97 				 struct ib_udata *udata)
98 {
99 	struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
100 	struct mana_ib_dev *mdev =
101 		container_of(pd->device, struct mana_ib_dev, ib_dev);
102 	struct ib_rwq_ind_table *ind_tbl = attr->rwq_ind_tbl;
103 	struct mana_ib_create_qp_rss_resp resp = {};
104 	struct mana_ib_create_qp_rss ucmd = {};
105 	struct gdma_dev *gd = mdev->gdma_dev;
106 	mana_handle_t *mana_ind_table;
107 	struct mana_port_context *mpc;
108 	struct mana_context *mc;
109 	struct net_device *ndev;
110 	struct mana_ib_cq *cq;
111 	struct mana_ib_wq *wq;
112 	unsigned int ind_tbl_size;
113 	struct ib_cq *ibcq;
114 	struct ib_wq *ibwq;
115 	int i = 0;
116 	u32 port;
117 	int ret;
118 
119 	mc = gd->driver_data;
120 
121 	if (!udata || udata->inlen < sizeof(ucmd))
122 		return -EINVAL;
123 
124 	ret = ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen));
125 	if (ret) {
126 		ibdev_dbg(&mdev->ib_dev,
127 			  "Failed copy from udata for create rss-qp, err %d\n",
128 			  ret);
129 		return ret;
130 	}
131 
132 	if (attr->cap.max_recv_wr > MAX_SEND_BUFFERS_PER_QUEUE) {
133 		ibdev_dbg(&mdev->ib_dev,
134 			  "Requested max_recv_wr %d exceeding limit\n",
135 			  attr->cap.max_recv_wr);
136 		return -EINVAL;
137 	}
138 
139 	if (attr->cap.max_recv_sge > MAX_RX_WQE_SGL_ENTRIES) {
140 		ibdev_dbg(&mdev->ib_dev,
141 			  "Requested max_recv_sge %d exceeding limit\n",
142 			  attr->cap.max_recv_sge);
143 		return -EINVAL;
144 	}
145 
146 	ind_tbl_size = 1 << ind_tbl->log_ind_tbl_size;
147 	if (ind_tbl_size > MANA_INDIRECT_TABLE_SIZE) {
148 		ibdev_dbg(&mdev->ib_dev,
149 			  "Indirect table size %d exceeding limit\n",
150 			  ind_tbl_size);
151 		return -EINVAL;
152 	}
153 
154 	if (ucmd.rx_hash_function != MANA_IB_RX_HASH_FUNC_TOEPLITZ) {
155 		ibdev_dbg(&mdev->ib_dev,
156 			  "RX Hash function is not supported, %d\n",
157 			  ucmd.rx_hash_function);
158 		return -EINVAL;
159 	}
160 
161 	/* IB ports start with 1, MANA start with 0 */
162 	port = ucmd.port;
163 	if (port < 1 || port > mc->num_ports) {
164 		ibdev_dbg(&mdev->ib_dev, "Invalid port %u in creating qp\n",
165 			  port);
166 		return -EINVAL;
167 	}
168 	ndev = mc->ports[port - 1];
169 	mpc = netdev_priv(ndev);
170 
171 	ibdev_dbg(&mdev->ib_dev, "rx_hash_function %d port %d\n",
172 		  ucmd.rx_hash_function, port);
173 
174 	mana_ind_table = kcalloc(ind_tbl_size, sizeof(mana_handle_t),
175 				 GFP_KERNEL);
176 	if (!mana_ind_table) {
177 		ret = -ENOMEM;
178 		goto fail;
179 	}
180 
181 	qp->port = port;
182 
183 	for (i = 0; i < ind_tbl_size; i++) {
184 		struct mana_obj_spec wq_spec = {};
185 		struct mana_obj_spec cq_spec = {};
186 
187 		ibwq = ind_tbl->ind_tbl[i];
188 		wq = container_of(ibwq, struct mana_ib_wq, ibwq);
189 
190 		ibcq = ibwq->cq;
191 		cq = container_of(ibcq, struct mana_ib_cq, ibcq);
192 
193 		wq_spec.gdma_region = wq->gdma_region;
194 		wq_spec.queue_size = wq->wq_buf_size;
195 
196 		cq_spec.gdma_region = cq->gdma_region;
197 		cq_spec.queue_size = cq->cqe * COMP_ENTRY_SIZE;
198 		cq_spec.modr_ctx_id = 0;
199 		cq_spec.attached_eq = GDMA_CQ_NO_EQ;
200 
201 		ret = mana_create_wq_obj(mpc, mpc->port_handle, GDMA_RQ,
202 					 &wq_spec, &cq_spec, &wq->rx_object);
203 		if (ret)
204 			goto fail;
205 
206 		/* The GDMA regions are now owned by the WQ object */
207 		wq->gdma_region = GDMA_INVALID_DMA_REGION;
208 		cq->gdma_region = GDMA_INVALID_DMA_REGION;
209 
210 		wq->id = wq_spec.queue_index;
211 		cq->id = cq_spec.queue_index;
212 
213 		ibdev_dbg(&mdev->ib_dev,
214 			  "ret %d rx_object 0x%llx wq id %llu cq id %llu\n",
215 			  ret, wq->rx_object, wq->id, cq->id);
216 
217 		resp.entries[i].cqid = cq->id;
218 		resp.entries[i].wqid = wq->id;
219 
220 		mana_ind_table[i] = wq->rx_object;
221 	}
222 	resp.num_entries = i;
223 
224 	ret = mana_ib_cfg_vport_steering(mdev, ndev, wq->rx_object,
225 					 mana_ind_table,
226 					 ind_tbl->log_ind_tbl_size,
227 					 ucmd.rx_hash_key_len,
228 					 ucmd.rx_hash_key);
229 	if (ret)
230 		goto fail;
231 
232 	ret = ib_copy_to_udata(udata, &resp, sizeof(resp));
233 	if (ret) {
234 		ibdev_dbg(&mdev->ib_dev,
235 			  "Failed to copy to udata create rss-qp, %d\n",
236 			  ret);
237 		goto fail;
238 	}
239 
240 	kfree(mana_ind_table);
241 
242 	return 0;
243 
244 fail:
245 	while (i-- > 0) {
246 		ibwq = ind_tbl->ind_tbl[i];
247 		wq = container_of(ibwq, struct mana_ib_wq, ibwq);
248 		mana_destroy_wq_obj(mpc, GDMA_RQ, wq->rx_object);
249 	}
250 
251 	kfree(mana_ind_table);
252 
253 	return ret;
254 }
255 
256 static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd,
257 				 struct ib_qp_init_attr *attr,
258 				 struct ib_udata *udata)
259 {
260 	struct mana_ib_pd *pd = container_of(ibpd, struct mana_ib_pd, ibpd);
261 	struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
262 	struct mana_ib_dev *mdev =
263 		container_of(ibpd->device, struct mana_ib_dev, ib_dev);
264 	struct mana_ib_cq *send_cq =
265 		container_of(attr->send_cq, struct mana_ib_cq, ibcq);
266 	struct mana_ib_ucontext *mana_ucontext =
267 		rdma_udata_to_drv_context(udata, struct mana_ib_ucontext,
268 					  ibucontext);
269 	struct mana_ib_create_qp_resp resp = {};
270 	struct gdma_dev *gd = mdev->gdma_dev;
271 	struct mana_ib_create_qp ucmd = {};
272 	struct mana_obj_spec wq_spec = {};
273 	struct mana_obj_spec cq_spec = {};
274 	struct mana_port_context *mpc;
275 	struct mana_context *mc;
276 	struct net_device *ndev;
277 	struct ib_umem *umem;
278 	int err;
279 	u32 port;
280 
281 	mc = gd->driver_data;
282 
283 	if (!mana_ucontext || udata->inlen < sizeof(ucmd))
284 		return -EINVAL;
285 
286 	err = ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen));
287 	if (err) {
288 		ibdev_dbg(&mdev->ib_dev,
289 			  "Failed to copy from udata create qp-raw, %d\n", err);
290 		return err;
291 	}
292 
293 	/* IB ports start with 1, MANA Ethernet ports start with 0 */
294 	port = ucmd.port;
295 	if (port < 1 || port > mc->num_ports)
296 		return -EINVAL;
297 
298 	if (attr->cap.max_send_wr > MAX_SEND_BUFFERS_PER_QUEUE) {
299 		ibdev_dbg(&mdev->ib_dev,
300 			  "Requested max_send_wr %d exceeding limit\n",
301 			  attr->cap.max_send_wr);
302 		return -EINVAL;
303 	}
304 
305 	if (attr->cap.max_send_sge > MAX_TX_WQE_SGL_ENTRIES) {
306 		ibdev_dbg(&mdev->ib_dev,
307 			  "Requested max_send_sge %d exceeding limit\n",
308 			  attr->cap.max_send_sge);
309 		return -EINVAL;
310 	}
311 
312 	ndev = mc->ports[port - 1];
313 	mpc = netdev_priv(ndev);
314 	ibdev_dbg(&mdev->ib_dev, "port %u ndev %p mpc %p\n", port, ndev, mpc);
315 
316 	err = mana_ib_cfg_vport(mdev, port - 1, pd, mana_ucontext->doorbell);
317 	if (err)
318 		return -ENODEV;
319 
320 	qp->port = port;
321 
322 	ibdev_dbg(&mdev->ib_dev, "ucmd sq_buf_addr 0x%llx port %u\n",
323 		  ucmd.sq_buf_addr, ucmd.port);
324 
325 	umem = ib_umem_get(ibpd->device, ucmd.sq_buf_addr, ucmd.sq_buf_size,
326 			   IB_ACCESS_LOCAL_WRITE);
327 	if (IS_ERR(umem)) {
328 		err = PTR_ERR(umem);
329 		ibdev_dbg(&mdev->ib_dev,
330 			  "Failed to get umem for create qp-raw, err %d\n",
331 			  err);
332 		goto err_free_vport;
333 	}
334 	qp->sq_umem = umem;
335 
336 	err = mana_ib_gd_create_dma_region(mdev, qp->sq_umem,
337 					   &qp->sq_gdma_region);
338 	if (err) {
339 		ibdev_dbg(&mdev->ib_dev,
340 			  "Failed to create dma region for create qp-raw, %d\n",
341 			  err);
342 		goto err_release_umem;
343 	}
344 
345 	ibdev_dbg(&mdev->ib_dev,
346 		  "mana_ib_gd_create_dma_region ret %d gdma_region 0x%llx\n",
347 		  err, qp->sq_gdma_region);
348 
349 	/* Create a WQ on the same port handle used by the Ethernet */
350 	wq_spec.gdma_region = qp->sq_gdma_region;
351 	wq_spec.queue_size = ucmd.sq_buf_size;
352 
353 	cq_spec.gdma_region = send_cq->gdma_region;
354 	cq_spec.queue_size = send_cq->cqe * COMP_ENTRY_SIZE;
355 	cq_spec.modr_ctx_id = 0;
356 	cq_spec.attached_eq = GDMA_CQ_NO_EQ;
357 
358 	err = mana_create_wq_obj(mpc, mpc->port_handle, GDMA_SQ, &wq_spec,
359 				 &cq_spec, &qp->tx_object);
360 	if (err) {
361 		ibdev_dbg(&mdev->ib_dev,
362 			  "Failed to create wq for create raw-qp, err %d\n",
363 			  err);
364 		goto err_destroy_dma_region;
365 	}
366 
367 	/* The GDMA regions are now owned by the WQ object */
368 	qp->sq_gdma_region = GDMA_INVALID_DMA_REGION;
369 	send_cq->gdma_region = GDMA_INVALID_DMA_REGION;
370 
371 	qp->sq_id = wq_spec.queue_index;
372 	send_cq->id = cq_spec.queue_index;
373 
374 	ibdev_dbg(&mdev->ib_dev,
375 		  "ret %d qp->tx_object 0x%llx sq id %llu cq id %llu\n", err,
376 		  qp->tx_object, qp->sq_id, send_cq->id);
377 
378 	resp.sqid = qp->sq_id;
379 	resp.cqid = send_cq->id;
380 	resp.tx_vp_offset = pd->tx_vp_offset;
381 
382 	err = ib_copy_to_udata(udata, &resp, sizeof(resp));
383 	if (err) {
384 		ibdev_dbg(&mdev->ib_dev,
385 			  "Failed copy udata for create qp-raw, %d\n",
386 			  err);
387 		goto err_destroy_wq_obj;
388 	}
389 
390 	return 0;
391 
392 err_destroy_wq_obj:
393 	mana_destroy_wq_obj(mpc, GDMA_SQ, qp->tx_object);
394 
395 err_destroy_dma_region:
396 	mana_ib_gd_destroy_dma_region(mdev, qp->sq_gdma_region);
397 
398 err_release_umem:
399 	ib_umem_release(umem);
400 
401 err_free_vport:
402 	mana_ib_uncfg_vport(mdev, pd, port - 1);
403 
404 	return err;
405 }
406 
407 int mana_ib_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
408 		      struct ib_udata *udata)
409 {
410 	switch (attr->qp_type) {
411 	case IB_QPT_RAW_PACKET:
412 		/* When rwq_ind_tbl is used, it's for creating WQs for RSS */
413 		if (attr->rwq_ind_tbl)
414 			return mana_ib_create_qp_rss(ibqp, ibqp->pd, attr,
415 						     udata);
416 
417 		return mana_ib_create_qp_raw(ibqp, ibqp->pd, attr, udata);
418 	default:
419 		/* Creating QP other than IB_QPT_RAW_PACKET is not supported */
420 		ibdev_dbg(ibqp->device, "Creating QP type %u not supported\n",
421 			  attr->qp_type);
422 	}
423 
424 	return -EINVAL;
425 }
426 
427 int mana_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
428 		      int attr_mask, struct ib_udata *udata)
429 {
430 	/* modify_qp is not supported by this version of the driver */
431 	return -EOPNOTSUPP;
432 }
433 
434 static int mana_ib_destroy_qp_rss(struct mana_ib_qp *qp,
435 				  struct ib_rwq_ind_table *ind_tbl,
436 				  struct ib_udata *udata)
437 {
438 	struct mana_ib_dev *mdev =
439 		container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev);
440 	struct gdma_dev *gd = mdev->gdma_dev;
441 	struct mana_port_context *mpc;
442 	struct mana_context *mc;
443 	struct net_device *ndev;
444 	struct mana_ib_wq *wq;
445 	struct ib_wq *ibwq;
446 	int i;
447 
448 	mc = gd->driver_data;
449 	ndev = mc->ports[qp->port - 1];
450 	mpc = netdev_priv(ndev);
451 
452 	for (i = 0; i < (1 << ind_tbl->log_ind_tbl_size); i++) {
453 		ibwq = ind_tbl->ind_tbl[i];
454 		wq = container_of(ibwq, struct mana_ib_wq, ibwq);
455 		ibdev_dbg(&mdev->ib_dev, "destroying wq->rx_object %llu\n",
456 			  wq->rx_object);
457 		mana_destroy_wq_obj(mpc, GDMA_RQ, wq->rx_object);
458 	}
459 
460 	return 0;
461 }
462 
463 static int mana_ib_destroy_qp_raw(struct mana_ib_qp *qp, struct ib_udata *udata)
464 {
465 	struct mana_ib_dev *mdev =
466 		container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev);
467 	struct gdma_dev *gd = mdev->gdma_dev;
468 	struct ib_pd *ibpd = qp->ibqp.pd;
469 	struct mana_port_context *mpc;
470 	struct mana_context *mc;
471 	struct net_device *ndev;
472 	struct mana_ib_pd *pd;
473 
474 	mc = gd->driver_data;
475 	ndev = mc->ports[qp->port - 1];
476 	mpc = netdev_priv(ndev);
477 	pd = container_of(ibpd, struct mana_ib_pd, ibpd);
478 
479 	mana_destroy_wq_obj(mpc, GDMA_SQ, qp->tx_object);
480 
481 	if (qp->sq_umem) {
482 		mana_ib_gd_destroy_dma_region(mdev, qp->sq_gdma_region);
483 		ib_umem_release(qp->sq_umem);
484 	}
485 
486 	mana_ib_uncfg_vport(mdev, pd, qp->port - 1);
487 
488 	return 0;
489 }
490 
491 int mana_ib_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
492 {
493 	struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
494 
495 	switch (ibqp->qp_type) {
496 	case IB_QPT_RAW_PACKET:
497 		if (ibqp->rwq_ind_tbl)
498 			return mana_ib_destroy_qp_rss(qp, ibqp->rwq_ind_tbl,
499 						      udata);
500 
501 		return mana_ib_destroy_qp_raw(qp, udata);
502 
503 	default:
504 		ibdev_dbg(ibqp->device, "Unexpected QP type %u\n",
505 			  ibqp->qp_type);
506 	}
507 
508 	return -ENOENT;
509 }
510