1 /*******************************************************************
2  * This file is part of the Emulex RoCE Device Driver for          *
3  * RoCE (RDMA over Converged Ethernet) adapters.                   *
4  * Copyright (C) 2008-2012 Emulex. All rights reserved.            *
5  * EMULEX and SLI are trademarks of Emulex.                        *
6  * www.emulex.com                                                  *
7  *                                                                 *
8  * This program is free software; you can redistribute it and/or   *
9  * modify it under the terms of version 2 of the GNU General       *
10  * Public License as published by the Free Software Foundation.    *
11  * This program is distributed in the hope that it will be useful. *
12  * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND          *
13  * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY,  *
14  * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE      *
15  * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *
16  * TO BE LEGALLY INVALID.  See the GNU General Public License for  *
17  * more details, a copy of which can be found in the file COPYING  *
18  * included with this package.                                     *
19  *
20  * Contact Information:
21  * linux-drivers@emulex.com
22  *
23  * Emulex
24  * 3333 Susan Street
25  * Costa Mesa, CA 92626
26  *******************************************************************/
27 
28 #include <linux/dma-mapping.h>
29 #include <rdma/ib_verbs.h>
30 #include <rdma/ib_user_verbs.h>
31 #include <rdma/iw_cm.h>
32 #include <rdma/ib_umem.h>
33 #include <rdma/ib_addr.h>
34 
35 #include "ocrdma.h"
36 #include "ocrdma_hw.h"
37 #include "ocrdma_verbs.h"
38 #include "ocrdma_abi.h"
39 
40 int ocrdma_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
41 {
42 	if (index > 1)
43 		return -EINVAL;
44 
45 	*pkey = 0xffff;
46 	return 0;
47 }
48 
49 int ocrdma_query_gid(struct ib_device *ibdev, u8 port,
50 		     int index, union ib_gid *sgid)
51 {
52 	struct ocrdma_dev *dev;
53 
54 	dev = get_ocrdma_dev(ibdev);
55 	memset(sgid, 0, sizeof(*sgid));
56 	if (index > OCRDMA_MAX_SGID)
57 		return -EINVAL;
58 
59 	memcpy(sgid, &dev->sgid_tbl[index], sizeof(*sgid));
60 
61 	return 0;
62 }
63 
64 int ocrdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr)
65 {
66 	struct ocrdma_dev *dev = get_ocrdma_dev(ibdev);
67 
68 	memset(attr, 0, sizeof *attr);
69 	memcpy(&attr->fw_ver, &dev->attr.fw_ver[0],
70 	       min(sizeof(dev->attr.fw_ver), sizeof(attr->fw_ver)));
71 	ocrdma_get_guid(dev, (u8 *)&attr->sys_image_guid);
72 	attr->max_mr_size = dev->attr.max_mr_size;
73 	attr->page_size_cap = 0xffff000;
74 	attr->vendor_id = dev->nic_info.pdev->vendor;
75 	attr->vendor_part_id = dev->nic_info.pdev->device;
76 	attr->hw_ver = dev->asic_id;
77 	attr->max_qp = dev->attr.max_qp;
78 	attr->max_ah = OCRDMA_MAX_AH;
79 	attr->max_qp_wr = dev->attr.max_wqe;
80 
81 	attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
82 					IB_DEVICE_RC_RNR_NAK_GEN |
83 					IB_DEVICE_SHUTDOWN_PORT |
84 					IB_DEVICE_SYS_IMAGE_GUID |
85 					IB_DEVICE_LOCAL_DMA_LKEY |
86 					IB_DEVICE_MEM_MGT_EXTENSIONS;
87 	attr->max_sge = min(dev->attr.max_send_sge, dev->attr.max_srq_sge);
88 	attr->max_sge_rd = 0;
89 	attr->max_cq = dev->attr.max_cq;
90 	attr->max_cqe = dev->attr.max_cqe;
91 	attr->max_mr = dev->attr.max_mr;
92 	attr->max_mw = dev->attr.max_mw;
93 	attr->max_pd = dev->attr.max_pd;
94 	attr->atomic_cap = 0;
95 	attr->max_fmr = 0;
96 	attr->max_map_per_fmr = 0;
97 	attr->max_qp_rd_atom =
98 	    min(dev->attr.max_ord_per_qp, dev->attr.max_ird_per_qp);
99 	attr->max_qp_init_rd_atom = dev->attr.max_ord_per_qp;
100 	attr->max_srq = dev->attr.max_srq;
101 	attr->max_srq_sge = dev->attr.max_srq_sge;
102 	attr->max_srq_wr = dev->attr.max_rqe;
103 	attr->local_ca_ack_delay = dev->attr.local_ca_ack_delay;
104 	attr->max_fast_reg_page_list_len = dev->attr.max_pages_per_frmr;
105 	attr->max_pkeys = 1;
106 	return 0;
107 }
108 
109 static inline void get_link_speed_and_width(struct ocrdma_dev *dev,
110 					    u8 *ib_speed, u8 *ib_width)
111 {
112 	int status;
113 	u8 speed;
114 
115 	status = ocrdma_mbx_get_link_speed(dev, &speed);
116 	if (status)
117 		speed = OCRDMA_PHYS_LINK_SPEED_ZERO;
118 
119 	switch (speed) {
120 	case OCRDMA_PHYS_LINK_SPEED_1GBPS:
121 		*ib_speed = IB_SPEED_SDR;
122 		*ib_width = IB_WIDTH_1X;
123 		break;
124 
125 	case OCRDMA_PHYS_LINK_SPEED_10GBPS:
126 		*ib_speed = IB_SPEED_QDR;
127 		*ib_width = IB_WIDTH_1X;
128 		break;
129 
130 	case OCRDMA_PHYS_LINK_SPEED_20GBPS:
131 		*ib_speed = IB_SPEED_DDR;
132 		*ib_width = IB_WIDTH_4X;
133 		break;
134 
135 	case OCRDMA_PHYS_LINK_SPEED_40GBPS:
136 		*ib_speed = IB_SPEED_QDR;
137 		*ib_width = IB_WIDTH_4X;
138 		break;
139 
140 	default:
141 		/* Unsupported */
142 		*ib_speed = IB_SPEED_SDR;
143 		*ib_width = IB_WIDTH_1X;
144 	}
145 }
146 
147 int ocrdma_query_port(struct ib_device *ibdev,
148 		      u8 port, struct ib_port_attr *props)
149 {
150 	enum ib_port_state port_state;
151 	struct ocrdma_dev *dev;
152 	struct net_device *netdev;
153 
154 	dev = get_ocrdma_dev(ibdev);
155 	if (port > 1) {
156 		pr_err("%s(%d) invalid_port=0x%x\n", __func__,
157 		       dev->id, port);
158 		return -EINVAL;
159 	}
160 	netdev = dev->nic_info.netdev;
161 	if (netif_running(netdev) && netif_oper_up(netdev)) {
162 		port_state = IB_PORT_ACTIVE;
163 		props->phys_state = 5;
164 	} else {
165 		port_state = IB_PORT_DOWN;
166 		props->phys_state = 3;
167 	}
168 	props->max_mtu = IB_MTU_4096;
169 	props->active_mtu = iboe_get_mtu(netdev->mtu);
170 	props->lid = 0;
171 	props->lmc = 0;
172 	props->sm_lid = 0;
173 	props->sm_sl = 0;
174 	props->state = port_state;
175 	props->port_cap_flags =
176 	    IB_PORT_CM_SUP |
177 	    IB_PORT_REINIT_SUP |
178 	    IB_PORT_DEVICE_MGMT_SUP | IB_PORT_VENDOR_CLASS_SUP | IB_PORT_IP_BASED_GIDS;
179 	props->gid_tbl_len = OCRDMA_MAX_SGID;
180 	props->pkey_tbl_len = 1;
181 	props->bad_pkey_cntr = 0;
182 	props->qkey_viol_cntr = 0;
183 	get_link_speed_and_width(dev, &props->active_speed,
184 				 &props->active_width);
185 	props->max_msg_sz = 0x80000000;
186 	props->max_vl_num = 4;
187 	return 0;
188 }
189 
190 int ocrdma_modify_port(struct ib_device *ibdev, u8 port, int mask,
191 		       struct ib_port_modify *props)
192 {
193 	struct ocrdma_dev *dev;
194 
195 	dev = get_ocrdma_dev(ibdev);
196 	if (port > 1) {
197 		pr_err("%s(%d) invalid_port=0x%x\n", __func__, dev->id, port);
198 		return -EINVAL;
199 	}
200 	return 0;
201 }
202 
203 static int ocrdma_add_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr,
204 			   unsigned long len)
205 {
206 	struct ocrdma_mm *mm;
207 
208 	mm = kzalloc(sizeof(*mm), GFP_KERNEL);
209 	if (mm == NULL)
210 		return -ENOMEM;
211 	mm->key.phy_addr = phy_addr;
212 	mm->key.len = len;
213 	INIT_LIST_HEAD(&mm->entry);
214 
215 	mutex_lock(&uctx->mm_list_lock);
216 	list_add_tail(&mm->entry, &uctx->mm_head);
217 	mutex_unlock(&uctx->mm_list_lock);
218 	return 0;
219 }
220 
221 static void ocrdma_del_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr,
222 			    unsigned long len)
223 {
224 	struct ocrdma_mm *mm, *tmp;
225 
226 	mutex_lock(&uctx->mm_list_lock);
227 	list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
228 		if (len != mm->key.len && phy_addr != mm->key.phy_addr)
229 			continue;
230 
231 		list_del(&mm->entry);
232 		kfree(mm);
233 		break;
234 	}
235 	mutex_unlock(&uctx->mm_list_lock);
236 }
237 
238 static bool ocrdma_search_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr,
239 			      unsigned long len)
240 {
241 	bool found = false;
242 	struct ocrdma_mm *mm;
243 
244 	mutex_lock(&uctx->mm_list_lock);
245 	list_for_each_entry(mm, &uctx->mm_head, entry) {
246 		if (len != mm->key.len && phy_addr != mm->key.phy_addr)
247 			continue;
248 
249 		found = true;
250 		break;
251 	}
252 	mutex_unlock(&uctx->mm_list_lock);
253 	return found;
254 }
255 
256 static struct ocrdma_pd *_ocrdma_alloc_pd(struct ocrdma_dev *dev,
257 					  struct ocrdma_ucontext *uctx,
258 					  struct ib_udata *udata)
259 {
260 	struct ocrdma_pd *pd = NULL;
261 	int status = 0;
262 
263 	pd = kzalloc(sizeof(*pd), GFP_KERNEL);
264 	if (!pd)
265 		return ERR_PTR(-ENOMEM);
266 
267 	if (udata && uctx) {
268 		pd->dpp_enabled =
269 			ocrdma_get_asic_type(dev) == OCRDMA_ASIC_GEN_SKH_R;
270 		pd->num_dpp_qp =
271 			pd->dpp_enabled ? (dev->nic_info.db_page_size /
272 					   dev->attr.wqe_size) : 0;
273 	}
274 
275 retry:
276 	status = ocrdma_mbx_alloc_pd(dev, pd);
277 	if (status) {
278 		if (pd->dpp_enabled) {
279 			pd->dpp_enabled = false;
280 			pd->num_dpp_qp = 0;
281 			goto retry;
282 		} else {
283 			kfree(pd);
284 			return ERR_PTR(status);
285 		}
286 	}
287 
288 	return pd;
289 }
290 
291 static inline int is_ucontext_pd(struct ocrdma_ucontext *uctx,
292 				 struct ocrdma_pd *pd)
293 {
294 	return (uctx->cntxt_pd == pd ? true : false);
295 }
296 
297 static int _ocrdma_dealloc_pd(struct ocrdma_dev *dev,
298 			      struct ocrdma_pd *pd)
299 {
300 	int status = 0;
301 
302 	status = ocrdma_mbx_dealloc_pd(dev, pd);
303 	kfree(pd);
304 	return status;
305 }
306 
307 static int ocrdma_alloc_ucontext_pd(struct ocrdma_dev *dev,
308 				    struct ocrdma_ucontext *uctx,
309 				    struct ib_udata *udata)
310 {
311 	int status = 0;
312 
313 	uctx->cntxt_pd = _ocrdma_alloc_pd(dev, uctx, udata);
314 	if (IS_ERR(uctx->cntxt_pd)) {
315 		status = PTR_ERR(uctx->cntxt_pd);
316 		uctx->cntxt_pd = NULL;
317 		goto err;
318 	}
319 
320 	uctx->cntxt_pd->uctx = uctx;
321 	uctx->cntxt_pd->ibpd.device = &dev->ibdev;
322 err:
323 	return status;
324 }
325 
326 static int ocrdma_dealloc_ucontext_pd(struct ocrdma_ucontext *uctx)
327 {
328 	int status = 0;
329 	struct ocrdma_pd *pd = uctx->cntxt_pd;
330 	struct ocrdma_dev *dev = get_ocrdma_dev(pd->ibpd.device);
331 
332 	if (uctx->pd_in_use) {
333 		pr_err("%s(%d) Freeing in use pdid=0x%x.\n",
334 		       __func__, dev->id, pd->id);
335 	}
336 	uctx->cntxt_pd = NULL;
337 	status = _ocrdma_dealloc_pd(dev, pd);
338 	return status;
339 }
340 
341 static struct ocrdma_pd *ocrdma_get_ucontext_pd(struct ocrdma_ucontext *uctx)
342 {
343 	struct ocrdma_pd *pd = NULL;
344 
345 	mutex_lock(&uctx->mm_list_lock);
346 	if (!uctx->pd_in_use) {
347 		uctx->pd_in_use = true;
348 		pd = uctx->cntxt_pd;
349 	}
350 	mutex_unlock(&uctx->mm_list_lock);
351 
352 	return pd;
353 }
354 
355 static void ocrdma_release_ucontext_pd(struct ocrdma_ucontext *uctx)
356 {
357 	mutex_lock(&uctx->mm_list_lock);
358 	uctx->pd_in_use = false;
359 	mutex_unlock(&uctx->mm_list_lock);
360 }
361 
362 struct ib_ucontext *ocrdma_alloc_ucontext(struct ib_device *ibdev,
363 					  struct ib_udata *udata)
364 {
365 	int status;
366 	struct ocrdma_ucontext *ctx;
367 	struct ocrdma_alloc_ucontext_resp resp;
368 	struct ocrdma_dev *dev = get_ocrdma_dev(ibdev);
369 	struct pci_dev *pdev = dev->nic_info.pdev;
370 	u32 map_len = roundup(sizeof(u32) * 2048, PAGE_SIZE);
371 
372 	if (!udata)
373 		return ERR_PTR(-EFAULT);
374 	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
375 	if (!ctx)
376 		return ERR_PTR(-ENOMEM);
377 	INIT_LIST_HEAD(&ctx->mm_head);
378 	mutex_init(&ctx->mm_list_lock);
379 
380 	ctx->ah_tbl.va = dma_alloc_coherent(&pdev->dev, map_len,
381 					    &ctx->ah_tbl.pa, GFP_KERNEL);
382 	if (!ctx->ah_tbl.va) {
383 		kfree(ctx);
384 		return ERR_PTR(-ENOMEM);
385 	}
386 	memset(ctx->ah_tbl.va, 0, map_len);
387 	ctx->ah_tbl.len = map_len;
388 
389 	memset(&resp, 0, sizeof(resp));
390 	resp.ah_tbl_len = ctx->ah_tbl.len;
391 	resp.ah_tbl_page = virt_to_phys(ctx->ah_tbl.va);
392 
393 	status = ocrdma_add_mmap(ctx, resp.ah_tbl_page, resp.ah_tbl_len);
394 	if (status)
395 		goto map_err;
396 
397 	status = ocrdma_alloc_ucontext_pd(dev, ctx, udata);
398 	if (status)
399 		goto pd_err;
400 
401 	resp.dev_id = dev->id;
402 	resp.max_inline_data = dev->attr.max_inline_data;
403 	resp.wqe_size = dev->attr.wqe_size;
404 	resp.rqe_size = dev->attr.rqe_size;
405 	resp.dpp_wqe_size = dev->attr.wqe_size;
406 
407 	memcpy(resp.fw_ver, dev->attr.fw_ver, sizeof(resp.fw_ver));
408 	status = ib_copy_to_udata(udata, &resp, sizeof(resp));
409 	if (status)
410 		goto cpy_err;
411 	return &ctx->ibucontext;
412 
413 cpy_err:
414 pd_err:
415 	ocrdma_del_mmap(ctx, ctx->ah_tbl.pa, ctx->ah_tbl.len);
416 map_err:
417 	dma_free_coherent(&pdev->dev, ctx->ah_tbl.len, ctx->ah_tbl.va,
418 			  ctx->ah_tbl.pa);
419 	kfree(ctx);
420 	return ERR_PTR(status);
421 }
422 
423 int ocrdma_dealloc_ucontext(struct ib_ucontext *ibctx)
424 {
425 	int status = 0;
426 	struct ocrdma_mm *mm, *tmp;
427 	struct ocrdma_ucontext *uctx = get_ocrdma_ucontext(ibctx);
428 	struct ocrdma_dev *dev = get_ocrdma_dev(ibctx->device);
429 	struct pci_dev *pdev = dev->nic_info.pdev;
430 
431 	status = ocrdma_dealloc_ucontext_pd(uctx);
432 
433 	ocrdma_del_mmap(uctx, uctx->ah_tbl.pa, uctx->ah_tbl.len);
434 	dma_free_coherent(&pdev->dev, uctx->ah_tbl.len, uctx->ah_tbl.va,
435 			  uctx->ah_tbl.pa);
436 
437 	list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
438 		list_del(&mm->entry);
439 		kfree(mm);
440 	}
441 	kfree(uctx);
442 	return status;
443 }
444 
445 int ocrdma_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
446 {
447 	struct ocrdma_ucontext *ucontext = get_ocrdma_ucontext(context);
448 	struct ocrdma_dev *dev = get_ocrdma_dev(context->device);
449 	unsigned long vm_page = vma->vm_pgoff << PAGE_SHIFT;
450 	u64 unmapped_db = (u64) dev->nic_info.unmapped_db;
451 	unsigned long len = (vma->vm_end - vma->vm_start);
452 	int status = 0;
453 	bool found;
454 
455 	if (vma->vm_start & (PAGE_SIZE - 1))
456 		return -EINVAL;
457 	found = ocrdma_search_mmap(ucontext, vma->vm_pgoff << PAGE_SHIFT, len);
458 	if (!found)
459 		return -EINVAL;
460 
461 	if ((vm_page >= unmapped_db) && (vm_page <= (unmapped_db +
462 		dev->nic_info.db_total_size)) &&
463 		(len <=	dev->nic_info.db_page_size)) {
464 		if (vma->vm_flags & VM_READ)
465 			return -EPERM;
466 
467 		vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
468 		status = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
469 					    len, vma->vm_page_prot);
470 	} else if (dev->nic_info.dpp_unmapped_len &&
471 		(vm_page >= (u64) dev->nic_info.dpp_unmapped_addr) &&
472 		(vm_page <= (u64) (dev->nic_info.dpp_unmapped_addr +
473 			dev->nic_info.dpp_unmapped_len)) &&
474 		(len <= dev->nic_info.dpp_unmapped_len)) {
475 		if (vma->vm_flags & VM_READ)
476 			return -EPERM;
477 
478 		vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
479 		status = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
480 					    len, vma->vm_page_prot);
481 	} else {
482 		status = remap_pfn_range(vma, vma->vm_start,
483 					 vma->vm_pgoff, len, vma->vm_page_prot);
484 	}
485 	return status;
486 }
487 
488 static int ocrdma_copy_pd_uresp(struct ocrdma_dev *dev, struct ocrdma_pd *pd,
489 				struct ib_ucontext *ib_ctx,
490 				struct ib_udata *udata)
491 {
492 	int status;
493 	u64 db_page_addr;
494 	u64 dpp_page_addr = 0;
495 	u32 db_page_size;
496 	struct ocrdma_alloc_pd_uresp rsp;
497 	struct ocrdma_ucontext *uctx = get_ocrdma_ucontext(ib_ctx);
498 
499 	memset(&rsp, 0, sizeof(rsp));
500 	rsp.id = pd->id;
501 	rsp.dpp_enabled = pd->dpp_enabled;
502 	db_page_addr = ocrdma_get_db_addr(dev, pd->id);
503 	db_page_size = dev->nic_info.db_page_size;
504 
505 	status = ocrdma_add_mmap(uctx, db_page_addr, db_page_size);
506 	if (status)
507 		return status;
508 
509 	if (pd->dpp_enabled) {
510 		dpp_page_addr = dev->nic_info.dpp_unmapped_addr +
511 				(pd->id * PAGE_SIZE);
512 		status = ocrdma_add_mmap(uctx, dpp_page_addr,
513 				 PAGE_SIZE);
514 		if (status)
515 			goto dpp_map_err;
516 		rsp.dpp_page_addr_hi = upper_32_bits(dpp_page_addr);
517 		rsp.dpp_page_addr_lo = dpp_page_addr;
518 	}
519 
520 	status = ib_copy_to_udata(udata, &rsp, sizeof(rsp));
521 	if (status)
522 		goto ucopy_err;
523 
524 	pd->uctx = uctx;
525 	return 0;
526 
527 ucopy_err:
528 	if (pd->dpp_enabled)
529 		ocrdma_del_mmap(pd->uctx, dpp_page_addr, PAGE_SIZE);
530 dpp_map_err:
531 	ocrdma_del_mmap(pd->uctx, db_page_addr, db_page_size);
532 	return status;
533 }
534 
535 struct ib_pd *ocrdma_alloc_pd(struct ib_device *ibdev,
536 			      struct ib_ucontext *context,
537 			      struct ib_udata *udata)
538 {
539 	struct ocrdma_dev *dev = get_ocrdma_dev(ibdev);
540 	struct ocrdma_pd *pd;
541 	struct ocrdma_ucontext *uctx = NULL;
542 	int status;
543 	u8 is_uctx_pd = false;
544 
545 	if (udata && context) {
546 		uctx = get_ocrdma_ucontext(context);
547 		pd = ocrdma_get_ucontext_pd(uctx);
548 		if (pd) {
549 			is_uctx_pd = true;
550 			goto pd_mapping;
551 		}
552 	}
553 
554 	pd = _ocrdma_alloc_pd(dev, uctx, udata);
555 	if (IS_ERR(pd)) {
556 		status = PTR_ERR(pd);
557 		goto exit;
558 	}
559 
560 pd_mapping:
561 	if (udata && context) {
562 		status = ocrdma_copy_pd_uresp(dev, pd, context, udata);
563 		if (status)
564 			goto err;
565 	}
566 	return &pd->ibpd;
567 
568 err:
569 	if (is_uctx_pd) {
570 		ocrdma_release_ucontext_pd(uctx);
571 	} else {
572 		status = ocrdma_mbx_dealloc_pd(dev, pd);
573 		kfree(pd);
574 	}
575 exit:
576 	return ERR_PTR(status);
577 }
578 
579 int ocrdma_dealloc_pd(struct ib_pd *ibpd)
580 {
581 	struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);
582 	struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device);
583 	struct ocrdma_ucontext *uctx = NULL;
584 	int status = 0;
585 	u64 usr_db;
586 
587 	uctx = pd->uctx;
588 	if (uctx) {
589 		u64 dpp_db = dev->nic_info.dpp_unmapped_addr +
590 			(pd->id * PAGE_SIZE);
591 		if (pd->dpp_enabled)
592 			ocrdma_del_mmap(pd->uctx, dpp_db, PAGE_SIZE);
593 		usr_db = ocrdma_get_db_addr(dev, pd->id);
594 		ocrdma_del_mmap(pd->uctx, usr_db, dev->nic_info.db_page_size);
595 
596 		if (is_ucontext_pd(uctx, pd)) {
597 			ocrdma_release_ucontext_pd(uctx);
598 			return status;
599 		}
600 	}
601 	status = _ocrdma_dealloc_pd(dev, pd);
602 	return status;
603 }
604 
605 static int ocrdma_alloc_lkey(struct ocrdma_dev *dev, struct ocrdma_mr *mr,
606 			    u32 pdid, int acc, u32 num_pbls, u32 addr_check)
607 {
608 	int status;
609 
610 	mr->hwmr.fr_mr = 0;
611 	mr->hwmr.local_rd = 1;
612 	mr->hwmr.remote_rd = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
613 	mr->hwmr.remote_wr = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
614 	mr->hwmr.local_wr = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
615 	mr->hwmr.mw_bind = (acc & IB_ACCESS_MW_BIND) ? 1 : 0;
616 	mr->hwmr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
617 	mr->hwmr.num_pbls = num_pbls;
618 
619 	status = ocrdma_mbx_alloc_lkey(dev, &mr->hwmr, pdid, addr_check);
620 	if (status)
621 		return status;
622 
623 	mr->ibmr.lkey = mr->hwmr.lkey;
624 	if (mr->hwmr.remote_wr || mr->hwmr.remote_rd)
625 		mr->ibmr.rkey = mr->hwmr.lkey;
626 	return 0;
627 }
628 
629 struct ib_mr *ocrdma_get_dma_mr(struct ib_pd *ibpd, int acc)
630 {
631 	int status;
632 	struct ocrdma_mr *mr;
633 	struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);
634 	struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device);
635 
636 	if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) {
637 		pr_err("%s err, invalid access rights\n", __func__);
638 		return ERR_PTR(-EINVAL);
639 	}
640 
641 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
642 	if (!mr)
643 		return ERR_PTR(-ENOMEM);
644 
645 	status = ocrdma_alloc_lkey(dev, mr, pd->id, acc, 0,
646 				   OCRDMA_ADDR_CHECK_DISABLE);
647 	if (status) {
648 		kfree(mr);
649 		return ERR_PTR(status);
650 	}
651 
652 	return &mr->ibmr;
653 }
654 
655 static void ocrdma_free_mr_pbl_tbl(struct ocrdma_dev *dev,
656 				   struct ocrdma_hw_mr *mr)
657 {
658 	struct pci_dev *pdev = dev->nic_info.pdev;
659 	int i = 0;
660 
661 	if (mr->pbl_table) {
662 		for (i = 0; i < mr->num_pbls; i++) {
663 			if (!mr->pbl_table[i].va)
664 				continue;
665 			dma_free_coherent(&pdev->dev, mr->pbl_size,
666 					  mr->pbl_table[i].va,
667 					  mr->pbl_table[i].pa);
668 		}
669 		kfree(mr->pbl_table);
670 		mr->pbl_table = NULL;
671 	}
672 }
673 
674 static int ocrdma_get_pbl_info(struct ocrdma_dev *dev, struct ocrdma_mr *mr,
675 			      u32 num_pbes)
676 {
677 	u32 num_pbls = 0;
678 	u32 idx = 0;
679 	int status = 0;
680 	u32 pbl_size;
681 
682 	do {
683 		pbl_size = OCRDMA_MIN_HPAGE_SIZE * (1 << idx);
684 		if (pbl_size > MAX_OCRDMA_PBL_SIZE) {
685 			status = -EFAULT;
686 			break;
687 		}
688 		num_pbls = roundup(num_pbes, (pbl_size / sizeof(u64)));
689 		num_pbls = num_pbls / (pbl_size / sizeof(u64));
690 		idx++;
691 	} while (num_pbls >= dev->attr.max_num_mr_pbl);
692 
693 	mr->hwmr.num_pbes = num_pbes;
694 	mr->hwmr.num_pbls = num_pbls;
695 	mr->hwmr.pbl_size = pbl_size;
696 	return status;
697 }
698 
699 static int ocrdma_build_pbl_tbl(struct ocrdma_dev *dev, struct ocrdma_hw_mr *mr)
700 {
701 	int status = 0;
702 	int i;
703 	u32 dma_len = mr->pbl_size;
704 	struct pci_dev *pdev = dev->nic_info.pdev;
705 	void *va;
706 	dma_addr_t pa;
707 
708 	mr->pbl_table = kzalloc(sizeof(struct ocrdma_pbl) *
709 				mr->num_pbls, GFP_KERNEL);
710 
711 	if (!mr->pbl_table)
712 		return -ENOMEM;
713 
714 	for (i = 0; i < mr->num_pbls; i++) {
715 		va = dma_alloc_coherent(&pdev->dev, dma_len, &pa, GFP_KERNEL);
716 		if (!va) {
717 			ocrdma_free_mr_pbl_tbl(dev, mr);
718 			status = -ENOMEM;
719 			break;
720 		}
721 		memset(va, 0, dma_len);
722 		mr->pbl_table[i].va = va;
723 		mr->pbl_table[i].pa = pa;
724 	}
725 	return status;
726 }
727 
728 static void build_user_pbes(struct ocrdma_dev *dev, struct ocrdma_mr *mr,
729 			    u32 num_pbes)
730 {
731 	struct ocrdma_pbe *pbe;
732 	struct scatterlist *sg;
733 	struct ocrdma_pbl *pbl_tbl = mr->hwmr.pbl_table;
734 	struct ib_umem *umem = mr->umem;
735 	int shift, pg_cnt, pages, pbe_cnt, entry, total_num_pbes = 0;
736 
737 	if (!mr->hwmr.num_pbes)
738 		return;
739 
740 	pbe = (struct ocrdma_pbe *)pbl_tbl->va;
741 	pbe_cnt = 0;
742 
743 	shift = ilog2(umem->page_size);
744 
745 	for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
746 		pages = sg_dma_len(sg) >> shift;
747 		for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) {
748 			/* store the page address in pbe */
749 			pbe->pa_lo =
750 			    cpu_to_le32(sg_dma_address
751 					(sg) +
752 					(umem->page_size * pg_cnt));
753 			pbe->pa_hi =
754 			    cpu_to_le32(upper_32_bits
755 					((sg_dma_address
756 					  (sg) +
757 					  umem->page_size * pg_cnt)));
758 			pbe_cnt += 1;
759 			total_num_pbes += 1;
760 			pbe++;
761 
762 			/* if done building pbes, issue the mbx cmd. */
763 			if (total_num_pbes == num_pbes)
764 				return;
765 
766 			/* if the given pbl is full storing the pbes,
767 			 * move to next pbl.
768 			 */
769 			if (pbe_cnt ==
770 				(mr->hwmr.pbl_size / sizeof(u64))) {
771 				pbl_tbl++;
772 				pbe = (struct ocrdma_pbe *)pbl_tbl->va;
773 				pbe_cnt = 0;
774 			}
775 
776 		}
777 	}
778 }
779 
780 struct ib_mr *ocrdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
781 				 u64 usr_addr, int acc, struct ib_udata *udata)
782 {
783 	int status = -ENOMEM;
784 	struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device);
785 	struct ocrdma_mr *mr;
786 	struct ocrdma_pd *pd;
787 	u32 num_pbes;
788 
789 	pd = get_ocrdma_pd(ibpd);
790 
791 	if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
792 		return ERR_PTR(-EINVAL);
793 
794 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
795 	if (!mr)
796 		return ERR_PTR(status);
797 	mr->umem = ib_umem_get(ibpd->uobject->context, start, len, acc, 0);
798 	if (IS_ERR(mr->umem)) {
799 		status = -EFAULT;
800 		goto umem_err;
801 	}
802 	num_pbes = ib_umem_page_count(mr->umem);
803 	status = ocrdma_get_pbl_info(dev, mr, num_pbes);
804 	if (status)
805 		goto umem_err;
806 
807 	mr->hwmr.pbe_size = mr->umem->page_size;
808 	mr->hwmr.fbo = ib_umem_offset(mr->umem);
809 	mr->hwmr.va = usr_addr;
810 	mr->hwmr.len = len;
811 	mr->hwmr.remote_wr = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
812 	mr->hwmr.remote_rd = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
813 	mr->hwmr.local_wr = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
814 	mr->hwmr.local_rd = 1;
815 	mr->hwmr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
816 	status = ocrdma_build_pbl_tbl(dev, &mr->hwmr);
817 	if (status)
818 		goto umem_err;
819 	build_user_pbes(dev, mr, num_pbes);
820 	status = ocrdma_reg_mr(dev, &mr->hwmr, pd->id, acc);
821 	if (status)
822 		goto mbx_err;
823 	mr->ibmr.lkey = mr->hwmr.lkey;
824 	if (mr->hwmr.remote_wr || mr->hwmr.remote_rd)
825 		mr->ibmr.rkey = mr->hwmr.lkey;
826 
827 	return &mr->ibmr;
828 
829 mbx_err:
830 	ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr);
831 umem_err:
832 	kfree(mr);
833 	return ERR_PTR(status);
834 }
835 
836 int ocrdma_dereg_mr(struct ib_mr *ib_mr)
837 {
838 	struct ocrdma_mr *mr = get_ocrdma_mr(ib_mr);
839 	struct ocrdma_dev *dev = get_ocrdma_dev(ib_mr->device);
840 	int status;
841 
842 	status = ocrdma_mbx_dealloc_lkey(dev, mr->hwmr.fr_mr, mr->hwmr.lkey);
843 
844 	ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr);
845 
846 	/* it could be user registered memory. */
847 	if (mr->umem)
848 		ib_umem_release(mr->umem);
849 	kfree(mr);
850 
851 	/* Don't stop cleanup, in case FW is unresponsive */
852 	if (dev->mqe_ctx.fw_error_state) {
853 		status = 0;
854 		pr_err("%s(%d) fw not responding.\n",
855 		       __func__, dev->id);
856 	}
857 	return status;
858 }
859 
860 static int ocrdma_copy_cq_uresp(struct ocrdma_dev *dev, struct ocrdma_cq *cq,
861 				struct ib_udata *udata,
862 				struct ib_ucontext *ib_ctx)
863 {
864 	int status;
865 	struct ocrdma_ucontext *uctx = get_ocrdma_ucontext(ib_ctx);
866 	struct ocrdma_create_cq_uresp uresp;
867 
868 	memset(&uresp, 0, sizeof(uresp));
869 	uresp.cq_id = cq->id;
870 	uresp.page_size = PAGE_ALIGN(cq->len);
871 	uresp.num_pages = 1;
872 	uresp.max_hw_cqe = cq->max_hw_cqe;
873 	uresp.page_addr[0] = virt_to_phys(cq->va);
874 	uresp.db_page_addr =  ocrdma_get_db_addr(dev, uctx->cntxt_pd->id);
875 	uresp.db_page_size = dev->nic_info.db_page_size;
876 	uresp.phase_change = cq->phase_change ? 1 : 0;
877 	status = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
878 	if (status) {
879 		pr_err("%s(%d) copy error cqid=0x%x.\n",
880 		       __func__, dev->id, cq->id);
881 		goto err;
882 	}
883 	status = ocrdma_add_mmap(uctx, uresp.db_page_addr, uresp.db_page_size);
884 	if (status)
885 		goto err;
886 	status = ocrdma_add_mmap(uctx, uresp.page_addr[0], uresp.page_size);
887 	if (status) {
888 		ocrdma_del_mmap(uctx, uresp.db_page_addr, uresp.db_page_size);
889 		goto err;
890 	}
891 	cq->ucontext = uctx;
892 err:
893 	return status;
894 }
895 
896 struct ib_cq *ocrdma_create_cq(struct ib_device *ibdev, int entries, int vector,
897 			       struct ib_ucontext *ib_ctx,
898 			       struct ib_udata *udata)
899 {
900 	struct ocrdma_cq *cq;
901 	struct ocrdma_dev *dev = get_ocrdma_dev(ibdev);
902 	struct ocrdma_ucontext *uctx = NULL;
903 	u16 pd_id = 0;
904 	int status;
905 	struct ocrdma_create_cq_ureq ureq;
906 
907 	if (udata) {
908 		if (ib_copy_from_udata(&ureq, udata, sizeof(ureq)))
909 			return ERR_PTR(-EFAULT);
910 	} else
911 		ureq.dpp_cq = 0;
912 	cq = kzalloc(sizeof(*cq), GFP_KERNEL);
913 	if (!cq)
914 		return ERR_PTR(-ENOMEM);
915 
916 	spin_lock_init(&cq->cq_lock);
917 	spin_lock_init(&cq->comp_handler_lock);
918 	INIT_LIST_HEAD(&cq->sq_head);
919 	INIT_LIST_HEAD(&cq->rq_head);
920 	cq->first_arm = true;
921 
922 	if (ib_ctx) {
923 		uctx = get_ocrdma_ucontext(ib_ctx);
924 		pd_id = uctx->cntxt_pd->id;
925 	}
926 
927 	status = ocrdma_mbx_create_cq(dev, cq, entries, ureq.dpp_cq, pd_id);
928 	if (status) {
929 		kfree(cq);
930 		return ERR_PTR(status);
931 	}
932 	if (ib_ctx) {
933 		status = ocrdma_copy_cq_uresp(dev, cq, udata, ib_ctx);
934 		if (status)
935 			goto ctx_err;
936 	}
937 	cq->phase = OCRDMA_CQE_VALID;
938 	dev->cq_tbl[cq->id] = cq;
939 	return &cq->ibcq;
940 
941 ctx_err:
942 	ocrdma_mbx_destroy_cq(dev, cq);
943 	kfree(cq);
944 	return ERR_PTR(status);
945 }
946 
947 int ocrdma_resize_cq(struct ib_cq *ibcq, int new_cnt,
948 		     struct ib_udata *udata)
949 {
950 	int status = 0;
951 	struct ocrdma_cq *cq = get_ocrdma_cq(ibcq);
952 
953 	if (new_cnt < 1 || new_cnt > cq->max_hw_cqe) {
954 		status = -EINVAL;
955 		return status;
956 	}
957 	ibcq->cqe = new_cnt;
958 	return status;
959 }
960 
961 static void ocrdma_flush_cq(struct ocrdma_cq *cq)
962 {
963 	int cqe_cnt;
964 	int valid_count = 0;
965 	unsigned long flags;
966 
967 	struct ocrdma_dev *dev = get_ocrdma_dev(cq->ibcq.device);
968 	struct ocrdma_cqe *cqe = NULL;
969 
970 	cqe = cq->va;
971 	cqe_cnt = cq->cqe_cnt;
972 
973 	/* Last irq might have scheduled a polling thread
974 	 * sync-up with it before hard flushing.
975 	 */
976 	spin_lock_irqsave(&cq->cq_lock, flags);
977 	while (cqe_cnt) {
978 		if (is_cqe_valid(cq, cqe))
979 			valid_count++;
980 		cqe++;
981 		cqe_cnt--;
982 	}
983 	ocrdma_ring_cq_db(dev, cq->id, false, false, valid_count);
984 	spin_unlock_irqrestore(&cq->cq_lock, flags);
985 }
986 
987 int ocrdma_destroy_cq(struct ib_cq *ibcq)
988 {
989 	int status;
990 	struct ocrdma_cq *cq = get_ocrdma_cq(ibcq);
991 	struct ocrdma_eq *eq = NULL;
992 	struct ocrdma_dev *dev = get_ocrdma_dev(ibcq->device);
993 	int pdid = 0;
994 	u32 irq, indx;
995 
996 	dev->cq_tbl[cq->id] = NULL;
997 	indx = ocrdma_get_eq_table_index(dev, cq->eqn);
998 	if (indx == -EINVAL)
999 		BUG();
1000 
1001 	eq = &dev->eq_tbl[indx];
1002 	irq = ocrdma_get_irq(dev, eq);
1003 	synchronize_irq(irq);
1004 	ocrdma_flush_cq(cq);
1005 
1006 	status = ocrdma_mbx_destroy_cq(dev, cq);
1007 	if (cq->ucontext) {
1008 		pdid = cq->ucontext->cntxt_pd->id;
1009 		ocrdma_del_mmap(cq->ucontext, (u64) cq->pa,
1010 				PAGE_ALIGN(cq->len));
1011 		ocrdma_del_mmap(cq->ucontext,
1012 				ocrdma_get_db_addr(dev, pdid),
1013 				dev->nic_info.db_page_size);
1014 	}
1015 
1016 	kfree(cq);
1017 	return status;
1018 }
1019 
1020 static int ocrdma_add_qpn_map(struct ocrdma_dev *dev, struct ocrdma_qp *qp)
1021 {
1022 	int status = -EINVAL;
1023 
1024 	if (qp->id < OCRDMA_MAX_QP && dev->qp_tbl[qp->id] == NULL) {
1025 		dev->qp_tbl[qp->id] = qp;
1026 		status = 0;
1027 	}
1028 	return status;
1029 }
1030 
1031 static void ocrdma_del_qpn_map(struct ocrdma_dev *dev, struct ocrdma_qp *qp)
1032 {
1033 	dev->qp_tbl[qp->id] = NULL;
1034 }
1035 
1036 static int ocrdma_check_qp_params(struct ib_pd *ibpd, struct ocrdma_dev *dev,
1037 				  struct ib_qp_init_attr *attrs)
1038 {
1039 	if ((attrs->qp_type != IB_QPT_GSI) &&
1040 	    (attrs->qp_type != IB_QPT_RC) &&
1041 	    (attrs->qp_type != IB_QPT_UC) &&
1042 	    (attrs->qp_type != IB_QPT_UD)) {
1043 		pr_err("%s(%d) unsupported qp type=0x%x requested\n",
1044 		       __func__, dev->id, attrs->qp_type);
1045 		return -EINVAL;
1046 	}
1047 	/* Skip the check for QP1 to support CM size of 128 */
1048 	if ((attrs->qp_type != IB_QPT_GSI) &&
1049 	    (attrs->cap.max_send_wr > dev->attr.max_wqe)) {
1050 		pr_err("%s(%d) unsupported send_wr=0x%x requested\n",
1051 		       __func__, dev->id, attrs->cap.max_send_wr);
1052 		pr_err("%s(%d) supported send_wr=0x%x\n",
1053 		       __func__, dev->id, dev->attr.max_wqe);
1054 		return -EINVAL;
1055 	}
1056 	if (!attrs->srq && (attrs->cap.max_recv_wr > dev->attr.max_rqe)) {
1057 		pr_err("%s(%d) unsupported recv_wr=0x%x requested\n",
1058 		       __func__, dev->id, attrs->cap.max_recv_wr);
1059 		pr_err("%s(%d) supported recv_wr=0x%x\n",
1060 		       __func__, dev->id, dev->attr.max_rqe);
1061 		return -EINVAL;
1062 	}
1063 	if (attrs->cap.max_inline_data > dev->attr.max_inline_data) {
1064 		pr_err("%s(%d) unsupported inline data size=0x%x requested\n",
1065 		       __func__, dev->id, attrs->cap.max_inline_data);
1066 		pr_err("%s(%d) supported inline data size=0x%x\n",
1067 		       __func__, dev->id, dev->attr.max_inline_data);
1068 		return -EINVAL;
1069 	}
1070 	if (attrs->cap.max_send_sge > dev->attr.max_send_sge) {
1071 		pr_err("%s(%d) unsupported send_sge=0x%x requested\n",
1072 		       __func__, dev->id, attrs->cap.max_send_sge);
1073 		pr_err("%s(%d) supported send_sge=0x%x\n",
1074 		       __func__, dev->id, dev->attr.max_send_sge);
1075 		return -EINVAL;
1076 	}
1077 	if (attrs->cap.max_recv_sge > dev->attr.max_recv_sge) {
1078 		pr_err("%s(%d) unsupported recv_sge=0x%x requested\n",
1079 		       __func__, dev->id, attrs->cap.max_recv_sge);
1080 		pr_err("%s(%d) supported recv_sge=0x%x\n",
1081 		       __func__, dev->id, dev->attr.max_recv_sge);
1082 		return -EINVAL;
1083 	}
1084 	/* unprivileged user space cannot create special QP */
1085 	if (ibpd->uobject && attrs->qp_type == IB_QPT_GSI) {
1086 		pr_err
1087 		    ("%s(%d) Userspace can't create special QPs of type=0x%x\n",
1088 		     __func__, dev->id, attrs->qp_type);
1089 		return -EINVAL;
1090 	}
1091 	/* allow creating only one GSI type of QP */
1092 	if (attrs->qp_type == IB_QPT_GSI && dev->gsi_qp_created) {
1093 		pr_err("%s(%d) GSI special QPs already created.\n",
1094 		       __func__, dev->id);
1095 		return -EINVAL;
1096 	}
1097 	/* verify consumer QPs are not trying to use GSI QP's CQ */
1098 	if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created)) {
1099 		if ((dev->gsi_sqcq == get_ocrdma_cq(attrs->send_cq)) ||
1100 			(dev->gsi_rqcq == get_ocrdma_cq(attrs->recv_cq))) {
1101 			pr_err("%s(%d) Consumer QP cannot use GSI CQs.\n",
1102 				__func__, dev->id);
1103 			return -EINVAL;
1104 		}
1105 	}
1106 	return 0;
1107 }
1108 
1109 static int ocrdma_copy_qp_uresp(struct ocrdma_qp *qp,
1110 				struct ib_udata *udata, int dpp_offset,
1111 				int dpp_credit_lmt, int srq)
1112 {
1113 	int status = 0;
1114 	u64 usr_db;
1115 	struct ocrdma_create_qp_uresp uresp;
1116 	struct ocrdma_dev *dev = qp->dev;
1117 	struct ocrdma_pd *pd = qp->pd;
1118 
1119 	memset(&uresp, 0, sizeof(uresp));
1120 	usr_db = dev->nic_info.unmapped_db +
1121 			(pd->id * dev->nic_info.db_page_size);
1122 	uresp.qp_id = qp->id;
1123 	uresp.sq_dbid = qp->sq.dbid;
1124 	uresp.num_sq_pages = 1;
1125 	uresp.sq_page_size = PAGE_ALIGN(qp->sq.len);
1126 	uresp.sq_page_addr[0] = virt_to_phys(qp->sq.va);
1127 	uresp.num_wqe_allocated = qp->sq.max_cnt;
1128 	if (!srq) {
1129 		uresp.rq_dbid = qp->rq.dbid;
1130 		uresp.num_rq_pages = 1;
1131 		uresp.rq_page_size = PAGE_ALIGN(qp->rq.len);
1132 		uresp.rq_page_addr[0] = virt_to_phys(qp->rq.va);
1133 		uresp.num_rqe_allocated = qp->rq.max_cnt;
1134 	}
1135 	uresp.db_page_addr = usr_db;
1136 	uresp.db_page_size = dev->nic_info.db_page_size;
1137 	uresp.db_sq_offset = OCRDMA_DB_GEN2_SQ_OFFSET;
1138 	uresp.db_rq_offset = OCRDMA_DB_GEN2_RQ_OFFSET;
1139 	uresp.db_shift = OCRDMA_DB_RQ_SHIFT;
1140 
1141 	if (qp->dpp_enabled) {
1142 		uresp.dpp_credit = dpp_credit_lmt;
1143 		uresp.dpp_offset = dpp_offset;
1144 	}
1145 	status = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1146 	if (status) {
1147 		pr_err("%s(%d) user copy error.\n", __func__, dev->id);
1148 		goto err;
1149 	}
1150 	status = ocrdma_add_mmap(pd->uctx, uresp.sq_page_addr[0],
1151 				 uresp.sq_page_size);
1152 	if (status)
1153 		goto err;
1154 
1155 	if (!srq) {
1156 		status = ocrdma_add_mmap(pd->uctx, uresp.rq_page_addr[0],
1157 					 uresp.rq_page_size);
1158 		if (status)
1159 			goto rq_map_err;
1160 	}
1161 	return status;
1162 rq_map_err:
1163 	ocrdma_del_mmap(pd->uctx, uresp.sq_page_addr[0], uresp.sq_page_size);
1164 err:
1165 	return status;
1166 }
1167 
1168 static void ocrdma_set_qp_db(struct ocrdma_dev *dev, struct ocrdma_qp *qp,
1169 			     struct ocrdma_pd *pd)
1170 {
1171 	if (ocrdma_get_asic_type(dev) == OCRDMA_ASIC_GEN_SKH_R) {
1172 		qp->sq_db = dev->nic_info.db +
1173 			(pd->id * dev->nic_info.db_page_size) +
1174 			OCRDMA_DB_GEN2_SQ_OFFSET;
1175 		qp->rq_db = dev->nic_info.db +
1176 			(pd->id * dev->nic_info.db_page_size) +
1177 			OCRDMA_DB_GEN2_RQ_OFFSET;
1178 	} else {
1179 		qp->sq_db = dev->nic_info.db +
1180 			(pd->id * dev->nic_info.db_page_size) +
1181 			OCRDMA_DB_SQ_OFFSET;
1182 		qp->rq_db = dev->nic_info.db +
1183 			(pd->id * dev->nic_info.db_page_size) +
1184 			OCRDMA_DB_RQ_OFFSET;
1185 	}
1186 }
1187 
1188 static int ocrdma_alloc_wr_id_tbl(struct ocrdma_qp *qp)
1189 {
1190 	qp->wqe_wr_id_tbl =
1191 	    kzalloc(sizeof(*(qp->wqe_wr_id_tbl)) * qp->sq.max_cnt,
1192 		    GFP_KERNEL);
1193 	if (qp->wqe_wr_id_tbl == NULL)
1194 		return -ENOMEM;
1195 	qp->rqe_wr_id_tbl =
1196 	    kzalloc(sizeof(u64) * qp->rq.max_cnt, GFP_KERNEL);
1197 	if (qp->rqe_wr_id_tbl == NULL)
1198 		return -ENOMEM;
1199 
1200 	return 0;
1201 }
1202 
1203 static void ocrdma_set_qp_init_params(struct ocrdma_qp *qp,
1204 				      struct ocrdma_pd *pd,
1205 				      struct ib_qp_init_attr *attrs)
1206 {
1207 	qp->pd = pd;
1208 	spin_lock_init(&qp->q_lock);
1209 	INIT_LIST_HEAD(&qp->sq_entry);
1210 	INIT_LIST_HEAD(&qp->rq_entry);
1211 
1212 	qp->qp_type = attrs->qp_type;
1213 	qp->cap_flags = OCRDMA_QP_INB_RD | OCRDMA_QP_INB_WR;
1214 	qp->max_inline_data = attrs->cap.max_inline_data;
1215 	qp->sq.max_sges = attrs->cap.max_send_sge;
1216 	qp->rq.max_sges = attrs->cap.max_recv_sge;
1217 	qp->state = OCRDMA_QPS_RST;
1218 	qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
1219 }
1220 
1221 static void ocrdma_store_gsi_qp_cq(struct ocrdma_dev *dev,
1222 				   struct ib_qp_init_attr *attrs)
1223 {
1224 	if (attrs->qp_type == IB_QPT_GSI) {
1225 		dev->gsi_qp_created = 1;
1226 		dev->gsi_sqcq = get_ocrdma_cq(attrs->send_cq);
1227 		dev->gsi_rqcq = get_ocrdma_cq(attrs->recv_cq);
1228 	}
1229 }
1230 
1231 struct ib_qp *ocrdma_create_qp(struct ib_pd *ibpd,
1232 			       struct ib_qp_init_attr *attrs,
1233 			       struct ib_udata *udata)
1234 {
1235 	int status;
1236 	struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);
1237 	struct ocrdma_qp *qp;
1238 	struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device);
1239 	struct ocrdma_create_qp_ureq ureq;
1240 	u16 dpp_credit_lmt, dpp_offset;
1241 
1242 	status = ocrdma_check_qp_params(ibpd, dev, attrs);
1243 	if (status)
1244 		goto gen_err;
1245 
1246 	memset(&ureq, 0, sizeof(ureq));
1247 	if (udata) {
1248 		if (ib_copy_from_udata(&ureq, udata, sizeof(ureq)))
1249 			return ERR_PTR(-EFAULT);
1250 	}
1251 	qp = kzalloc(sizeof(*qp), GFP_KERNEL);
1252 	if (!qp) {
1253 		status = -ENOMEM;
1254 		goto gen_err;
1255 	}
1256 	qp->dev = dev;
1257 	ocrdma_set_qp_init_params(qp, pd, attrs);
1258 	if (udata == NULL)
1259 		qp->cap_flags |= (OCRDMA_QP_MW_BIND | OCRDMA_QP_LKEY0 |
1260 					OCRDMA_QP_FAST_REG);
1261 
1262 	mutex_lock(&dev->dev_lock);
1263 	status = ocrdma_mbx_create_qp(qp, attrs, ureq.enable_dpp_cq,
1264 					ureq.dpp_cq_id,
1265 					&dpp_offset, &dpp_credit_lmt);
1266 	if (status)
1267 		goto mbx_err;
1268 
1269 	/* user space QP's wr_id table are managed in library */
1270 	if (udata == NULL) {
1271 		status = ocrdma_alloc_wr_id_tbl(qp);
1272 		if (status)
1273 			goto map_err;
1274 	}
1275 
1276 	status = ocrdma_add_qpn_map(dev, qp);
1277 	if (status)
1278 		goto map_err;
1279 	ocrdma_set_qp_db(dev, qp, pd);
1280 	if (udata) {
1281 		status = ocrdma_copy_qp_uresp(qp, udata, dpp_offset,
1282 					      dpp_credit_lmt,
1283 					      (attrs->srq != NULL));
1284 		if (status)
1285 			goto cpy_err;
1286 	}
1287 	ocrdma_store_gsi_qp_cq(dev, attrs);
1288 	qp->ibqp.qp_num = qp->id;
1289 	mutex_unlock(&dev->dev_lock);
1290 	return &qp->ibqp;
1291 
1292 cpy_err:
1293 	ocrdma_del_qpn_map(dev, qp);
1294 map_err:
1295 	ocrdma_mbx_destroy_qp(dev, qp);
1296 mbx_err:
1297 	mutex_unlock(&dev->dev_lock);
1298 	kfree(qp->wqe_wr_id_tbl);
1299 	kfree(qp->rqe_wr_id_tbl);
1300 	kfree(qp);
1301 	pr_err("%s(%d) error=%d\n", __func__, dev->id, status);
1302 gen_err:
1303 	return ERR_PTR(status);
1304 }
1305 
1306 int _ocrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1307 		      int attr_mask)
1308 {
1309 	int status = 0;
1310 	struct ocrdma_qp *qp;
1311 	struct ocrdma_dev *dev;
1312 	enum ib_qp_state old_qps;
1313 
1314 	qp = get_ocrdma_qp(ibqp);
1315 	dev = qp->dev;
1316 	if (attr_mask & IB_QP_STATE)
1317 		status = ocrdma_qp_state_change(qp, attr->qp_state, &old_qps);
1318 	/* if new and previous states are same hw doesn't need to
1319 	 * know about it.
1320 	 */
1321 	if (status < 0)
1322 		return status;
1323 	status = ocrdma_mbx_modify_qp(dev, qp, attr, attr_mask);
1324 
1325 	return status;
1326 }
1327 
1328 int ocrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1329 		     int attr_mask, struct ib_udata *udata)
1330 {
1331 	unsigned long flags;
1332 	int status = -EINVAL;
1333 	struct ocrdma_qp *qp;
1334 	struct ocrdma_dev *dev;
1335 	enum ib_qp_state old_qps, new_qps;
1336 
1337 	qp = get_ocrdma_qp(ibqp);
1338 	dev = qp->dev;
1339 
1340 	/* syncronize with multiple context trying to change, retrive qps */
1341 	mutex_lock(&dev->dev_lock);
1342 	/* syncronize with wqe, rqe posting and cqe processing contexts */
1343 	spin_lock_irqsave(&qp->q_lock, flags);
1344 	old_qps = get_ibqp_state(qp->state);
1345 	if (attr_mask & IB_QP_STATE)
1346 		new_qps = attr->qp_state;
1347 	else
1348 		new_qps = old_qps;
1349 	spin_unlock_irqrestore(&qp->q_lock, flags);
1350 
1351 	if (!ib_modify_qp_is_ok(old_qps, new_qps, ibqp->qp_type, attr_mask,
1352 				IB_LINK_LAYER_ETHERNET)) {
1353 		pr_err("%s(%d) invalid attribute mask=0x%x specified for\n"
1354 		       "qpn=0x%x of type=0x%x old_qps=0x%x, new_qps=0x%x\n",
1355 		       __func__, dev->id, attr_mask, qp->id, ibqp->qp_type,
1356 		       old_qps, new_qps);
1357 		goto param_err;
1358 	}
1359 
1360 	status = _ocrdma_modify_qp(ibqp, attr, attr_mask);
1361 	if (status > 0)
1362 		status = 0;
1363 param_err:
1364 	mutex_unlock(&dev->dev_lock);
1365 	return status;
1366 }
1367 
1368 static enum ib_mtu ocrdma_mtu_int_to_enum(u16 mtu)
1369 {
1370 	switch (mtu) {
1371 	case 256:
1372 		return IB_MTU_256;
1373 	case 512:
1374 		return IB_MTU_512;
1375 	case 1024:
1376 		return IB_MTU_1024;
1377 	case 2048:
1378 		return IB_MTU_2048;
1379 	case 4096:
1380 		return IB_MTU_4096;
1381 	default:
1382 		return IB_MTU_1024;
1383 	}
1384 }
1385 
1386 static int ocrdma_to_ib_qp_acc_flags(int qp_cap_flags)
1387 {
1388 	int ib_qp_acc_flags = 0;
1389 
1390 	if (qp_cap_flags & OCRDMA_QP_INB_WR)
1391 		ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
1392 	if (qp_cap_flags & OCRDMA_QP_INB_RD)
1393 		ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
1394 	return ib_qp_acc_flags;
1395 }
1396 
1397 int ocrdma_query_qp(struct ib_qp *ibqp,
1398 		    struct ib_qp_attr *qp_attr,
1399 		    int attr_mask, struct ib_qp_init_attr *qp_init_attr)
1400 {
1401 	int status;
1402 	u32 qp_state;
1403 	struct ocrdma_qp_params params;
1404 	struct ocrdma_qp *qp = get_ocrdma_qp(ibqp);
1405 	struct ocrdma_dev *dev = qp->dev;
1406 
1407 	memset(&params, 0, sizeof(params));
1408 	mutex_lock(&dev->dev_lock);
1409 	status = ocrdma_mbx_query_qp(dev, qp, &params);
1410 	mutex_unlock(&dev->dev_lock);
1411 	if (status)
1412 		goto mbx_err;
1413 	if (qp->qp_type == IB_QPT_UD)
1414 		qp_attr->qkey = params.qkey;
1415 	qp_attr->qp_state = get_ibqp_state(IB_QPS_INIT);
1416 	qp_attr->cur_qp_state = get_ibqp_state(IB_QPS_INIT);
1417 	qp_attr->path_mtu =
1418 		ocrdma_mtu_int_to_enum(params.path_mtu_pkey_indx &
1419 				OCRDMA_QP_PARAMS_PATH_MTU_MASK) >>
1420 				OCRDMA_QP_PARAMS_PATH_MTU_SHIFT;
1421 	qp_attr->path_mig_state = IB_MIG_MIGRATED;
1422 	qp_attr->rq_psn = params.hop_lmt_rq_psn & OCRDMA_QP_PARAMS_RQ_PSN_MASK;
1423 	qp_attr->sq_psn = params.tclass_sq_psn & OCRDMA_QP_PARAMS_SQ_PSN_MASK;
1424 	qp_attr->dest_qp_num =
1425 	    params.ack_to_rnr_rtc_dest_qpn & OCRDMA_QP_PARAMS_DEST_QPN_MASK;
1426 
1427 	qp_attr->qp_access_flags = ocrdma_to_ib_qp_acc_flags(qp->cap_flags);
1428 	qp_attr->cap.max_send_wr = qp->sq.max_cnt - 1;
1429 	qp_attr->cap.max_recv_wr = qp->rq.max_cnt - 1;
1430 	qp_attr->cap.max_send_sge = qp->sq.max_sges;
1431 	qp_attr->cap.max_recv_sge = qp->rq.max_sges;
1432 	qp_attr->cap.max_inline_data = qp->max_inline_data;
1433 	qp_init_attr->cap = qp_attr->cap;
1434 	memcpy(&qp_attr->ah_attr.grh.dgid, &params.dgid[0],
1435 	       sizeof(params.dgid));
1436 	qp_attr->ah_attr.grh.flow_label = params.rnt_rc_sl_fl &
1437 	    OCRDMA_QP_PARAMS_FLOW_LABEL_MASK;
1438 	qp_attr->ah_attr.grh.sgid_index = qp->sgid_idx;
1439 	qp_attr->ah_attr.grh.hop_limit = (params.hop_lmt_rq_psn &
1440 					  OCRDMA_QP_PARAMS_HOP_LMT_MASK) >>
1441 						OCRDMA_QP_PARAMS_HOP_LMT_SHIFT;
1442 	qp_attr->ah_attr.grh.traffic_class = (params.tclass_sq_psn &
1443 					      OCRDMA_QP_PARAMS_TCLASS_MASK) >>
1444 						OCRDMA_QP_PARAMS_TCLASS_SHIFT;
1445 
1446 	qp_attr->ah_attr.ah_flags = IB_AH_GRH;
1447 	qp_attr->ah_attr.port_num = 1;
1448 	qp_attr->ah_attr.sl = (params.rnt_rc_sl_fl &
1449 			       OCRDMA_QP_PARAMS_SL_MASK) >>
1450 				OCRDMA_QP_PARAMS_SL_SHIFT;
1451 	qp_attr->timeout = (params.ack_to_rnr_rtc_dest_qpn &
1452 			    OCRDMA_QP_PARAMS_ACK_TIMEOUT_MASK) >>
1453 				OCRDMA_QP_PARAMS_ACK_TIMEOUT_SHIFT;
1454 	qp_attr->rnr_retry = (params.ack_to_rnr_rtc_dest_qpn &
1455 			      OCRDMA_QP_PARAMS_RNR_RETRY_CNT_MASK) >>
1456 				OCRDMA_QP_PARAMS_RNR_RETRY_CNT_SHIFT;
1457 	qp_attr->retry_cnt =
1458 	    (params.rnt_rc_sl_fl & OCRDMA_QP_PARAMS_RETRY_CNT_MASK) >>
1459 		OCRDMA_QP_PARAMS_RETRY_CNT_SHIFT;
1460 	qp_attr->min_rnr_timer = 0;
1461 	qp_attr->pkey_index = 0;
1462 	qp_attr->port_num = 1;
1463 	qp_attr->ah_attr.src_path_bits = 0;
1464 	qp_attr->ah_attr.static_rate = 0;
1465 	qp_attr->alt_pkey_index = 0;
1466 	qp_attr->alt_port_num = 0;
1467 	qp_attr->alt_timeout = 0;
1468 	memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
1469 	qp_state = (params.max_sge_recv_flags & OCRDMA_QP_PARAMS_STATE_MASK) >>
1470 		    OCRDMA_QP_PARAMS_STATE_SHIFT;
1471 	qp_attr->sq_draining = (qp_state == OCRDMA_QPS_SQ_DRAINING) ? 1 : 0;
1472 	qp_attr->max_dest_rd_atomic =
1473 	    params.max_ord_ird >> OCRDMA_QP_PARAMS_MAX_ORD_SHIFT;
1474 	qp_attr->max_rd_atomic =
1475 	    params.max_ord_ird & OCRDMA_QP_PARAMS_MAX_IRD_MASK;
1476 	qp_attr->en_sqd_async_notify = (params.max_sge_recv_flags &
1477 				OCRDMA_QP_PARAMS_FLAGS_SQD_ASYNC) ? 1 : 0;
1478 mbx_err:
1479 	return status;
1480 }
1481 
1482 static void ocrdma_srq_toggle_bit(struct ocrdma_srq *srq, int idx)
1483 {
1484 	int i = idx / 32;
1485 	unsigned int mask = (1 << (idx % 32));
1486 
1487 	if (srq->idx_bit_fields[i] & mask)
1488 		srq->idx_bit_fields[i] &= ~mask;
1489 	else
1490 		srq->idx_bit_fields[i] |= mask;
1491 }
1492 
1493 static int ocrdma_hwq_free_cnt(struct ocrdma_qp_hwq_info *q)
1494 {
1495 	return ((q->max_wqe_idx - q->head) + q->tail) % q->max_cnt;
1496 }
1497 
1498 static int is_hw_sq_empty(struct ocrdma_qp *qp)
1499 {
1500 	return (qp->sq.tail == qp->sq.head);
1501 }
1502 
1503 static int is_hw_rq_empty(struct ocrdma_qp *qp)
1504 {
1505 	return (qp->rq.tail == qp->rq.head);
1506 }
1507 
1508 static void *ocrdma_hwq_head(struct ocrdma_qp_hwq_info *q)
1509 {
1510 	return q->va + (q->head * q->entry_size);
1511 }
1512 
1513 static void *ocrdma_hwq_head_from_idx(struct ocrdma_qp_hwq_info *q,
1514 				      u32 idx)
1515 {
1516 	return q->va + (idx * q->entry_size);
1517 }
1518 
1519 static void ocrdma_hwq_inc_head(struct ocrdma_qp_hwq_info *q)
1520 {
1521 	q->head = (q->head + 1) & q->max_wqe_idx;
1522 }
1523 
1524 static void ocrdma_hwq_inc_tail(struct ocrdma_qp_hwq_info *q)
1525 {
1526 	q->tail = (q->tail + 1) & q->max_wqe_idx;
1527 }
1528 
1529 /* discard the cqe for a given QP */
1530 static void ocrdma_discard_cqes(struct ocrdma_qp *qp, struct ocrdma_cq *cq)
1531 {
1532 	unsigned long cq_flags;
1533 	unsigned long flags;
1534 	int discard_cnt = 0;
1535 	u32 cur_getp, stop_getp;
1536 	struct ocrdma_cqe *cqe;
1537 	u32 qpn = 0, wqe_idx = 0;
1538 
1539 	spin_lock_irqsave(&cq->cq_lock, cq_flags);
1540 
1541 	/* traverse through the CQEs in the hw CQ,
1542 	 * find the matching CQE for a given qp,
1543 	 * mark the matching one discarded by clearing qpn.
1544 	 * ring the doorbell in the poll_cq() as
1545 	 * we don't complete out of order cqe.
1546 	 */
1547 
1548 	cur_getp = cq->getp;
1549 	/* find upto when do we reap the cq. */
1550 	stop_getp = cur_getp;
1551 	do {
1552 		if (is_hw_sq_empty(qp) && (!qp->srq && is_hw_rq_empty(qp)))
1553 			break;
1554 
1555 		cqe = cq->va + cur_getp;
1556 		/* if (a) done reaping whole hw cq, or
1557 		 *    (b) qp_xq becomes empty.
1558 		 * then exit
1559 		 */
1560 		qpn = cqe->cmn.qpn & OCRDMA_CQE_QPN_MASK;
1561 		/* if previously discarded cqe found, skip that too. */
1562 		/* check for matching qp */
1563 		if (qpn == 0 || qpn != qp->id)
1564 			goto skip_cqe;
1565 
1566 		if (is_cqe_for_sq(cqe)) {
1567 			ocrdma_hwq_inc_tail(&qp->sq);
1568 		} else {
1569 			if (qp->srq) {
1570 				wqe_idx = (le32_to_cpu(cqe->rq.buftag_qpn) >>
1571 					OCRDMA_CQE_BUFTAG_SHIFT) &
1572 					qp->srq->rq.max_wqe_idx;
1573 				if (wqe_idx < 1)
1574 					BUG();
1575 				spin_lock_irqsave(&qp->srq->q_lock, flags);
1576 				ocrdma_hwq_inc_tail(&qp->srq->rq);
1577 				ocrdma_srq_toggle_bit(qp->srq, wqe_idx - 1);
1578 				spin_unlock_irqrestore(&qp->srq->q_lock, flags);
1579 
1580 			} else {
1581 				ocrdma_hwq_inc_tail(&qp->rq);
1582 			}
1583 		}
1584 		/* mark cqe discarded so that it is not picked up later
1585 		 * in the poll_cq().
1586 		 */
1587 		discard_cnt += 1;
1588 		cqe->cmn.qpn = 0;
1589 skip_cqe:
1590 		cur_getp = (cur_getp + 1) % cq->max_hw_cqe;
1591 	} while (cur_getp != stop_getp);
1592 	spin_unlock_irqrestore(&cq->cq_lock, cq_flags);
1593 }
1594 
1595 void ocrdma_del_flush_qp(struct ocrdma_qp *qp)
1596 {
1597 	int found = false;
1598 	unsigned long flags;
1599 	struct ocrdma_dev *dev = qp->dev;
1600 	/* sync with any active CQ poll */
1601 
1602 	spin_lock_irqsave(&dev->flush_q_lock, flags);
1603 	found = ocrdma_is_qp_in_sq_flushlist(qp->sq_cq, qp);
1604 	if (found)
1605 		list_del(&qp->sq_entry);
1606 	if (!qp->srq) {
1607 		found = ocrdma_is_qp_in_rq_flushlist(qp->rq_cq, qp);
1608 		if (found)
1609 			list_del(&qp->rq_entry);
1610 	}
1611 	spin_unlock_irqrestore(&dev->flush_q_lock, flags);
1612 }
1613 
1614 int ocrdma_destroy_qp(struct ib_qp *ibqp)
1615 {
1616 	int status;
1617 	struct ocrdma_pd *pd;
1618 	struct ocrdma_qp *qp;
1619 	struct ocrdma_dev *dev;
1620 	struct ib_qp_attr attrs;
1621 	int attr_mask = IB_QP_STATE;
1622 	unsigned long flags;
1623 
1624 	qp = get_ocrdma_qp(ibqp);
1625 	dev = qp->dev;
1626 
1627 	attrs.qp_state = IB_QPS_ERR;
1628 	pd = qp->pd;
1629 
1630 	/* change the QP state to ERROR */
1631 	_ocrdma_modify_qp(ibqp, &attrs, attr_mask);
1632 
1633 	/* ensure that CQEs for newly created QP (whose id may be same with
1634 	 * one which just getting destroyed are same), dont get
1635 	 * discarded until the old CQEs are discarded.
1636 	 */
1637 	mutex_lock(&dev->dev_lock);
1638 	status = ocrdma_mbx_destroy_qp(dev, qp);
1639 
1640 	/*
1641 	 * acquire CQ lock while destroy is in progress, in order to
1642 	 * protect against proessing in-flight CQEs for this QP.
1643 	 */
1644 	spin_lock_irqsave(&qp->sq_cq->cq_lock, flags);
1645 	if (qp->rq_cq && (qp->rq_cq != qp->sq_cq))
1646 		spin_lock(&qp->rq_cq->cq_lock);
1647 
1648 	ocrdma_del_qpn_map(dev, qp);
1649 
1650 	if (qp->rq_cq && (qp->rq_cq != qp->sq_cq))
1651 		spin_unlock(&qp->rq_cq->cq_lock);
1652 	spin_unlock_irqrestore(&qp->sq_cq->cq_lock, flags);
1653 
1654 	if (!pd->uctx) {
1655 		ocrdma_discard_cqes(qp, qp->sq_cq);
1656 		ocrdma_discard_cqes(qp, qp->rq_cq);
1657 	}
1658 	mutex_unlock(&dev->dev_lock);
1659 
1660 	if (pd->uctx) {
1661 		ocrdma_del_mmap(pd->uctx, (u64) qp->sq.pa,
1662 				PAGE_ALIGN(qp->sq.len));
1663 		if (!qp->srq)
1664 			ocrdma_del_mmap(pd->uctx, (u64) qp->rq.pa,
1665 					PAGE_ALIGN(qp->rq.len));
1666 	}
1667 
1668 	ocrdma_del_flush_qp(qp);
1669 
1670 	kfree(qp->wqe_wr_id_tbl);
1671 	kfree(qp->rqe_wr_id_tbl);
1672 	kfree(qp);
1673 	return status;
1674 }
1675 
1676 static int ocrdma_copy_srq_uresp(struct ocrdma_dev *dev, struct ocrdma_srq *srq,
1677 				struct ib_udata *udata)
1678 {
1679 	int status;
1680 	struct ocrdma_create_srq_uresp uresp;
1681 
1682 	memset(&uresp, 0, sizeof(uresp));
1683 	uresp.rq_dbid = srq->rq.dbid;
1684 	uresp.num_rq_pages = 1;
1685 	uresp.rq_page_addr[0] = virt_to_phys(srq->rq.va);
1686 	uresp.rq_page_size = srq->rq.len;
1687 	uresp.db_page_addr = dev->nic_info.unmapped_db +
1688 	    (srq->pd->id * dev->nic_info.db_page_size);
1689 	uresp.db_page_size = dev->nic_info.db_page_size;
1690 	uresp.num_rqe_allocated = srq->rq.max_cnt;
1691 	if (ocrdma_get_asic_type(dev) == OCRDMA_ASIC_GEN_SKH_R) {
1692 		uresp.db_rq_offset = OCRDMA_DB_GEN2_RQ_OFFSET;
1693 		uresp.db_shift = 24;
1694 	} else {
1695 		uresp.db_rq_offset = OCRDMA_DB_RQ_OFFSET;
1696 		uresp.db_shift = 16;
1697 	}
1698 
1699 	status = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1700 	if (status)
1701 		return status;
1702 	status = ocrdma_add_mmap(srq->pd->uctx, uresp.rq_page_addr[0],
1703 				 uresp.rq_page_size);
1704 	if (status)
1705 		return status;
1706 	return status;
1707 }
1708 
1709 struct ib_srq *ocrdma_create_srq(struct ib_pd *ibpd,
1710 				 struct ib_srq_init_attr *init_attr,
1711 				 struct ib_udata *udata)
1712 {
1713 	int status = -ENOMEM;
1714 	struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);
1715 	struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device);
1716 	struct ocrdma_srq *srq;
1717 
1718 	if (init_attr->attr.max_sge > dev->attr.max_recv_sge)
1719 		return ERR_PTR(-EINVAL);
1720 	if (init_attr->attr.max_wr > dev->attr.max_rqe)
1721 		return ERR_PTR(-EINVAL);
1722 
1723 	srq = kzalloc(sizeof(*srq), GFP_KERNEL);
1724 	if (!srq)
1725 		return ERR_PTR(status);
1726 
1727 	spin_lock_init(&srq->q_lock);
1728 	srq->pd = pd;
1729 	srq->db = dev->nic_info.db + (pd->id * dev->nic_info.db_page_size);
1730 	status = ocrdma_mbx_create_srq(dev, srq, init_attr, pd);
1731 	if (status)
1732 		goto err;
1733 
1734 	if (udata == NULL) {
1735 		srq->rqe_wr_id_tbl = kzalloc(sizeof(u64) * srq->rq.max_cnt,
1736 			    GFP_KERNEL);
1737 		if (srq->rqe_wr_id_tbl == NULL)
1738 			goto arm_err;
1739 
1740 		srq->bit_fields_len = (srq->rq.max_cnt / 32) +
1741 		    (srq->rq.max_cnt % 32 ? 1 : 0);
1742 		srq->idx_bit_fields =
1743 		    kmalloc(srq->bit_fields_len * sizeof(u32), GFP_KERNEL);
1744 		if (srq->idx_bit_fields == NULL)
1745 			goto arm_err;
1746 		memset(srq->idx_bit_fields, 0xff,
1747 		       srq->bit_fields_len * sizeof(u32));
1748 	}
1749 
1750 	if (init_attr->attr.srq_limit) {
1751 		status = ocrdma_mbx_modify_srq(srq, &init_attr->attr);
1752 		if (status)
1753 			goto arm_err;
1754 	}
1755 
1756 	if (udata) {
1757 		status = ocrdma_copy_srq_uresp(dev, srq, udata);
1758 		if (status)
1759 			goto arm_err;
1760 	}
1761 
1762 	return &srq->ibsrq;
1763 
1764 arm_err:
1765 	ocrdma_mbx_destroy_srq(dev, srq);
1766 err:
1767 	kfree(srq->rqe_wr_id_tbl);
1768 	kfree(srq->idx_bit_fields);
1769 	kfree(srq);
1770 	return ERR_PTR(status);
1771 }
1772 
1773 int ocrdma_modify_srq(struct ib_srq *ibsrq,
1774 		      struct ib_srq_attr *srq_attr,
1775 		      enum ib_srq_attr_mask srq_attr_mask,
1776 		      struct ib_udata *udata)
1777 {
1778 	int status = 0;
1779 	struct ocrdma_srq *srq;
1780 
1781 	srq = get_ocrdma_srq(ibsrq);
1782 	if (srq_attr_mask & IB_SRQ_MAX_WR)
1783 		status = -EINVAL;
1784 	else
1785 		status = ocrdma_mbx_modify_srq(srq, srq_attr);
1786 	return status;
1787 }
1788 
1789 int ocrdma_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
1790 {
1791 	int status;
1792 	struct ocrdma_srq *srq;
1793 
1794 	srq = get_ocrdma_srq(ibsrq);
1795 	status = ocrdma_mbx_query_srq(srq, srq_attr);
1796 	return status;
1797 }
1798 
1799 int ocrdma_destroy_srq(struct ib_srq *ibsrq)
1800 {
1801 	int status;
1802 	struct ocrdma_srq *srq;
1803 	struct ocrdma_dev *dev = get_ocrdma_dev(ibsrq->device);
1804 
1805 	srq = get_ocrdma_srq(ibsrq);
1806 
1807 	status = ocrdma_mbx_destroy_srq(dev, srq);
1808 
1809 	if (srq->pd->uctx)
1810 		ocrdma_del_mmap(srq->pd->uctx, (u64) srq->rq.pa,
1811 				PAGE_ALIGN(srq->rq.len));
1812 
1813 	kfree(srq->idx_bit_fields);
1814 	kfree(srq->rqe_wr_id_tbl);
1815 	kfree(srq);
1816 	return status;
1817 }
1818 
1819 /* unprivileged verbs and their support functions. */
1820 static void ocrdma_build_ud_hdr(struct ocrdma_qp *qp,
1821 				struct ocrdma_hdr_wqe *hdr,
1822 				struct ib_send_wr *wr)
1823 {
1824 	struct ocrdma_ewqe_ud_hdr *ud_hdr =
1825 		(struct ocrdma_ewqe_ud_hdr *)(hdr + 1);
1826 	struct ocrdma_ah *ah = get_ocrdma_ah(wr->wr.ud.ah);
1827 
1828 	ud_hdr->rsvd_dest_qpn = wr->wr.ud.remote_qpn;
1829 	if (qp->qp_type == IB_QPT_GSI)
1830 		ud_hdr->qkey = qp->qkey;
1831 	else
1832 		ud_hdr->qkey = wr->wr.ud.remote_qkey;
1833 	ud_hdr->rsvd_ahid = ah->id;
1834 }
1835 
1836 static void ocrdma_build_sges(struct ocrdma_hdr_wqe *hdr,
1837 			      struct ocrdma_sge *sge, int num_sge,
1838 			      struct ib_sge *sg_list)
1839 {
1840 	int i;
1841 
1842 	for (i = 0; i < num_sge; i++) {
1843 		sge[i].lrkey = sg_list[i].lkey;
1844 		sge[i].addr_lo = sg_list[i].addr;
1845 		sge[i].addr_hi = upper_32_bits(sg_list[i].addr);
1846 		sge[i].len = sg_list[i].length;
1847 		hdr->total_len += sg_list[i].length;
1848 	}
1849 	if (num_sge == 0)
1850 		memset(sge, 0, sizeof(*sge));
1851 }
1852 
1853 static inline uint32_t ocrdma_sglist_len(struct ib_sge *sg_list, int num_sge)
1854 {
1855 	uint32_t total_len = 0, i;
1856 
1857 	for (i = 0; i < num_sge; i++)
1858 		total_len += sg_list[i].length;
1859 	return total_len;
1860 }
1861 
1862 
1863 static int ocrdma_build_inline_sges(struct ocrdma_qp *qp,
1864 				    struct ocrdma_hdr_wqe *hdr,
1865 				    struct ocrdma_sge *sge,
1866 				    struct ib_send_wr *wr, u32 wqe_size)
1867 {
1868 	int i;
1869 	char *dpp_addr;
1870 
1871 	if (wr->send_flags & IB_SEND_INLINE && qp->qp_type != IB_QPT_UD) {
1872 		hdr->total_len = ocrdma_sglist_len(wr->sg_list, wr->num_sge);
1873 		if (unlikely(hdr->total_len > qp->max_inline_data)) {
1874 			pr_err("%s() supported_len=0x%x,\n"
1875 			       " unsupported len req=0x%x\n", __func__,
1876 				qp->max_inline_data, hdr->total_len);
1877 			return -EINVAL;
1878 		}
1879 		dpp_addr = (char *)sge;
1880 		for (i = 0; i < wr->num_sge; i++) {
1881 			memcpy(dpp_addr,
1882 			       (void *)(unsigned long)wr->sg_list[i].addr,
1883 			       wr->sg_list[i].length);
1884 			dpp_addr += wr->sg_list[i].length;
1885 		}
1886 
1887 		wqe_size += roundup(hdr->total_len, OCRDMA_WQE_ALIGN_BYTES);
1888 		if (0 == hdr->total_len)
1889 			wqe_size += sizeof(struct ocrdma_sge);
1890 		hdr->cw |= (OCRDMA_TYPE_INLINE << OCRDMA_WQE_TYPE_SHIFT);
1891 	} else {
1892 		ocrdma_build_sges(hdr, sge, wr->num_sge, wr->sg_list);
1893 		if (wr->num_sge)
1894 			wqe_size += (wr->num_sge * sizeof(struct ocrdma_sge));
1895 		else
1896 			wqe_size += sizeof(struct ocrdma_sge);
1897 		hdr->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT);
1898 	}
1899 	hdr->cw |= ((wqe_size / OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT);
1900 	return 0;
1901 }
1902 
1903 static int ocrdma_build_send(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr,
1904 			     struct ib_send_wr *wr)
1905 {
1906 	int status;
1907 	struct ocrdma_sge *sge;
1908 	u32 wqe_size = sizeof(*hdr);
1909 
1910 	if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) {
1911 		ocrdma_build_ud_hdr(qp, hdr, wr);
1912 		sge = (struct ocrdma_sge *)(hdr + 2);
1913 		wqe_size += sizeof(struct ocrdma_ewqe_ud_hdr);
1914 	} else {
1915 		sge = (struct ocrdma_sge *)(hdr + 1);
1916 	}
1917 
1918 	status = ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size);
1919 	return status;
1920 }
1921 
1922 static int ocrdma_build_write(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr,
1923 			      struct ib_send_wr *wr)
1924 {
1925 	int status;
1926 	struct ocrdma_sge *ext_rw = (struct ocrdma_sge *)(hdr + 1);
1927 	struct ocrdma_sge *sge = ext_rw + 1;
1928 	u32 wqe_size = sizeof(*hdr) + sizeof(*ext_rw);
1929 
1930 	status = ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size);
1931 	if (status)
1932 		return status;
1933 	ext_rw->addr_lo = wr->wr.rdma.remote_addr;
1934 	ext_rw->addr_hi = upper_32_bits(wr->wr.rdma.remote_addr);
1935 	ext_rw->lrkey = wr->wr.rdma.rkey;
1936 	ext_rw->len = hdr->total_len;
1937 	return 0;
1938 }
1939 
1940 static void ocrdma_build_read(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr,
1941 			      struct ib_send_wr *wr)
1942 {
1943 	struct ocrdma_sge *ext_rw = (struct ocrdma_sge *)(hdr + 1);
1944 	struct ocrdma_sge *sge = ext_rw + 1;
1945 	u32 wqe_size = ((wr->num_sge + 1) * sizeof(struct ocrdma_sge)) +
1946 	    sizeof(struct ocrdma_hdr_wqe);
1947 
1948 	ocrdma_build_sges(hdr, sge, wr->num_sge, wr->sg_list);
1949 	hdr->cw |= ((wqe_size / OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT);
1950 	hdr->cw |= (OCRDMA_READ << OCRDMA_WQE_OPCODE_SHIFT);
1951 	hdr->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT);
1952 
1953 	ext_rw->addr_lo = wr->wr.rdma.remote_addr;
1954 	ext_rw->addr_hi = upper_32_bits(wr->wr.rdma.remote_addr);
1955 	ext_rw->lrkey = wr->wr.rdma.rkey;
1956 	ext_rw->len = hdr->total_len;
1957 }
1958 
1959 static void build_frmr_pbes(struct ib_send_wr *wr, struct ocrdma_pbl *pbl_tbl,
1960 			    struct ocrdma_hw_mr *hwmr)
1961 {
1962 	int i;
1963 	u64 buf_addr = 0;
1964 	int num_pbes;
1965 	struct ocrdma_pbe *pbe;
1966 
1967 	pbe = (struct ocrdma_pbe *)pbl_tbl->va;
1968 	num_pbes = 0;
1969 
1970 	/* go through the OS phy regions & fill hw pbe entries into pbls. */
1971 	for (i = 0; i < wr->wr.fast_reg.page_list_len; i++) {
1972 		/* number of pbes can be more for one OS buf, when
1973 		 * buffers are of different sizes.
1974 		 * split the ib_buf to one or more pbes.
1975 		 */
1976 		buf_addr = wr->wr.fast_reg.page_list->page_list[i];
1977 		pbe->pa_lo = cpu_to_le32((u32) (buf_addr & PAGE_MASK));
1978 		pbe->pa_hi = cpu_to_le32((u32) upper_32_bits(buf_addr));
1979 		num_pbes += 1;
1980 		pbe++;
1981 
1982 		/* if the pbl is full storing the pbes,
1983 		 * move to next pbl.
1984 		*/
1985 		if (num_pbes == (hwmr->pbl_size/sizeof(u64))) {
1986 			pbl_tbl++;
1987 			pbe = (struct ocrdma_pbe *)pbl_tbl->va;
1988 		}
1989 	}
1990 	return;
1991 }
1992 
1993 static int get_encoded_page_size(int pg_sz)
1994 {
1995 	/* Max size is 256M 4096 << 16 */
1996 	int i = 0;
1997 	for (; i < 17; i++)
1998 		if (pg_sz == (4096 << i))
1999 			break;
2000 	return i;
2001 }
2002 
2003 
2004 static int ocrdma_build_fr(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr,
2005 			   struct ib_send_wr *wr)
2006 {
2007 	u64 fbo;
2008 	struct ocrdma_ewqe_fr *fast_reg = (struct ocrdma_ewqe_fr *)(hdr + 1);
2009 	struct ocrdma_mr *mr;
2010 	u32 wqe_size = sizeof(*fast_reg) + sizeof(*hdr);
2011 
2012 	wqe_size = roundup(wqe_size, OCRDMA_WQE_ALIGN_BYTES);
2013 
2014 	if (wr->wr.fast_reg.page_list_len > qp->dev->attr.max_pages_per_frmr)
2015 		return -EINVAL;
2016 
2017 	hdr->cw |= (OCRDMA_FR_MR << OCRDMA_WQE_OPCODE_SHIFT);
2018 	hdr->cw |= ((wqe_size / OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT);
2019 
2020 	if (wr->wr.fast_reg.page_list_len == 0)
2021 		BUG();
2022 	if (wr->wr.fast_reg.access_flags & IB_ACCESS_LOCAL_WRITE)
2023 		hdr->rsvd_lkey_flags |= OCRDMA_LKEY_FLAG_LOCAL_WR;
2024 	if (wr->wr.fast_reg.access_flags & IB_ACCESS_REMOTE_WRITE)
2025 		hdr->rsvd_lkey_flags |= OCRDMA_LKEY_FLAG_REMOTE_WR;
2026 	if (wr->wr.fast_reg.access_flags & IB_ACCESS_REMOTE_READ)
2027 		hdr->rsvd_lkey_flags |= OCRDMA_LKEY_FLAG_REMOTE_RD;
2028 	hdr->lkey = wr->wr.fast_reg.rkey;
2029 	hdr->total_len = wr->wr.fast_reg.length;
2030 
2031 	fbo = wr->wr.fast_reg.iova_start -
2032 	    (wr->wr.fast_reg.page_list->page_list[0] & PAGE_MASK);
2033 
2034 	fast_reg->va_hi = upper_32_bits(wr->wr.fast_reg.iova_start);
2035 	fast_reg->va_lo = (u32) (wr->wr.fast_reg.iova_start & 0xffffffff);
2036 	fast_reg->fbo_hi = upper_32_bits(fbo);
2037 	fast_reg->fbo_lo = (u32) fbo & 0xffffffff;
2038 	fast_reg->num_sges = wr->wr.fast_reg.page_list_len;
2039 	fast_reg->size_sge =
2040 		get_encoded_page_size(1 << wr->wr.fast_reg.page_shift);
2041 	mr = (struct ocrdma_mr *) (unsigned long)
2042 		qp->dev->stag_arr[(hdr->lkey >> 8) & (OCRDMA_MAX_STAG - 1)];
2043 	build_frmr_pbes(wr, mr->hwmr.pbl_table, &mr->hwmr);
2044 	return 0;
2045 }
2046 
2047 static void ocrdma_ring_sq_db(struct ocrdma_qp *qp)
2048 {
2049 	u32 val = qp->sq.dbid | (1 << OCRDMA_DB_SQ_SHIFT);
2050 
2051 	iowrite32(val, qp->sq_db);
2052 }
2053 
2054 int ocrdma_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
2055 		     struct ib_send_wr **bad_wr)
2056 {
2057 	int status = 0;
2058 	struct ocrdma_qp *qp = get_ocrdma_qp(ibqp);
2059 	struct ocrdma_hdr_wqe *hdr;
2060 	unsigned long flags;
2061 
2062 	spin_lock_irqsave(&qp->q_lock, flags);
2063 	if (qp->state != OCRDMA_QPS_RTS && qp->state != OCRDMA_QPS_SQD) {
2064 		spin_unlock_irqrestore(&qp->q_lock, flags);
2065 		*bad_wr = wr;
2066 		return -EINVAL;
2067 	}
2068 
2069 	while (wr) {
2070 		if (qp->qp_type == IB_QPT_UD &&
2071 		    (wr->opcode != IB_WR_SEND &&
2072 		     wr->opcode != IB_WR_SEND_WITH_IMM)) {
2073 			*bad_wr = wr;
2074 			status = -EINVAL;
2075 			break;
2076 		}
2077 		if (ocrdma_hwq_free_cnt(&qp->sq) == 0 ||
2078 		    wr->num_sge > qp->sq.max_sges) {
2079 			*bad_wr = wr;
2080 			status = -ENOMEM;
2081 			break;
2082 		}
2083 		hdr = ocrdma_hwq_head(&qp->sq);
2084 		hdr->cw = 0;
2085 		if (wr->send_flags & IB_SEND_SIGNALED || qp->signaled)
2086 			hdr->cw |= (OCRDMA_FLAG_SIG << OCRDMA_WQE_FLAGS_SHIFT);
2087 		if (wr->send_flags & IB_SEND_FENCE)
2088 			hdr->cw |=
2089 			    (OCRDMA_FLAG_FENCE_L << OCRDMA_WQE_FLAGS_SHIFT);
2090 		if (wr->send_flags & IB_SEND_SOLICITED)
2091 			hdr->cw |=
2092 			    (OCRDMA_FLAG_SOLICIT << OCRDMA_WQE_FLAGS_SHIFT);
2093 		hdr->total_len = 0;
2094 		switch (wr->opcode) {
2095 		case IB_WR_SEND_WITH_IMM:
2096 			hdr->cw |= (OCRDMA_FLAG_IMM << OCRDMA_WQE_FLAGS_SHIFT);
2097 			hdr->immdt = ntohl(wr->ex.imm_data);
2098 		case IB_WR_SEND:
2099 			hdr->cw |= (OCRDMA_SEND << OCRDMA_WQE_OPCODE_SHIFT);
2100 			ocrdma_build_send(qp, hdr, wr);
2101 			break;
2102 		case IB_WR_SEND_WITH_INV:
2103 			hdr->cw |= (OCRDMA_FLAG_INV << OCRDMA_WQE_FLAGS_SHIFT);
2104 			hdr->cw |= (OCRDMA_SEND << OCRDMA_WQE_OPCODE_SHIFT);
2105 			hdr->lkey = wr->ex.invalidate_rkey;
2106 			status = ocrdma_build_send(qp, hdr, wr);
2107 			break;
2108 		case IB_WR_RDMA_WRITE_WITH_IMM:
2109 			hdr->cw |= (OCRDMA_FLAG_IMM << OCRDMA_WQE_FLAGS_SHIFT);
2110 			hdr->immdt = ntohl(wr->ex.imm_data);
2111 		case IB_WR_RDMA_WRITE:
2112 			hdr->cw |= (OCRDMA_WRITE << OCRDMA_WQE_OPCODE_SHIFT);
2113 			status = ocrdma_build_write(qp, hdr, wr);
2114 			break;
2115 		case IB_WR_RDMA_READ_WITH_INV:
2116 			hdr->cw |= (OCRDMA_FLAG_INV << OCRDMA_WQE_FLAGS_SHIFT);
2117 		case IB_WR_RDMA_READ:
2118 			ocrdma_build_read(qp, hdr, wr);
2119 			break;
2120 		case IB_WR_LOCAL_INV:
2121 			hdr->cw |=
2122 			    (OCRDMA_LKEY_INV << OCRDMA_WQE_OPCODE_SHIFT);
2123 			hdr->cw |= ((sizeof(struct ocrdma_hdr_wqe) +
2124 					sizeof(struct ocrdma_sge)) /
2125 				OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT;
2126 			hdr->lkey = wr->ex.invalidate_rkey;
2127 			break;
2128 		case IB_WR_FAST_REG_MR:
2129 			status = ocrdma_build_fr(qp, hdr, wr);
2130 			break;
2131 		default:
2132 			status = -EINVAL;
2133 			break;
2134 		}
2135 		if (status) {
2136 			*bad_wr = wr;
2137 			break;
2138 		}
2139 		if (wr->send_flags & IB_SEND_SIGNALED || qp->signaled)
2140 			qp->wqe_wr_id_tbl[qp->sq.head].signaled = 1;
2141 		else
2142 			qp->wqe_wr_id_tbl[qp->sq.head].signaled = 0;
2143 		qp->wqe_wr_id_tbl[qp->sq.head].wrid = wr->wr_id;
2144 		ocrdma_cpu_to_le32(hdr, ((hdr->cw >> OCRDMA_WQE_SIZE_SHIFT) &
2145 				   OCRDMA_WQE_SIZE_MASK) * OCRDMA_WQE_STRIDE);
2146 		/* make sure wqe is written before adapter can access it */
2147 		wmb();
2148 		/* inform hw to start processing it */
2149 		ocrdma_ring_sq_db(qp);
2150 
2151 		/* update pointer, counter for next wr */
2152 		ocrdma_hwq_inc_head(&qp->sq);
2153 		wr = wr->next;
2154 	}
2155 	spin_unlock_irqrestore(&qp->q_lock, flags);
2156 	return status;
2157 }
2158 
2159 static void ocrdma_ring_rq_db(struct ocrdma_qp *qp)
2160 {
2161 	u32 val = qp->rq.dbid | (1 << OCRDMA_DB_RQ_SHIFT);
2162 
2163 	iowrite32(val, qp->rq_db);
2164 }
2165 
2166 static void ocrdma_build_rqe(struct ocrdma_hdr_wqe *rqe, struct ib_recv_wr *wr,
2167 			     u16 tag)
2168 {
2169 	u32 wqe_size = 0;
2170 	struct ocrdma_sge *sge;
2171 	if (wr->num_sge)
2172 		wqe_size = (wr->num_sge * sizeof(*sge)) + sizeof(*rqe);
2173 	else
2174 		wqe_size = sizeof(*sge) + sizeof(*rqe);
2175 
2176 	rqe->cw = ((wqe_size / OCRDMA_WQE_STRIDE) <<
2177 				OCRDMA_WQE_SIZE_SHIFT);
2178 	rqe->cw |= (OCRDMA_FLAG_SIG << OCRDMA_WQE_FLAGS_SHIFT);
2179 	rqe->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT);
2180 	rqe->total_len = 0;
2181 	rqe->rsvd_tag = tag;
2182 	sge = (struct ocrdma_sge *)(rqe + 1);
2183 	ocrdma_build_sges(rqe, sge, wr->num_sge, wr->sg_list);
2184 	ocrdma_cpu_to_le32(rqe, wqe_size);
2185 }
2186 
2187 int ocrdma_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
2188 		     struct ib_recv_wr **bad_wr)
2189 {
2190 	int status = 0;
2191 	unsigned long flags;
2192 	struct ocrdma_qp *qp = get_ocrdma_qp(ibqp);
2193 	struct ocrdma_hdr_wqe *rqe;
2194 
2195 	spin_lock_irqsave(&qp->q_lock, flags);
2196 	if (qp->state == OCRDMA_QPS_RST || qp->state == OCRDMA_QPS_ERR) {
2197 		spin_unlock_irqrestore(&qp->q_lock, flags);
2198 		*bad_wr = wr;
2199 		return -EINVAL;
2200 	}
2201 	while (wr) {
2202 		if (ocrdma_hwq_free_cnt(&qp->rq) == 0 ||
2203 		    wr->num_sge > qp->rq.max_sges) {
2204 			*bad_wr = wr;
2205 			status = -ENOMEM;
2206 			break;
2207 		}
2208 		rqe = ocrdma_hwq_head(&qp->rq);
2209 		ocrdma_build_rqe(rqe, wr, 0);
2210 
2211 		qp->rqe_wr_id_tbl[qp->rq.head] = wr->wr_id;
2212 		/* make sure rqe is written before adapter can access it */
2213 		wmb();
2214 
2215 		/* inform hw to start processing it */
2216 		ocrdma_ring_rq_db(qp);
2217 
2218 		/* update pointer, counter for next wr */
2219 		ocrdma_hwq_inc_head(&qp->rq);
2220 		wr = wr->next;
2221 	}
2222 	spin_unlock_irqrestore(&qp->q_lock, flags);
2223 	return status;
2224 }
2225 
2226 /* cqe for srq's rqe can potentially arrive out of order.
2227  * index gives the entry in the shadow table where to store
2228  * the wr_id. tag/index is returned in cqe to reference back
2229  * for a given rqe.
2230  */
2231 static int ocrdma_srq_get_idx(struct ocrdma_srq *srq)
2232 {
2233 	int row = 0;
2234 	int indx = 0;
2235 
2236 	for (row = 0; row < srq->bit_fields_len; row++) {
2237 		if (srq->idx_bit_fields[row]) {
2238 			indx = ffs(srq->idx_bit_fields[row]);
2239 			indx = (row * 32) + (indx - 1);
2240 			if (indx >= srq->rq.max_cnt)
2241 				BUG();
2242 			ocrdma_srq_toggle_bit(srq, indx);
2243 			break;
2244 		}
2245 	}
2246 
2247 	if (row == srq->bit_fields_len)
2248 		BUG();
2249 	return indx + 1; /* Use from index 1 */
2250 }
2251 
2252 static void ocrdma_ring_srq_db(struct ocrdma_srq *srq)
2253 {
2254 	u32 val = srq->rq.dbid | (1 << 16);
2255 
2256 	iowrite32(val, srq->db + OCRDMA_DB_GEN2_SRQ_OFFSET);
2257 }
2258 
2259 int ocrdma_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr,
2260 			 struct ib_recv_wr **bad_wr)
2261 {
2262 	int status = 0;
2263 	unsigned long flags;
2264 	struct ocrdma_srq *srq;
2265 	struct ocrdma_hdr_wqe *rqe;
2266 	u16 tag;
2267 
2268 	srq = get_ocrdma_srq(ibsrq);
2269 
2270 	spin_lock_irqsave(&srq->q_lock, flags);
2271 	while (wr) {
2272 		if (ocrdma_hwq_free_cnt(&srq->rq) == 0 ||
2273 		    wr->num_sge > srq->rq.max_sges) {
2274 			status = -ENOMEM;
2275 			*bad_wr = wr;
2276 			break;
2277 		}
2278 		tag = ocrdma_srq_get_idx(srq);
2279 		rqe = ocrdma_hwq_head(&srq->rq);
2280 		ocrdma_build_rqe(rqe, wr, tag);
2281 
2282 		srq->rqe_wr_id_tbl[tag] = wr->wr_id;
2283 		/* make sure rqe is written before adapter can perform DMA */
2284 		wmb();
2285 		/* inform hw to start processing it */
2286 		ocrdma_ring_srq_db(srq);
2287 		/* update pointer, counter for next wr */
2288 		ocrdma_hwq_inc_head(&srq->rq);
2289 		wr = wr->next;
2290 	}
2291 	spin_unlock_irqrestore(&srq->q_lock, flags);
2292 	return status;
2293 }
2294 
2295 static enum ib_wc_status ocrdma_to_ibwc_err(u16 status)
2296 {
2297 	enum ib_wc_status ibwc_status;
2298 
2299 	switch (status) {
2300 	case OCRDMA_CQE_GENERAL_ERR:
2301 		ibwc_status = IB_WC_GENERAL_ERR;
2302 		break;
2303 	case OCRDMA_CQE_LOC_LEN_ERR:
2304 		ibwc_status = IB_WC_LOC_LEN_ERR;
2305 		break;
2306 	case OCRDMA_CQE_LOC_QP_OP_ERR:
2307 		ibwc_status = IB_WC_LOC_QP_OP_ERR;
2308 		break;
2309 	case OCRDMA_CQE_LOC_EEC_OP_ERR:
2310 		ibwc_status = IB_WC_LOC_EEC_OP_ERR;
2311 		break;
2312 	case OCRDMA_CQE_LOC_PROT_ERR:
2313 		ibwc_status = IB_WC_LOC_PROT_ERR;
2314 		break;
2315 	case OCRDMA_CQE_WR_FLUSH_ERR:
2316 		ibwc_status = IB_WC_WR_FLUSH_ERR;
2317 		break;
2318 	case OCRDMA_CQE_MW_BIND_ERR:
2319 		ibwc_status = IB_WC_MW_BIND_ERR;
2320 		break;
2321 	case OCRDMA_CQE_BAD_RESP_ERR:
2322 		ibwc_status = IB_WC_BAD_RESP_ERR;
2323 		break;
2324 	case OCRDMA_CQE_LOC_ACCESS_ERR:
2325 		ibwc_status = IB_WC_LOC_ACCESS_ERR;
2326 		break;
2327 	case OCRDMA_CQE_REM_INV_REQ_ERR:
2328 		ibwc_status = IB_WC_REM_INV_REQ_ERR;
2329 		break;
2330 	case OCRDMA_CQE_REM_ACCESS_ERR:
2331 		ibwc_status = IB_WC_REM_ACCESS_ERR;
2332 		break;
2333 	case OCRDMA_CQE_REM_OP_ERR:
2334 		ibwc_status = IB_WC_REM_OP_ERR;
2335 		break;
2336 	case OCRDMA_CQE_RETRY_EXC_ERR:
2337 		ibwc_status = IB_WC_RETRY_EXC_ERR;
2338 		break;
2339 	case OCRDMA_CQE_RNR_RETRY_EXC_ERR:
2340 		ibwc_status = IB_WC_RNR_RETRY_EXC_ERR;
2341 		break;
2342 	case OCRDMA_CQE_LOC_RDD_VIOL_ERR:
2343 		ibwc_status = IB_WC_LOC_RDD_VIOL_ERR;
2344 		break;
2345 	case OCRDMA_CQE_REM_INV_RD_REQ_ERR:
2346 		ibwc_status = IB_WC_REM_INV_RD_REQ_ERR;
2347 		break;
2348 	case OCRDMA_CQE_REM_ABORT_ERR:
2349 		ibwc_status = IB_WC_REM_ABORT_ERR;
2350 		break;
2351 	case OCRDMA_CQE_INV_EECN_ERR:
2352 		ibwc_status = IB_WC_INV_EECN_ERR;
2353 		break;
2354 	case OCRDMA_CQE_INV_EEC_STATE_ERR:
2355 		ibwc_status = IB_WC_INV_EEC_STATE_ERR;
2356 		break;
2357 	case OCRDMA_CQE_FATAL_ERR:
2358 		ibwc_status = IB_WC_FATAL_ERR;
2359 		break;
2360 	case OCRDMA_CQE_RESP_TIMEOUT_ERR:
2361 		ibwc_status = IB_WC_RESP_TIMEOUT_ERR;
2362 		break;
2363 	default:
2364 		ibwc_status = IB_WC_GENERAL_ERR;
2365 		break;
2366 	}
2367 	return ibwc_status;
2368 }
2369 
2370 static void ocrdma_update_wc(struct ocrdma_qp *qp, struct ib_wc *ibwc,
2371 		      u32 wqe_idx)
2372 {
2373 	struct ocrdma_hdr_wqe *hdr;
2374 	struct ocrdma_sge *rw;
2375 	int opcode;
2376 
2377 	hdr = ocrdma_hwq_head_from_idx(&qp->sq, wqe_idx);
2378 
2379 	ibwc->wr_id = qp->wqe_wr_id_tbl[wqe_idx].wrid;
2380 	/* Undo the hdr->cw swap */
2381 	opcode = le32_to_cpu(hdr->cw) & OCRDMA_WQE_OPCODE_MASK;
2382 	switch (opcode) {
2383 	case OCRDMA_WRITE:
2384 		ibwc->opcode = IB_WC_RDMA_WRITE;
2385 		break;
2386 	case OCRDMA_READ:
2387 		rw = (struct ocrdma_sge *)(hdr + 1);
2388 		ibwc->opcode = IB_WC_RDMA_READ;
2389 		ibwc->byte_len = rw->len;
2390 		break;
2391 	case OCRDMA_SEND:
2392 		ibwc->opcode = IB_WC_SEND;
2393 		break;
2394 	case OCRDMA_FR_MR:
2395 		ibwc->opcode = IB_WC_FAST_REG_MR;
2396 		break;
2397 	case OCRDMA_LKEY_INV:
2398 		ibwc->opcode = IB_WC_LOCAL_INV;
2399 		break;
2400 	default:
2401 		ibwc->status = IB_WC_GENERAL_ERR;
2402 		pr_err("%s() invalid opcode received = 0x%x\n",
2403 		       __func__, hdr->cw & OCRDMA_WQE_OPCODE_MASK);
2404 		break;
2405 	}
2406 }
2407 
2408 static void ocrdma_set_cqe_status_flushed(struct ocrdma_qp *qp,
2409 						struct ocrdma_cqe *cqe)
2410 {
2411 	if (is_cqe_for_sq(cqe)) {
2412 		cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(
2413 				cqe->flags_status_srcqpn) &
2414 					~OCRDMA_CQE_STATUS_MASK);
2415 		cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(
2416 				cqe->flags_status_srcqpn) |
2417 				(OCRDMA_CQE_WR_FLUSH_ERR <<
2418 					OCRDMA_CQE_STATUS_SHIFT));
2419 	} else {
2420 		if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) {
2421 			cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(
2422 					cqe->flags_status_srcqpn) &
2423 						~OCRDMA_CQE_UD_STATUS_MASK);
2424 			cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(
2425 					cqe->flags_status_srcqpn) |
2426 					(OCRDMA_CQE_WR_FLUSH_ERR <<
2427 						OCRDMA_CQE_UD_STATUS_SHIFT));
2428 		} else {
2429 			cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(
2430 					cqe->flags_status_srcqpn) &
2431 						~OCRDMA_CQE_STATUS_MASK);
2432 			cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(
2433 					cqe->flags_status_srcqpn) |
2434 					(OCRDMA_CQE_WR_FLUSH_ERR <<
2435 						OCRDMA_CQE_STATUS_SHIFT));
2436 		}
2437 	}
2438 }
2439 
2440 static bool ocrdma_update_err_cqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe,
2441 				  struct ocrdma_qp *qp, int status)
2442 {
2443 	bool expand = false;
2444 
2445 	ibwc->byte_len = 0;
2446 	ibwc->qp = &qp->ibqp;
2447 	ibwc->status = ocrdma_to_ibwc_err(status);
2448 
2449 	ocrdma_flush_qp(qp);
2450 	ocrdma_qp_state_change(qp, IB_QPS_ERR, NULL);
2451 
2452 	/* if wqe/rqe pending for which cqe needs to be returned,
2453 	 * trigger inflating it.
2454 	 */
2455 	if (!is_hw_rq_empty(qp) || !is_hw_sq_empty(qp)) {
2456 		expand = true;
2457 		ocrdma_set_cqe_status_flushed(qp, cqe);
2458 	}
2459 	return expand;
2460 }
2461 
2462 static int ocrdma_update_err_rcqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe,
2463 				  struct ocrdma_qp *qp, int status)
2464 {
2465 	ibwc->opcode = IB_WC_RECV;
2466 	ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail];
2467 	ocrdma_hwq_inc_tail(&qp->rq);
2468 
2469 	return ocrdma_update_err_cqe(ibwc, cqe, qp, status);
2470 }
2471 
2472 static int ocrdma_update_err_scqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe,
2473 				  struct ocrdma_qp *qp, int status)
2474 {
2475 	ocrdma_update_wc(qp, ibwc, qp->sq.tail);
2476 	ocrdma_hwq_inc_tail(&qp->sq);
2477 
2478 	return ocrdma_update_err_cqe(ibwc, cqe, qp, status);
2479 }
2480 
2481 
2482 static bool ocrdma_poll_err_scqe(struct ocrdma_qp *qp,
2483 				 struct ocrdma_cqe *cqe, struct ib_wc *ibwc,
2484 				 bool *polled, bool *stop)
2485 {
2486 	bool expand;
2487 	int status = (le32_to_cpu(cqe->flags_status_srcqpn) &
2488 		OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT;
2489 
2490 	/* when hw sq is empty, but rq is not empty, so we continue
2491 	 * to keep the cqe in order to get the cq event again.
2492 	 */
2493 	if (is_hw_sq_empty(qp) && !is_hw_rq_empty(qp)) {
2494 		/* when cq for rq and sq is same, it is safe to return
2495 		 * flush cqe for RQEs.
2496 		 */
2497 		if (!qp->srq && (qp->sq_cq == qp->rq_cq)) {
2498 			*polled = true;
2499 			status = OCRDMA_CQE_WR_FLUSH_ERR;
2500 			expand = ocrdma_update_err_rcqe(ibwc, cqe, qp, status);
2501 		} else {
2502 			/* stop processing further cqe as this cqe is used for
2503 			 * triggering cq event on buddy cq of RQ.
2504 			 * When QP is destroyed, this cqe will be removed
2505 			 * from the cq's hardware q.
2506 			 */
2507 			*polled = false;
2508 			*stop = true;
2509 			expand = false;
2510 		}
2511 	} else if (is_hw_sq_empty(qp)) {
2512 		/* Do nothing */
2513 		expand = false;
2514 		*polled = false;
2515 		*stop = false;
2516 	} else {
2517 		*polled = true;
2518 		expand = ocrdma_update_err_scqe(ibwc, cqe, qp, status);
2519 	}
2520 	return expand;
2521 }
2522 
2523 static bool ocrdma_poll_success_scqe(struct ocrdma_qp *qp,
2524 				     struct ocrdma_cqe *cqe,
2525 				     struct ib_wc *ibwc, bool *polled)
2526 {
2527 	bool expand = false;
2528 	int tail = qp->sq.tail;
2529 	u32 wqe_idx;
2530 
2531 	if (!qp->wqe_wr_id_tbl[tail].signaled) {
2532 		*polled = false;    /* WC cannot be consumed yet */
2533 	} else {
2534 		ibwc->status = IB_WC_SUCCESS;
2535 		ibwc->wc_flags = 0;
2536 		ibwc->qp = &qp->ibqp;
2537 		ocrdma_update_wc(qp, ibwc, tail);
2538 		*polled = true;
2539 	}
2540 	wqe_idx = (le32_to_cpu(cqe->wq.wqeidx) &
2541 			OCRDMA_CQE_WQEIDX_MASK) & qp->sq.max_wqe_idx;
2542 	if (tail != wqe_idx)
2543 		expand = true; /* Coalesced CQE can't be consumed yet */
2544 
2545 	ocrdma_hwq_inc_tail(&qp->sq);
2546 	return expand;
2547 }
2548 
2549 static bool ocrdma_poll_scqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe,
2550 			     struct ib_wc *ibwc, bool *polled, bool *stop)
2551 {
2552 	int status;
2553 	bool expand;
2554 
2555 	status = (le32_to_cpu(cqe->flags_status_srcqpn) &
2556 		OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT;
2557 
2558 	if (status == OCRDMA_CQE_SUCCESS)
2559 		expand = ocrdma_poll_success_scqe(qp, cqe, ibwc, polled);
2560 	else
2561 		expand = ocrdma_poll_err_scqe(qp, cqe, ibwc, polled, stop);
2562 	return expand;
2563 }
2564 
2565 static int ocrdma_update_ud_rcqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe)
2566 {
2567 	int status;
2568 
2569 	status = (le32_to_cpu(cqe->flags_status_srcqpn) &
2570 		OCRDMA_CQE_UD_STATUS_MASK) >> OCRDMA_CQE_UD_STATUS_SHIFT;
2571 	ibwc->src_qp = le32_to_cpu(cqe->flags_status_srcqpn) &
2572 						OCRDMA_CQE_SRCQP_MASK;
2573 	ibwc->pkey_index = le32_to_cpu(cqe->ud.rxlen_pkey) &
2574 						OCRDMA_CQE_PKEY_MASK;
2575 	ibwc->wc_flags = IB_WC_GRH;
2576 	ibwc->byte_len = (le32_to_cpu(cqe->ud.rxlen_pkey) >>
2577 					OCRDMA_CQE_UD_XFER_LEN_SHIFT);
2578 	return status;
2579 }
2580 
2581 static void ocrdma_update_free_srq_cqe(struct ib_wc *ibwc,
2582 				       struct ocrdma_cqe *cqe,
2583 				       struct ocrdma_qp *qp)
2584 {
2585 	unsigned long flags;
2586 	struct ocrdma_srq *srq;
2587 	u32 wqe_idx;
2588 
2589 	srq = get_ocrdma_srq(qp->ibqp.srq);
2590 	wqe_idx = (le32_to_cpu(cqe->rq.buftag_qpn) >>
2591 		OCRDMA_CQE_BUFTAG_SHIFT) & srq->rq.max_wqe_idx;
2592 	if (wqe_idx < 1)
2593 		BUG();
2594 
2595 	ibwc->wr_id = srq->rqe_wr_id_tbl[wqe_idx];
2596 	spin_lock_irqsave(&srq->q_lock, flags);
2597 	ocrdma_srq_toggle_bit(srq, wqe_idx - 1);
2598 	spin_unlock_irqrestore(&srq->q_lock, flags);
2599 	ocrdma_hwq_inc_tail(&srq->rq);
2600 }
2601 
2602 static bool ocrdma_poll_err_rcqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe,
2603 				struct ib_wc *ibwc, bool *polled, bool *stop,
2604 				int status)
2605 {
2606 	bool expand;
2607 
2608 	/* when hw_rq is empty, but wq is not empty, so continue
2609 	 * to keep the cqe to get the cq event again.
2610 	 */
2611 	if (is_hw_rq_empty(qp) && !is_hw_sq_empty(qp)) {
2612 		if (!qp->srq && (qp->sq_cq == qp->rq_cq)) {
2613 			*polled = true;
2614 			status = OCRDMA_CQE_WR_FLUSH_ERR;
2615 			expand = ocrdma_update_err_scqe(ibwc, cqe, qp, status);
2616 		} else {
2617 			*polled = false;
2618 			*stop = true;
2619 			expand = false;
2620 		}
2621 	} else if (is_hw_rq_empty(qp)) {
2622 		/* Do nothing */
2623 		expand = false;
2624 		*polled = false;
2625 		*stop = false;
2626 	} else {
2627 		*polled = true;
2628 		expand = ocrdma_update_err_rcqe(ibwc, cqe, qp, status);
2629 	}
2630 	return expand;
2631 }
2632 
2633 static void ocrdma_poll_success_rcqe(struct ocrdma_qp *qp,
2634 				     struct ocrdma_cqe *cqe, struct ib_wc *ibwc)
2635 {
2636 	ibwc->opcode = IB_WC_RECV;
2637 	ibwc->qp = &qp->ibqp;
2638 	ibwc->status = IB_WC_SUCCESS;
2639 
2640 	if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI)
2641 		ocrdma_update_ud_rcqe(ibwc, cqe);
2642 	else
2643 		ibwc->byte_len = le32_to_cpu(cqe->rq.rxlen);
2644 
2645 	if (is_cqe_imm(cqe)) {
2646 		ibwc->ex.imm_data = htonl(le32_to_cpu(cqe->rq.lkey_immdt));
2647 		ibwc->wc_flags |= IB_WC_WITH_IMM;
2648 	} else if (is_cqe_wr_imm(cqe)) {
2649 		ibwc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
2650 		ibwc->ex.imm_data = htonl(le32_to_cpu(cqe->rq.lkey_immdt));
2651 		ibwc->wc_flags |= IB_WC_WITH_IMM;
2652 	} else if (is_cqe_invalidated(cqe)) {
2653 		ibwc->ex.invalidate_rkey = le32_to_cpu(cqe->rq.lkey_immdt);
2654 		ibwc->wc_flags |= IB_WC_WITH_INVALIDATE;
2655 	}
2656 	if (qp->ibqp.srq) {
2657 		ocrdma_update_free_srq_cqe(ibwc, cqe, qp);
2658 	} else {
2659 		ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail];
2660 		ocrdma_hwq_inc_tail(&qp->rq);
2661 	}
2662 }
2663 
2664 static bool ocrdma_poll_rcqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe,
2665 			     struct ib_wc *ibwc, bool *polled, bool *stop)
2666 {
2667 	int status;
2668 	bool expand = false;
2669 
2670 	ibwc->wc_flags = 0;
2671 	if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) {
2672 		status = (le32_to_cpu(cqe->flags_status_srcqpn) &
2673 					OCRDMA_CQE_UD_STATUS_MASK) >>
2674 					OCRDMA_CQE_UD_STATUS_SHIFT;
2675 	} else {
2676 		status = (le32_to_cpu(cqe->flags_status_srcqpn) &
2677 			     OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT;
2678 	}
2679 
2680 	if (status == OCRDMA_CQE_SUCCESS) {
2681 		*polled = true;
2682 		ocrdma_poll_success_rcqe(qp, cqe, ibwc);
2683 	} else {
2684 		expand = ocrdma_poll_err_rcqe(qp, cqe, ibwc, polled, stop,
2685 					      status);
2686 	}
2687 	return expand;
2688 }
2689 
2690 static void ocrdma_change_cq_phase(struct ocrdma_cq *cq, struct ocrdma_cqe *cqe,
2691 				   u16 cur_getp)
2692 {
2693 	if (cq->phase_change) {
2694 		if (cur_getp == 0)
2695 			cq->phase = (~cq->phase & OCRDMA_CQE_VALID);
2696 	} else {
2697 		/* clear valid bit */
2698 		cqe->flags_status_srcqpn = 0;
2699 	}
2700 }
2701 
2702 static int ocrdma_poll_hwcq(struct ocrdma_cq *cq, int num_entries,
2703 			    struct ib_wc *ibwc)
2704 {
2705 	u16 qpn = 0;
2706 	int i = 0;
2707 	bool expand = false;
2708 	int polled_hw_cqes = 0;
2709 	struct ocrdma_qp *qp = NULL;
2710 	struct ocrdma_dev *dev = get_ocrdma_dev(cq->ibcq.device);
2711 	struct ocrdma_cqe *cqe;
2712 	u16 cur_getp; bool polled = false; bool stop = false;
2713 
2714 	cur_getp = cq->getp;
2715 	while (num_entries) {
2716 		cqe = cq->va + cur_getp;
2717 		/* check whether valid cqe or not */
2718 		if (!is_cqe_valid(cq, cqe))
2719 			break;
2720 		qpn = (le32_to_cpu(cqe->cmn.qpn) & OCRDMA_CQE_QPN_MASK);
2721 		/* ignore discarded cqe */
2722 		if (qpn == 0)
2723 			goto skip_cqe;
2724 		qp = dev->qp_tbl[qpn];
2725 		BUG_ON(qp == NULL);
2726 
2727 		if (is_cqe_for_sq(cqe)) {
2728 			expand = ocrdma_poll_scqe(qp, cqe, ibwc, &polled,
2729 						  &stop);
2730 		} else {
2731 			expand = ocrdma_poll_rcqe(qp, cqe, ibwc, &polled,
2732 						  &stop);
2733 		}
2734 		if (expand)
2735 			goto expand_cqe;
2736 		if (stop)
2737 			goto stop_cqe;
2738 		/* clear qpn to avoid duplicate processing by discard_cqe() */
2739 		cqe->cmn.qpn = 0;
2740 skip_cqe:
2741 		polled_hw_cqes += 1;
2742 		cur_getp = (cur_getp + 1) % cq->max_hw_cqe;
2743 		ocrdma_change_cq_phase(cq, cqe, cur_getp);
2744 expand_cqe:
2745 		if (polled) {
2746 			num_entries -= 1;
2747 			i += 1;
2748 			ibwc = ibwc + 1;
2749 			polled = false;
2750 		}
2751 	}
2752 stop_cqe:
2753 	cq->getp = cur_getp;
2754 	if (cq->deferred_arm) {
2755 		ocrdma_ring_cq_db(dev, cq->id, true, cq->deferred_sol,
2756 				  polled_hw_cqes);
2757 		cq->deferred_arm = false;
2758 		cq->deferred_sol = false;
2759 	} else {
2760 		/* We need to pop the CQE. No need to arm */
2761 		ocrdma_ring_cq_db(dev, cq->id, false, cq->deferred_sol,
2762 				  polled_hw_cqes);
2763 		cq->deferred_sol = false;
2764 	}
2765 
2766 	return i;
2767 }
2768 
2769 /* insert error cqe if the QP's SQ or RQ's CQ matches the CQ under poll. */
2770 static int ocrdma_add_err_cqe(struct ocrdma_cq *cq, int num_entries,
2771 			      struct ocrdma_qp *qp, struct ib_wc *ibwc)
2772 {
2773 	int err_cqes = 0;
2774 
2775 	while (num_entries) {
2776 		if (is_hw_sq_empty(qp) && is_hw_rq_empty(qp))
2777 			break;
2778 		if (!is_hw_sq_empty(qp) && qp->sq_cq == cq) {
2779 			ocrdma_update_wc(qp, ibwc, qp->sq.tail);
2780 			ocrdma_hwq_inc_tail(&qp->sq);
2781 		} else if (!is_hw_rq_empty(qp) && qp->rq_cq == cq) {
2782 			ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail];
2783 			ocrdma_hwq_inc_tail(&qp->rq);
2784 		} else {
2785 			return err_cqes;
2786 		}
2787 		ibwc->byte_len = 0;
2788 		ibwc->status = IB_WC_WR_FLUSH_ERR;
2789 		ibwc = ibwc + 1;
2790 		err_cqes += 1;
2791 		num_entries -= 1;
2792 	}
2793 	return err_cqes;
2794 }
2795 
2796 int ocrdma_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
2797 {
2798 	int cqes_to_poll = num_entries;
2799 	struct ocrdma_cq *cq = get_ocrdma_cq(ibcq);
2800 	struct ocrdma_dev *dev = get_ocrdma_dev(ibcq->device);
2801 	int num_os_cqe = 0, err_cqes = 0;
2802 	struct ocrdma_qp *qp;
2803 	unsigned long flags;
2804 
2805 	/* poll cqes from adapter CQ */
2806 	spin_lock_irqsave(&cq->cq_lock, flags);
2807 	num_os_cqe = ocrdma_poll_hwcq(cq, cqes_to_poll, wc);
2808 	spin_unlock_irqrestore(&cq->cq_lock, flags);
2809 	cqes_to_poll -= num_os_cqe;
2810 
2811 	if (cqes_to_poll) {
2812 		wc = wc + num_os_cqe;
2813 		/* adapter returns single error cqe when qp moves to
2814 		 * error state. So insert error cqes with wc_status as
2815 		 * FLUSHED for pending WQEs and RQEs of QP's SQ and RQ
2816 		 * respectively which uses this CQ.
2817 		 */
2818 		spin_lock_irqsave(&dev->flush_q_lock, flags);
2819 		list_for_each_entry(qp, &cq->sq_head, sq_entry) {
2820 			if (cqes_to_poll == 0)
2821 				break;
2822 			err_cqes = ocrdma_add_err_cqe(cq, cqes_to_poll, qp, wc);
2823 			cqes_to_poll -= err_cqes;
2824 			num_os_cqe += err_cqes;
2825 			wc = wc + err_cqes;
2826 		}
2827 		spin_unlock_irqrestore(&dev->flush_q_lock, flags);
2828 	}
2829 	return num_os_cqe;
2830 }
2831 
2832 int ocrdma_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags cq_flags)
2833 {
2834 	struct ocrdma_cq *cq = get_ocrdma_cq(ibcq);
2835 	struct ocrdma_dev *dev = get_ocrdma_dev(ibcq->device);
2836 	u16 cq_id;
2837 	unsigned long flags;
2838 	bool arm_needed = false, sol_needed = false;
2839 
2840 	cq_id = cq->id;
2841 
2842 	spin_lock_irqsave(&cq->cq_lock, flags);
2843 	if (cq_flags & IB_CQ_NEXT_COMP || cq_flags & IB_CQ_SOLICITED)
2844 		arm_needed = true;
2845 	if (cq_flags & IB_CQ_SOLICITED)
2846 		sol_needed = true;
2847 
2848 	if (cq->first_arm) {
2849 		ocrdma_ring_cq_db(dev, cq_id, arm_needed, sol_needed, 0);
2850 		cq->first_arm = false;
2851 	}
2852 
2853 	cq->deferred_arm = true;
2854 	cq->deferred_sol = sol_needed;
2855 	spin_unlock_irqrestore(&cq->cq_lock, flags);
2856 
2857 	return 0;
2858 }
2859 
2860 struct ib_mr *ocrdma_alloc_frmr(struct ib_pd *ibpd, int max_page_list_len)
2861 {
2862 	int status;
2863 	struct ocrdma_mr *mr;
2864 	struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);
2865 	struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device);
2866 
2867 	if (max_page_list_len > dev->attr.max_pages_per_frmr)
2868 		return ERR_PTR(-EINVAL);
2869 
2870 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2871 	if (!mr)
2872 		return ERR_PTR(-ENOMEM);
2873 
2874 	status = ocrdma_get_pbl_info(dev, mr, max_page_list_len);
2875 	if (status)
2876 		goto pbl_err;
2877 	mr->hwmr.fr_mr = 1;
2878 	mr->hwmr.remote_rd = 0;
2879 	mr->hwmr.remote_wr = 0;
2880 	mr->hwmr.local_rd = 0;
2881 	mr->hwmr.local_wr = 0;
2882 	mr->hwmr.mw_bind = 0;
2883 	status = ocrdma_build_pbl_tbl(dev, &mr->hwmr);
2884 	if (status)
2885 		goto pbl_err;
2886 	status = ocrdma_reg_mr(dev, &mr->hwmr, pd->id, 0);
2887 	if (status)
2888 		goto mbx_err;
2889 	mr->ibmr.rkey = mr->hwmr.lkey;
2890 	mr->ibmr.lkey = mr->hwmr.lkey;
2891 	dev->stag_arr[(mr->hwmr.lkey >> 8) & (OCRDMA_MAX_STAG - 1)] =
2892 		(unsigned long) mr;
2893 	return &mr->ibmr;
2894 mbx_err:
2895 	ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr);
2896 pbl_err:
2897 	kfree(mr);
2898 	return ERR_PTR(-ENOMEM);
2899 }
2900 
2901 struct ib_fast_reg_page_list *ocrdma_alloc_frmr_page_list(struct ib_device
2902 							  *ibdev,
2903 							  int page_list_len)
2904 {
2905 	struct ib_fast_reg_page_list *frmr_list;
2906 	int size;
2907 
2908 	size = sizeof(*frmr_list) + (page_list_len * sizeof(u64));
2909 	frmr_list = kzalloc(size, GFP_KERNEL);
2910 	if (!frmr_list)
2911 		return ERR_PTR(-ENOMEM);
2912 	frmr_list->page_list = (u64 *)(frmr_list + 1);
2913 	return frmr_list;
2914 }
2915 
2916 void ocrdma_free_frmr_page_list(struct ib_fast_reg_page_list *page_list)
2917 {
2918 	kfree(page_list);
2919 }
2920 
2921 #define MAX_KERNEL_PBE_SIZE 65536
2922 static inline int count_kernel_pbes(struct ib_phys_buf *buf_list,
2923 				    int buf_cnt, u32 *pbe_size)
2924 {
2925 	u64 total_size = 0;
2926 	u64 buf_size = 0;
2927 	int i;
2928 	*pbe_size = roundup(buf_list[0].size, PAGE_SIZE);
2929 	*pbe_size = roundup_pow_of_two(*pbe_size);
2930 
2931 	/* find the smallest PBE size that we can have */
2932 	for (i = 0; i < buf_cnt; i++) {
2933 		/* first addr may not be page aligned, so ignore checking */
2934 		if ((i != 0) && ((buf_list[i].addr & ~PAGE_MASK) ||
2935 				 (buf_list[i].size & ~PAGE_MASK))) {
2936 			return 0;
2937 		}
2938 
2939 		/* if configured PBE size is greater then the chosen one,
2940 		 * reduce the PBE size.
2941 		 */
2942 		buf_size = roundup(buf_list[i].size, PAGE_SIZE);
2943 		/* pbe_size has to be even multiple of 4K 1,2,4,8...*/
2944 		buf_size = roundup_pow_of_two(buf_size);
2945 		if (*pbe_size > buf_size)
2946 			*pbe_size = buf_size;
2947 
2948 		total_size += buf_size;
2949 	}
2950 	*pbe_size = *pbe_size > MAX_KERNEL_PBE_SIZE ?
2951 	    (MAX_KERNEL_PBE_SIZE) : (*pbe_size);
2952 
2953 	/* num_pbes = total_size / (*pbe_size);  this is implemented below. */
2954 
2955 	return total_size >> ilog2(*pbe_size);
2956 }
2957 
2958 static void build_kernel_pbes(struct ib_phys_buf *buf_list, int ib_buf_cnt,
2959 			      u32 pbe_size, struct ocrdma_pbl *pbl_tbl,
2960 			      struct ocrdma_hw_mr *hwmr)
2961 {
2962 	int i;
2963 	int idx;
2964 	int pbes_per_buf = 0;
2965 	u64 buf_addr = 0;
2966 	int num_pbes;
2967 	struct ocrdma_pbe *pbe;
2968 	int total_num_pbes = 0;
2969 
2970 	if (!hwmr->num_pbes)
2971 		return;
2972 
2973 	pbe = (struct ocrdma_pbe *)pbl_tbl->va;
2974 	num_pbes = 0;
2975 
2976 	/* go through the OS phy regions & fill hw pbe entries into pbls. */
2977 	for (i = 0; i < ib_buf_cnt; i++) {
2978 		buf_addr = buf_list[i].addr;
2979 		pbes_per_buf =
2980 		    roundup_pow_of_two(roundup(buf_list[i].size, PAGE_SIZE)) /
2981 		    pbe_size;
2982 		hwmr->len += buf_list[i].size;
2983 		/* number of pbes can be more for one OS buf, when
2984 		 * buffers are of different sizes.
2985 		 * split the ib_buf to one or more pbes.
2986 		 */
2987 		for (idx = 0; idx < pbes_per_buf; idx++) {
2988 			/* we program always page aligned addresses,
2989 			 * first unaligned address is taken care by fbo.
2990 			 */
2991 			if (i == 0) {
2992 				/* for non zero fbo, assign the
2993 				 * start of the page.
2994 				 */
2995 				pbe->pa_lo =
2996 				    cpu_to_le32((u32) (buf_addr & PAGE_MASK));
2997 				pbe->pa_hi =
2998 				    cpu_to_le32((u32) upper_32_bits(buf_addr));
2999 			} else {
3000 				pbe->pa_lo =
3001 				    cpu_to_le32((u32) (buf_addr & 0xffffffff));
3002 				pbe->pa_hi =
3003 				    cpu_to_le32((u32) upper_32_bits(buf_addr));
3004 			}
3005 			buf_addr += pbe_size;
3006 			num_pbes += 1;
3007 			total_num_pbes += 1;
3008 			pbe++;
3009 
3010 			if (total_num_pbes == hwmr->num_pbes)
3011 				goto mr_tbl_done;
3012 			/* if the pbl is full storing the pbes,
3013 			 * move to next pbl.
3014 			 */
3015 			if (num_pbes == (hwmr->pbl_size/sizeof(u64))) {
3016 				pbl_tbl++;
3017 				pbe = (struct ocrdma_pbe *)pbl_tbl->va;
3018 				num_pbes = 0;
3019 			}
3020 		}
3021 	}
3022 mr_tbl_done:
3023 	return;
3024 }
3025 
3026 struct ib_mr *ocrdma_reg_kernel_mr(struct ib_pd *ibpd,
3027 				   struct ib_phys_buf *buf_list,
3028 				   int buf_cnt, int acc, u64 *iova_start)
3029 {
3030 	int status = -ENOMEM;
3031 	struct ocrdma_mr *mr;
3032 	struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);
3033 	struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device);
3034 	u32 num_pbes;
3035 	u32 pbe_size = 0;
3036 
3037 	if ((acc & IB_ACCESS_REMOTE_WRITE) && !(acc & IB_ACCESS_LOCAL_WRITE))
3038 		return ERR_PTR(-EINVAL);
3039 
3040 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
3041 	if (!mr)
3042 		return ERR_PTR(status);
3043 
3044 	num_pbes = count_kernel_pbes(buf_list, buf_cnt, &pbe_size);
3045 	if (num_pbes == 0) {
3046 		status = -EINVAL;
3047 		goto pbl_err;
3048 	}
3049 	status = ocrdma_get_pbl_info(dev, mr, num_pbes);
3050 	if (status)
3051 		goto pbl_err;
3052 
3053 	mr->hwmr.pbe_size = pbe_size;
3054 	mr->hwmr.fbo = *iova_start - (buf_list[0].addr & PAGE_MASK);
3055 	mr->hwmr.va = *iova_start;
3056 	mr->hwmr.local_rd = 1;
3057 	mr->hwmr.remote_wr = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
3058 	mr->hwmr.remote_rd = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
3059 	mr->hwmr.local_wr = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
3060 	mr->hwmr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
3061 	mr->hwmr.mw_bind = (acc & IB_ACCESS_MW_BIND) ? 1 : 0;
3062 
3063 	status = ocrdma_build_pbl_tbl(dev, &mr->hwmr);
3064 	if (status)
3065 		goto pbl_err;
3066 	build_kernel_pbes(buf_list, buf_cnt, pbe_size, mr->hwmr.pbl_table,
3067 			  &mr->hwmr);
3068 	status = ocrdma_reg_mr(dev, &mr->hwmr, pd->id, acc);
3069 	if (status)
3070 		goto mbx_err;
3071 
3072 	mr->ibmr.lkey = mr->hwmr.lkey;
3073 	if (mr->hwmr.remote_wr || mr->hwmr.remote_rd)
3074 		mr->ibmr.rkey = mr->hwmr.lkey;
3075 	return &mr->ibmr;
3076 
3077 mbx_err:
3078 	ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr);
3079 pbl_err:
3080 	kfree(mr);
3081 	return ERR_PTR(status);
3082 }
3083