1 // SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
2 /* Copyright (c) 2015 - 2021 Intel Corporation */
3 #include "main.h"
4 
5 /**
6  * irdma_arp_table -manage arp table
7  * @rf: RDMA PCI function
8  * @ip_addr: ip address for device
9  * @ipv4: IPv4 flag
10  * @mac_addr: mac address ptr
11  * @action: modify, delete or add
12  */
13 int irdma_arp_table(struct irdma_pci_f *rf, u32 *ip_addr, bool ipv4,
14 		    const u8 *mac_addr, u32 action)
15 {
16 	unsigned long flags;
17 	int arp_index;
18 	u32 ip[4] = {};
19 
20 	if (ipv4)
21 		ip[0] = *ip_addr;
22 	else
23 		memcpy(ip, ip_addr, sizeof(ip));
24 
25 	spin_lock_irqsave(&rf->arp_lock, flags);
26 	for (arp_index = 0; (u32)arp_index < rf->arp_table_size; arp_index++) {
27 		if (!memcmp(rf->arp_table[arp_index].ip_addr, ip, sizeof(ip)))
28 			break;
29 	}
30 
31 	switch (action) {
32 	case IRDMA_ARP_ADD:
33 		if (arp_index != rf->arp_table_size) {
34 			arp_index = -1;
35 			break;
36 		}
37 
38 		arp_index = 0;
39 		if (irdma_alloc_rsrc(rf, rf->allocated_arps, rf->arp_table_size,
40 				     (u32 *)&arp_index, &rf->next_arp_index)) {
41 			arp_index = -1;
42 			break;
43 		}
44 
45 		memcpy(rf->arp_table[arp_index].ip_addr, ip,
46 		       sizeof(rf->arp_table[arp_index].ip_addr));
47 		ether_addr_copy(rf->arp_table[arp_index].mac_addr, mac_addr);
48 		break;
49 	case IRDMA_ARP_RESOLVE:
50 		if (arp_index == rf->arp_table_size)
51 			arp_index = -1;
52 		break;
53 	case IRDMA_ARP_DELETE:
54 		if (arp_index == rf->arp_table_size) {
55 			arp_index = -1;
56 			break;
57 		}
58 
59 		memset(rf->arp_table[arp_index].ip_addr, 0,
60 		       sizeof(rf->arp_table[arp_index].ip_addr));
61 		eth_zero_addr(rf->arp_table[arp_index].mac_addr);
62 		irdma_free_rsrc(rf, rf->allocated_arps, arp_index);
63 		break;
64 	default:
65 		arp_index = -1;
66 		break;
67 	}
68 
69 	spin_unlock_irqrestore(&rf->arp_lock, flags);
70 	return arp_index;
71 }
72 
73 /**
74  * irdma_add_arp - add a new arp entry if needed
75  * @rf: RDMA function
76  * @ip: IP address
77  * @ipv4: IPv4 flag
78  * @mac: MAC address
79  */
80 int irdma_add_arp(struct irdma_pci_f *rf, u32 *ip, bool ipv4, const u8 *mac)
81 {
82 	int arpidx;
83 
84 	arpidx = irdma_arp_table(rf, &ip[0], ipv4, NULL, IRDMA_ARP_RESOLVE);
85 	if (arpidx >= 0) {
86 		if (ether_addr_equal(rf->arp_table[arpidx].mac_addr, mac))
87 			return arpidx;
88 
89 		irdma_manage_arp_cache(rf, rf->arp_table[arpidx].mac_addr, ip,
90 				       ipv4, IRDMA_ARP_DELETE);
91 	}
92 
93 	irdma_manage_arp_cache(rf, mac, ip, ipv4, IRDMA_ARP_ADD);
94 
95 	return irdma_arp_table(rf, ip, ipv4, NULL, IRDMA_ARP_RESOLVE);
96 }
97 
98 /**
99  * wr32 - write 32 bits to hw register
100  * @hw: hardware information including registers
101  * @reg: register offset
102  * @val: value to write to register
103  */
104 inline void wr32(struct irdma_hw *hw, u32 reg, u32 val)
105 {
106 	writel(val, hw->hw_addr + reg);
107 }
108 
109 /**
110  * rd32 - read a 32 bit hw register
111  * @hw: hardware information including registers
112  * @reg: register offset
113  *
114  * Return value of register content
115  */
116 inline u32 rd32(struct irdma_hw *hw, u32 reg)
117 {
118 	return readl(hw->hw_addr + reg);
119 }
120 
121 /**
122  * rd64 - read a 64 bit hw register
123  * @hw: hardware information including registers
124  * @reg: register offset
125  *
126  * Return value of register content
127  */
128 inline u64 rd64(struct irdma_hw *hw, u32 reg)
129 {
130 	return readq(hw->hw_addr + reg);
131 }
132 
133 static void irdma_gid_change_event(struct ib_device *ibdev)
134 {
135 	struct ib_event ib_event;
136 
137 	ib_event.event = IB_EVENT_GID_CHANGE;
138 	ib_event.device = ibdev;
139 	ib_event.element.port_num = 1;
140 	ib_dispatch_event(&ib_event);
141 }
142 
143 /**
144  * irdma_inetaddr_event - system notifier for ipv4 addr events
145  * @notifier: not used
146  * @event: event for notifier
147  * @ptr: if address
148  */
149 int irdma_inetaddr_event(struct notifier_block *notifier, unsigned long event,
150 			 void *ptr)
151 {
152 	struct in_ifaddr *ifa = ptr;
153 	struct net_device *netdev = ifa->ifa_dev->dev;
154 	struct irdma_device *iwdev;
155 	struct ib_device *ibdev;
156 	u32 local_ipaddr;
157 
158 	ibdev = ib_device_get_by_netdev(netdev, RDMA_DRIVER_IRDMA);
159 	if (!ibdev)
160 		return NOTIFY_DONE;
161 
162 	iwdev = to_iwdev(ibdev);
163 	local_ipaddr = ntohl(ifa->ifa_address);
164 	ibdev_dbg(&iwdev->ibdev,
165 		  "DEV: netdev %p event %lu local_ip=%pI4 MAC=%pM\n", netdev,
166 		  event, &local_ipaddr, netdev->dev_addr);
167 	switch (event) {
168 	case NETDEV_DOWN:
169 		irdma_manage_arp_cache(iwdev->rf, netdev->dev_addr,
170 				       &local_ipaddr, true, IRDMA_ARP_DELETE);
171 		irdma_if_notify(iwdev, netdev, &local_ipaddr, true, false);
172 		irdma_gid_change_event(&iwdev->ibdev);
173 		break;
174 	case NETDEV_UP:
175 	case NETDEV_CHANGEADDR:
176 		irdma_add_arp(iwdev->rf, &local_ipaddr, true, netdev->dev_addr);
177 		irdma_if_notify(iwdev, netdev, &local_ipaddr, true, true);
178 		irdma_gid_change_event(&iwdev->ibdev);
179 		break;
180 	default:
181 		break;
182 	}
183 
184 	ib_device_put(ibdev);
185 
186 	return NOTIFY_DONE;
187 }
188 
189 /**
190  * irdma_inet6addr_event - system notifier for ipv6 addr events
191  * @notifier: not used
192  * @event: event for notifier
193  * @ptr: if address
194  */
195 int irdma_inet6addr_event(struct notifier_block *notifier, unsigned long event,
196 			  void *ptr)
197 {
198 	struct inet6_ifaddr *ifa = ptr;
199 	struct net_device *netdev = ifa->idev->dev;
200 	struct irdma_device *iwdev;
201 	struct ib_device *ibdev;
202 	u32 local_ipaddr6[4];
203 
204 	ibdev = ib_device_get_by_netdev(netdev, RDMA_DRIVER_IRDMA);
205 	if (!ibdev)
206 		return NOTIFY_DONE;
207 
208 	iwdev = to_iwdev(ibdev);
209 	irdma_copy_ip_ntohl(local_ipaddr6, ifa->addr.in6_u.u6_addr32);
210 	ibdev_dbg(&iwdev->ibdev,
211 		  "DEV: netdev %p event %lu local_ip=%pI6 MAC=%pM\n", netdev,
212 		  event, local_ipaddr6, netdev->dev_addr);
213 	switch (event) {
214 	case NETDEV_DOWN:
215 		irdma_manage_arp_cache(iwdev->rf, netdev->dev_addr,
216 				       local_ipaddr6, false, IRDMA_ARP_DELETE);
217 		irdma_if_notify(iwdev, netdev, local_ipaddr6, false, false);
218 		irdma_gid_change_event(&iwdev->ibdev);
219 		break;
220 	case NETDEV_UP:
221 	case NETDEV_CHANGEADDR:
222 		irdma_add_arp(iwdev->rf, local_ipaddr6, false,
223 			      netdev->dev_addr);
224 		irdma_if_notify(iwdev, netdev, local_ipaddr6, false, true);
225 		irdma_gid_change_event(&iwdev->ibdev);
226 		break;
227 	default:
228 		break;
229 	}
230 
231 	ib_device_put(ibdev);
232 
233 	return NOTIFY_DONE;
234 }
235 
236 /**
237  * irdma_net_event - system notifier for net events
238  * @notifier: not used
239  * @event: event for notifier
240  * @ptr: neighbor
241  */
242 int irdma_net_event(struct notifier_block *notifier, unsigned long event,
243 		    void *ptr)
244 {
245 	struct neighbour *neigh = ptr;
246 	struct irdma_device *iwdev;
247 	struct ib_device *ibdev;
248 	__be32 *p;
249 	u32 local_ipaddr[4] = {};
250 	bool ipv4 = true;
251 
252 	ibdev = ib_device_get_by_netdev((struct net_device *)neigh->dev,
253 					RDMA_DRIVER_IRDMA);
254 	if (!ibdev)
255 		return NOTIFY_DONE;
256 
257 	iwdev = to_iwdev(ibdev);
258 
259 	switch (event) {
260 	case NETEVENT_NEIGH_UPDATE:
261 		p = (__be32 *)neigh->primary_key;
262 		if (neigh->tbl->family == AF_INET6) {
263 			ipv4 = false;
264 			irdma_copy_ip_ntohl(local_ipaddr, p);
265 		} else {
266 			local_ipaddr[0] = ntohl(*p);
267 		}
268 
269 		ibdev_dbg(&iwdev->ibdev,
270 			  "DEV: netdev %p state %d local_ip=%pI4 MAC=%pM\n",
271 			  iwdev->netdev, neigh->nud_state, local_ipaddr,
272 			  neigh->ha);
273 
274 		if (neigh->nud_state & NUD_VALID)
275 			irdma_add_arp(iwdev->rf, local_ipaddr, ipv4, neigh->ha);
276 
277 		else
278 			irdma_manage_arp_cache(iwdev->rf, neigh->ha,
279 					       local_ipaddr, ipv4,
280 					       IRDMA_ARP_DELETE);
281 		break;
282 	default:
283 		break;
284 	}
285 
286 	ib_device_put(ibdev);
287 
288 	return NOTIFY_DONE;
289 }
290 
291 /**
292  * irdma_netdevice_event - system notifier for netdev events
293  * @notifier: not used
294  * @event: event for notifier
295  * @ptr: netdev
296  */
297 int irdma_netdevice_event(struct notifier_block *notifier, unsigned long event,
298 			  void *ptr)
299 {
300 	struct irdma_device *iwdev;
301 	struct ib_device *ibdev;
302 	struct net_device *netdev = netdev_notifier_info_to_dev(ptr);
303 
304 	ibdev = ib_device_get_by_netdev(netdev, RDMA_DRIVER_IRDMA);
305 	if (!ibdev)
306 		return NOTIFY_DONE;
307 
308 	iwdev = to_iwdev(ibdev);
309 	iwdev->iw_status = 1;
310 	switch (event) {
311 	case NETDEV_DOWN:
312 		iwdev->iw_status = 0;
313 		fallthrough;
314 	case NETDEV_UP:
315 		irdma_port_ibevent(iwdev);
316 		break;
317 	default:
318 		break;
319 	}
320 	ib_device_put(ibdev);
321 
322 	return NOTIFY_DONE;
323 }
324 
325 /**
326  * irdma_add_ipv6_addr - add ipv6 address to the hw arp table
327  * @iwdev: irdma device
328  */
329 static void irdma_add_ipv6_addr(struct irdma_device *iwdev)
330 {
331 	struct net_device *ip_dev;
332 	struct inet6_dev *idev;
333 	struct inet6_ifaddr *ifp, *tmp;
334 	u32 local_ipaddr6[4];
335 
336 	rcu_read_lock();
337 	for_each_netdev_rcu (&init_net, ip_dev) {
338 		if (((rdma_vlan_dev_vlan_id(ip_dev) < 0xFFFF &&
339 		      rdma_vlan_dev_real_dev(ip_dev) == iwdev->netdev) ||
340 		      ip_dev == iwdev->netdev) &&
341 		      (READ_ONCE(ip_dev->flags) & IFF_UP)) {
342 			idev = __in6_dev_get(ip_dev);
343 			if (!idev) {
344 				ibdev_err(&iwdev->ibdev, "ipv6 inet device not found\n");
345 				break;
346 			}
347 			list_for_each_entry_safe (ifp, tmp, &idev->addr_list,
348 						  if_list) {
349 				ibdev_dbg(&iwdev->ibdev,
350 					  "INIT: IP=%pI6, vlan_id=%d, MAC=%pM\n",
351 					  &ifp->addr,
352 					  rdma_vlan_dev_vlan_id(ip_dev),
353 					  ip_dev->dev_addr);
354 
355 				irdma_copy_ip_ntohl(local_ipaddr6,
356 						    ifp->addr.in6_u.u6_addr32);
357 				irdma_manage_arp_cache(iwdev->rf,
358 						       ip_dev->dev_addr,
359 						       local_ipaddr6, false,
360 						       IRDMA_ARP_ADD);
361 			}
362 		}
363 	}
364 	rcu_read_unlock();
365 }
366 
367 /**
368  * irdma_add_ipv4_addr - add ipv4 address to the hw arp table
369  * @iwdev: irdma device
370  */
371 static void irdma_add_ipv4_addr(struct irdma_device *iwdev)
372 {
373 	struct net_device *dev;
374 	struct in_device *idev;
375 	u32 ip_addr;
376 
377 	rcu_read_lock();
378 	for_each_netdev_rcu (&init_net, dev) {
379 		if (((rdma_vlan_dev_vlan_id(dev) < 0xFFFF &&
380 		      rdma_vlan_dev_real_dev(dev) == iwdev->netdev) ||
381 		      dev == iwdev->netdev) && (READ_ONCE(dev->flags) & IFF_UP)) {
382 			const struct in_ifaddr *ifa;
383 
384 			idev = __in_dev_get_rcu(dev);
385 			if (!idev)
386 				continue;
387 
388 			in_dev_for_each_ifa_rcu(ifa, idev) {
389 				ibdev_dbg(&iwdev->ibdev, "CM: IP=%pI4, vlan_id=%d, MAC=%pM\n",
390 					  &ifa->ifa_address, rdma_vlan_dev_vlan_id(dev),
391 					  dev->dev_addr);
392 
393 				ip_addr = ntohl(ifa->ifa_address);
394 				irdma_manage_arp_cache(iwdev->rf, dev->dev_addr,
395 						       &ip_addr, true,
396 						       IRDMA_ARP_ADD);
397 			}
398 		}
399 	}
400 	rcu_read_unlock();
401 }
402 
403 /**
404  * irdma_add_ip - add ip addresses
405  * @iwdev: irdma device
406  *
407  * Add ipv4/ipv6 addresses to the arp cache
408  */
409 void irdma_add_ip(struct irdma_device *iwdev)
410 {
411 	irdma_add_ipv4_addr(iwdev);
412 	irdma_add_ipv6_addr(iwdev);
413 }
414 
415 /**
416  * irdma_alloc_and_get_cqp_request - get cqp struct
417  * @cqp: device cqp ptr
418  * @wait: cqp to be used in wait mode
419  */
420 struct irdma_cqp_request *irdma_alloc_and_get_cqp_request(struct irdma_cqp *cqp,
421 							  bool wait)
422 {
423 	struct irdma_cqp_request *cqp_request = NULL;
424 	unsigned long flags;
425 
426 	spin_lock_irqsave(&cqp->req_lock, flags);
427 	if (!list_empty(&cqp->cqp_avail_reqs)) {
428 		cqp_request = list_first_entry(&cqp->cqp_avail_reqs,
429 					       struct irdma_cqp_request, list);
430 		list_del_init(&cqp_request->list);
431 	}
432 	spin_unlock_irqrestore(&cqp->req_lock, flags);
433 	if (!cqp_request) {
434 		cqp_request = kzalloc(sizeof(*cqp_request), GFP_ATOMIC);
435 		if (cqp_request) {
436 			cqp_request->dynamic = true;
437 			if (wait)
438 				init_waitqueue_head(&cqp_request->waitq);
439 		}
440 	}
441 	if (!cqp_request) {
442 		ibdev_dbg(to_ibdev(cqp->sc_cqp.dev), "ERR: CQP Request Fail: No Memory");
443 		return NULL;
444 	}
445 
446 	cqp_request->waiting = wait;
447 	refcount_set(&cqp_request->refcnt, 1);
448 	memset(&cqp_request->compl_info, 0, sizeof(cqp_request->compl_info));
449 
450 	return cqp_request;
451 }
452 
453 /**
454  * irdma_get_cqp_request - increase refcount for cqp_request
455  * @cqp_request: pointer to cqp_request instance
456  */
457 static inline void irdma_get_cqp_request(struct irdma_cqp_request *cqp_request)
458 {
459 	refcount_inc(&cqp_request->refcnt);
460 }
461 
462 /**
463  * irdma_free_cqp_request - free cqp request
464  * @cqp: cqp ptr
465  * @cqp_request: to be put back in cqp list
466  */
467 void irdma_free_cqp_request(struct irdma_cqp *cqp,
468 			    struct irdma_cqp_request *cqp_request)
469 {
470 	unsigned long flags;
471 
472 	if (cqp_request->dynamic) {
473 		kfree(cqp_request);
474 	} else {
475 		cqp_request->request_done = false;
476 		cqp_request->callback_fcn = NULL;
477 		cqp_request->waiting = false;
478 
479 		spin_lock_irqsave(&cqp->req_lock, flags);
480 		list_add_tail(&cqp_request->list, &cqp->cqp_avail_reqs);
481 		spin_unlock_irqrestore(&cqp->req_lock, flags);
482 	}
483 	wake_up(&cqp->remove_wq);
484 }
485 
486 /**
487  * irdma_put_cqp_request - dec ref count and free if 0
488  * @cqp: cqp ptr
489  * @cqp_request: to be put back in cqp list
490  */
491 void irdma_put_cqp_request(struct irdma_cqp *cqp,
492 			   struct irdma_cqp_request *cqp_request)
493 {
494 	if (refcount_dec_and_test(&cqp_request->refcnt))
495 		irdma_free_cqp_request(cqp, cqp_request);
496 }
497 
498 /**
499  * irdma_free_pending_cqp_request -free pending cqp request objs
500  * @cqp: cqp ptr
501  * @cqp_request: to be put back in cqp list
502  */
503 static void
504 irdma_free_pending_cqp_request(struct irdma_cqp *cqp,
505 			       struct irdma_cqp_request *cqp_request)
506 {
507 	if (cqp_request->waiting) {
508 		cqp_request->compl_info.error = true;
509 		cqp_request->request_done = true;
510 		wake_up(&cqp_request->waitq);
511 	}
512 	wait_event_timeout(cqp->remove_wq,
513 			   refcount_read(&cqp_request->refcnt) == 1, 1000);
514 	irdma_put_cqp_request(cqp, cqp_request);
515 }
516 
517 /**
518  * irdma_cleanup_pending_cqp_op - clean-up cqp with no
519  * completions
520  * @rf: RDMA PCI function
521  */
522 void irdma_cleanup_pending_cqp_op(struct irdma_pci_f *rf)
523 {
524 	struct irdma_sc_dev *dev = &rf->sc_dev;
525 	struct irdma_cqp *cqp = &rf->cqp;
526 	struct irdma_cqp_request *cqp_request = NULL;
527 	struct cqp_cmds_info *pcmdinfo = NULL;
528 	u32 i, pending_work, wqe_idx;
529 
530 	pending_work = IRDMA_RING_USED_QUANTA(cqp->sc_cqp.sq_ring);
531 	wqe_idx = IRDMA_RING_CURRENT_TAIL(cqp->sc_cqp.sq_ring);
532 	for (i = 0; i < pending_work; i++) {
533 		cqp_request = (struct irdma_cqp_request *)(unsigned long)
534 				      cqp->scratch_array[wqe_idx];
535 		if (cqp_request)
536 			irdma_free_pending_cqp_request(cqp, cqp_request);
537 		wqe_idx = (wqe_idx + 1) % IRDMA_RING_SIZE(cqp->sc_cqp.sq_ring);
538 	}
539 
540 	while (!list_empty(&dev->cqp_cmd_head)) {
541 		pcmdinfo = irdma_remove_cqp_head(dev);
542 		cqp_request =
543 			container_of(pcmdinfo, struct irdma_cqp_request, info);
544 		if (cqp_request)
545 			irdma_free_pending_cqp_request(cqp, cqp_request);
546 	}
547 }
548 
549 /**
550  * irdma_wait_event - wait for completion
551  * @rf: RDMA PCI function
552  * @cqp_request: cqp request to wait
553  */
554 static enum irdma_status_code irdma_wait_event(struct irdma_pci_f *rf,
555 					       struct irdma_cqp_request *cqp_request)
556 {
557 	struct irdma_cqp_timeout cqp_timeout = {};
558 	bool cqp_error = false;
559 	enum irdma_status_code err_code = 0;
560 
561 	cqp_timeout.compl_cqp_cmds = rf->sc_dev.cqp_cmd_stats[IRDMA_OP_CMPL_CMDS];
562 	do {
563 		irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq);
564 		if (wait_event_timeout(cqp_request->waitq,
565 				       cqp_request->request_done,
566 				       msecs_to_jiffies(CQP_COMPL_WAIT_TIME_MS)))
567 			break;
568 
569 		irdma_check_cqp_progress(&cqp_timeout, &rf->sc_dev);
570 
571 		if (cqp_timeout.count < CQP_TIMEOUT_THRESHOLD)
572 			continue;
573 
574 		if (!rf->reset) {
575 			rf->reset = true;
576 			rf->gen_ops.request_reset(rf);
577 		}
578 		return IRDMA_ERR_TIMEOUT;
579 	} while (1);
580 
581 	cqp_error = cqp_request->compl_info.error;
582 	if (cqp_error) {
583 		err_code = IRDMA_ERR_CQP_COMPL_ERROR;
584 		if (cqp_request->compl_info.maj_err_code == 0xFFFF &&
585 		    cqp_request->compl_info.min_err_code == 0x8029) {
586 			if (!rf->reset) {
587 				rf->reset = true;
588 				rf->gen_ops.request_reset(rf);
589 			}
590 		}
591 	}
592 
593 	return err_code;
594 }
595 
596 static const char *const irdma_cqp_cmd_names[IRDMA_MAX_CQP_OPS] = {
597 	[IRDMA_OP_CEQ_DESTROY] = "Destroy CEQ Cmd",
598 	[IRDMA_OP_AEQ_DESTROY] = "Destroy AEQ Cmd",
599 	[IRDMA_OP_DELETE_ARP_CACHE_ENTRY] = "Delete ARP Cache Cmd",
600 	[IRDMA_OP_MANAGE_APBVT_ENTRY] = "Manage APBV Table Entry Cmd",
601 	[IRDMA_OP_CEQ_CREATE] = "CEQ Create Cmd",
602 	[IRDMA_OP_AEQ_CREATE] = "AEQ Destroy Cmd",
603 	[IRDMA_OP_MANAGE_QHASH_TABLE_ENTRY] = "Manage Quad Hash Table Entry Cmd",
604 	[IRDMA_OP_QP_MODIFY] = "Modify QP Cmd",
605 	[IRDMA_OP_QP_UPLOAD_CONTEXT] = "Upload Context Cmd",
606 	[IRDMA_OP_CQ_CREATE] = "Create CQ Cmd",
607 	[IRDMA_OP_CQ_DESTROY] = "Destroy CQ Cmd",
608 	[IRDMA_OP_QP_CREATE] = "Create QP Cmd",
609 	[IRDMA_OP_QP_DESTROY] = "Destroy QP Cmd",
610 	[IRDMA_OP_ALLOC_STAG] = "Allocate STag Cmd",
611 	[IRDMA_OP_MR_REG_NON_SHARED] = "Register Non-Shared MR Cmd",
612 	[IRDMA_OP_DEALLOC_STAG] = "Deallocate STag Cmd",
613 	[IRDMA_OP_MW_ALLOC] = "Allocate Memory Window Cmd",
614 	[IRDMA_OP_QP_FLUSH_WQES] = "Flush QP Cmd",
615 	[IRDMA_OP_ADD_ARP_CACHE_ENTRY] = "Add ARP Cache Cmd",
616 	[IRDMA_OP_MANAGE_PUSH_PAGE] = "Manage Push Page Cmd",
617 	[IRDMA_OP_UPDATE_PE_SDS] = "Update PE SDs Cmd",
618 	[IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE] = "Manage HMC PM Function Table Cmd",
619 	[IRDMA_OP_SUSPEND] = "Suspend QP Cmd",
620 	[IRDMA_OP_RESUME] = "Resume QP Cmd",
621 	[IRDMA_OP_MANAGE_VF_PBLE_BP] = "Manage VF PBLE Backing Pages Cmd",
622 	[IRDMA_OP_QUERY_FPM_VAL] = "Query FPM Values Cmd",
623 	[IRDMA_OP_COMMIT_FPM_VAL] = "Commit FPM Values Cmd",
624 	[IRDMA_OP_AH_CREATE] = "Create Address Handle Cmd",
625 	[IRDMA_OP_AH_MODIFY] = "Modify Address Handle Cmd",
626 	[IRDMA_OP_AH_DESTROY] = "Destroy Address Handle Cmd",
627 	[IRDMA_OP_MC_CREATE] = "Create Multicast Group Cmd",
628 	[IRDMA_OP_MC_DESTROY] = "Destroy Multicast Group Cmd",
629 	[IRDMA_OP_MC_MODIFY] = "Modify Multicast Group Cmd",
630 	[IRDMA_OP_STATS_ALLOCATE] = "Add Statistics Instance Cmd",
631 	[IRDMA_OP_STATS_FREE] = "Free Statistics Instance Cmd",
632 	[IRDMA_OP_STATS_GATHER] = "Gather Statistics Cmd",
633 	[IRDMA_OP_WS_ADD_NODE] = "Add Work Scheduler Node Cmd",
634 	[IRDMA_OP_WS_MODIFY_NODE] = "Modify Work Scheduler Node Cmd",
635 	[IRDMA_OP_WS_DELETE_NODE] = "Delete Work Scheduler Node Cmd",
636 	[IRDMA_OP_SET_UP_MAP] = "Set UP-UP Mapping Cmd",
637 	[IRDMA_OP_GEN_AE] = "Generate AE Cmd",
638 	[IRDMA_OP_QUERY_RDMA_FEATURES] = "RDMA Get Features Cmd",
639 	[IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY] = "Allocate Local MAC Entry Cmd",
640 	[IRDMA_OP_ADD_LOCAL_MAC_ENTRY] = "Add Local MAC Entry Cmd",
641 	[IRDMA_OP_DELETE_LOCAL_MAC_ENTRY] = "Delete Local MAC Entry Cmd",
642 	[IRDMA_OP_CQ_MODIFY] = "CQ Modify Cmd",
643 };
644 
645 static const struct irdma_cqp_err_info irdma_noncrit_err_list[] = {
646 	{0xffff, 0x8006, "Flush No Wqe Pending"},
647 	{0xffff, 0x8007, "Modify QP Bad Close"},
648 	{0xffff, 0x8009, "LLP Closed"},
649 	{0xffff, 0x800a, "Reset Not Sent"}
650 };
651 
652 /**
653  * irdma_cqp_crit_err - check if CQP error is critical
654  * @dev: pointer to dev structure
655  * @cqp_cmd: code for last CQP operation
656  * @maj_err_code: major error code
657  * @min_err_code: minot error code
658  */
659 bool irdma_cqp_crit_err(struct irdma_sc_dev *dev, u8 cqp_cmd,
660 			u16 maj_err_code, u16 min_err_code)
661 {
662 	int i;
663 
664 	for (i = 0; i < ARRAY_SIZE(irdma_noncrit_err_list); ++i) {
665 		if (maj_err_code == irdma_noncrit_err_list[i].maj &&
666 		    min_err_code == irdma_noncrit_err_list[i].min) {
667 			ibdev_dbg(to_ibdev(dev),
668 				  "CQP: [%s Error][%s] maj=0x%x min=0x%x\n",
669 				  irdma_noncrit_err_list[i].desc,
670 				  irdma_cqp_cmd_names[cqp_cmd], maj_err_code,
671 				  min_err_code);
672 			return false;
673 		}
674 	}
675 	return true;
676 }
677 
678 /**
679  * irdma_handle_cqp_op - process cqp command
680  * @rf: RDMA PCI function
681  * @cqp_request: cqp request to process
682  */
683 enum irdma_status_code irdma_handle_cqp_op(struct irdma_pci_f *rf,
684 					   struct irdma_cqp_request *cqp_request)
685 {
686 	struct irdma_sc_dev *dev = &rf->sc_dev;
687 	struct cqp_cmds_info *info = &cqp_request->info;
688 	enum irdma_status_code status;
689 	bool put_cqp_request = true;
690 
691 	if (rf->reset)
692 		return IRDMA_ERR_NOT_READY;
693 
694 	irdma_get_cqp_request(cqp_request);
695 	status = irdma_process_cqp_cmd(dev, info);
696 	if (status)
697 		goto err;
698 
699 	if (cqp_request->waiting) {
700 		put_cqp_request = false;
701 		status = irdma_wait_event(rf, cqp_request);
702 		if (status)
703 			goto err;
704 	}
705 
706 	return 0;
707 
708 err:
709 	if (irdma_cqp_crit_err(dev, info->cqp_cmd,
710 			       cqp_request->compl_info.maj_err_code,
711 			       cqp_request->compl_info.min_err_code))
712 		ibdev_err(&rf->iwdev->ibdev,
713 			  "[%s Error][op_code=%d] status=%d waiting=%d completion_err=%d maj=0x%x min=0x%x\n",
714 			  irdma_cqp_cmd_names[info->cqp_cmd], info->cqp_cmd, status, cqp_request->waiting,
715 			  cqp_request->compl_info.error, cqp_request->compl_info.maj_err_code,
716 			  cqp_request->compl_info.min_err_code);
717 
718 	if (put_cqp_request)
719 		irdma_put_cqp_request(&rf->cqp, cqp_request);
720 
721 	return status;
722 }
723 
724 void irdma_qp_add_ref(struct ib_qp *ibqp)
725 {
726 	struct irdma_qp *iwqp = (struct irdma_qp *)ibqp;
727 
728 	refcount_inc(&iwqp->refcnt);
729 }
730 
731 void irdma_qp_rem_ref(struct ib_qp *ibqp)
732 {
733 	struct irdma_qp *iwqp = to_iwqp(ibqp);
734 	struct irdma_device *iwdev = iwqp->iwdev;
735 	u32 qp_num;
736 	unsigned long flags;
737 
738 	spin_lock_irqsave(&iwdev->rf->qptable_lock, flags);
739 	if (!refcount_dec_and_test(&iwqp->refcnt)) {
740 		spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags);
741 		return;
742 	}
743 
744 	qp_num = iwqp->ibqp.qp_num;
745 	iwdev->rf->qp_table[qp_num] = NULL;
746 	spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags);
747 	complete(&iwqp->free_qp);
748 }
749 
750 struct ib_device *to_ibdev(struct irdma_sc_dev *dev)
751 {
752 	return &(container_of(dev, struct irdma_pci_f, sc_dev))->iwdev->ibdev;
753 }
754 
755 /**
756  * irdma_get_qp - get qp address
757  * @device: iwarp device
758  * @qpn: qp number
759  */
760 struct ib_qp *irdma_get_qp(struct ib_device *device, int qpn)
761 {
762 	struct irdma_device *iwdev = to_iwdev(device);
763 
764 	if (qpn < IW_FIRST_QPN || qpn >= iwdev->rf->max_qp)
765 		return NULL;
766 
767 	return &iwdev->rf->qp_table[qpn]->ibqp;
768 }
769 
770 /**
771  * irdma_remove_cqp_head - return head entry and remove
772  * @dev: device
773  */
774 void *irdma_remove_cqp_head(struct irdma_sc_dev *dev)
775 {
776 	struct list_head *entry;
777 	struct list_head *list = &dev->cqp_cmd_head;
778 
779 	if (list_empty(list))
780 		return NULL;
781 
782 	entry = list->next;
783 	list_del(entry);
784 
785 	return entry;
786 }
787 
788 /**
789  * irdma_cqp_sds_cmd - create cqp command for sd
790  * @dev: hardware control device structure
791  * @sdinfo: information for sd cqp
792  *
793  */
794 enum irdma_status_code irdma_cqp_sds_cmd(struct irdma_sc_dev *dev,
795 					 struct irdma_update_sds_info *sdinfo)
796 {
797 	struct irdma_cqp_request *cqp_request;
798 	struct cqp_cmds_info *cqp_info;
799 	struct irdma_pci_f *rf = dev_to_rf(dev);
800 	enum irdma_status_code status;
801 
802 	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
803 	if (!cqp_request)
804 		return IRDMA_ERR_NO_MEMORY;
805 
806 	cqp_info = &cqp_request->info;
807 	memcpy(&cqp_info->in.u.update_pe_sds.info, sdinfo,
808 	       sizeof(cqp_info->in.u.update_pe_sds.info));
809 	cqp_info->cqp_cmd = IRDMA_OP_UPDATE_PE_SDS;
810 	cqp_info->post_sq = 1;
811 	cqp_info->in.u.update_pe_sds.dev = dev;
812 	cqp_info->in.u.update_pe_sds.scratch = (uintptr_t)cqp_request;
813 
814 	status = irdma_handle_cqp_op(rf, cqp_request);
815 	irdma_put_cqp_request(&rf->cqp, cqp_request);
816 
817 	return status;
818 }
819 
820 /**
821  * irdma_cqp_qp_suspend_resume - cqp command for suspend/resume
822  * @qp: hardware control qp
823  * @op: suspend or resume
824  */
825 enum irdma_status_code irdma_cqp_qp_suspend_resume(struct irdma_sc_qp *qp,
826 						   u8 op)
827 {
828 	struct irdma_sc_dev *dev = qp->dev;
829 	struct irdma_cqp_request *cqp_request;
830 	struct irdma_sc_cqp *cqp = dev->cqp;
831 	struct cqp_cmds_info *cqp_info;
832 	struct irdma_pci_f *rf = dev_to_rf(dev);
833 	enum irdma_status_code status;
834 
835 	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false);
836 	if (!cqp_request)
837 		return IRDMA_ERR_NO_MEMORY;
838 
839 	cqp_info = &cqp_request->info;
840 	cqp_info->cqp_cmd = op;
841 	cqp_info->in.u.suspend_resume.cqp = cqp;
842 	cqp_info->in.u.suspend_resume.qp = qp;
843 	cqp_info->in.u.suspend_resume.scratch = (uintptr_t)cqp_request;
844 
845 	status = irdma_handle_cqp_op(rf, cqp_request);
846 	irdma_put_cqp_request(&rf->cqp, cqp_request);
847 
848 	return status;
849 }
850 
851 /**
852  * irdma_term_modify_qp - modify qp for term message
853  * @qp: hardware control qp
854  * @next_state: qp's next state
855  * @term: terminate code
856  * @term_len: length
857  */
858 void irdma_term_modify_qp(struct irdma_sc_qp *qp, u8 next_state, u8 term,
859 			  u8 term_len)
860 {
861 	struct irdma_qp *iwqp;
862 
863 	iwqp = qp->qp_uk.back_qp;
864 	irdma_next_iw_state(iwqp, next_state, 0, term, term_len);
865 };
866 
867 /**
868  * irdma_terminate_done - after terminate is completed
869  * @qp: hardware control qp
870  * @timeout_occurred: indicates if terminate timer expired
871  */
872 void irdma_terminate_done(struct irdma_sc_qp *qp, int timeout_occurred)
873 {
874 	struct irdma_qp *iwqp;
875 	u8 hte = 0;
876 	bool first_time;
877 	unsigned long flags;
878 
879 	iwqp = qp->qp_uk.back_qp;
880 	spin_lock_irqsave(&iwqp->lock, flags);
881 	if (iwqp->hte_added) {
882 		iwqp->hte_added = 0;
883 		hte = 1;
884 	}
885 	first_time = !(qp->term_flags & IRDMA_TERM_DONE);
886 	qp->term_flags |= IRDMA_TERM_DONE;
887 	spin_unlock_irqrestore(&iwqp->lock, flags);
888 	if (first_time) {
889 		if (!timeout_occurred)
890 			irdma_terminate_del_timer(qp);
891 
892 		irdma_next_iw_state(iwqp, IRDMA_QP_STATE_ERROR, hte, 0, 0);
893 		irdma_cm_disconn(iwqp);
894 	}
895 }
896 
897 static void irdma_terminate_timeout(struct timer_list *t)
898 {
899 	struct irdma_qp *iwqp = from_timer(iwqp, t, terminate_timer);
900 	struct irdma_sc_qp *qp = &iwqp->sc_qp;
901 
902 	irdma_terminate_done(qp, 1);
903 	irdma_qp_rem_ref(&iwqp->ibqp);
904 }
905 
906 /**
907  * irdma_terminate_start_timer - start terminate timeout
908  * @qp: hardware control qp
909  */
910 void irdma_terminate_start_timer(struct irdma_sc_qp *qp)
911 {
912 	struct irdma_qp *iwqp;
913 
914 	iwqp = qp->qp_uk.back_qp;
915 	irdma_qp_add_ref(&iwqp->ibqp);
916 	timer_setup(&iwqp->terminate_timer, irdma_terminate_timeout, 0);
917 	iwqp->terminate_timer.expires = jiffies + HZ;
918 
919 	add_timer(&iwqp->terminate_timer);
920 }
921 
922 /**
923  * irdma_terminate_del_timer - delete terminate timeout
924  * @qp: hardware control qp
925  */
926 void irdma_terminate_del_timer(struct irdma_sc_qp *qp)
927 {
928 	struct irdma_qp *iwqp;
929 	int ret;
930 
931 	iwqp = qp->qp_uk.back_qp;
932 	ret = del_timer(&iwqp->terminate_timer);
933 	if (ret)
934 		irdma_qp_rem_ref(&iwqp->ibqp);
935 }
936 
937 /**
938  * irdma_cqp_query_fpm_val_cmd - send cqp command for fpm
939  * @dev: function device struct
940  * @val_mem: buffer for fpm
941  * @hmc_fn_id: function id for fpm
942  */
943 enum irdma_status_code
944 irdma_cqp_query_fpm_val_cmd(struct irdma_sc_dev *dev,
945 			    struct irdma_dma_mem *val_mem, u8 hmc_fn_id)
946 {
947 	struct irdma_cqp_request *cqp_request;
948 	struct cqp_cmds_info *cqp_info;
949 	struct irdma_pci_f *rf = dev_to_rf(dev);
950 	enum irdma_status_code status;
951 
952 	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
953 	if (!cqp_request)
954 		return IRDMA_ERR_NO_MEMORY;
955 
956 	cqp_info = &cqp_request->info;
957 	cqp_request->param = NULL;
958 	cqp_info->in.u.query_fpm_val.cqp = dev->cqp;
959 	cqp_info->in.u.query_fpm_val.fpm_val_pa = val_mem->pa;
960 	cqp_info->in.u.query_fpm_val.fpm_val_va = val_mem->va;
961 	cqp_info->in.u.query_fpm_val.hmc_fn_id = hmc_fn_id;
962 	cqp_info->cqp_cmd = IRDMA_OP_QUERY_FPM_VAL;
963 	cqp_info->post_sq = 1;
964 	cqp_info->in.u.query_fpm_val.scratch = (uintptr_t)cqp_request;
965 
966 	status = irdma_handle_cqp_op(rf, cqp_request);
967 	irdma_put_cqp_request(&rf->cqp, cqp_request);
968 
969 	return status;
970 }
971 
972 /**
973  * irdma_cqp_commit_fpm_val_cmd - commit fpm values in hw
974  * @dev: hardware control device structure
975  * @val_mem: buffer with fpm values
976  * @hmc_fn_id: function id for fpm
977  */
978 enum irdma_status_code
979 irdma_cqp_commit_fpm_val_cmd(struct irdma_sc_dev *dev,
980 			     struct irdma_dma_mem *val_mem, u8 hmc_fn_id)
981 {
982 	struct irdma_cqp_request *cqp_request;
983 	struct cqp_cmds_info *cqp_info;
984 	struct irdma_pci_f *rf = dev_to_rf(dev);
985 	enum irdma_status_code status;
986 
987 	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
988 	if (!cqp_request)
989 		return IRDMA_ERR_NO_MEMORY;
990 
991 	cqp_info = &cqp_request->info;
992 	cqp_request->param = NULL;
993 	cqp_info->in.u.commit_fpm_val.cqp = dev->cqp;
994 	cqp_info->in.u.commit_fpm_val.fpm_val_pa = val_mem->pa;
995 	cqp_info->in.u.commit_fpm_val.fpm_val_va = val_mem->va;
996 	cqp_info->in.u.commit_fpm_val.hmc_fn_id = hmc_fn_id;
997 	cqp_info->cqp_cmd = IRDMA_OP_COMMIT_FPM_VAL;
998 	cqp_info->post_sq = 1;
999 	cqp_info->in.u.commit_fpm_val.scratch = (uintptr_t)cqp_request;
1000 
1001 	status = irdma_handle_cqp_op(rf, cqp_request);
1002 	irdma_put_cqp_request(&rf->cqp, cqp_request);
1003 
1004 	return status;
1005 }
1006 
1007 /**
1008  * irdma_cqp_cq_create_cmd - create a cq for the cqp
1009  * @dev: device pointer
1010  * @cq: pointer to created cq
1011  */
1012 enum irdma_status_code irdma_cqp_cq_create_cmd(struct irdma_sc_dev *dev,
1013 					       struct irdma_sc_cq *cq)
1014 {
1015 	struct irdma_pci_f *rf = dev_to_rf(dev);
1016 	struct irdma_cqp *iwcqp = &rf->cqp;
1017 	struct irdma_cqp_request *cqp_request;
1018 	struct cqp_cmds_info *cqp_info;
1019 	enum irdma_status_code status;
1020 
1021 	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);
1022 	if (!cqp_request)
1023 		return IRDMA_ERR_NO_MEMORY;
1024 
1025 	cqp_info = &cqp_request->info;
1026 	cqp_info->cqp_cmd = IRDMA_OP_CQ_CREATE;
1027 	cqp_info->post_sq = 1;
1028 	cqp_info->in.u.cq_create.cq = cq;
1029 	cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request;
1030 
1031 	status = irdma_handle_cqp_op(rf, cqp_request);
1032 	irdma_put_cqp_request(iwcqp, cqp_request);
1033 
1034 	return status;
1035 }
1036 
1037 /**
1038  * irdma_cqp_qp_create_cmd - create a qp for the cqp
1039  * @dev: device pointer
1040  * @qp: pointer to created qp
1041  */
1042 enum irdma_status_code irdma_cqp_qp_create_cmd(struct irdma_sc_dev *dev,
1043 					       struct irdma_sc_qp *qp)
1044 {
1045 	struct irdma_pci_f *rf = dev_to_rf(dev);
1046 	struct irdma_cqp *iwcqp = &rf->cqp;
1047 	struct irdma_cqp_request *cqp_request;
1048 	struct cqp_cmds_info *cqp_info;
1049 	struct irdma_create_qp_info *qp_info;
1050 	enum irdma_status_code status;
1051 
1052 	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);
1053 	if (!cqp_request)
1054 		return IRDMA_ERR_NO_MEMORY;
1055 
1056 	cqp_info = &cqp_request->info;
1057 	qp_info = &cqp_request->info.in.u.qp_create.info;
1058 	memset(qp_info, 0, sizeof(*qp_info));
1059 	qp_info->cq_num_valid = true;
1060 	qp_info->next_iwarp_state = IRDMA_QP_STATE_RTS;
1061 	cqp_info->cqp_cmd = IRDMA_OP_QP_CREATE;
1062 	cqp_info->post_sq = 1;
1063 	cqp_info->in.u.qp_create.qp = qp;
1064 	cqp_info->in.u.qp_create.scratch = (uintptr_t)cqp_request;
1065 
1066 	status = irdma_handle_cqp_op(rf, cqp_request);
1067 	irdma_put_cqp_request(iwcqp, cqp_request);
1068 
1069 	return status;
1070 }
1071 
1072 /**
1073  * irdma_dealloc_push_page - free a push page for qp
1074  * @rf: RDMA PCI function
1075  * @qp: hardware control qp
1076  */
1077 static void irdma_dealloc_push_page(struct irdma_pci_f *rf,
1078 				    struct irdma_sc_qp *qp)
1079 {
1080 	struct irdma_cqp_request *cqp_request;
1081 	struct cqp_cmds_info *cqp_info;
1082 	enum irdma_status_code status;
1083 
1084 	if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX)
1085 		return;
1086 
1087 	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false);
1088 	if (!cqp_request)
1089 		return;
1090 
1091 	cqp_info = &cqp_request->info;
1092 	cqp_info->cqp_cmd = IRDMA_OP_MANAGE_PUSH_PAGE;
1093 	cqp_info->post_sq = 1;
1094 	cqp_info->in.u.manage_push_page.info.push_idx = qp->push_idx;
1095 	cqp_info->in.u.manage_push_page.info.qs_handle = qp->qs_handle;
1096 	cqp_info->in.u.manage_push_page.info.free_page = 1;
1097 	cqp_info->in.u.manage_push_page.info.push_page_type = 0;
1098 	cqp_info->in.u.manage_push_page.cqp = &rf->cqp.sc_cqp;
1099 	cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request;
1100 	status = irdma_handle_cqp_op(rf, cqp_request);
1101 	if (!status)
1102 		qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX;
1103 	irdma_put_cqp_request(&rf->cqp, cqp_request);
1104 }
1105 
1106 /**
1107  * irdma_free_qp_rsrc - free up memory resources for qp
1108  * @iwqp: qp ptr (user or kernel)
1109  */
1110 void irdma_free_qp_rsrc(struct irdma_qp *iwqp)
1111 {
1112 	struct irdma_device *iwdev = iwqp->iwdev;
1113 	struct irdma_pci_f *rf = iwdev->rf;
1114 	u32 qp_num = iwqp->ibqp.qp_num;
1115 
1116 	irdma_ieq_cleanup_qp(iwdev->vsi.ieq, &iwqp->sc_qp);
1117 	irdma_dealloc_push_page(rf, &iwqp->sc_qp);
1118 	if (iwqp->sc_qp.vsi) {
1119 		irdma_qp_rem_qos(&iwqp->sc_qp);
1120 		iwqp->sc_qp.dev->ws_remove(iwqp->sc_qp.vsi,
1121 					   iwqp->sc_qp.user_pri);
1122 	}
1123 
1124 	if (qp_num > 2)
1125 		irdma_free_rsrc(rf, rf->allocated_qps, qp_num);
1126 	dma_free_coherent(rf->sc_dev.hw->device, iwqp->q2_ctx_mem.size,
1127 			  iwqp->q2_ctx_mem.va, iwqp->q2_ctx_mem.pa);
1128 	iwqp->q2_ctx_mem.va = NULL;
1129 	dma_free_coherent(rf->sc_dev.hw->device, iwqp->kqp.dma_mem.size,
1130 			  iwqp->kqp.dma_mem.va, iwqp->kqp.dma_mem.pa);
1131 	iwqp->kqp.dma_mem.va = NULL;
1132 	kfree(iwqp->kqp.sq_wrid_mem);
1133 	kfree(iwqp->kqp.rq_wrid_mem);
1134 }
1135 
1136 /**
1137  * irdma_cq_wq_destroy - send cq destroy cqp
1138  * @rf: RDMA PCI function
1139  * @cq: hardware control cq
1140  */
1141 void irdma_cq_wq_destroy(struct irdma_pci_f *rf, struct irdma_sc_cq *cq)
1142 {
1143 	struct irdma_cqp_request *cqp_request;
1144 	struct cqp_cmds_info *cqp_info;
1145 
1146 	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
1147 	if (!cqp_request)
1148 		return;
1149 
1150 	cqp_info = &cqp_request->info;
1151 	cqp_info->cqp_cmd = IRDMA_OP_CQ_DESTROY;
1152 	cqp_info->post_sq = 1;
1153 	cqp_info->in.u.cq_destroy.cq = cq;
1154 	cqp_info->in.u.cq_destroy.scratch = (uintptr_t)cqp_request;
1155 
1156 	irdma_handle_cqp_op(rf, cqp_request);
1157 	irdma_put_cqp_request(&rf->cqp, cqp_request);
1158 }
1159 
1160 /**
1161  * irdma_hw_modify_qp_callback - handle state for modifyQPs that don't wait
1162  * @cqp_request: modify QP completion
1163  */
1164 static void irdma_hw_modify_qp_callback(struct irdma_cqp_request *cqp_request)
1165 {
1166 	struct cqp_cmds_info *cqp_info;
1167 	struct irdma_qp *iwqp;
1168 
1169 	cqp_info = &cqp_request->info;
1170 	iwqp = cqp_info->in.u.qp_modify.qp->qp_uk.back_qp;
1171 	atomic_dec(&iwqp->hw_mod_qp_pend);
1172 	wake_up(&iwqp->mod_qp_waitq);
1173 }
1174 
1175 /**
1176  * irdma_hw_modify_qp - setup cqp for modify qp
1177  * @iwdev: RDMA device
1178  * @iwqp: qp ptr (user or kernel)
1179  * @info: info for modify qp
1180  * @wait: flag to wait or not for modify qp completion
1181  */
1182 enum irdma_status_code irdma_hw_modify_qp(struct irdma_device *iwdev,
1183 					  struct irdma_qp *iwqp,
1184 					  struct irdma_modify_qp_info *info,
1185 					  bool wait)
1186 {
1187 	enum irdma_status_code status;
1188 	struct irdma_pci_f *rf = iwdev->rf;
1189 	struct irdma_cqp_request *cqp_request;
1190 	struct cqp_cmds_info *cqp_info;
1191 	struct irdma_modify_qp_info *m_info;
1192 
1193 	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait);
1194 	if (!cqp_request)
1195 		return IRDMA_ERR_NO_MEMORY;
1196 
1197 	if (!wait) {
1198 		cqp_request->callback_fcn = irdma_hw_modify_qp_callback;
1199 		atomic_inc(&iwqp->hw_mod_qp_pend);
1200 	}
1201 	cqp_info = &cqp_request->info;
1202 	m_info = &cqp_info->in.u.qp_modify.info;
1203 	memcpy(m_info, info, sizeof(*m_info));
1204 	cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY;
1205 	cqp_info->post_sq = 1;
1206 	cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp;
1207 	cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request;
1208 	status = irdma_handle_cqp_op(rf, cqp_request);
1209 	irdma_put_cqp_request(&rf->cqp, cqp_request);
1210 	if (status) {
1211 		if (rdma_protocol_roce(&iwdev->ibdev, 1))
1212 			return status;
1213 
1214 		switch (m_info->next_iwarp_state) {
1215 			struct irdma_gen_ae_info ae_info;
1216 
1217 		case IRDMA_QP_STATE_RTS:
1218 		case IRDMA_QP_STATE_IDLE:
1219 		case IRDMA_QP_STATE_TERMINATE:
1220 		case IRDMA_QP_STATE_CLOSING:
1221 			if (info->curr_iwarp_state == IRDMA_QP_STATE_IDLE)
1222 				irdma_send_reset(iwqp->cm_node);
1223 			else
1224 				iwqp->sc_qp.term_flags = IRDMA_TERM_DONE;
1225 			if (!wait) {
1226 				ae_info.ae_code = IRDMA_AE_BAD_CLOSE;
1227 				ae_info.ae_src = 0;
1228 				irdma_gen_ae(rf, &iwqp->sc_qp, &ae_info, false);
1229 			} else {
1230 				cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp,
1231 									      wait);
1232 				if (!cqp_request)
1233 					return IRDMA_ERR_NO_MEMORY;
1234 
1235 				cqp_info = &cqp_request->info;
1236 				m_info = &cqp_info->in.u.qp_modify.info;
1237 				memcpy(m_info, info, sizeof(*m_info));
1238 				cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY;
1239 				cqp_info->post_sq = 1;
1240 				cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp;
1241 				cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request;
1242 				m_info->next_iwarp_state = IRDMA_QP_STATE_ERROR;
1243 				m_info->reset_tcp_conn = true;
1244 				irdma_handle_cqp_op(rf, cqp_request);
1245 				irdma_put_cqp_request(&rf->cqp, cqp_request);
1246 			}
1247 			break;
1248 		case IRDMA_QP_STATE_ERROR:
1249 		default:
1250 			break;
1251 		}
1252 	}
1253 
1254 	return status;
1255 }
1256 
1257 /**
1258  * irdma_cqp_cq_destroy_cmd - destroy the cqp cq
1259  * @dev: device pointer
1260  * @cq: pointer to cq
1261  */
1262 void irdma_cqp_cq_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq)
1263 {
1264 	struct irdma_pci_f *rf = dev_to_rf(dev);
1265 
1266 	irdma_cq_wq_destroy(rf, cq);
1267 }
1268 
1269 /**
1270  * irdma_cqp_qp_destroy_cmd - destroy the cqp
1271  * @dev: device pointer
1272  * @qp: pointer to qp
1273  */
1274 enum irdma_status_code irdma_cqp_qp_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)
1275 {
1276 	struct irdma_pci_f *rf = dev_to_rf(dev);
1277 	struct irdma_cqp *iwcqp = &rf->cqp;
1278 	struct irdma_cqp_request *cqp_request;
1279 	struct cqp_cmds_info *cqp_info;
1280 	enum irdma_status_code status;
1281 
1282 	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);
1283 	if (!cqp_request)
1284 		return IRDMA_ERR_NO_MEMORY;
1285 
1286 	cqp_info = &cqp_request->info;
1287 	memset(cqp_info, 0, sizeof(*cqp_info));
1288 	cqp_info->cqp_cmd = IRDMA_OP_QP_DESTROY;
1289 	cqp_info->post_sq = 1;
1290 	cqp_info->in.u.qp_destroy.qp = qp;
1291 	cqp_info->in.u.qp_destroy.scratch = (uintptr_t)cqp_request;
1292 	cqp_info->in.u.qp_destroy.remove_hash_idx = true;
1293 
1294 	status = irdma_handle_cqp_op(rf, cqp_request);
1295 	irdma_put_cqp_request(&rf->cqp, cqp_request);
1296 
1297 	return status;
1298 }
1299 
1300 /**
1301  * irdma_ieq_mpa_crc_ae - generate AE for crc error
1302  * @dev: hardware control device structure
1303  * @qp: hardware control qp
1304  */
1305 void irdma_ieq_mpa_crc_ae(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)
1306 {
1307 	struct irdma_gen_ae_info info = {};
1308 	struct irdma_pci_f *rf = dev_to_rf(dev);
1309 
1310 	ibdev_dbg(&rf->iwdev->ibdev, "AEQ: Generate MPA CRC AE\n");
1311 	info.ae_code = IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR;
1312 	info.ae_src = IRDMA_AE_SOURCE_RQ;
1313 	irdma_gen_ae(rf, qp, &info, false);
1314 }
1315 
1316 /**
1317  * irdma_init_hash_desc - initialize hash for crc calculation
1318  * @desc: cryption type
1319  */
1320 enum irdma_status_code irdma_init_hash_desc(struct shash_desc **desc)
1321 {
1322 	struct crypto_shash *tfm;
1323 	struct shash_desc *tdesc;
1324 
1325 	tfm = crypto_alloc_shash("crc32c", 0, 0);
1326 	if (IS_ERR(tfm))
1327 		return IRDMA_ERR_MPA_CRC;
1328 
1329 	tdesc = kzalloc(sizeof(*tdesc) + crypto_shash_descsize(tfm),
1330 			GFP_KERNEL);
1331 	if (!tdesc) {
1332 		crypto_free_shash(tfm);
1333 		return IRDMA_ERR_MPA_CRC;
1334 	}
1335 
1336 	tdesc->tfm = tfm;
1337 	*desc = tdesc;
1338 
1339 	return 0;
1340 }
1341 
1342 /**
1343  * irdma_free_hash_desc - free hash desc
1344  * @desc: to be freed
1345  */
1346 void irdma_free_hash_desc(struct shash_desc *desc)
1347 {
1348 	if (desc) {
1349 		crypto_free_shash(desc->tfm);
1350 		kfree(desc);
1351 	}
1352 }
1353 
1354 /**
1355  * irdma_ieq_check_mpacrc - check if mpa crc is OK
1356  * @desc: desc for hash
1357  * @addr: address of buffer for crc
1358  * @len: length of buffer
1359  * @val: value to be compared
1360  */
1361 enum irdma_status_code irdma_ieq_check_mpacrc(struct shash_desc *desc,
1362 					      void *addr, u32 len, u32 val)
1363 {
1364 	u32 crc = 0;
1365 	int ret;
1366 	enum irdma_status_code ret_code = 0;
1367 
1368 	crypto_shash_init(desc);
1369 	ret = crypto_shash_update(desc, addr, len);
1370 	if (!ret)
1371 		crypto_shash_final(desc, (u8 *)&crc);
1372 	if (crc != val)
1373 		ret_code = IRDMA_ERR_MPA_CRC;
1374 
1375 	return ret_code;
1376 }
1377 
1378 /**
1379  * irdma_ieq_get_qp - get qp based on quad in puda buffer
1380  * @dev: hardware control device structure
1381  * @buf: receive puda buffer on exception q
1382  */
1383 struct irdma_sc_qp *irdma_ieq_get_qp(struct irdma_sc_dev *dev,
1384 				     struct irdma_puda_buf *buf)
1385 {
1386 	struct irdma_qp *iwqp;
1387 	struct irdma_cm_node *cm_node;
1388 	struct irdma_device *iwdev = buf->vsi->back_vsi;
1389 	u32 loc_addr[4] = {};
1390 	u32 rem_addr[4] = {};
1391 	u16 loc_port, rem_port;
1392 	struct ipv6hdr *ip6h;
1393 	struct iphdr *iph = (struct iphdr *)buf->iph;
1394 	struct tcphdr *tcph = (struct tcphdr *)buf->tcph;
1395 
1396 	if (iph->version == 4) {
1397 		loc_addr[0] = ntohl(iph->daddr);
1398 		rem_addr[0] = ntohl(iph->saddr);
1399 	} else {
1400 		ip6h = (struct ipv6hdr *)buf->iph;
1401 		irdma_copy_ip_ntohl(loc_addr, ip6h->daddr.in6_u.u6_addr32);
1402 		irdma_copy_ip_ntohl(rem_addr, ip6h->saddr.in6_u.u6_addr32);
1403 	}
1404 	loc_port = ntohs(tcph->dest);
1405 	rem_port = ntohs(tcph->source);
1406 	cm_node = irdma_find_node(&iwdev->cm_core, rem_port, rem_addr, loc_port,
1407 				  loc_addr, buf->vlan_valid ? buf->vlan_id : 0xFFFF);
1408 	if (!cm_node)
1409 		return NULL;
1410 
1411 	iwqp = cm_node->iwqp;
1412 	irdma_rem_ref_cm_node(cm_node);
1413 
1414 	return &iwqp->sc_qp;
1415 }
1416 
1417 /**
1418  * irdma_send_ieq_ack - ACKs for duplicate or OOO partials FPDUs
1419  * @qp: qp ptr
1420  */
1421 void irdma_send_ieq_ack(struct irdma_sc_qp *qp)
1422 {
1423 	struct irdma_cm_node *cm_node = ((struct irdma_qp *)qp->qp_uk.back_qp)->cm_node;
1424 	struct irdma_puda_buf *buf = qp->pfpdu.lastrcv_buf;
1425 	struct tcphdr *tcph = (struct tcphdr *)buf->tcph;
1426 
1427 	cm_node->tcp_cntxt.rcv_nxt = qp->pfpdu.nextseqnum;
1428 	cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->ack_seq);
1429 
1430 	irdma_send_ack(cm_node);
1431 }
1432 
1433 /**
1434  * irdma_puda_ieq_get_ah_info - get AH info from IEQ buffer
1435  * @qp: qp pointer
1436  * @ah_info: AH info pointer
1437  */
1438 void irdma_puda_ieq_get_ah_info(struct irdma_sc_qp *qp,
1439 				struct irdma_ah_info *ah_info)
1440 {
1441 	struct irdma_puda_buf *buf = qp->pfpdu.ah_buf;
1442 	struct iphdr *iph;
1443 	struct ipv6hdr *ip6h;
1444 
1445 	memset(ah_info, 0, sizeof(*ah_info));
1446 	ah_info->do_lpbk = true;
1447 	ah_info->vlan_tag = buf->vlan_id;
1448 	ah_info->insert_vlan_tag = buf->vlan_valid;
1449 	ah_info->ipv4_valid = buf->ipv4;
1450 	ah_info->vsi = qp->vsi;
1451 
1452 	if (buf->smac_valid)
1453 		ether_addr_copy(ah_info->mac_addr, buf->smac);
1454 
1455 	if (buf->ipv4) {
1456 		ah_info->ipv4_valid = true;
1457 		iph = (struct iphdr *)buf->iph;
1458 		ah_info->hop_ttl = iph->ttl;
1459 		ah_info->tc_tos = iph->tos;
1460 		ah_info->dest_ip_addr[0] = ntohl(iph->daddr);
1461 		ah_info->src_ip_addr[0] = ntohl(iph->saddr);
1462 	} else {
1463 		ip6h = (struct ipv6hdr *)buf->iph;
1464 		ah_info->hop_ttl = ip6h->hop_limit;
1465 		ah_info->tc_tos = ip6h->priority;
1466 		irdma_copy_ip_ntohl(ah_info->dest_ip_addr,
1467 				    ip6h->daddr.in6_u.u6_addr32);
1468 		irdma_copy_ip_ntohl(ah_info->src_ip_addr,
1469 				    ip6h->saddr.in6_u.u6_addr32);
1470 	}
1471 
1472 	ah_info->dst_arpindex = irdma_arp_table(dev_to_rf(qp->dev),
1473 						ah_info->dest_ip_addr,
1474 						ah_info->ipv4_valid,
1475 						NULL, IRDMA_ARP_RESOLVE);
1476 }
1477 
1478 /**
1479  * irdma_gen1_ieq_update_tcpip_info - update tcpip in the buffer
1480  * @buf: puda to update
1481  * @len: length of buffer
1482  * @seqnum: seq number for tcp
1483  */
1484 static void irdma_gen1_ieq_update_tcpip_info(struct irdma_puda_buf *buf,
1485 					     u16 len, u32 seqnum)
1486 {
1487 	struct tcphdr *tcph;
1488 	struct iphdr *iph;
1489 	u16 iphlen;
1490 	u16 pktsize;
1491 	u8 *addr = buf->mem.va;
1492 
1493 	iphlen = (buf->ipv4) ? 20 : 40;
1494 	iph = (struct iphdr *)(addr + buf->maclen);
1495 	tcph = (struct tcphdr *)(addr + buf->maclen + iphlen);
1496 	pktsize = len + buf->tcphlen + iphlen;
1497 	iph->tot_len = htons(pktsize);
1498 	tcph->seq = htonl(seqnum);
1499 }
1500 
1501 /**
1502  * irdma_ieq_update_tcpip_info - update tcpip in the buffer
1503  * @buf: puda to update
1504  * @len: length of buffer
1505  * @seqnum: seq number for tcp
1506  */
1507 void irdma_ieq_update_tcpip_info(struct irdma_puda_buf *buf, u16 len,
1508 				 u32 seqnum)
1509 {
1510 	struct tcphdr *tcph;
1511 	u8 *addr;
1512 
1513 	if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
1514 		return irdma_gen1_ieq_update_tcpip_info(buf, len, seqnum);
1515 
1516 	addr = buf->mem.va;
1517 	tcph = (struct tcphdr *)addr;
1518 	tcph->seq = htonl(seqnum);
1519 }
1520 
1521 /**
1522  * irdma_gen1_puda_get_tcpip_info - get tcpip info from puda
1523  * buffer
1524  * @info: to get information
1525  * @buf: puda buffer
1526  */
1527 static enum irdma_status_code
1528 irdma_gen1_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info,
1529 			       struct irdma_puda_buf *buf)
1530 {
1531 	struct iphdr *iph;
1532 	struct ipv6hdr *ip6h;
1533 	struct tcphdr *tcph;
1534 	u16 iphlen;
1535 	u16 pkt_len;
1536 	u8 *mem = buf->mem.va;
1537 	struct ethhdr *ethh = buf->mem.va;
1538 
1539 	if (ethh->h_proto == htons(0x8100)) {
1540 		info->vlan_valid = true;
1541 		buf->vlan_id = ntohs(((struct vlan_ethhdr *)ethh)->h_vlan_TCI) &
1542 			       VLAN_VID_MASK;
1543 	}
1544 
1545 	buf->maclen = (info->vlan_valid) ? 18 : 14;
1546 	iphlen = (info->l3proto) ? 40 : 20;
1547 	buf->ipv4 = (info->l3proto) ? false : true;
1548 	buf->iph = mem + buf->maclen;
1549 	iph = (struct iphdr *)buf->iph;
1550 	buf->tcph = buf->iph + iphlen;
1551 	tcph = (struct tcphdr *)buf->tcph;
1552 
1553 	if (buf->ipv4) {
1554 		pkt_len = ntohs(iph->tot_len);
1555 	} else {
1556 		ip6h = (struct ipv6hdr *)buf->iph;
1557 		pkt_len = ntohs(ip6h->payload_len) + iphlen;
1558 	}
1559 
1560 	buf->totallen = pkt_len + buf->maclen;
1561 
1562 	if (info->payload_len < buf->totallen) {
1563 		ibdev_dbg(to_ibdev(buf->vsi->dev),
1564 			  "ERR: payload_len = 0x%x totallen expected0x%x\n",
1565 			  info->payload_len, buf->totallen);
1566 		return IRDMA_ERR_INVALID_SIZE;
1567 	}
1568 
1569 	buf->tcphlen = tcph->doff << 2;
1570 	buf->datalen = pkt_len - iphlen - buf->tcphlen;
1571 	buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL;
1572 	buf->hdrlen = buf->maclen + iphlen + buf->tcphlen;
1573 	buf->seqnum = ntohl(tcph->seq);
1574 
1575 	return 0;
1576 }
1577 
1578 /**
1579  * irdma_puda_get_tcpip_info - get tcpip info from puda buffer
1580  * @info: to get information
1581  * @buf: puda buffer
1582  */
1583 enum irdma_status_code
1584 irdma_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info,
1585 			  struct irdma_puda_buf *buf)
1586 {
1587 	struct tcphdr *tcph;
1588 	u32 pkt_len;
1589 	u8 *mem;
1590 
1591 	if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
1592 		return irdma_gen1_puda_get_tcpip_info(info, buf);
1593 
1594 	mem = buf->mem.va;
1595 	buf->vlan_valid = info->vlan_valid;
1596 	if (info->vlan_valid)
1597 		buf->vlan_id = info->vlan;
1598 
1599 	buf->ipv4 = info->ipv4;
1600 	if (buf->ipv4)
1601 		buf->iph = mem + IRDMA_IPV4_PAD;
1602 	else
1603 		buf->iph = mem;
1604 
1605 	buf->tcph = mem + IRDMA_TCP_OFFSET;
1606 	tcph = (struct tcphdr *)buf->tcph;
1607 	pkt_len = info->payload_len;
1608 	buf->totallen = pkt_len;
1609 	buf->tcphlen = tcph->doff << 2;
1610 	buf->datalen = pkt_len - IRDMA_TCP_OFFSET - buf->tcphlen;
1611 	buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL;
1612 	buf->hdrlen = IRDMA_TCP_OFFSET + buf->tcphlen;
1613 	buf->seqnum = ntohl(tcph->seq);
1614 
1615 	if (info->smac_valid) {
1616 		ether_addr_copy(buf->smac, info->smac);
1617 		buf->smac_valid = true;
1618 	}
1619 
1620 	return 0;
1621 }
1622 
1623 /**
1624  * irdma_hw_stats_timeout - Stats timer-handler which updates all HW stats
1625  * @t: timer_list pointer
1626  */
1627 static void irdma_hw_stats_timeout(struct timer_list *t)
1628 {
1629 	struct irdma_vsi_pestat *pf_devstat =
1630 		from_timer(pf_devstat, t, stats_timer);
1631 	struct irdma_sc_vsi *sc_vsi = pf_devstat->vsi;
1632 
1633 	if (sc_vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
1634 		irdma_cqp_gather_stats_gen1(sc_vsi->dev, sc_vsi->pestat);
1635 	else
1636 		irdma_cqp_gather_stats_cmd(sc_vsi->dev, sc_vsi->pestat, false);
1637 
1638 	mod_timer(&pf_devstat->stats_timer,
1639 		  jiffies + msecs_to_jiffies(STATS_TIMER_DELAY));
1640 }
1641 
1642 /**
1643  * irdma_hw_stats_start_timer - Start periodic stats timer
1644  * @vsi: vsi structure pointer
1645  */
1646 void irdma_hw_stats_start_timer(struct irdma_sc_vsi *vsi)
1647 {
1648 	struct irdma_vsi_pestat *devstat = vsi->pestat;
1649 
1650 	timer_setup(&devstat->stats_timer, irdma_hw_stats_timeout, 0);
1651 	mod_timer(&devstat->stats_timer,
1652 		  jiffies + msecs_to_jiffies(STATS_TIMER_DELAY));
1653 }
1654 
1655 /**
1656  * irdma_hw_stats_stop_timer - Delete periodic stats timer
1657  * @vsi: pointer to vsi structure
1658  */
1659 void irdma_hw_stats_stop_timer(struct irdma_sc_vsi *vsi)
1660 {
1661 	struct irdma_vsi_pestat *devstat = vsi->pestat;
1662 
1663 	del_timer_sync(&devstat->stats_timer);
1664 }
1665 
1666 /**
1667  * irdma_process_stats - Checking for wrap and update stats
1668  * @pestat: stats structure pointer
1669  */
1670 static inline void irdma_process_stats(struct irdma_vsi_pestat *pestat)
1671 {
1672 	sc_vsi_update_stats(pestat->vsi);
1673 }
1674 
1675 /**
1676  * irdma_cqp_gather_stats_gen1 - Gather stats
1677  * @dev: pointer to device structure
1678  * @pestat: statistics structure
1679  */
1680 void irdma_cqp_gather_stats_gen1(struct irdma_sc_dev *dev,
1681 				 struct irdma_vsi_pestat *pestat)
1682 {
1683 	struct irdma_gather_stats *gather_stats =
1684 		pestat->gather_info.gather_stats_va;
1685 	u32 stats_inst_offset_32;
1686 	u32 stats_inst_offset_64;
1687 
1688 	stats_inst_offset_32 = (pestat->gather_info.use_stats_inst) ?
1689 				       pestat->gather_info.stats_inst_index :
1690 				       pestat->hw->hmc.hmc_fn_id;
1691 	stats_inst_offset_32 *= 4;
1692 	stats_inst_offset_64 = stats_inst_offset_32 * 2;
1693 
1694 	gather_stats->rxvlanerr =
1695 		rd32(dev->hw,
1696 		     dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_RXVLANERR]
1697 		     + stats_inst_offset_32);
1698 	gather_stats->ip4rxdiscard =
1699 		rd32(dev->hw,
1700 		     dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP4RXDISCARD]
1701 		     + stats_inst_offset_32);
1702 	gather_stats->ip4rxtrunc =
1703 		rd32(dev->hw,
1704 		     dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP4RXTRUNC]
1705 		     + stats_inst_offset_32);
1706 	gather_stats->ip4txnoroute =
1707 		rd32(dev->hw,
1708 		     dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP4TXNOROUTE]
1709 		     + stats_inst_offset_32);
1710 	gather_stats->ip6rxdiscard =
1711 		rd32(dev->hw,
1712 		     dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP6RXDISCARD]
1713 		     + stats_inst_offset_32);
1714 	gather_stats->ip6rxtrunc =
1715 		rd32(dev->hw,
1716 		     dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP6RXTRUNC]
1717 		     + stats_inst_offset_32);
1718 	gather_stats->ip6txnoroute =
1719 		rd32(dev->hw,
1720 		     dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP6TXNOROUTE]
1721 		     + stats_inst_offset_32);
1722 	gather_stats->tcprtxseg =
1723 		rd32(dev->hw,
1724 		     dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_TCPRTXSEG]
1725 		     + stats_inst_offset_32);
1726 	gather_stats->tcprxopterr =
1727 		rd32(dev->hw,
1728 		     dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_TCPRXOPTERR]
1729 		     + stats_inst_offset_32);
1730 
1731 	gather_stats->ip4rxocts =
1732 		rd64(dev->hw,
1733 		     dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4RXOCTS]
1734 		     + stats_inst_offset_64);
1735 	gather_stats->ip4rxpkts =
1736 		rd64(dev->hw,
1737 		     dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4RXPKTS]
1738 		     + stats_inst_offset_64);
1739 	gather_stats->ip4txfrag =
1740 		rd64(dev->hw,
1741 		     dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4RXFRAGS]
1742 		     + stats_inst_offset_64);
1743 	gather_stats->ip4rxmcpkts =
1744 		rd64(dev->hw,
1745 		     dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4RXMCPKTS]
1746 		     + stats_inst_offset_64);
1747 	gather_stats->ip4txocts =
1748 		rd64(dev->hw,
1749 		     dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4TXOCTS]
1750 		     + stats_inst_offset_64);
1751 	gather_stats->ip4txpkts =
1752 		rd64(dev->hw,
1753 		     dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4TXPKTS]
1754 		     + stats_inst_offset_64);
1755 	gather_stats->ip4txfrag =
1756 		rd64(dev->hw,
1757 		     dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4TXFRAGS]
1758 		     + stats_inst_offset_64);
1759 	gather_stats->ip4txmcpkts =
1760 		rd64(dev->hw,
1761 		     dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4TXMCPKTS]
1762 		     + stats_inst_offset_64);
1763 	gather_stats->ip6rxocts =
1764 		rd64(dev->hw,
1765 		     dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6RXOCTS]
1766 		     + stats_inst_offset_64);
1767 	gather_stats->ip6rxpkts =
1768 		rd64(dev->hw,
1769 		     dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6RXPKTS]
1770 		     + stats_inst_offset_64);
1771 	gather_stats->ip6txfrags =
1772 		rd64(dev->hw,
1773 		     dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6RXFRAGS]
1774 		     + stats_inst_offset_64);
1775 	gather_stats->ip6rxmcpkts =
1776 		rd64(dev->hw,
1777 		     dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6RXMCPKTS]
1778 		     + stats_inst_offset_64);
1779 	gather_stats->ip6txocts =
1780 		rd64(dev->hw,
1781 		     dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6TXOCTS]
1782 		     + stats_inst_offset_64);
1783 	gather_stats->ip6txpkts =
1784 		rd64(dev->hw,
1785 		     dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6TXPKTS]
1786 		     + stats_inst_offset_64);
1787 	gather_stats->ip6txfrags =
1788 		rd64(dev->hw,
1789 		     dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6TXFRAGS]
1790 		     + stats_inst_offset_64);
1791 	gather_stats->ip6txmcpkts =
1792 		rd64(dev->hw,
1793 		     dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6TXMCPKTS]
1794 		     + stats_inst_offset_64);
1795 	gather_stats->tcprxsegs =
1796 		rd64(dev->hw,
1797 		     dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_TCPRXSEGS]
1798 		     + stats_inst_offset_64);
1799 	gather_stats->tcptxsegs =
1800 		rd64(dev->hw,
1801 		     dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_TCPTXSEG]
1802 		     + stats_inst_offset_64);
1803 	gather_stats->rdmarxrds =
1804 		rd64(dev->hw,
1805 		     dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMARXRDS]
1806 		     + stats_inst_offset_64);
1807 	gather_stats->rdmarxsnds =
1808 		rd64(dev->hw,
1809 		     dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMARXSNDS]
1810 		     + stats_inst_offset_64);
1811 	gather_stats->rdmarxwrs =
1812 		rd64(dev->hw,
1813 		     dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMARXWRS]
1814 		     + stats_inst_offset_64);
1815 	gather_stats->rdmatxrds =
1816 		rd64(dev->hw,
1817 		     dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMATXRDS]
1818 		     + stats_inst_offset_64);
1819 	gather_stats->rdmatxsnds =
1820 		rd64(dev->hw,
1821 		     dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMATXSNDS]
1822 		     + stats_inst_offset_64);
1823 	gather_stats->rdmatxwrs =
1824 		rd64(dev->hw,
1825 		     dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMATXWRS]
1826 		     + stats_inst_offset_64);
1827 	gather_stats->rdmavbn =
1828 		rd64(dev->hw,
1829 		     dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMAVBND]
1830 		     + stats_inst_offset_64);
1831 	gather_stats->rdmavinv =
1832 		rd64(dev->hw,
1833 		     dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMAVINV]
1834 		     + stats_inst_offset_64);
1835 	gather_stats->udprxpkts =
1836 		rd64(dev->hw,
1837 		     dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_UDPRXPKTS]
1838 		     + stats_inst_offset_64);
1839 	gather_stats->udptxpkts =
1840 		rd64(dev->hw,
1841 		     dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_UDPTXPKTS]
1842 		     + stats_inst_offset_64);
1843 
1844 	irdma_process_stats(pestat);
1845 }
1846 
1847 /**
1848  * irdma_process_cqp_stats - Checking for wrap and update stats
1849  * @cqp_request: cqp_request structure pointer
1850  */
1851 static void irdma_process_cqp_stats(struct irdma_cqp_request *cqp_request)
1852 {
1853 	struct irdma_vsi_pestat *pestat = cqp_request->param;
1854 
1855 	irdma_process_stats(pestat);
1856 }
1857 
1858 /**
1859  * irdma_cqp_gather_stats_cmd - Gather stats
1860  * @dev: pointer to device structure
1861  * @pestat: pointer to stats info
1862  * @wait: flag to wait or not wait for stats
1863  */
1864 enum irdma_status_code
1865 irdma_cqp_gather_stats_cmd(struct irdma_sc_dev *dev,
1866 			   struct irdma_vsi_pestat *pestat, bool wait)
1867 
1868 {
1869 	struct irdma_pci_f *rf = dev_to_rf(dev);
1870 	struct irdma_cqp *iwcqp = &rf->cqp;
1871 	struct irdma_cqp_request *cqp_request;
1872 	struct cqp_cmds_info *cqp_info;
1873 	enum irdma_status_code status;
1874 
1875 	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, wait);
1876 	if (!cqp_request)
1877 		return IRDMA_ERR_NO_MEMORY;
1878 
1879 	cqp_info = &cqp_request->info;
1880 	memset(cqp_info, 0, sizeof(*cqp_info));
1881 	cqp_info->cqp_cmd = IRDMA_OP_STATS_GATHER;
1882 	cqp_info->post_sq = 1;
1883 	cqp_info->in.u.stats_gather.info = pestat->gather_info;
1884 	cqp_info->in.u.stats_gather.scratch = (uintptr_t)cqp_request;
1885 	cqp_info->in.u.stats_gather.cqp = &rf->cqp.sc_cqp;
1886 	cqp_request->param = pestat;
1887 	if (!wait)
1888 		cqp_request->callback_fcn = irdma_process_cqp_stats;
1889 	status = irdma_handle_cqp_op(rf, cqp_request);
1890 	if (wait)
1891 		irdma_process_stats(pestat);
1892 	irdma_put_cqp_request(&rf->cqp, cqp_request);
1893 
1894 	return status;
1895 }
1896 
1897 /**
1898  * irdma_cqp_stats_inst_cmd - Allocate/free stats instance
1899  * @vsi: pointer to vsi structure
1900  * @cmd: command to allocate or free
1901  * @stats_info: pointer to allocate stats info
1902  */
1903 enum irdma_status_code
1904 irdma_cqp_stats_inst_cmd(struct irdma_sc_vsi *vsi, u8 cmd,
1905 			 struct irdma_stats_inst_info *stats_info)
1906 {
1907 	struct irdma_pci_f *rf = dev_to_rf(vsi->dev);
1908 	struct irdma_cqp *iwcqp = &rf->cqp;
1909 	struct irdma_cqp_request *cqp_request;
1910 	struct cqp_cmds_info *cqp_info;
1911 	enum irdma_status_code status;
1912 	bool wait = false;
1913 
1914 	if (cmd == IRDMA_OP_STATS_ALLOCATE)
1915 		wait = true;
1916 	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, wait);
1917 	if (!cqp_request)
1918 		return IRDMA_ERR_NO_MEMORY;
1919 
1920 	cqp_info = &cqp_request->info;
1921 	memset(cqp_info, 0, sizeof(*cqp_info));
1922 	cqp_info->cqp_cmd = cmd;
1923 	cqp_info->post_sq = 1;
1924 	cqp_info->in.u.stats_manage.info = *stats_info;
1925 	cqp_info->in.u.stats_manage.scratch = (uintptr_t)cqp_request;
1926 	cqp_info->in.u.stats_manage.cqp = &rf->cqp.sc_cqp;
1927 	status = irdma_handle_cqp_op(rf, cqp_request);
1928 	if (wait)
1929 		stats_info->stats_idx = cqp_request->compl_info.op_ret_val;
1930 	irdma_put_cqp_request(iwcqp, cqp_request);
1931 
1932 	return status;
1933 }
1934 
1935 /**
1936  * irdma_cqp_ceq_cmd - Create/Destroy CEQ's after CEQ 0
1937  * @dev: pointer to device info
1938  * @sc_ceq: pointer to ceq structure
1939  * @op: Create or Destroy
1940  */
1941 enum irdma_status_code irdma_cqp_ceq_cmd(struct irdma_sc_dev *dev,
1942 					 struct irdma_sc_ceq *sc_ceq, u8 op)
1943 {
1944 	struct irdma_cqp_request *cqp_request;
1945 	struct cqp_cmds_info *cqp_info;
1946 	struct irdma_pci_f *rf = dev_to_rf(dev);
1947 	enum irdma_status_code status;
1948 
1949 	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
1950 	if (!cqp_request)
1951 		return IRDMA_ERR_NO_MEMORY;
1952 
1953 	cqp_info = &cqp_request->info;
1954 	cqp_info->post_sq = 1;
1955 	cqp_info->cqp_cmd = op;
1956 	cqp_info->in.u.ceq_create.ceq = sc_ceq;
1957 	cqp_info->in.u.ceq_create.scratch = (uintptr_t)cqp_request;
1958 
1959 	status = irdma_handle_cqp_op(rf, cqp_request);
1960 	irdma_put_cqp_request(&rf->cqp, cqp_request);
1961 
1962 	return status;
1963 }
1964 
1965 /**
1966  * irdma_cqp_aeq_cmd - Create/Destroy AEQ
1967  * @dev: pointer to device info
1968  * @sc_aeq: pointer to aeq structure
1969  * @op: Create or Destroy
1970  */
1971 enum irdma_status_code irdma_cqp_aeq_cmd(struct irdma_sc_dev *dev,
1972 					 struct irdma_sc_aeq *sc_aeq, u8 op)
1973 {
1974 	struct irdma_cqp_request *cqp_request;
1975 	struct cqp_cmds_info *cqp_info;
1976 	struct irdma_pci_f *rf = dev_to_rf(dev);
1977 	enum irdma_status_code status;
1978 
1979 	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
1980 	if (!cqp_request)
1981 		return IRDMA_ERR_NO_MEMORY;
1982 
1983 	cqp_info = &cqp_request->info;
1984 	cqp_info->post_sq = 1;
1985 	cqp_info->cqp_cmd = op;
1986 	cqp_info->in.u.aeq_create.aeq = sc_aeq;
1987 	cqp_info->in.u.aeq_create.scratch = (uintptr_t)cqp_request;
1988 
1989 	status = irdma_handle_cqp_op(rf, cqp_request);
1990 	irdma_put_cqp_request(&rf->cqp, cqp_request);
1991 
1992 	return status;
1993 }
1994 
1995 /**
1996  * irdma_cqp_ws_node_cmd - Add/modify/delete ws node
1997  * @dev: pointer to device structure
1998  * @cmd: Add, modify or delete
1999  * @node_info: pointer to ws node info
2000  */
2001 enum irdma_status_code
2002 irdma_cqp_ws_node_cmd(struct irdma_sc_dev *dev, u8 cmd,
2003 		      struct irdma_ws_node_info *node_info)
2004 {
2005 	struct irdma_pci_f *rf = dev_to_rf(dev);
2006 	struct irdma_cqp *iwcqp = &rf->cqp;
2007 	struct irdma_sc_cqp *cqp = &iwcqp->sc_cqp;
2008 	struct irdma_cqp_request *cqp_request;
2009 	struct cqp_cmds_info *cqp_info;
2010 	enum irdma_status_code status;
2011 	bool poll;
2012 
2013 	if (!rf->sc_dev.ceq_valid)
2014 		poll = true;
2015 	else
2016 		poll = false;
2017 
2018 	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, !poll);
2019 	if (!cqp_request)
2020 		return IRDMA_ERR_NO_MEMORY;
2021 
2022 	cqp_info = &cqp_request->info;
2023 	memset(cqp_info, 0, sizeof(*cqp_info));
2024 	cqp_info->cqp_cmd = cmd;
2025 	cqp_info->post_sq = 1;
2026 	cqp_info->in.u.ws_node.info = *node_info;
2027 	cqp_info->in.u.ws_node.cqp = cqp;
2028 	cqp_info->in.u.ws_node.scratch = (uintptr_t)cqp_request;
2029 	status = irdma_handle_cqp_op(rf, cqp_request);
2030 	if (status)
2031 		goto exit;
2032 
2033 	if (poll) {
2034 		struct irdma_ccq_cqe_info compl_info;
2035 
2036 		status = irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_WORK_SCHED_NODE,
2037 						       &compl_info);
2038 		node_info->qs_handle = compl_info.op_ret_val;
2039 		ibdev_dbg(&rf->iwdev->ibdev, "DCB: opcode=%d, compl_info.retval=%d\n",
2040 			  compl_info.op_code, compl_info.op_ret_val);
2041 	} else {
2042 		node_info->qs_handle = cqp_request->compl_info.op_ret_val;
2043 	}
2044 
2045 exit:
2046 	irdma_put_cqp_request(&rf->cqp, cqp_request);
2047 
2048 	return status;
2049 }
2050 
2051 /**
2052  * irdma_ah_cqp_op - perform an AH cqp operation
2053  * @rf: RDMA PCI function
2054  * @sc_ah: address handle
2055  * @cmd: AH operation
2056  * @wait: wait if true
2057  * @callback_fcn: Callback function on CQP op completion
2058  * @cb_param: parameter for callback function
2059  *
2060  * returns errno
2061  */
2062 int irdma_ah_cqp_op(struct irdma_pci_f *rf, struct irdma_sc_ah *sc_ah, u8 cmd,
2063 		    bool wait,
2064 		    void (*callback_fcn)(struct irdma_cqp_request *),
2065 		    void *cb_param)
2066 {
2067 	struct irdma_cqp_request *cqp_request;
2068 	struct cqp_cmds_info *cqp_info;
2069 	enum irdma_status_code status;
2070 
2071 	if (cmd != IRDMA_OP_AH_CREATE && cmd != IRDMA_OP_AH_DESTROY)
2072 		return -EINVAL;
2073 
2074 	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait);
2075 	if (!cqp_request)
2076 		return -ENOMEM;
2077 
2078 	cqp_info = &cqp_request->info;
2079 	cqp_info->cqp_cmd = cmd;
2080 	cqp_info->post_sq = 1;
2081 	if (cmd == IRDMA_OP_AH_CREATE) {
2082 		cqp_info->in.u.ah_create.info = sc_ah->ah_info;
2083 		cqp_info->in.u.ah_create.scratch = (uintptr_t)cqp_request;
2084 		cqp_info->in.u.ah_create.cqp = &rf->cqp.sc_cqp;
2085 	} else if (cmd == IRDMA_OP_AH_DESTROY) {
2086 		cqp_info->in.u.ah_destroy.info = sc_ah->ah_info;
2087 		cqp_info->in.u.ah_destroy.scratch = (uintptr_t)cqp_request;
2088 		cqp_info->in.u.ah_destroy.cqp = &rf->cqp.sc_cqp;
2089 	}
2090 
2091 	if (!wait) {
2092 		cqp_request->callback_fcn = callback_fcn;
2093 		cqp_request->param = cb_param;
2094 	}
2095 	status = irdma_handle_cqp_op(rf, cqp_request);
2096 	irdma_put_cqp_request(&rf->cqp, cqp_request);
2097 
2098 	if (status)
2099 		return -ENOMEM;
2100 
2101 	if (wait)
2102 		sc_ah->ah_info.ah_valid = (cmd == IRDMA_OP_AH_CREATE);
2103 
2104 	return 0;
2105 }
2106 
2107 /**
2108  * irdma_ieq_ah_cb - callback after creation of AH for IEQ
2109  * @cqp_request: pointer to cqp_request of create AH
2110  */
2111 static void irdma_ieq_ah_cb(struct irdma_cqp_request *cqp_request)
2112 {
2113 	struct irdma_sc_qp *qp = cqp_request->param;
2114 	struct irdma_sc_ah *sc_ah = qp->pfpdu.ah;
2115 	unsigned long flags;
2116 
2117 	spin_lock_irqsave(&qp->pfpdu.lock, flags);
2118 	if (!cqp_request->compl_info.op_ret_val) {
2119 		sc_ah->ah_info.ah_valid = true;
2120 		irdma_ieq_process_fpdus(qp, qp->vsi->ieq);
2121 	} else {
2122 		sc_ah->ah_info.ah_valid = false;
2123 		irdma_ieq_cleanup_qp(qp->vsi->ieq, qp);
2124 	}
2125 	spin_unlock_irqrestore(&qp->pfpdu.lock, flags);
2126 }
2127 
2128 /**
2129  * irdma_ilq_ah_cb - callback after creation of AH for ILQ
2130  * @cqp_request: pointer to cqp_request of create AH
2131  */
2132 static void irdma_ilq_ah_cb(struct irdma_cqp_request *cqp_request)
2133 {
2134 	struct irdma_cm_node *cm_node = cqp_request->param;
2135 	struct irdma_sc_ah *sc_ah = cm_node->ah;
2136 
2137 	sc_ah->ah_info.ah_valid = !cqp_request->compl_info.op_ret_val;
2138 	irdma_add_conn_est_qh(cm_node);
2139 }
2140 
2141 /**
2142  * irdma_puda_create_ah - create AH for ILQ/IEQ qp's
2143  * @dev: device pointer
2144  * @ah_info: Address handle info
2145  * @wait: When true will wait for operation to complete
2146  * @type: ILQ/IEQ
2147  * @cb_param: Callback param when not waiting
2148  * @ah_ret: Returned pointer to address handle if created
2149  *
2150  */
2151 enum irdma_status_code irdma_puda_create_ah(struct irdma_sc_dev *dev,
2152 					    struct irdma_ah_info *ah_info,
2153 					    bool wait, enum puda_rsrc_type type,
2154 					    void *cb_param,
2155 					    struct irdma_sc_ah **ah_ret)
2156 {
2157 	struct irdma_sc_ah *ah;
2158 	struct irdma_pci_f *rf = dev_to_rf(dev);
2159 	int err;
2160 
2161 	ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
2162 	*ah_ret = ah;
2163 	if (!ah)
2164 		return IRDMA_ERR_NO_MEMORY;
2165 
2166 	err = irdma_alloc_rsrc(rf, rf->allocated_ahs, rf->max_ah,
2167 			       &ah_info->ah_idx, &rf->next_ah);
2168 	if (err)
2169 		goto err_free;
2170 
2171 	ah->dev = dev;
2172 	ah->ah_info = *ah_info;
2173 
2174 	if (type == IRDMA_PUDA_RSRC_TYPE_ILQ)
2175 		err = irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_CREATE, wait,
2176 				      irdma_ilq_ah_cb, cb_param);
2177 	else
2178 		err = irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_CREATE, wait,
2179 				      irdma_ieq_ah_cb, cb_param);
2180 
2181 	if (err)
2182 		goto error;
2183 	return 0;
2184 
2185 error:
2186 	irdma_free_rsrc(rf, rf->allocated_ahs, ah->ah_info.ah_idx);
2187 err_free:
2188 	kfree(ah);
2189 	*ah_ret = NULL;
2190 	return IRDMA_ERR_NO_MEMORY;
2191 }
2192 
2193 /**
2194  * irdma_puda_free_ah - free a puda address handle
2195  * @dev: device pointer
2196  * @ah: The address handle to free
2197  */
2198 void irdma_puda_free_ah(struct irdma_sc_dev *dev, struct irdma_sc_ah *ah)
2199 {
2200 	struct irdma_pci_f *rf = dev_to_rf(dev);
2201 
2202 	if (!ah)
2203 		return;
2204 
2205 	if (ah->ah_info.ah_valid) {
2206 		irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_DESTROY, false, NULL, NULL);
2207 		irdma_free_rsrc(rf, rf->allocated_ahs, ah->ah_info.ah_idx);
2208 	}
2209 
2210 	kfree(ah);
2211 }
2212 
2213 /**
2214  * irdma_gsi_ud_qp_ah_cb - callback after creation of AH for GSI/ID QP
2215  * @cqp_request: pointer to cqp_request of create AH
2216  */
2217 void irdma_gsi_ud_qp_ah_cb(struct irdma_cqp_request *cqp_request)
2218 {
2219 	struct irdma_sc_ah *sc_ah = cqp_request->param;
2220 
2221 	if (!cqp_request->compl_info.op_ret_val)
2222 		sc_ah->ah_info.ah_valid = true;
2223 	else
2224 		sc_ah->ah_info.ah_valid = false;
2225 }
2226 
2227 /**
2228  * irdma_prm_add_pble_mem - add moemory to pble resources
2229  * @pprm: pble resource manager
2230  * @pchunk: chunk of memory to add
2231  */
2232 enum irdma_status_code irdma_prm_add_pble_mem(struct irdma_pble_prm *pprm,
2233 					      struct irdma_chunk *pchunk)
2234 {
2235 	u64 sizeofbitmap;
2236 
2237 	if (pchunk->size & 0xfff)
2238 		return IRDMA_ERR_PARAM;
2239 
2240 	sizeofbitmap = (u64)pchunk->size >> pprm->pble_shift;
2241 
2242 	pchunk->bitmapbuf = bitmap_zalloc(sizeofbitmap, GFP_KERNEL);
2243 	if (!pchunk->bitmapbuf)
2244 		return IRDMA_ERR_NO_MEMORY;
2245 
2246 	pchunk->sizeofbitmap = sizeofbitmap;
2247 	/* each pble is 8 bytes hence shift by 3 */
2248 	pprm->total_pble_alloc += pchunk->size >> 3;
2249 	pprm->free_pble_cnt += pchunk->size >> 3;
2250 
2251 	return 0;
2252 }
2253 
2254 /**
2255  * irdma_prm_get_pbles - get pble's from prm
2256  * @pprm: pble resource manager
2257  * @chunkinfo: nformation about chunk where pble's were acquired
2258  * @mem_size: size of pble memory needed
2259  * @vaddr: returns virtual address of pble memory
2260  * @fpm_addr: returns fpm address of pble memory
2261  */
2262 enum irdma_status_code
2263 irdma_prm_get_pbles(struct irdma_pble_prm *pprm,
2264 		    struct irdma_pble_chunkinfo *chunkinfo, u64 mem_size,
2265 		    u64 **vaddr, u64 *fpm_addr)
2266 {
2267 	u64 bits_needed;
2268 	u64 bit_idx = PBLE_INVALID_IDX;
2269 	struct irdma_chunk *pchunk = NULL;
2270 	struct list_head *chunk_entry = pprm->clist.next;
2271 	u32 offset;
2272 	unsigned long flags;
2273 	*vaddr = NULL;
2274 	*fpm_addr = 0;
2275 
2276 	bits_needed = DIV_ROUND_UP_ULL(mem_size, BIT_ULL(pprm->pble_shift));
2277 
2278 	spin_lock_irqsave(&pprm->prm_lock, flags);
2279 	while (chunk_entry != &pprm->clist) {
2280 		pchunk = (struct irdma_chunk *)chunk_entry;
2281 		bit_idx = bitmap_find_next_zero_area(pchunk->bitmapbuf,
2282 						     pchunk->sizeofbitmap, 0,
2283 						     bits_needed, 0);
2284 		if (bit_idx < pchunk->sizeofbitmap)
2285 			break;
2286 
2287 		/* list.next used macro */
2288 		chunk_entry = pchunk->list.next;
2289 	}
2290 
2291 	if (!pchunk || bit_idx >= pchunk->sizeofbitmap) {
2292 		spin_unlock_irqrestore(&pprm->prm_lock, flags);
2293 		return IRDMA_ERR_NO_MEMORY;
2294 	}
2295 
2296 	bitmap_set(pchunk->bitmapbuf, bit_idx, bits_needed);
2297 	offset = bit_idx << pprm->pble_shift;
2298 	*vaddr = pchunk->vaddr + offset;
2299 	*fpm_addr = pchunk->fpm_addr + offset;
2300 
2301 	chunkinfo->pchunk = pchunk;
2302 	chunkinfo->bit_idx = bit_idx;
2303 	chunkinfo->bits_used = bits_needed;
2304 	/* 3 is sizeof pble divide */
2305 	pprm->free_pble_cnt -= chunkinfo->bits_used << (pprm->pble_shift - 3);
2306 	spin_unlock_irqrestore(&pprm->prm_lock, flags);
2307 
2308 	return 0;
2309 }
2310 
2311 /**
2312  * irdma_prm_return_pbles - return pbles back to prm
2313  * @pprm: pble resource manager
2314  * @chunkinfo: chunk where pble's were acquired and to be freed
2315  */
2316 void irdma_prm_return_pbles(struct irdma_pble_prm *pprm,
2317 			    struct irdma_pble_chunkinfo *chunkinfo)
2318 {
2319 	unsigned long flags;
2320 
2321 	spin_lock_irqsave(&pprm->prm_lock, flags);
2322 	pprm->free_pble_cnt += chunkinfo->bits_used << (pprm->pble_shift - 3);
2323 	bitmap_clear(chunkinfo->pchunk->bitmapbuf, chunkinfo->bit_idx,
2324 		     chunkinfo->bits_used);
2325 	spin_unlock_irqrestore(&pprm->prm_lock, flags);
2326 }
2327 
2328 enum irdma_status_code irdma_map_vm_page_list(struct irdma_hw *hw, void *va,
2329 					      dma_addr_t *pg_dma, u32 pg_cnt)
2330 {
2331 	struct page *vm_page;
2332 	int i;
2333 	u8 *addr;
2334 
2335 	addr = (u8 *)(uintptr_t)va;
2336 	for (i = 0; i < pg_cnt; i++) {
2337 		vm_page = vmalloc_to_page(addr);
2338 		if (!vm_page)
2339 			goto err;
2340 
2341 		pg_dma[i] = dma_map_page(hw->device, vm_page, 0, PAGE_SIZE,
2342 					 DMA_BIDIRECTIONAL);
2343 		if (dma_mapping_error(hw->device, pg_dma[i]))
2344 			goto err;
2345 
2346 		addr += PAGE_SIZE;
2347 	}
2348 
2349 	return 0;
2350 
2351 err:
2352 	irdma_unmap_vm_page_list(hw, pg_dma, i);
2353 	return IRDMA_ERR_NO_MEMORY;
2354 }
2355 
2356 void irdma_unmap_vm_page_list(struct irdma_hw *hw, dma_addr_t *pg_dma, u32 pg_cnt)
2357 {
2358 	int i;
2359 
2360 	for (i = 0; i < pg_cnt; i++)
2361 		dma_unmap_page(hw->device, pg_dma[i], PAGE_SIZE, DMA_BIDIRECTIONAL);
2362 }
2363 
2364 /**
2365  * irdma_pble_free_paged_mem - free virtual paged memory
2366  * @chunk: chunk to free with paged memory
2367  */
2368 void irdma_pble_free_paged_mem(struct irdma_chunk *chunk)
2369 {
2370 	if (!chunk->pg_cnt)
2371 		goto done;
2372 
2373 	irdma_unmap_vm_page_list(chunk->dev->hw, chunk->dmainfo.dmaaddrs,
2374 				 chunk->pg_cnt);
2375 
2376 done:
2377 	kfree(chunk->dmainfo.dmaaddrs);
2378 	chunk->dmainfo.dmaaddrs = NULL;
2379 	vfree(chunk->vaddr);
2380 	chunk->vaddr = NULL;
2381 	chunk->type = 0;
2382 }
2383 
2384 /**
2385  * irdma_pble_get_paged_mem -allocate paged memory for pbles
2386  * @chunk: chunk to add for paged memory
2387  * @pg_cnt: number of pages needed
2388  */
2389 enum irdma_status_code irdma_pble_get_paged_mem(struct irdma_chunk *chunk,
2390 						u32 pg_cnt)
2391 {
2392 	u32 size;
2393 	void *va;
2394 
2395 	chunk->dmainfo.dmaaddrs = kzalloc(pg_cnt << 3, GFP_KERNEL);
2396 	if (!chunk->dmainfo.dmaaddrs)
2397 		return IRDMA_ERR_NO_MEMORY;
2398 
2399 	size = PAGE_SIZE * pg_cnt;
2400 	va = vmalloc(size);
2401 	if (!va)
2402 		goto err;
2403 
2404 	if (irdma_map_vm_page_list(chunk->dev->hw, va, chunk->dmainfo.dmaaddrs,
2405 				   pg_cnt)) {
2406 		vfree(va);
2407 		goto err;
2408 	}
2409 	chunk->vaddr = va;
2410 	chunk->size = size;
2411 	chunk->pg_cnt = pg_cnt;
2412 	chunk->type = PBLE_SD_PAGED;
2413 
2414 	return 0;
2415 err:
2416 	kfree(chunk->dmainfo.dmaaddrs);
2417 	chunk->dmainfo.dmaaddrs = NULL;
2418 
2419 	return IRDMA_ERR_NO_MEMORY;
2420 }
2421 
2422 /**
2423  * irdma_alloc_ws_node_id - Allocate a tx scheduler node ID
2424  * @dev: device pointer
2425  */
2426 u16 irdma_alloc_ws_node_id(struct irdma_sc_dev *dev)
2427 {
2428 	struct irdma_pci_f *rf = dev_to_rf(dev);
2429 	u32 next = 1;
2430 	u32 node_id;
2431 
2432 	if (irdma_alloc_rsrc(rf, rf->allocated_ws_nodes, rf->max_ws_node_id,
2433 			     &node_id, &next))
2434 		return IRDMA_WS_NODE_INVALID;
2435 
2436 	return (u16)node_id;
2437 }
2438 
2439 /**
2440  * irdma_free_ws_node_id - Free a tx scheduler node ID
2441  * @dev: device pointer
2442  * @node_id: Work scheduler node ID
2443  */
2444 void irdma_free_ws_node_id(struct irdma_sc_dev *dev, u16 node_id)
2445 {
2446 	struct irdma_pci_f *rf = dev_to_rf(dev);
2447 
2448 	irdma_free_rsrc(rf, rf->allocated_ws_nodes, (u32)node_id);
2449 }
2450 
2451 /**
2452  * irdma_modify_qp_to_err - Modify a QP to error
2453  * @sc_qp: qp structure
2454  */
2455 void irdma_modify_qp_to_err(struct irdma_sc_qp *sc_qp)
2456 {
2457 	struct irdma_qp *qp = sc_qp->qp_uk.back_qp;
2458 	struct ib_qp_attr attr;
2459 
2460 	if (qp->iwdev->rf->reset)
2461 		return;
2462 	attr.qp_state = IB_QPS_ERR;
2463 
2464 	if (rdma_protocol_roce(qp->ibqp.device, 1))
2465 		irdma_modify_qp_roce(&qp->ibqp, &attr, IB_QP_STATE, NULL);
2466 	else
2467 		irdma_modify_qp(&qp->ibqp, &attr, IB_QP_STATE, NULL);
2468 }
2469 
2470 void irdma_ib_qp_event(struct irdma_qp *iwqp, enum irdma_qp_event_type event)
2471 {
2472 	struct ib_event ibevent;
2473 
2474 	if (!iwqp->ibqp.event_handler)
2475 		return;
2476 
2477 	switch (event) {
2478 	case IRDMA_QP_EVENT_CATASTROPHIC:
2479 		ibevent.event = IB_EVENT_QP_FATAL;
2480 		break;
2481 	case IRDMA_QP_EVENT_ACCESS_ERR:
2482 		ibevent.event = IB_EVENT_QP_ACCESS_ERR;
2483 		break;
2484 	}
2485 	ibevent.device = iwqp->ibqp.device;
2486 	ibevent.element.qp = &iwqp->ibqp;
2487 	iwqp->ibqp.event_handler(&ibevent, iwqp->ibqp.qp_context);
2488 }
2489 
2490 bool irdma_cq_empty(struct irdma_cq *iwcq)
2491 {
2492 	struct irdma_cq_uk *ukcq;
2493 	u64 qword3;
2494 	__le64 *cqe;
2495 	u8 polarity;
2496 
2497 	ukcq  = &iwcq->sc_cq.cq_uk;
2498 	cqe = IRDMA_GET_CURRENT_CQ_ELEM(ukcq);
2499 	get_64bit_val(cqe, 24, &qword3);
2500 	polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword3);
2501 
2502 	return polarity != ukcq->polarity;
2503 }
2504