xref: /openbmc/linux/drivers/scsi/lpfc/lpfc_sli.c (revision e330fb14)
1 /*******************************************************************
2  * This file is part of the Emulex Linux Device Driver for         *
3  * Fibre Channel Host Bus Adapters.                                *
4  * Copyright (C) 2017-2021 Broadcom. All Rights Reserved. The term *
5  * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.     *
6  * Copyright (C) 2004-2016 Emulex.  All rights reserved.           *
7  * EMULEX and SLI are trademarks of Emulex.                        *
8  * www.broadcom.com                                                *
9  * Portions Copyright (C) 2004-2005 Christoph Hellwig              *
10  *                                                                 *
11  * This program is free software; you can redistribute it and/or   *
12  * modify it under the terms of version 2 of the GNU General       *
13  * Public License as published by the Free Software Foundation.    *
14  * This program is distributed in the hope that it will be useful. *
15  * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND          *
16  * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY,  *
17  * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE      *
18  * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *
19  * TO BE LEGALLY INVALID.  See the GNU General Public License for  *
20  * more details, a copy of which can be found in the file COPYING  *
21  * included with this package.                                     *
22  *******************************************************************/
23 
24 #include <linux/blkdev.h>
25 #include <linux/pci.h>
26 #include <linux/interrupt.h>
27 #include <linux/delay.h>
28 #include <linux/slab.h>
29 #include <linux/lockdep.h>
30 
31 #include <scsi/scsi.h>
32 #include <scsi/scsi_cmnd.h>
33 #include <scsi/scsi_device.h>
34 #include <scsi/scsi_host.h>
35 #include <scsi/scsi_transport_fc.h>
36 #include <scsi/fc/fc_fs.h>
37 #include <linux/aer.h>
38 #include <linux/crash_dump.h>
39 #ifdef CONFIG_X86
40 #include <asm/set_memory.h>
41 #endif
42 
43 #include "lpfc_hw4.h"
44 #include "lpfc_hw.h"
45 #include "lpfc_sli.h"
46 #include "lpfc_sli4.h"
47 #include "lpfc_nl.h"
48 #include "lpfc_disc.h"
49 #include "lpfc.h"
50 #include "lpfc_scsi.h"
51 #include "lpfc_nvme.h"
52 #include "lpfc_crtn.h"
53 #include "lpfc_logmsg.h"
54 #include "lpfc_compat.h"
55 #include "lpfc_debugfs.h"
56 #include "lpfc_vport.h"
57 #include "lpfc_version.h"
58 
59 /* There are only four IOCB completion types. */
60 typedef enum _lpfc_iocb_type {
61 	LPFC_UNKNOWN_IOCB,
62 	LPFC_UNSOL_IOCB,
63 	LPFC_SOL_IOCB,
64 	LPFC_ABORT_IOCB
65 } lpfc_iocb_type;
66 
67 
68 /* Provide function prototypes local to this module. */
69 static int lpfc_sli_issue_mbox_s4(struct lpfc_hba *, LPFC_MBOXQ_t *,
70 				  uint32_t);
71 static int lpfc_sli4_read_rev(struct lpfc_hba *, LPFC_MBOXQ_t *,
72 			      uint8_t *, uint32_t *);
73 static struct lpfc_iocbq *lpfc_sli4_els_wcqe_to_rspiocbq(struct lpfc_hba *,
74 							 struct lpfc_iocbq *);
75 static void lpfc_sli4_send_seq_to_ulp(struct lpfc_vport *,
76 				      struct hbq_dmabuf *);
77 static void lpfc_sli4_handle_mds_loopback(struct lpfc_vport *vport,
78 					  struct hbq_dmabuf *dmabuf);
79 static bool lpfc_sli4_fp_handle_cqe(struct lpfc_hba *phba,
80 				   struct lpfc_queue *cq, struct lpfc_cqe *cqe);
81 static int lpfc_sli4_post_sgl_list(struct lpfc_hba *, struct list_head *,
82 				       int);
83 static void lpfc_sli4_hba_handle_eqe(struct lpfc_hba *phba,
84 				     struct lpfc_queue *eq,
85 				     struct lpfc_eqe *eqe);
86 static bool lpfc_sli4_mbox_completions_pending(struct lpfc_hba *phba);
87 static bool lpfc_sli4_process_missed_mbox_completions(struct lpfc_hba *phba);
88 static struct lpfc_cqe *lpfc_sli4_cq_get(struct lpfc_queue *q);
89 static void __lpfc_sli4_consume_cqe(struct lpfc_hba *phba,
90 				    struct lpfc_queue *cq,
91 				    struct lpfc_cqe *cqe);
92 
93 union lpfc_wqe128 lpfc_iread_cmd_template;
94 union lpfc_wqe128 lpfc_iwrite_cmd_template;
95 union lpfc_wqe128 lpfc_icmnd_cmd_template;
96 
97 static IOCB_t *
98 lpfc_get_iocb_from_iocbq(struct lpfc_iocbq *iocbq)
99 {
100 	return &iocbq->iocb;
101 }
102 
103 /* Setup WQE templates for IOs */
104 void lpfc_wqe_cmd_template(void)
105 {
106 	union lpfc_wqe128 *wqe;
107 
108 	/* IREAD template */
109 	wqe = &lpfc_iread_cmd_template;
110 	memset(wqe, 0, sizeof(union lpfc_wqe128));
111 
112 	/* Word 0, 1, 2 - BDE is variable */
113 
114 	/* Word 3 - cmd_buff_len, payload_offset_len is zero */
115 
116 	/* Word 4 - total_xfer_len is variable */
117 
118 	/* Word 5 - is zero */
119 
120 	/* Word 6 - ctxt_tag, xri_tag is variable */
121 
122 	/* Word 7 */
123 	bf_set(wqe_cmnd, &wqe->fcp_iread.wqe_com, CMD_FCP_IREAD64_WQE);
124 	bf_set(wqe_pu, &wqe->fcp_iread.wqe_com, PARM_READ_CHECK);
125 	bf_set(wqe_class, &wqe->fcp_iread.wqe_com, CLASS3);
126 	bf_set(wqe_ct, &wqe->fcp_iread.wqe_com, SLI4_CT_RPI);
127 
128 	/* Word 8 - abort_tag is variable */
129 
130 	/* Word 9  - reqtag is variable */
131 
132 	/* Word 10 - dbde, wqes is variable */
133 	bf_set(wqe_qosd, &wqe->fcp_iread.wqe_com, 0);
134 	bf_set(wqe_iod, &wqe->fcp_iread.wqe_com, LPFC_WQE_IOD_READ);
135 	bf_set(wqe_lenloc, &wqe->fcp_iread.wqe_com, LPFC_WQE_LENLOC_WORD4);
136 	bf_set(wqe_dbde, &wqe->fcp_iread.wqe_com, 0);
137 	bf_set(wqe_wqes, &wqe->fcp_iread.wqe_com, 1);
138 
139 	/* Word 11 - pbde is variable */
140 	bf_set(wqe_cmd_type, &wqe->fcp_iread.wqe_com, COMMAND_DATA_IN);
141 	bf_set(wqe_cqid, &wqe->fcp_iread.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
142 	bf_set(wqe_pbde, &wqe->fcp_iread.wqe_com, 0);
143 
144 	/* Word 12 - is zero */
145 
146 	/* Word 13, 14, 15 - PBDE is variable */
147 
148 	/* IWRITE template */
149 	wqe = &lpfc_iwrite_cmd_template;
150 	memset(wqe, 0, sizeof(union lpfc_wqe128));
151 
152 	/* Word 0, 1, 2 - BDE is variable */
153 
154 	/* Word 3 - cmd_buff_len, payload_offset_len is zero */
155 
156 	/* Word 4 - total_xfer_len is variable */
157 
158 	/* Word 5 - initial_xfer_len is variable */
159 
160 	/* Word 6 - ctxt_tag, xri_tag is variable */
161 
162 	/* Word 7 */
163 	bf_set(wqe_cmnd, &wqe->fcp_iwrite.wqe_com, CMD_FCP_IWRITE64_WQE);
164 	bf_set(wqe_pu, &wqe->fcp_iwrite.wqe_com, PARM_READ_CHECK);
165 	bf_set(wqe_class, &wqe->fcp_iwrite.wqe_com, CLASS3);
166 	bf_set(wqe_ct, &wqe->fcp_iwrite.wqe_com, SLI4_CT_RPI);
167 
168 	/* Word 8 - abort_tag is variable */
169 
170 	/* Word 9  - reqtag is variable */
171 
172 	/* Word 10 - dbde, wqes is variable */
173 	bf_set(wqe_qosd, &wqe->fcp_iwrite.wqe_com, 0);
174 	bf_set(wqe_iod, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_IOD_WRITE);
175 	bf_set(wqe_lenloc, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_LENLOC_WORD4);
176 	bf_set(wqe_dbde, &wqe->fcp_iwrite.wqe_com, 0);
177 	bf_set(wqe_wqes, &wqe->fcp_iwrite.wqe_com, 1);
178 
179 	/* Word 11 - pbde is variable */
180 	bf_set(wqe_cmd_type, &wqe->fcp_iwrite.wqe_com, COMMAND_DATA_OUT);
181 	bf_set(wqe_cqid, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
182 	bf_set(wqe_pbde, &wqe->fcp_iwrite.wqe_com, 0);
183 
184 	/* Word 12 - is zero */
185 
186 	/* Word 13, 14, 15 - PBDE is variable */
187 
188 	/* ICMND template */
189 	wqe = &lpfc_icmnd_cmd_template;
190 	memset(wqe, 0, sizeof(union lpfc_wqe128));
191 
192 	/* Word 0, 1, 2 - BDE is variable */
193 
194 	/* Word 3 - payload_offset_len is variable */
195 
196 	/* Word 4, 5 - is zero */
197 
198 	/* Word 6 - ctxt_tag, xri_tag is variable */
199 
200 	/* Word 7 */
201 	bf_set(wqe_cmnd, &wqe->fcp_icmd.wqe_com, CMD_FCP_ICMND64_WQE);
202 	bf_set(wqe_pu, &wqe->fcp_icmd.wqe_com, 0);
203 	bf_set(wqe_class, &wqe->fcp_icmd.wqe_com, CLASS3);
204 	bf_set(wqe_ct, &wqe->fcp_icmd.wqe_com, SLI4_CT_RPI);
205 
206 	/* Word 8 - abort_tag is variable */
207 
208 	/* Word 9  - reqtag is variable */
209 
210 	/* Word 10 - dbde, wqes is variable */
211 	bf_set(wqe_qosd, &wqe->fcp_icmd.wqe_com, 1);
212 	bf_set(wqe_iod, &wqe->fcp_icmd.wqe_com, LPFC_WQE_IOD_NONE);
213 	bf_set(wqe_lenloc, &wqe->fcp_icmd.wqe_com, LPFC_WQE_LENLOC_NONE);
214 	bf_set(wqe_dbde, &wqe->fcp_icmd.wqe_com, 0);
215 	bf_set(wqe_wqes, &wqe->fcp_icmd.wqe_com, 1);
216 
217 	/* Word 11 */
218 	bf_set(wqe_cmd_type, &wqe->fcp_icmd.wqe_com, COMMAND_DATA_IN);
219 	bf_set(wqe_cqid, &wqe->fcp_icmd.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
220 	bf_set(wqe_pbde, &wqe->fcp_icmd.wqe_com, 0);
221 
222 	/* Word 12, 13, 14, 15 - is zero */
223 }
224 
225 #if defined(CONFIG_64BIT) && defined(__LITTLE_ENDIAN)
226 /**
227  * lpfc_sli4_pcimem_bcopy - SLI4 memory copy function
228  * @srcp: Source memory pointer.
229  * @destp: Destination memory pointer.
230  * @cnt: Number of words required to be copied.
231  *       Must be a multiple of sizeof(uint64_t)
232  *
233  * This function is used for copying data between driver memory
234  * and the SLI WQ. This function also changes the endianness
235  * of each word if native endianness is different from SLI
236  * endianness. This function can be called with or without
237  * lock.
238  **/
239 static void
240 lpfc_sli4_pcimem_bcopy(void *srcp, void *destp, uint32_t cnt)
241 {
242 	uint64_t *src = srcp;
243 	uint64_t *dest = destp;
244 	int i;
245 
246 	for (i = 0; i < (int)cnt; i += sizeof(uint64_t))
247 		*dest++ = *src++;
248 }
249 #else
250 #define lpfc_sli4_pcimem_bcopy(a, b, c) lpfc_sli_pcimem_bcopy(a, b, c)
251 #endif
252 
253 /**
254  * lpfc_sli4_wq_put - Put a Work Queue Entry on an Work Queue
255  * @q: The Work Queue to operate on.
256  * @wqe: The work Queue Entry to put on the Work queue.
257  *
258  * This routine will copy the contents of @wqe to the next available entry on
259  * the @q. This function will then ring the Work Queue Doorbell to signal the
260  * HBA to start processing the Work Queue Entry. This function returns 0 if
261  * successful. If no entries are available on @q then this function will return
262  * -ENOMEM.
263  * The caller is expected to hold the hbalock when calling this routine.
264  **/
265 static int
266 lpfc_sli4_wq_put(struct lpfc_queue *q, union lpfc_wqe128 *wqe)
267 {
268 	union lpfc_wqe *temp_wqe;
269 	struct lpfc_register doorbell;
270 	uint32_t host_index;
271 	uint32_t idx;
272 	uint32_t i = 0;
273 	uint8_t *tmp;
274 	u32 if_type;
275 
276 	/* sanity check on queue memory */
277 	if (unlikely(!q))
278 		return -ENOMEM;
279 
280 	temp_wqe = lpfc_sli4_qe(q, q->host_index);
281 
282 	/* If the host has not yet processed the next entry then we are done */
283 	idx = ((q->host_index + 1) % q->entry_count);
284 	if (idx == q->hba_index) {
285 		q->WQ_overflow++;
286 		return -EBUSY;
287 	}
288 	q->WQ_posted++;
289 	/* set consumption flag every once in a while */
290 	if (!((q->host_index + 1) % q->notify_interval))
291 		bf_set(wqe_wqec, &wqe->generic.wqe_com, 1);
292 	else
293 		bf_set(wqe_wqec, &wqe->generic.wqe_com, 0);
294 	if (q->phba->sli3_options & LPFC_SLI4_PHWQ_ENABLED)
295 		bf_set(wqe_wqid, &wqe->generic.wqe_com, q->queue_id);
296 	lpfc_sli4_pcimem_bcopy(wqe, temp_wqe, q->entry_size);
297 	if (q->dpp_enable && q->phba->cfg_enable_dpp) {
298 		/* write to DPP aperture taking advatage of Combined Writes */
299 		tmp = (uint8_t *)temp_wqe;
300 #ifdef __raw_writeq
301 		for (i = 0; i < q->entry_size; i += sizeof(uint64_t))
302 			__raw_writeq(*((uint64_t *)(tmp + i)),
303 					q->dpp_regaddr + i);
304 #else
305 		for (i = 0; i < q->entry_size; i += sizeof(uint32_t))
306 			__raw_writel(*((uint32_t *)(tmp + i)),
307 					q->dpp_regaddr + i);
308 #endif
309 	}
310 	/* ensure WQE bcopy and DPP flushed before doorbell write */
311 	wmb();
312 
313 	/* Update the host index before invoking device */
314 	host_index = q->host_index;
315 
316 	q->host_index = idx;
317 
318 	/* Ring Doorbell */
319 	doorbell.word0 = 0;
320 	if (q->db_format == LPFC_DB_LIST_FORMAT) {
321 		if (q->dpp_enable && q->phba->cfg_enable_dpp) {
322 			bf_set(lpfc_if6_wq_db_list_fm_num_posted, &doorbell, 1);
323 			bf_set(lpfc_if6_wq_db_list_fm_dpp, &doorbell, 1);
324 			bf_set(lpfc_if6_wq_db_list_fm_dpp_id, &doorbell,
325 			    q->dpp_id);
326 			bf_set(lpfc_if6_wq_db_list_fm_id, &doorbell,
327 			    q->queue_id);
328 		} else {
329 			bf_set(lpfc_wq_db_list_fm_num_posted, &doorbell, 1);
330 			bf_set(lpfc_wq_db_list_fm_id, &doorbell, q->queue_id);
331 
332 			/* Leave bits <23:16> clear for if_type 6 dpp */
333 			if_type = bf_get(lpfc_sli_intf_if_type,
334 					 &q->phba->sli4_hba.sli_intf);
335 			if (if_type != LPFC_SLI_INTF_IF_TYPE_6)
336 				bf_set(lpfc_wq_db_list_fm_index, &doorbell,
337 				       host_index);
338 		}
339 	} else if (q->db_format == LPFC_DB_RING_FORMAT) {
340 		bf_set(lpfc_wq_db_ring_fm_num_posted, &doorbell, 1);
341 		bf_set(lpfc_wq_db_ring_fm_id, &doorbell, q->queue_id);
342 	} else {
343 		return -EINVAL;
344 	}
345 	writel(doorbell.word0, q->db_regaddr);
346 
347 	return 0;
348 }
349 
350 /**
351  * lpfc_sli4_wq_release - Updates internal hba index for WQ
352  * @q: The Work Queue to operate on.
353  * @index: The index to advance the hba index to.
354  *
355  * This routine will update the HBA index of a queue to reflect consumption of
356  * Work Queue Entries by the HBA. When the HBA indicates that it has consumed
357  * an entry the host calls this function to update the queue's internal
358  * pointers.
359  **/
360 static void
361 lpfc_sli4_wq_release(struct lpfc_queue *q, uint32_t index)
362 {
363 	/* sanity check on queue memory */
364 	if (unlikely(!q))
365 		return;
366 
367 	q->hba_index = index;
368 }
369 
370 /**
371  * lpfc_sli4_mq_put - Put a Mailbox Queue Entry on an Mailbox Queue
372  * @q: The Mailbox Queue to operate on.
373  * @mqe: The Mailbox Queue Entry to put on the Work queue.
374  *
375  * This routine will copy the contents of @mqe to the next available entry on
376  * the @q. This function will then ring the Work Queue Doorbell to signal the
377  * HBA to start processing the Work Queue Entry. This function returns 0 if
378  * successful. If no entries are available on @q then this function will return
379  * -ENOMEM.
380  * The caller is expected to hold the hbalock when calling this routine.
381  **/
382 static uint32_t
383 lpfc_sli4_mq_put(struct lpfc_queue *q, struct lpfc_mqe *mqe)
384 {
385 	struct lpfc_mqe *temp_mqe;
386 	struct lpfc_register doorbell;
387 
388 	/* sanity check on queue memory */
389 	if (unlikely(!q))
390 		return -ENOMEM;
391 	temp_mqe = lpfc_sli4_qe(q, q->host_index);
392 
393 	/* If the host has not yet processed the next entry then we are done */
394 	if (((q->host_index + 1) % q->entry_count) == q->hba_index)
395 		return -ENOMEM;
396 	lpfc_sli4_pcimem_bcopy(mqe, temp_mqe, q->entry_size);
397 	/* Save off the mailbox pointer for completion */
398 	q->phba->mbox = (MAILBOX_t *)temp_mqe;
399 
400 	/* Update the host index before invoking device */
401 	q->host_index = ((q->host_index + 1) % q->entry_count);
402 
403 	/* Ring Doorbell */
404 	doorbell.word0 = 0;
405 	bf_set(lpfc_mq_doorbell_num_posted, &doorbell, 1);
406 	bf_set(lpfc_mq_doorbell_id, &doorbell, q->queue_id);
407 	writel(doorbell.word0, q->phba->sli4_hba.MQDBregaddr);
408 	return 0;
409 }
410 
411 /**
412  * lpfc_sli4_mq_release - Updates internal hba index for MQ
413  * @q: The Mailbox Queue to operate on.
414  *
415  * This routine will update the HBA index of a queue to reflect consumption of
416  * a Mailbox Queue Entry by the HBA. When the HBA indicates that it has consumed
417  * an entry the host calls this function to update the queue's internal
418  * pointers. This routine returns the number of entries that were consumed by
419  * the HBA.
420  **/
421 static uint32_t
422 lpfc_sli4_mq_release(struct lpfc_queue *q)
423 {
424 	/* sanity check on queue memory */
425 	if (unlikely(!q))
426 		return 0;
427 
428 	/* Clear the mailbox pointer for completion */
429 	q->phba->mbox = NULL;
430 	q->hba_index = ((q->hba_index + 1) % q->entry_count);
431 	return 1;
432 }
433 
434 /**
435  * lpfc_sli4_eq_get - Gets the next valid EQE from a EQ
436  * @q: The Event Queue to get the first valid EQE from
437  *
438  * This routine will get the first valid Event Queue Entry from @q, update
439  * the queue's internal hba index, and return the EQE. If no valid EQEs are in
440  * the Queue (no more work to do), or the Queue is full of EQEs that have been
441  * processed, but not popped back to the HBA then this routine will return NULL.
442  **/
443 static struct lpfc_eqe *
444 lpfc_sli4_eq_get(struct lpfc_queue *q)
445 {
446 	struct lpfc_eqe *eqe;
447 
448 	/* sanity check on queue memory */
449 	if (unlikely(!q))
450 		return NULL;
451 	eqe = lpfc_sli4_qe(q, q->host_index);
452 
453 	/* If the next EQE is not valid then we are done */
454 	if (bf_get_le32(lpfc_eqe_valid, eqe) != q->qe_valid)
455 		return NULL;
456 
457 	/*
458 	 * insert barrier for instruction interlock : data from the hardware
459 	 * must have the valid bit checked before it can be copied and acted
460 	 * upon. Speculative instructions were allowing a bcopy at the start
461 	 * of lpfc_sli4_fp_handle_wcqe(), which is called immediately
462 	 * after our return, to copy data before the valid bit check above
463 	 * was done. As such, some of the copied data was stale. The barrier
464 	 * ensures the check is before any data is copied.
465 	 */
466 	mb();
467 	return eqe;
468 }
469 
470 /**
471  * lpfc_sli4_eq_clr_intr - Turn off interrupts from this EQ
472  * @q: The Event Queue to disable interrupts
473  *
474  **/
475 void
476 lpfc_sli4_eq_clr_intr(struct lpfc_queue *q)
477 {
478 	struct lpfc_register doorbell;
479 
480 	doorbell.word0 = 0;
481 	bf_set(lpfc_eqcq_doorbell_eqci, &doorbell, 1);
482 	bf_set(lpfc_eqcq_doorbell_qt, &doorbell, LPFC_QUEUE_TYPE_EVENT);
483 	bf_set(lpfc_eqcq_doorbell_eqid_hi, &doorbell,
484 		(q->queue_id >> LPFC_EQID_HI_FIELD_SHIFT));
485 	bf_set(lpfc_eqcq_doorbell_eqid_lo, &doorbell, q->queue_id);
486 	writel(doorbell.word0, q->phba->sli4_hba.EQDBregaddr);
487 }
488 
489 /**
490  * lpfc_sli4_if6_eq_clr_intr - Turn off interrupts from this EQ
491  * @q: The Event Queue to disable interrupts
492  *
493  **/
494 void
495 lpfc_sli4_if6_eq_clr_intr(struct lpfc_queue *q)
496 {
497 	struct lpfc_register doorbell;
498 
499 	doorbell.word0 = 0;
500 	bf_set(lpfc_if6_eq_doorbell_eqid, &doorbell, q->queue_id);
501 	writel(doorbell.word0, q->phba->sli4_hba.EQDBregaddr);
502 }
503 
504 /**
505  * lpfc_sli4_write_eq_db - write EQ DB for eqe's consumed or arm state
506  * @phba: adapter with EQ
507  * @q: The Event Queue that the host has completed processing for.
508  * @count: Number of elements that have been consumed
509  * @arm: Indicates whether the host wants to arms this CQ.
510  *
511  * This routine will notify the HBA, by ringing the doorbell, that count
512  * number of EQEs have been processed. The @arm parameter indicates whether
513  * the queue should be rearmed when ringing the doorbell.
514  **/
515 void
516 lpfc_sli4_write_eq_db(struct lpfc_hba *phba, struct lpfc_queue *q,
517 		     uint32_t count, bool arm)
518 {
519 	struct lpfc_register doorbell;
520 
521 	/* sanity check on queue memory */
522 	if (unlikely(!q || (count == 0 && !arm)))
523 		return;
524 
525 	/* ring doorbell for number popped */
526 	doorbell.word0 = 0;
527 	if (arm) {
528 		bf_set(lpfc_eqcq_doorbell_arm, &doorbell, 1);
529 		bf_set(lpfc_eqcq_doorbell_eqci, &doorbell, 1);
530 	}
531 	bf_set(lpfc_eqcq_doorbell_num_released, &doorbell, count);
532 	bf_set(lpfc_eqcq_doorbell_qt, &doorbell, LPFC_QUEUE_TYPE_EVENT);
533 	bf_set(lpfc_eqcq_doorbell_eqid_hi, &doorbell,
534 			(q->queue_id >> LPFC_EQID_HI_FIELD_SHIFT));
535 	bf_set(lpfc_eqcq_doorbell_eqid_lo, &doorbell, q->queue_id);
536 	writel(doorbell.word0, q->phba->sli4_hba.EQDBregaddr);
537 	/* PCI read to flush PCI pipeline on re-arming for INTx mode */
538 	if ((q->phba->intr_type == INTx) && (arm == LPFC_QUEUE_REARM))
539 		readl(q->phba->sli4_hba.EQDBregaddr);
540 }
541 
542 /**
543  * lpfc_sli4_if6_write_eq_db - write EQ DB for eqe's consumed or arm state
544  * @phba: adapter with EQ
545  * @q: The Event Queue that the host has completed processing for.
546  * @count: Number of elements that have been consumed
547  * @arm: Indicates whether the host wants to arms this CQ.
548  *
549  * This routine will notify the HBA, by ringing the doorbell, that count
550  * number of EQEs have been processed. The @arm parameter indicates whether
551  * the queue should be rearmed when ringing the doorbell.
552  **/
553 void
554 lpfc_sli4_if6_write_eq_db(struct lpfc_hba *phba, struct lpfc_queue *q,
555 			  uint32_t count, bool arm)
556 {
557 	struct lpfc_register doorbell;
558 
559 	/* sanity check on queue memory */
560 	if (unlikely(!q || (count == 0 && !arm)))
561 		return;
562 
563 	/* ring doorbell for number popped */
564 	doorbell.word0 = 0;
565 	if (arm)
566 		bf_set(lpfc_if6_eq_doorbell_arm, &doorbell, 1);
567 	bf_set(lpfc_if6_eq_doorbell_num_released, &doorbell, count);
568 	bf_set(lpfc_if6_eq_doorbell_eqid, &doorbell, q->queue_id);
569 	writel(doorbell.word0, q->phba->sli4_hba.EQDBregaddr);
570 	/* PCI read to flush PCI pipeline on re-arming for INTx mode */
571 	if ((q->phba->intr_type == INTx) && (arm == LPFC_QUEUE_REARM))
572 		readl(q->phba->sli4_hba.EQDBregaddr);
573 }
574 
575 static void
576 __lpfc_sli4_consume_eqe(struct lpfc_hba *phba, struct lpfc_queue *eq,
577 			struct lpfc_eqe *eqe)
578 {
579 	if (!phba->sli4_hba.pc_sli4_params.eqav)
580 		bf_set_le32(lpfc_eqe_valid, eqe, 0);
581 
582 	eq->host_index = ((eq->host_index + 1) % eq->entry_count);
583 
584 	/* if the index wrapped around, toggle the valid bit */
585 	if (phba->sli4_hba.pc_sli4_params.eqav && !eq->host_index)
586 		eq->qe_valid = (eq->qe_valid) ? 0 : 1;
587 }
588 
589 static void
590 lpfc_sli4_eqcq_flush(struct lpfc_hba *phba, struct lpfc_queue *eq)
591 {
592 	struct lpfc_eqe *eqe = NULL;
593 	u32 eq_count = 0, cq_count = 0;
594 	struct lpfc_cqe *cqe = NULL;
595 	struct lpfc_queue *cq = NULL, *childq = NULL;
596 	int cqid = 0;
597 
598 	/* walk all the EQ entries and drop on the floor */
599 	eqe = lpfc_sli4_eq_get(eq);
600 	while (eqe) {
601 		/* Get the reference to the corresponding CQ */
602 		cqid = bf_get_le32(lpfc_eqe_resource_id, eqe);
603 		cq = NULL;
604 
605 		list_for_each_entry(childq, &eq->child_list, list) {
606 			if (childq->queue_id == cqid) {
607 				cq = childq;
608 				break;
609 			}
610 		}
611 		/* If CQ is valid, iterate through it and drop all the CQEs */
612 		if (cq) {
613 			cqe = lpfc_sli4_cq_get(cq);
614 			while (cqe) {
615 				__lpfc_sli4_consume_cqe(phba, cq, cqe);
616 				cq_count++;
617 				cqe = lpfc_sli4_cq_get(cq);
618 			}
619 			/* Clear and re-arm the CQ */
620 			phba->sli4_hba.sli4_write_cq_db(phba, cq, cq_count,
621 			    LPFC_QUEUE_REARM);
622 			cq_count = 0;
623 		}
624 		__lpfc_sli4_consume_eqe(phba, eq, eqe);
625 		eq_count++;
626 		eqe = lpfc_sli4_eq_get(eq);
627 	}
628 
629 	/* Clear and re-arm the EQ */
630 	phba->sli4_hba.sli4_write_eq_db(phba, eq, eq_count, LPFC_QUEUE_REARM);
631 }
632 
633 static int
634 lpfc_sli4_process_eq(struct lpfc_hba *phba, struct lpfc_queue *eq,
635 		     uint8_t rearm)
636 {
637 	struct lpfc_eqe *eqe;
638 	int count = 0, consumed = 0;
639 
640 	if (cmpxchg(&eq->queue_claimed, 0, 1) != 0)
641 		goto rearm_and_exit;
642 
643 	eqe = lpfc_sli4_eq_get(eq);
644 	while (eqe) {
645 		lpfc_sli4_hba_handle_eqe(phba, eq, eqe);
646 		__lpfc_sli4_consume_eqe(phba, eq, eqe);
647 
648 		consumed++;
649 		if (!(++count % eq->max_proc_limit))
650 			break;
651 
652 		if (!(count % eq->notify_interval)) {
653 			phba->sli4_hba.sli4_write_eq_db(phba, eq, consumed,
654 							LPFC_QUEUE_NOARM);
655 			consumed = 0;
656 		}
657 
658 		eqe = lpfc_sli4_eq_get(eq);
659 	}
660 	eq->EQ_processed += count;
661 
662 	/* Track the max number of EQEs processed in 1 intr */
663 	if (count > eq->EQ_max_eqe)
664 		eq->EQ_max_eqe = count;
665 
666 	xchg(&eq->queue_claimed, 0);
667 
668 rearm_and_exit:
669 	/* Always clear the EQ. */
670 	phba->sli4_hba.sli4_write_eq_db(phba, eq, consumed, rearm);
671 
672 	return count;
673 }
674 
675 /**
676  * lpfc_sli4_cq_get - Gets the next valid CQE from a CQ
677  * @q: The Completion Queue to get the first valid CQE from
678  *
679  * This routine will get the first valid Completion Queue Entry from @q, update
680  * the queue's internal hba index, and return the CQE. If no valid CQEs are in
681  * the Queue (no more work to do), or the Queue is full of CQEs that have been
682  * processed, but not popped back to the HBA then this routine will return NULL.
683  **/
684 static struct lpfc_cqe *
685 lpfc_sli4_cq_get(struct lpfc_queue *q)
686 {
687 	struct lpfc_cqe *cqe;
688 
689 	/* sanity check on queue memory */
690 	if (unlikely(!q))
691 		return NULL;
692 	cqe = lpfc_sli4_qe(q, q->host_index);
693 
694 	/* If the next CQE is not valid then we are done */
695 	if (bf_get_le32(lpfc_cqe_valid, cqe) != q->qe_valid)
696 		return NULL;
697 
698 	/*
699 	 * insert barrier for instruction interlock : data from the hardware
700 	 * must have the valid bit checked before it can be copied and acted
701 	 * upon. Given what was seen in lpfc_sli4_cq_get() of speculative
702 	 * instructions allowing action on content before valid bit checked,
703 	 * add barrier here as well. May not be needed as "content" is a
704 	 * single 32-bit entity here (vs multi word structure for cq's).
705 	 */
706 	mb();
707 	return cqe;
708 }
709 
710 static void
711 __lpfc_sli4_consume_cqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
712 			struct lpfc_cqe *cqe)
713 {
714 	if (!phba->sli4_hba.pc_sli4_params.cqav)
715 		bf_set_le32(lpfc_cqe_valid, cqe, 0);
716 
717 	cq->host_index = ((cq->host_index + 1) % cq->entry_count);
718 
719 	/* if the index wrapped around, toggle the valid bit */
720 	if (phba->sli4_hba.pc_sli4_params.cqav && !cq->host_index)
721 		cq->qe_valid = (cq->qe_valid) ? 0 : 1;
722 }
723 
724 /**
725  * lpfc_sli4_write_cq_db - write cq DB for entries consumed or arm state.
726  * @phba: the adapter with the CQ
727  * @q: The Completion Queue that the host has completed processing for.
728  * @count: the number of elements that were consumed
729  * @arm: Indicates whether the host wants to arms this CQ.
730  *
731  * This routine will notify the HBA, by ringing the doorbell, that the
732  * CQEs have been processed. The @arm parameter specifies whether the
733  * queue should be rearmed when ringing the doorbell.
734  **/
735 void
736 lpfc_sli4_write_cq_db(struct lpfc_hba *phba, struct lpfc_queue *q,
737 		     uint32_t count, bool arm)
738 {
739 	struct lpfc_register doorbell;
740 
741 	/* sanity check on queue memory */
742 	if (unlikely(!q || (count == 0 && !arm)))
743 		return;
744 
745 	/* ring doorbell for number popped */
746 	doorbell.word0 = 0;
747 	if (arm)
748 		bf_set(lpfc_eqcq_doorbell_arm, &doorbell, 1);
749 	bf_set(lpfc_eqcq_doorbell_num_released, &doorbell, count);
750 	bf_set(lpfc_eqcq_doorbell_qt, &doorbell, LPFC_QUEUE_TYPE_COMPLETION);
751 	bf_set(lpfc_eqcq_doorbell_cqid_hi, &doorbell,
752 			(q->queue_id >> LPFC_CQID_HI_FIELD_SHIFT));
753 	bf_set(lpfc_eqcq_doorbell_cqid_lo, &doorbell, q->queue_id);
754 	writel(doorbell.word0, q->phba->sli4_hba.CQDBregaddr);
755 }
756 
757 /**
758  * lpfc_sli4_if6_write_cq_db - write cq DB for entries consumed or arm state.
759  * @phba: the adapter with the CQ
760  * @q: The Completion Queue that the host has completed processing for.
761  * @count: the number of elements that were consumed
762  * @arm: Indicates whether the host wants to arms this CQ.
763  *
764  * This routine will notify the HBA, by ringing the doorbell, that the
765  * CQEs have been processed. The @arm parameter specifies whether the
766  * queue should be rearmed when ringing the doorbell.
767  **/
768 void
769 lpfc_sli4_if6_write_cq_db(struct lpfc_hba *phba, struct lpfc_queue *q,
770 			 uint32_t count, bool arm)
771 {
772 	struct lpfc_register doorbell;
773 
774 	/* sanity check on queue memory */
775 	if (unlikely(!q || (count == 0 && !arm)))
776 		return;
777 
778 	/* ring doorbell for number popped */
779 	doorbell.word0 = 0;
780 	if (arm)
781 		bf_set(lpfc_if6_cq_doorbell_arm, &doorbell, 1);
782 	bf_set(lpfc_if6_cq_doorbell_num_released, &doorbell, count);
783 	bf_set(lpfc_if6_cq_doorbell_cqid, &doorbell, q->queue_id);
784 	writel(doorbell.word0, q->phba->sli4_hba.CQDBregaddr);
785 }
786 
787 /*
788  * lpfc_sli4_rq_put - Put a Receive Buffer Queue Entry on a Receive Queue
789  *
790  * This routine will copy the contents of @wqe to the next available entry on
791  * the @q. This function will then ring the Receive Queue Doorbell to signal the
792  * HBA to start processing the Receive Queue Entry. This function returns the
793  * index that the rqe was copied to if successful. If no entries are available
794  * on @q then this function will return -ENOMEM.
795  * The caller is expected to hold the hbalock when calling this routine.
796  **/
797 int
798 lpfc_sli4_rq_put(struct lpfc_queue *hq, struct lpfc_queue *dq,
799 		 struct lpfc_rqe *hrqe, struct lpfc_rqe *drqe)
800 {
801 	struct lpfc_rqe *temp_hrqe;
802 	struct lpfc_rqe *temp_drqe;
803 	struct lpfc_register doorbell;
804 	int hq_put_index;
805 	int dq_put_index;
806 
807 	/* sanity check on queue memory */
808 	if (unlikely(!hq) || unlikely(!dq))
809 		return -ENOMEM;
810 	hq_put_index = hq->host_index;
811 	dq_put_index = dq->host_index;
812 	temp_hrqe = lpfc_sli4_qe(hq, hq_put_index);
813 	temp_drqe = lpfc_sli4_qe(dq, dq_put_index);
814 
815 	if (hq->type != LPFC_HRQ || dq->type != LPFC_DRQ)
816 		return -EINVAL;
817 	if (hq_put_index != dq_put_index)
818 		return -EINVAL;
819 	/* If the host has not yet processed the next entry then we are done */
820 	if (((hq_put_index + 1) % hq->entry_count) == hq->hba_index)
821 		return -EBUSY;
822 	lpfc_sli4_pcimem_bcopy(hrqe, temp_hrqe, hq->entry_size);
823 	lpfc_sli4_pcimem_bcopy(drqe, temp_drqe, dq->entry_size);
824 
825 	/* Update the host index to point to the next slot */
826 	hq->host_index = ((hq_put_index + 1) % hq->entry_count);
827 	dq->host_index = ((dq_put_index + 1) % dq->entry_count);
828 	hq->RQ_buf_posted++;
829 
830 	/* Ring The Header Receive Queue Doorbell */
831 	if (!(hq->host_index % hq->notify_interval)) {
832 		doorbell.word0 = 0;
833 		if (hq->db_format == LPFC_DB_RING_FORMAT) {
834 			bf_set(lpfc_rq_db_ring_fm_num_posted, &doorbell,
835 			       hq->notify_interval);
836 			bf_set(lpfc_rq_db_ring_fm_id, &doorbell, hq->queue_id);
837 		} else if (hq->db_format == LPFC_DB_LIST_FORMAT) {
838 			bf_set(lpfc_rq_db_list_fm_num_posted, &doorbell,
839 			       hq->notify_interval);
840 			bf_set(lpfc_rq_db_list_fm_index, &doorbell,
841 			       hq->host_index);
842 			bf_set(lpfc_rq_db_list_fm_id, &doorbell, hq->queue_id);
843 		} else {
844 			return -EINVAL;
845 		}
846 		writel(doorbell.word0, hq->db_regaddr);
847 	}
848 	return hq_put_index;
849 }
850 
851 /*
852  * lpfc_sli4_rq_release - Updates internal hba index for RQ
853  *
854  * This routine will update the HBA index of a queue to reflect consumption of
855  * one Receive Queue Entry by the HBA. When the HBA indicates that it has
856  * consumed an entry the host calls this function to update the queue's
857  * internal pointers. This routine returns the number of entries that were
858  * consumed by the HBA.
859  **/
860 static uint32_t
861 lpfc_sli4_rq_release(struct lpfc_queue *hq, struct lpfc_queue *dq)
862 {
863 	/* sanity check on queue memory */
864 	if (unlikely(!hq) || unlikely(!dq))
865 		return 0;
866 
867 	if ((hq->type != LPFC_HRQ) || (dq->type != LPFC_DRQ))
868 		return 0;
869 	hq->hba_index = ((hq->hba_index + 1) % hq->entry_count);
870 	dq->hba_index = ((dq->hba_index + 1) % dq->entry_count);
871 	return 1;
872 }
873 
874 /**
875  * lpfc_cmd_iocb - Get next command iocb entry in the ring
876  * @phba: Pointer to HBA context object.
877  * @pring: Pointer to driver SLI ring object.
878  *
879  * This function returns pointer to next command iocb entry
880  * in the command ring. The caller must hold hbalock to prevent
881  * other threads consume the next command iocb.
882  * SLI-2/SLI-3 provide different sized iocbs.
883  **/
884 static inline IOCB_t *
885 lpfc_cmd_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
886 {
887 	return (IOCB_t *) (((char *) pring->sli.sli3.cmdringaddr) +
888 			   pring->sli.sli3.cmdidx * phba->iocb_cmd_size);
889 }
890 
891 /**
892  * lpfc_resp_iocb - Get next response iocb entry in the ring
893  * @phba: Pointer to HBA context object.
894  * @pring: Pointer to driver SLI ring object.
895  *
896  * This function returns pointer to next response iocb entry
897  * in the response ring. The caller must hold hbalock to make sure
898  * that no other thread consume the next response iocb.
899  * SLI-2/SLI-3 provide different sized iocbs.
900  **/
901 static inline IOCB_t *
902 lpfc_resp_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
903 {
904 	return (IOCB_t *) (((char *) pring->sli.sli3.rspringaddr) +
905 			   pring->sli.sli3.rspidx * phba->iocb_rsp_size);
906 }
907 
908 /**
909  * __lpfc_sli_get_iocbq - Allocates an iocb object from iocb pool
910  * @phba: Pointer to HBA context object.
911  *
912  * This function is called with hbalock held. This function
913  * allocates a new driver iocb object from the iocb pool. If the
914  * allocation is successful, it returns pointer to the newly
915  * allocated iocb object else it returns NULL.
916  **/
917 struct lpfc_iocbq *
918 __lpfc_sli_get_iocbq(struct lpfc_hba *phba)
919 {
920 	struct list_head *lpfc_iocb_list = &phba->lpfc_iocb_list;
921 	struct lpfc_iocbq * iocbq = NULL;
922 
923 	lockdep_assert_held(&phba->hbalock);
924 
925 	list_remove_head(lpfc_iocb_list, iocbq, struct lpfc_iocbq, list);
926 	if (iocbq)
927 		phba->iocb_cnt++;
928 	if (phba->iocb_cnt > phba->iocb_max)
929 		phba->iocb_max = phba->iocb_cnt;
930 	return iocbq;
931 }
932 
933 /**
934  * __lpfc_clear_active_sglq - Remove the active sglq for this XRI.
935  * @phba: Pointer to HBA context object.
936  * @xritag: XRI value.
937  *
938  * This function clears the sglq pointer from the array of active
939  * sglq's. The xritag that is passed in is used to index into the
940  * array. Before the xritag can be used it needs to be adjusted
941  * by subtracting the xribase.
942  *
943  * Returns sglq ponter = success, NULL = Failure.
944  **/
945 struct lpfc_sglq *
946 __lpfc_clear_active_sglq(struct lpfc_hba *phba, uint16_t xritag)
947 {
948 	struct lpfc_sglq *sglq;
949 
950 	sglq = phba->sli4_hba.lpfc_sglq_active_list[xritag];
951 	phba->sli4_hba.lpfc_sglq_active_list[xritag] = NULL;
952 	return sglq;
953 }
954 
955 /**
956  * __lpfc_get_active_sglq - Get the active sglq for this XRI.
957  * @phba: Pointer to HBA context object.
958  * @xritag: XRI value.
959  *
960  * This function returns the sglq pointer from the array of active
961  * sglq's. The xritag that is passed in is used to index into the
962  * array. Before the xritag can be used it needs to be adjusted
963  * by subtracting the xribase.
964  *
965  * Returns sglq ponter = success, NULL = Failure.
966  **/
967 struct lpfc_sglq *
968 __lpfc_get_active_sglq(struct lpfc_hba *phba, uint16_t xritag)
969 {
970 	struct lpfc_sglq *sglq;
971 
972 	sglq =  phba->sli4_hba.lpfc_sglq_active_list[xritag];
973 	return sglq;
974 }
975 
976 /**
977  * lpfc_clr_rrq_active - Clears RRQ active bit in xri_bitmap.
978  * @phba: Pointer to HBA context object.
979  * @xritag: xri used in this exchange.
980  * @rrq: The RRQ to be cleared.
981  *
982  **/
983 void
984 lpfc_clr_rrq_active(struct lpfc_hba *phba,
985 		    uint16_t xritag,
986 		    struct lpfc_node_rrq *rrq)
987 {
988 	struct lpfc_nodelist *ndlp = NULL;
989 
990 	/* Lookup did to verify if did is still active on this vport */
991 	if (rrq->vport)
992 		ndlp = lpfc_findnode_did(rrq->vport, rrq->nlp_DID);
993 
994 	if (!ndlp)
995 		goto out;
996 
997 	if (test_and_clear_bit(xritag, ndlp->active_rrqs_xri_bitmap)) {
998 		rrq->send_rrq = 0;
999 		rrq->xritag = 0;
1000 		rrq->rrq_stop_time = 0;
1001 	}
1002 out:
1003 	mempool_free(rrq, phba->rrq_pool);
1004 }
1005 
1006 /**
1007  * lpfc_handle_rrq_active - Checks if RRQ has waithed RATOV.
1008  * @phba: Pointer to HBA context object.
1009  *
1010  * This function is called with hbalock held. This function
1011  * Checks if stop_time (ratov from setting rrq active) has
1012  * been reached, if it has and the send_rrq flag is set then
1013  * it will call lpfc_send_rrq. If the send_rrq flag is not set
1014  * then it will just call the routine to clear the rrq and
1015  * free the rrq resource.
1016  * The timer is set to the next rrq that is going to expire before
1017  * leaving the routine.
1018  *
1019  **/
1020 void
1021 lpfc_handle_rrq_active(struct lpfc_hba *phba)
1022 {
1023 	struct lpfc_node_rrq *rrq;
1024 	struct lpfc_node_rrq *nextrrq;
1025 	unsigned long next_time;
1026 	unsigned long iflags;
1027 	LIST_HEAD(send_rrq);
1028 
1029 	spin_lock_irqsave(&phba->hbalock, iflags);
1030 	phba->hba_flag &= ~HBA_RRQ_ACTIVE;
1031 	next_time = jiffies + msecs_to_jiffies(1000 * (phba->fc_ratov + 1));
1032 	list_for_each_entry_safe(rrq, nextrrq,
1033 				 &phba->active_rrq_list, list) {
1034 		if (time_after(jiffies, rrq->rrq_stop_time))
1035 			list_move(&rrq->list, &send_rrq);
1036 		else if (time_before(rrq->rrq_stop_time, next_time))
1037 			next_time = rrq->rrq_stop_time;
1038 	}
1039 	spin_unlock_irqrestore(&phba->hbalock, iflags);
1040 	if ((!list_empty(&phba->active_rrq_list)) &&
1041 	    (!(phba->pport->load_flag & FC_UNLOADING)))
1042 		mod_timer(&phba->rrq_tmr, next_time);
1043 	list_for_each_entry_safe(rrq, nextrrq, &send_rrq, list) {
1044 		list_del(&rrq->list);
1045 		if (!rrq->send_rrq) {
1046 			/* this call will free the rrq */
1047 			lpfc_clr_rrq_active(phba, rrq->xritag, rrq);
1048 		} else if (lpfc_send_rrq(phba, rrq)) {
1049 			/* if we send the rrq then the completion handler
1050 			*  will clear the bit in the xribitmap.
1051 			*/
1052 			lpfc_clr_rrq_active(phba, rrq->xritag,
1053 					    rrq);
1054 		}
1055 	}
1056 }
1057 
1058 /**
1059  * lpfc_get_active_rrq - Get the active RRQ for this exchange.
1060  * @vport: Pointer to vport context object.
1061  * @xri: The xri used in the exchange.
1062  * @did: The targets DID for this exchange.
1063  *
1064  * returns NULL = rrq not found in the phba->active_rrq_list.
1065  *         rrq = rrq for this xri and target.
1066  **/
1067 struct lpfc_node_rrq *
1068 lpfc_get_active_rrq(struct lpfc_vport *vport, uint16_t xri, uint32_t did)
1069 {
1070 	struct lpfc_hba *phba = vport->phba;
1071 	struct lpfc_node_rrq *rrq;
1072 	struct lpfc_node_rrq *nextrrq;
1073 	unsigned long iflags;
1074 
1075 	if (phba->sli_rev != LPFC_SLI_REV4)
1076 		return NULL;
1077 	spin_lock_irqsave(&phba->hbalock, iflags);
1078 	list_for_each_entry_safe(rrq, nextrrq, &phba->active_rrq_list, list) {
1079 		if (rrq->vport == vport && rrq->xritag == xri &&
1080 				rrq->nlp_DID == did){
1081 			list_del(&rrq->list);
1082 			spin_unlock_irqrestore(&phba->hbalock, iflags);
1083 			return rrq;
1084 		}
1085 	}
1086 	spin_unlock_irqrestore(&phba->hbalock, iflags);
1087 	return NULL;
1088 }
1089 
1090 /**
1091  * lpfc_cleanup_vports_rrqs - Remove and clear the active RRQ for this vport.
1092  * @vport: Pointer to vport context object.
1093  * @ndlp: Pointer to the lpfc_node_list structure.
1094  * If ndlp is NULL Remove all active RRQs for this vport from the
1095  * phba->active_rrq_list and clear the rrq.
1096  * If ndlp is not NULL then only remove rrqs for this vport & this ndlp.
1097  **/
1098 void
1099 lpfc_cleanup_vports_rrqs(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
1100 
1101 {
1102 	struct lpfc_hba *phba = vport->phba;
1103 	struct lpfc_node_rrq *rrq;
1104 	struct lpfc_node_rrq *nextrrq;
1105 	unsigned long iflags;
1106 	LIST_HEAD(rrq_list);
1107 
1108 	if (phba->sli_rev != LPFC_SLI_REV4)
1109 		return;
1110 	if (!ndlp) {
1111 		lpfc_sli4_vport_delete_els_xri_aborted(vport);
1112 		lpfc_sli4_vport_delete_fcp_xri_aborted(vport);
1113 	}
1114 	spin_lock_irqsave(&phba->hbalock, iflags);
1115 	list_for_each_entry_safe(rrq, nextrrq, &phba->active_rrq_list, list) {
1116 		if (rrq->vport != vport)
1117 			continue;
1118 
1119 		if (!ndlp || ndlp == lpfc_findnode_did(vport, rrq->nlp_DID))
1120 			list_move(&rrq->list, &rrq_list);
1121 
1122 	}
1123 	spin_unlock_irqrestore(&phba->hbalock, iflags);
1124 
1125 	list_for_each_entry_safe(rrq, nextrrq, &rrq_list, list) {
1126 		list_del(&rrq->list);
1127 		lpfc_clr_rrq_active(phba, rrq->xritag, rrq);
1128 	}
1129 }
1130 
1131 /**
1132  * lpfc_test_rrq_active - Test RRQ bit in xri_bitmap.
1133  * @phba: Pointer to HBA context object.
1134  * @ndlp: Targets nodelist pointer for this exchange.
1135  * @xritag: the xri in the bitmap to test.
1136  *
1137  * This function returns:
1138  * 0 = rrq not active for this xri
1139  * 1 = rrq is valid for this xri.
1140  **/
1141 int
1142 lpfc_test_rrq_active(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp,
1143 			uint16_t  xritag)
1144 {
1145 	if (!ndlp)
1146 		return 0;
1147 	if (!ndlp->active_rrqs_xri_bitmap)
1148 		return 0;
1149 	if (test_bit(xritag, ndlp->active_rrqs_xri_bitmap))
1150 		return 1;
1151 	else
1152 		return 0;
1153 }
1154 
1155 /**
1156  * lpfc_set_rrq_active - set RRQ active bit in xri_bitmap.
1157  * @phba: Pointer to HBA context object.
1158  * @ndlp: nodelist pointer for this target.
1159  * @xritag: xri used in this exchange.
1160  * @rxid: Remote Exchange ID.
1161  * @send_rrq: Flag used to determine if we should send rrq els cmd.
1162  *
1163  * This function takes the hbalock.
1164  * The active bit is always set in the active rrq xri_bitmap even
1165  * if there is no slot avaiable for the other rrq information.
1166  *
1167  * returns 0 rrq actived for this xri
1168  *         < 0 No memory or invalid ndlp.
1169  **/
1170 int
1171 lpfc_set_rrq_active(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp,
1172 		    uint16_t xritag, uint16_t rxid, uint16_t send_rrq)
1173 {
1174 	unsigned long iflags;
1175 	struct lpfc_node_rrq *rrq;
1176 	int empty;
1177 
1178 	if (!ndlp)
1179 		return -EINVAL;
1180 
1181 	if (!phba->cfg_enable_rrq)
1182 		return -EINVAL;
1183 
1184 	spin_lock_irqsave(&phba->hbalock, iflags);
1185 	if (phba->pport->load_flag & FC_UNLOADING) {
1186 		phba->hba_flag &= ~HBA_RRQ_ACTIVE;
1187 		goto out;
1188 	}
1189 
1190 	if (ndlp->vport && (ndlp->vport->load_flag & FC_UNLOADING))
1191 		goto out;
1192 
1193 	if (!ndlp->active_rrqs_xri_bitmap)
1194 		goto out;
1195 
1196 	if (test_and_set_bit(xritag, ndlp->active_rrqs_xri_bitmap))
1197 		goto out;
1198 
1199 	spin_unlock_irqrestore(&phba->hbalock, iflags);
1200 	rrq = mempool_alloc(phba->rrq_pool, GFP_ATOMIC);
1201 	if (!rrq) {
1202 		lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
1203 				"3155 Unable to allocate RRQ xri:0x%x rxid:0x%x"
1204 				" DID:0x%x Send:%d\n",
1205 				xritag, rxid, ndlp->nlp_DID, send_rrq);
1206 		return -EINVAL;
1207 	}
1208 	if (phba->cfg_enable_rrq == 1)
1209 		rrq->send_rrq = send_rrq;
1210 	else
1211 		rrq->send_rrq = 0;
1212 	rrq->xritag = xritag;
1213 	rrq->rrq_stop_time = jiffies +
1214 				msecs_to_jiffies(1000 * (phba->fc_ratov + 1));
1215 	rrq->nlp_DID = ndlp->nlp_DID;
1216 	rrq->vport = ndlp->vport;
1217 	rrq->rxid = rxid;
1218 	spin_lock_irqsave(&phba->hbalock, iflags);
1219 	empty = list_empty(&phba->active_rrq_list);
1220 	list_add_tail(&rrq->list, &phba->active_rrq_list);
1221 	phba->hba_flag |= HBA_RRQ_ACTIVE;
1222 	if (empty)
1223 		lpfc_worker_wake_up(phba);
1224 	spin_unlock_irqrestore(&phba->hbalock, iflags);
1225 	return 0;
1226 out:
1227 	spin_unlock_irqrestore(&phba->hbalock, iflags);
1228 	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
1229 			"2921 Can't set rrq active xri:0x%x rxid:0x%x"
1230 			" DID:0x%x Send:%d\n",
1231 			xritag, rxid, ndlp->nlp_DID, send_rrq);
1232 	return -EINVAL;
1233 }
1234 
1235 /**
1236  * __lpfc_sli_get_els_sglq - Allocates an iocb object from sgl pool
1237  * @phba: Pointer to HBA context object.
1238  * @piocbq: Pointer to the iocbq.
1239  *
1240  * The driver calls this function with either the nvme ls ring lock
1241  * or the fc els ring lock held depending on the iocb usage.  This function
1242  * gets a new driver sglq object from the sglq list. If the list is not empty
1243  * then it is successful, it returns pointer to the newly allocated sglq
1244  * object else it returns NULL.
1245  **/
1246 static struct lpfc_sglq *
1247 __lpfc_sli_get_els_sglq(struct lpfc_hba *phba, struct lpfc_iocbq *piocbq)
1248 {
1249 	struct list_head *lpfc_els_sgl_list = &phba->sli4_hba.lpfc_els_sgl_list;
1250 	struct lpfc_sglq *sglq = NULL;
1251 	struct lpfc_sglq *start_sglq = NULL;
1252 	struct lpfc_io_buf *lpfc_cmd;
1253 	struct lpfc_nodelist *ndlp;
1254 	struct lpfc_sli_ring *pring = NULL;
1255 	int found = 0;
1256 
1257 	if (piocbq->iocb_flag & LPFC_IO_NVME_LS)
1258 		pring =  phba->sli4_hba.nvmels_wq->pring;
1259 	else
1260 		pring = lpfc_phba_elsring(phba);
1261 
1262 	lockdep_assert_held(&pring->ring_lock);
1263 
1264 	if (piocbq->iocb_flag &  LPFC_IO_FCP) {
1265 		lpfc_cmd = (struct lpfc_io_buf *) piocbq->context1;
1266 		ndlp = lpfc_cmd->rdata->pnode;
1267 	} else  if ((piocbq->iocb.ulpCommand == CMD_GEN_REQUEST64_CR) &&
1268 			!(piocbq->iocb_flag & LPFC_IO_LIBDFC)) {
1269 		ndlp = piocbq->context_un.ndlp;
1270 	} else  if (piocbq->iocb_flag & LPFC_IO_LIBDFC) {
1271 		if (piocbq->iocb_flag & LPFC_IO_LOOPBACK)
1272 			ndlp = NULL;
1273 		else
1274 			ndlp = piocbq->context_un.ndlp;
1275 	} else {
1276 		ndlp = piocbq->context1;
1277 	}
1278 
1279 	spin_lock(&phba->sli4_hba.sgl_list_lock);
1280 	list_remove_head(lpfc_els_sgl_list, sglq, struct lpfc_sglq, list);
1281 	start_sglq = sglq;
1282 	while (!found) {
1283 		if (!sglq)
1284 			break;
1285 		if (ndlp && ndlp->active_rrqs_xri_bitmap &&
1286 		    test_bit(sglq->sli4_lxritag,
1287 		    ndlp->active_rrqs_xri_bitmap)) {
1288 			/* This xri has an rrq outstanding for this DID.
1289 			 * put it back in the list and get another xri.
1290 			 */
1291 			list_add_tail(&sglq->list, lpfc_els_sgl_list);
1292 			sglq = NULL;
1293 			list_remove_head(lpfc_els_sgl_list, sglq,
1294 						struct lpfc_sglq, list);
1295 			if (sglq == start_sglq) {
1296 				list_add_tail(&sglq->list, lpfc_els_sgl_list);
1297 				sglq = NULL;
1298 				break;
1299 			} else
1300 				continue;
1301 		}
1302 		sglq->ndlp = ndlp;
1303 		found = 1;
1304 		phba->sli4_hba.lpfc_sglq_active_list[sglq->sli4_lxritag] = sglq;
1305 		sglq->state = SGL_ALLOCATED;
1306 	}
1307 	spin_unlock(&phba->sli4_hba.sgl_list_lock);
1308 	return sglq;
1309 }
1310 
1311 /**
1312  * __lpfc_sli_get_nvmet_sglq - Allocates an iocb object from sgl pool
1313  * @phba: Pointer to HBA context object.
1314  * @piocbq: Pointer to the iocbq.
1315  *
1316  * This function is called with the sgl_list lock held. This function
1317  * gets a new driver sglq object from the sglq list. If the
1318  * list is not empty then it is successful, it returns pointer to the newly
1319  * allocated sglq object else it returns NULL.
1320  **/
1321 struct lpfc_sglq *
1322 __lpfc_sli_get_nvmet_sglq(struct lpfc_hba *phba, struct lpfc_iocbq *piocbq)
1323 {
1324 	struct list_head *lpfc_nvmet_sgl_list;
1325 	struct lpfc_sglq *sglq = NULL;
1326 
1327 	lpfc_nvmet_sgl_list = &phba->sli4_hba.lpfc_nvmet_sgl_list;
1328 
1329 	lockdep_assert_held(&phba->sli4_hba.sgl_list_lock);
1330 
1331 	list_remove_head(lpfc_nvmet_sgl_list, sglq, struct lpfc_sglq, list);
1332 	if (!sglq)
1333 		return NULL;
1334 	phba->sli4_hba.lpfc_sglq_active_list[sglq->sli4_lxritag] = sglq;
1335 	sglq->state = SGL_ALLOCATED;
1336 	return sglq;
1337 }
1338 
1339 /**
1340  * lpfc_sli_get_iocbq - Allocates an iocb object from iocb pool
1341  * @phba: Pointer to HBA context object.
1342  *
1343  * This function is called with no lock held. This function
1344  * allocates a new driver iocb object from the iocb pool. If the
1345  * allocation is successful, it returns pointer to the newly
1346  * allocated iocb object else it returns NULL.
1347  **/
1348 struct lpfc_iocbq *
1349 lpfc_sli_get_iocbq(struct lpfc_hba *phba)
1350 {
1351 	struct lpfc_iocbq * iocbq = NULL;
1352 	unsigned long iflags;
1353 
1354 	spin_lock_irqsave(&phba->hbalock, iflags);
1355 	iocbq = __lpfc_sli_get_iocbq(phba);
1356 	spin_unlock_irqrestore(&phba->hbalock, iflags);
1357 	return iocbq;
1358 }
1359 
1360 /**
1361  * __lpfc_sli_release_iocbq_s4 - Release iocb to the iocb pool
1362  * @phba: Pointer to HBA context object.
1363  * @iocbq: Pointer to driver iocb object.
1364  *
1365  * This function is called to release the driver iocb object
1366  * to the iocb pool. The iotag in the iocb object
1367  * does not change for each use of the iocb object. This function
1368  * clears all other fields of the iocb object when it is freed.
1369  * The sqlq structure that holds the xritag and phys and virtual
1370  * mappings for the scatter gather list is retrieved from the
1371  * active array of sglq. The get of the sglq pointer also clears
1372  * the entry in the array. If the status of the IO indiactes that
1373  * this IO was aborted then the sglq entry it put on the
1374  * lpfc_abts_els_sgl_list until the CQ_ABORTED_XRI is received. If the
1375  * IO has good status or fails for any other reason then the sglq
1376  * entry is added to the free list (lpfc_els_sgl_list). The hbalock is
1377  *  asserted held in the code path calling this routine.
1378  **/
1379 static void
1380 __lpfc_sli_release_iocbq_s4(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq)
1381 {
1382 	struct lpfc_sglq *sglq;
1383 	size_t start_clean = offsetof(struct lpfc_iocbq, iocb);
1384 	unsigned long iflag = 0;
1385 	struct lpfc_sli_ring *pring;
1386 
1387 	if (iocbq->sli4_xritag == NO_XRI)
1388 		sglq = NULL;
1389 	else
1390 		sglq = __lpfc_clear_active_sglq(phba, iocbq->sli4_lxritag);
1391 
1392 
1393 	if (sglq)  {
1394 		if (iocbq->iocb_flag & LPFC_IO_NVMET) {
1395 			spin_lock_irqsave(&phba->sli4_hba.sgl_list_lock,
1396 					  iflag);
1397 			sglq->state = SGL_FREED;
1398 			sglq->ndlp = NULL;
1399 			list_add_tail(&sglq->list,
1400 				      &phba->sli4_hba.lpfc_nvmet_sgl_list);
1401 			spin_unlock_irqrestore(
1402 				&phba->sli4_hba.sgl_list_lock, iflag);
1403 			goto out;
1404 		}
1405 
1406 		if ((iocbq->iocb_flag & LPFC_EXCHANGE_BUSY) &&
1407 			(sglq->state != SGL_XRI_ABORTED)) {
1408 			spin_lock_irqsave(&phba->sli4_hba.sgl_list_lock,
1409 					  iflag);
1410 
1411 			/* Check if we can get a reference on ndlp */
1412 			if (sglq->ndlp && !lpfc_nlp_get(sglq->ndlp))
1413 				sglq->ndlp = NULL;
1414 
1415 			list_add(&sglq->list,
1416 				 &phba->sli4_hba.lpfc_abts_els_sgl_list);
1417 			spin_unlock_irqrestore(
1418 				&phba->sli4_hba.sgl_list_lock, iflag);
1419 		} else {
1420 			spin_lock_irqsave(&phba->sli4_hba.sgl_list_lock,
1421 					  iflag);
1422 			sglq->state = SGL_FREED;
1423 			sglq->ndlp = NULL;
1424 			list_add_tail(&sglq->list,
1425 				      &phba->sli4_hba.lpfc_els_sgl_list);
1426 			spin_unlock_irqrestore(
1427 				&phba->sli4_hba.sgl_list_lock, iflag);
1428 			pring = lpfc_phba_elsring(phba);
1429 			/* Check if TXQ queue needs to be serviced */
1430 			if (pring && (!list_empty(&pring->txq)))
1431 				lpfc_worker_wake_up(phba);
1432 		}
1433 	}
1434 
1435 out:
1436 	/*
1437 	 * Clean all volatile data fields, preserve iotag and node struct.
1438 	 */
1439 	memset((char *)iocbq + start_clean, 0, sizeof(*iocbq) - start_clean);
1440 	iocbq->sli4_lxritag = NO_XRI;
1441 	iocbq->sli4_xritag = NO_XRI;
1442 	iocbq->iocb_flag &= ~(LPFC_IO_NVME | LPFC_IO_NVMET | LPFC_IO_CMF |
1443 			      LPFC_IO_NVME_LS);
1444 	list_add_tail(&iocbq->list, &phba->lpfc_iocb_list);
1445 }
1446 
1447 
1448 /**
1449  * __lpfc_sli_release_iocbq_s3 - Release iocb to the iocb pool
1450  * @phba: Pointer to HBA context object.
1451  * @iocbq: Pointer to driver iocb object.
1452  *
1453  * This function is called to release the driver iocb object to the
1454  * iocb pool. The iotag in the iocb object does not change for each
1455  * use of the iocb object. This function clears all other fields of
1456  * the iocb object when it is freed. The hbalock is asserted held in
1457  * the code path calling this routine.
1458  **/
1459 static void
1460 __lpfc_sli_release_iocbq_s3(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq)
1461 {
1462 	size_t start_clean = offsetof(struct lpfc_iocbq, iocb);
1463 
1464 	/*
1465 	 * Clean all volatile data fields, preserve iotag and node struct.
1466 	 */
1467 	memset((char*)iocbq + start_clean, 0, sizeof(*iocbq) - start_clean);
1468 	iocbq->sli4_xritag = NO_XRI;
1469 	list_add_tail(&iocbq->list, &phba->lpfc_iocb_list);
1470 }
1471 
1472 /**
1473  * __lpfc_sli_release_iocbq - Release iocb to the iocb pool
1474  * @phba: Pointer to HBA context object.
1475  * @iocbq: Pointer to driver iocb object.
1476  *
1477  * This function is called with hbalock held to release driver
1478  * iocb object to the iocb pool. The iotag in the iocb object
1479  * does not change for each use of the iocb object. This function
1480  * clears all other fields of the iocb object when it is freed.
1481  **/
1482 static void
1483 __lpfc_sli_release_iocbq(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq)
1484 {
1485 	lockdep_assert_held(&phba->hbalock);
1486 
1487 	phba->__lpfc_sli_release_iocbq(phba, iocbq);
1488 	phba->iocb_cnt--;
1489 }
1490 
1491 /**
1492  * lpfc_sli_release_iocbq - Release iocb to the iocb pool
1493  * @phba: Pointer to HBA context object.
1494  * @iocbq: Pointer to driver iocb object.
1495  *
1496  * This function is called with no lock held to release the iocb to
1497  * iocb pool.
1498  **/
1499 void
1500 lpfc_sli_release_iocbq(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq)
1501 {
1502 	unsigned long iflags;
1503 
1504 	/*
1505 	 * Clean all volatile data fields, preserve iotag and node struct.
1506 	 */
1507 	spin_lock_irqsave(&phba->hbalock, iflags);
1508 	__lpfc_sli_release_iocbq(phba, iocbq);
1509 	spin_unlock_irqrestore(&phba->hbalock, iflags);
1510 }
1511 
1512 /**
1513  * lpfc_sli_cancel_iocbs - Cancel all iocbs from a list.
1514  * @phba: Pointer to HBA context object.
1515  * @iocblist: List of IOCBs.
1516  * @ulpstatus: ULP status in IOCB command field.
1517  * @ulpWord4: ULP word-4 in IOCB command field.
1518  *
1519  * This function is called with a list of IOCBs to cancel. It cancels the IOCB
1520  * on the list by invoking the complete callback function associated with the
1521  * IOCB with the provided @ulpstatus and @ulpword4 set to the IOCB commond
1522  * fields.
1523  **/
1524 void
1525 lpfc_sli_cancel_iocbs(struct lpfc_hba *phba, struct list_head *iocblist,
1526 		      uint32_t ulpstatus, uint32_t ulpWord4)
1527 {
1528 	struct lpfc_iocbq *piocb;
1529 
1530 	while (!list_empty(iocblist)) {
1531 		list_remove_head(iocblist, piocb, struct lpfc_iocbq, list);
1532 		if (piocb->wqe_cmpl) {
1533 			if (piocb->iocb_flag & LPFC_IO_NVME)
1534 				lpfc_nvme_cancel_iocb(phba, piocb,
1535 						      ulpstatus, ulpWord4);
1536 			else
1537 				lpfc_sli_release_iocbq(phba, piocb);
1538 
1539 		} else if (piocb->iocb_cmpl) {
1540 			piocb->iocb.ulpStatus = ulpstatus;
1541 			piocb->iocb.un.ulpWord[4] = ulpWord4;
1542 			(piocb->iocb_cmpl) (phba, piocb, piocb);
1543 		} else {
1544 			lpfc_sli_release_iocbq(phba, piocb);
1545 		}
1546 	}
1547 	return;
1548 }
1549 
1550 /**
1551  * lpfc_sli_iocb_cmd_type - Get the iocb type
1552  * @iocb_cmnd: iocb command code.
1553  *
1554  * This function is called by ring event handler function to get the iocb type.
1555  * This function translates the iocb command to an iocb command type used to
1556  * decide the final disposition of each completed IOCB.
1557  * The function returns
1558  * LPFC_UNKNOWN_IOCB if it is an unsupported iocb
1559  * LPFC_SOL_IOCB     if it is a solicited iocb completion
1560  * LPFC_ABORT_IOCB   if it is an abort iocb
1561  * LPFC_UNSOL_IOCB   if it is an unsolicited iocb
1562  *
1563  * The caller is not required to hold any lock.
1564  **/
1565 static lpfc_iocb_type
1566 lpfc_sli_iocb_cmd_type(uint8_t iocb_cmnd)
1567 {
1568 	lpfc_iocb_type type = LPFC_UNKNOWN_IOCB;
1569 
1570 	if (iocb_cmnd > CMD_MAX_IOCB_CMD)
1571 		return 0;
1572 
1573 	switch (iocb_cmnd) {
1574 	case CMD_XMIT_SEQUENCE_CR:
1575 	case CMD_XMIT_SEQUENCE_CX:
1576 	case CMD_XMIT_BCAST_CN:
1577 	case CMD_XMIT_BCAST_CX:
1578 	case CMD_ELS_REQUEST_CR:
1579 	case CMD_ELS_REQUEST_CX:
1580 	case CMD_CREATE_XRI_CR:
1581 	case CMD_CREATE_XRI_CX:
1582 	case CMD_GET_RPI_CN:
1583 	case CMD_XMIT_ELS_RSP_CX:
1584 	case CMD_GET_RPI_CR:
1585 	case CMD_FCP_IWRITE_CR:
1586 	case CMD_FCP_IWRITE_CX:
1587 	case CMD_FCP_IREAD_CR:
1588 	case CMD_FCP_IREAD_CX:
1589 	case CMD_FCP_ICMND_CR:
1590 	case CMD_FCP_ICMND_CX:
1591 	case CMD_FCP_TSEND_CX:
1592 	case CMD_FCP_TRSP_CX:
1593 	case CMD_FCP_TRECEIVE_CX:
1594 	case CMD_FCP_AUTO_TRSP_CX:
1595 	case CMD_ADAPTER_MSG:
1596 	case CMD_ADAPTER_DUMP:
1597 	case CMD_XMIT_SEQUENCE64_CR:
1598 	case CMD_XMIT_SEQUENCE64_CX:
1599 	case CMD_XMIT_BCAST64_CN:
1600 	case CMD_XMIT_BCAST64_CX:
1601 	case CMD_ELS_REQUEST64_CR:
1602 	case CMD_ELS_REQUEST64_CX:
1603 	case CMD_FCP_IWRITE64_CR:
1604 	case CMD_FCP_IWRITE64_CX:
1605 	case CMD_FCP_IREAD64_CR:
1606 	case CMD_FCP_IREAD64_CX:
1607 	case CMD_FCP_ICMND64_CR:
1608 	case CMD_FCP_ICMND64_CX:
1609 	case CMD_FCP_TSEND64_CX:
1610 	case CMD_FCP_TRSP64_CX:
1611 	case CMD_FCP_TRECEIVE64_CX:
1612 	case CMD_GEN_REQUEST64_CR:
1613 	case CMD_GEN_REQUEST64_CX:
1614 	case CMD_XMIT_ELS_RSP64_CX:
1615 	case DSSCMD_IWRITE64_CR:
1616 	case DSSCMD_IWRITE64_CX:
1617 	case DSSCMD_IREAD64_CR:
1618 	case DSSCMD_IREAD64_CX:
1619 	case CMD_SEND_FRAME:
1620 		type = LPFC_SOL_IOCB;
1621 		break;
1622 	case CMD_ABORT_XRI_CN:
1623 	case CMD_ABORT_XRI_CX:
1624 	case CMD_CLOSE_XRI_CN:
1625 	case CMD_CLOSE_XRI_CX:
1626 	case CMD_XRI_ABORTED_CX:
1627 	case CMD_ABORT_MXRI64_CN:
1628 	case CMD_XMIT_BLS_RSP64_CX:
1629 		type = LPFC_ABORT_IOCB;
1630 		break;
1631 	case CMD_RCV_SEQUENCE_CX:
1632 	case CMD_RCV_ELS_REQ_CX:
1633 	case CMD_RCV_SEQUENCE64_CX:
1634 	case CMD_RCV_ELS_REQ64_CX:
1635 	case CMD_ASYNC_STATUS:
1636 	case CMD_IOCB_RCV_SEQ64_CX:
1637 	case CMD_IOCB_RCV_ELS64_CX:
1638 	case CMD_IOCB_RCV_CONT64_CX:
1639 	case CMD_IOCB_RET_XRI64_CX:
1640 		type = LPFC_UNSOL_IOCB;
1641 		break;
1642 	case CMD_IOCB_XMIT_MSEQ64_CR:
1643 	case CMD_IOCB_XMIT_MSEQ64_CX:
1644 	case CMD_IOCB_RCV_SEQ_LIST64_CX:
1645 	case CMD_IOCB_RCV_ELS_LIST64_CX:
1646 	case CMD_IOCB_CLOSE_EXTENDED_CN:
1647 	case CMD_IOCB_ABORT_EXTENDED_CN:
1648 	case CMD_IOCB_RET_HBQE64_CN:
1649 	case CMD_IOCB_FCP_IBIDIR64_CR:
1650 	case CMD_IOCB_FCP_IBIDIR64_CX:
1651 	case CMD_IOCB_FCP_ITASKMGT64_CX:
1652 	case CMD_IOCB_LOGENTRY_CN:
1653 	case CMD_IOCB_LOGENTRY_ASYNC_CN:
1654 		printk("%s - Unhandled SLI-3 Command x%x\n",
1655 				__func__, iocb_cmnd);
1656 		type = LPFC_UNKNOWN_IOCB;
1657 		break;
1658 	default:
1659 		type = LPFC_UNKNOWN_IOCB;
1660 		break;
1661 	}
1662 
1663 	return type;
1664 }
1665 
1666 /**
1667  * lpfc_sli_ring_map - Issue config_ring mbox for all rings
1668  * @phba: Pointer to HBA context object.
1669  *
1670  * This function is called from SLI initialization code
1671  * to configure every ring of the HBA's SLI interface. The
1672  * caller is not required to hold any lock. This function issues
1673  * a config_ring mailbox command for each ring.
1674  * This function returns zero if successful else returns a negative
1675  * error code.
1676  **/
1677 static int
1678 lpfc_sli_ring_map(struct lpfc_hba *phba)
1679 {
1680 	struct lpfc_sli *psli = &phba->sli;
1681 	LPFC_MBOXQ_t *pmb;
1682 	MAILBOX_t *pmbox;
1683 	int i, rc, ret = 0;
1684 
1685 	pmb = (LPFC_MBOXQ_t *) mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
1686 	if (!pmb)
1687 		return -ENOMEM;
1688 	pmbox = &pmb->u.mb;
1689 	phba->link_state = LPFC_INIT_MBX_CMDS;
1690 	for (i = 0; i < psli->num_rings; i++) {
1691 		lpfc_config_ring(phba, i, pmb);
1692 		rc = lpfc_sli_issue_mbox(phba, pmb, MBX_POLL);
1693 		if (rc != MBX_SUCCESS) {
1694 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1695 					"0446 Adapter failed to init (%d), "
1696 					"mbxCmd x%x CFG_RING, mbxStatus x%x, "
1697 					"ring %d\n",
1698 					rc, pmbox->mbxCommand,
1699 					pmbox->mbxStatus, i);
1700 			phba->link_state = LPFC_HBA_ERROR;
1701 			ret = -ENXIO;
1702 			break;
1703 		}
1704 	}
1705 	mempool_free(pmb, phba->mbox_mem_pool);
1706 	return ret;
1707 }
1708 
1709 /**
1710  * lpfc_sli_ringtxcmpl_put - Adds new iocb to the txcmplq
1711  * @phba: Pointer to HBA context object.
1712  * @pring: Pointer to driver SLI ring object.
1713  * @piocb: Pointer to the driver iocb object.
1714  *
1715  * The driver calls this function with the hbalock held for SLI3 ports or
1716  * the ring lock held for SLI4 ports. The function adds the
1717  * new iocb to txcmplq of the given ring. This function always returns
1718  * 0. If this function is called for ELS ring, this function checks if
1719  * there is a vport associated with the ELS command. This function also
1720  * starts els_tmofunc timer if this is an ELS command.
1721  **/
1722 static int
1723 lpfc_sli_ringtxcmpl_put(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
1724 			struct lpfc_iocbq *piocb)
1725 {
1726 	if (phba->sli_rev == LPFC_SLI_REV4)
1727 		lockdep_assert_held(&pring->ring_lock);
1728 	else
1729 		lockdep_assert_held(&phba->hbalock);
1730 
1731 	BUG_ON(!piocb);
1732 
1733 	list_add_tail(&piocb->list, &pring->txcmplq);
1734 	piocb->iocb_flag |= LPFC_IO_ON_TXCMPLQ;
1735 	pring->txcmplq_cnt++;
1736 
1737 	if ((unlikely(pring->ringno == LPFC_ELS_RING)) &&
1738 	   (piocb->iocb.ulpCommand != CMD_ABORT_XRI_CN) &&
1739 	   (piocb->iocb.ulpCommand != CMD_CLOSE_XRI_CN)) {
1740 		BUG_ON(!piocb->vport);
1741 		if (!(piocb->vport->load_flag & FC_UNLOADING))
1742 			mod_timer(&piocb->vport->els_tmofunc,
1743 				  jiffies +
1744 				  msecs_to_jiffies(1000 * (phba->fc_ratov << 1)));
1745 	}
1746 
1747 	return 0;
1748 }
1749 
1750 /**
1751  * lpfc_sli_ringtx_get - Get first element of the txq
1752  * @phba: Pointer to HBA context object.
1753  * @pring: Pointer to driver SLI ring object.
1754  *
1755  * This function is called with hbalock held to get next
1756  * iocb in txq of the given ring. If there is any iocb in
1757  * the txq, the function returns first iocb in the list after
1758  * removing the iocb from the list, else it returns NULL.
1759  **/
1760 struct lpfc_iocbq *
1761 lpfc_sli_ringtx_get(struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
1762 {
1763 	struct lpfc_iocbq *cmd_iocb;
1764 
1765 	lockdep_assert_held(&phba->hbalock);
1766 
1767 	list_remove_head((&pring->txq), cmd_iocb, struct lpfc_iocbq, list);
1768 	return cmd_iocb;
1769 }
1770 
1771 /**
1772  * lpfc_cmf_sync_cmpl - Process a CMF_SYNC_WQE cmpl
1773  * @phba: Pointer to HBA context object.
1774  * @cmdiocb: Pointer to driver command iocb object.
1775  * @cmf_cmpl: Pointer to completed WCQE.
1776  *
1777  * This routine will inform the driver of any BW adjustments we need
1778  * to make. These changes will be picked up during the next CMF
1779  * timer interrupt. In addition, any BW changes will be logged
1780  * with LOG_CGN_MGMT.
1781  **/
1782 static void
1783 lpfc_cmf_sync_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
1784 		   struct lpfc_wcqe_complete *cmf_cmpl)
1785 {
1786 	union lpfc_wqe128 *wqe;
1787 	uint32_t status, info;
1788 	uint64_t bw, bwdif, slop;
1789 	uint64_t pcent, bwpcent;
1790 	int asig, afpin, sigcnt, fpincnt;
1791 	int wsigmax, wfpinmax, cg, tdp;
1792 	char *s;
1793 
1794 	/* First check for error */
1795 	status = bf_get(lpfc_wcqe_c_status, cmf_cmpl);
1796 	if (status) {
1797 		lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT,
1798 				"6211 CMF_SYNC_WQE Error "
1799 				"req_tag x%x status x%x hwstatus x%x "
1800 				"tdatap x%x parm x%x\n",
1801 				bf_get(lpfc_wcqe_c_request_tag, cmf_cmpl),
1802 				bf_get(lpfc_wcqe_c_status, cmf_cmpl),
1803 				bf_get(lpfc_wcqe_c_hw_status, cmf_cmpl),
1804 				cmf_cmpl->total_data_placed,
1805 				cmf_cmpl->parameter);
1806 		goto out;
1807 	}
1808 
1809 	/* Gather congestion information on a successful cmpl */
1810 	info = cmf_cmpl->parameter;
1811 	phba->cmf_active_info = info;
1812 
1813 	/* See if firmware info count is valid or has changed */
1814 	if (info > LPFC_MAX_CMF_INFO || phba->cmf_info_per_interval == info)
1815 		info = 0;
1816 	else
1817 		phba->cmf_info_per_interval = info;
1818 
1819 	tdp = bf_get(lpfc_wcqe_c_cmf_bw, cmf_cmpl);
1820 	cg = bf_get(lpfc_wcqe_c_cmf_cg, cmf_cmpl);
1821 
1822 	/* Get BW requirement from firmware */
1823 	bw = (uint64_t)tdp * LPFC_CMF_BLK_SIZE;
1824 	if (!bw) {
1825 		lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT,
1826 				"6212 CMF_SYNC_WQE x%x: NULL bw\n",
1827 				bf_get(lpfc_wcqe_c_request_tag, cmf_cmpl));
1828 		goto out;
1829 	}
1830 
1831 	/* Gather information needed for logging if a BW change is required */
1832 	wqe = &cmdiocb->wqe;
1833 	asig = bf_get(cmf_sync_asig, &wqe->cmf_sync);
1834 	afpin = bf_get(cmf_sync_afpin, &wqe->cmf_sync);
1835 	fpincnt = bf_get(cmf_sync_wfpincnt, &wqe->cmf_sync);
1836 	sigcnt = bf_get(cmf_sync_wsigcnt, &wqe->cmf_sync);
1837 	if (phba->cmf_max_bytes_per_interval != bw ||
1838 	    (asig || afpin || sigcnt || fpincnt)) {
1839 		/* Are we increasing or decreasing BW */
1840 		if (phba->cmf_max_bytes_per_interval <  bw) {
1841 			bwdif = bw - phba->cmf_max_bytes_per_interval;
1842 			s = "Increase";
1843 		} else {
1844 			bwdif = phba->cmf_max_bytes_per_interval - bw;
1845 			s = "Decrease";
1846 		}
1847 
1848 		/* What is the change percentage */
1849 		slop = div_u64(phba->cmf_link_byte_count, 200); /*For rounding*/
1850 		pcent = div64_u64(bwdif * 100 + slop,
1851 				  phba->cmf_link_byte_count);
1852 		bwpcent = div64_u64(bw * 100 + slop,
1853 				    phba->cmf_link_byte_count);
1854 		if (asig) {
1855 			lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT,
1856 					"6237 BW Threshold %lld%% (%lld): "
1857 					"%lld%% %s: Signal Alarm: cg:%d "
1858 					"Info:%u\n",
1859 					bwpcent, bw, pcent, s, cg,
1860 					phba->cmf_active_info);
1861 		} else if (afpin) {
1862 			lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT,
1863 					"6238 BW Threshold %lld%% (%lld): "
1864 					"%lld%% %s: FPIN Alarm: cg:%d "
1865 					"Info:%u\n",
1866 					bwpcent, bw, pcent, s, cg,
1867 					phba->cmf_active_info);
1868 		} else if (sigcnt) {
1869 			wsigmax = bf_get(cmf_sync_wsigmax, &wqe->cmf_sync);
1870 			lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT,
1871 					"6239 BW Threshold %lld%% (%lld): "
1872 					"%lld%% %s: Signal Warning: "
1873 					"Cnt %d Max %d: cg:%d Info:%u\n",
1874 					bwpcent, bw, pcent, s, sigcnt,
1875 					wsigmax, cg, phba->cmf_active_info);
1876 		} else if (fpincnt) {
1877 			wfpinmax = bf_get(cmf_sync_wfpinmax, &wqe->cmf_sync);
1878 			lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT,
1879 					"6240 BW Threshold %lld%% (%lld): "
1880 					"%lld%% %s: FPIN Warning: "
1881 					"Cnt %d Max %d: cg:%d Info:%u\n",
1882 					bwpcent, bw, pcent, s, fpincnt,
1883 					wfpinmax, cg, phba->cmf_active_info);
1884 		} else {
1885 			lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT,
1886 					"6241 BW Threshold %lld%% (%lld): "
1887 					"CMF %lld%% %s: cg:%d Info:%u\n",
1888 					bwpcent, bw, pcent, s, cg,
1889 					phba->cmf_active_info);
1890 		}
1891 	} else if (info) {
1892 		lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT,
1893 				"6246 Info Threshold %u\n", info);
1894 	}
1895 
1896 	/* Save BW change to be picked up during next timer interrupt */
1897 	phba->cmf_last_sync_bw = bw;
1898 out:
1899 	lpfc_sli_release_iocbq(phba, cmdiocb);
1900 }
1901 
1902 /**
1903  * lpfc_issue_cmf_sync_wqe - Issue a CMF_SYNC_WQE
1904  * @phba: Pointer to HBA context object.
1905  * @ms:   ms to set in WQE interval, 0 means use init op
1906  * @total: Total rcv bytes for this interval
1907  *
1908  * This routine is called every CMF timer interrupt. Its purpose is
1909  * to issue a CMF_SYNC_WQE to the firmware to inform it of any events
1910  * that may indicate we have congestion (FPINs or Signals). Upon
1911  * completion, the firmware will indicate any BW restrictions the
1912  * driver may need to take.
1913  **/
1914 int
1915 lpfc_issue_cmf_sync_wqe(struct lpfc_hba *phba, u32 ms, u64 total)
1916 {
1917 	union lpfc_wqe128 *wqe;
1918 	struct lpfc_iocbq *sync_buf;
1919 	unsigned long iflags;
1920 	u32 ret_val;
1921 	u32 atot, wtot, max;
1922 
1923 	/* First address any alarm / warning activity */
1924 	atot = atomic_xchg(&phba->cgn_sync_alarm_cnt, 0);
1925 	wtot = atomic_xchg(&phba->cgn_sync_warn_cnt, 0);
1926 
1927 	/* ONLY Managed mode will send the CMF_SYNC_WQE to the HBA */
1928 	if (phba->cmf_active_mode != LPFC_CFG_MANAGED ||
1929 	    phba->link_state == LPFC_LINK_DOWN)
1930 		return 0;
1931 
1932 	spin_lock_irqsave(&phba->hbalock, iflags);
1933 	sync_buf = __lpfc_sli_get_iocbq(phba);
1934 	if (!sync_buf) {
1935 		lpfc_printf_log(phba, KERN_ERR, LOG_CGN_MGMT,
1936 				"6213 No available WQEs for CMF_SYNC_WQE\n");
1937 		ret_val = ENOMEM;
1938 		goto out_unlock;
1939 	}
1940 
1941 	wqe = &sync_buf->wqe;
1942 
1943 	/* WQEs are reused.  Clear stale data and set key fields to zero */
1944 	memset(wqe, 0, sizeof(*wqe));
1945 
1946 	/* If this is the very first CMF_SYNC_WQE, issue an init operation */
1947 	if (!ms) {
1948 		lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT,
1949 				"6441 CMF Init %d - CMF_SYNC_WQE\n",
1950 				phba->fc_eventTag);
1951 		bf_set(cmf_sync_op, &wqe->cmf_sync, 1); /* 1=init */
1952 		bf_set(cmf_sync_interval, &wqe->cmf_sync, LPFC_CMF_INTERVAL);
1953 		goto initpath;
1954 	}
1955 
1956 	bf_set(cmf_sync_op, &wqe->cmf_sync, 0); /* 0=recalc */
1957 	bf_set(cmf_sync_interval, &wqe->cmf_sync, ms);
1958 
1959 	/* Check for alarms / warnings */
1960 	if (atot) {
1961 		if (phba->cgn_reg_signal == EDC_CG_SIG_WARN_ALARM) {
1962 			/* We hit an Signal alarm condition */
1963 			bf_set(cmf_sync_asig, &wqe->cmf_sync, 1);
1964 		} else {
1965 			/* We hit a FPIN alarm condition */
1966 			bf_set(cmf_sync_afpin, &wqe->cmf_sync, 1);
1967 		}
1968 	} else if (wtot) {
1969 		if (phba->cgn_reg_signal == EDC_CG_SIG_WARN_ONLY ||
1970 		    phba->cgn_reg_signal == EDC_CG_SIG_WARN_ALARM) {
1971 			/* We hit an Signal warning condition */
1972 			max = LPFC_SEC_TO_MSEC / lpfc_fabric_cgn_frequency *
1973 				lpfc_acqe_cgn_frequency;
1974 			bf_set(cmf_sync_wsigmax, &wqe->cmf_sync, max);
1975 			bf_set(cmf_sync_wsigcnt, &wqe->cmf_sync, wtot);
1976 		} else {
1977 			/* We hit a FPIN warning condition */
1978 			bf_set(cmf_sync_wfpinmax, &wqe->cmf_sync, 1);
1979 			bf_set(cmf_sync_wfpincnt, &wqe->cmf_sync, 1);
1980 		}
1981 	}
1982 
1983 	/* Update total read blocks during previous timer interval */
1984 	wqe->cmf_sync.read_bytes = (u32)(total / LPFC_CMF_BLK_SIZE);
1985 
1986 initpath:
1987 	bf_set(cmf_sync_ver, &wqe->cmf_sync, LPFC_CMF_SYNC_VER);
1988 	wqe->cmf_sync.event_tag = phba->fc_eventTag;
1989 	bf_set(cmf_sync_cmnd, &wqe->cmf_sync, CMD_CMF_SYNC_WQE);
1990 
1991 	/* Setup reqtag to match the wqe completion. */
1992 	bf_set(cmf_sync_reqtag, &wqe->cmf_sync, sync_buf->iotag);
1993 
1994 	bf_set(cmf_sync_qosd, &wqe->cmf_sync, 1);
1995 
1996 	bf_set(cmf_sync_cmd_type, &wqe->cmf_sync, CMF_SYNC_COMMAND);
1997 	bf_set(cmf_sync_wqec, &wqe->cmf_sync, 1);
1998 	bf_set(cmf_sync_cqid, &wqe->cmf_sync, LPFC_WQE_CQ_ID_DEFAULT);
1999 
2000 	sync_buf->vport = phba->pport;
2001 	sync_buf->wqe_cmpl = lpfc_cmf_sync_cmpl;
2002 	sync_buf->iocb_cmpl = NULL;
2003 	sync_buf->context1 = NULL;
2004 	sync_buf->context2 = NULL;
2005 	sync_buf->context3 = NULL;
2006 	sync_buf->sli4_xritag = NO_XRI;
2007 
2008 	sync_buf->iocb_flag |= LPFC_IO_CMF;
2009 	ret_val = lpfc_sli4_issue_wqe(phba, &phba->sli4_hba.hdwq[0], sync_buf);
2010 	if (ret_val)
2011 		lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT,
2012 				"6214 Cannot issue CMF_SYNC_WQE: x%x\n",
2013 				ret_val);
2014 out_unlock:
2015 	spin_unlock_irqrestore(&phba->hbalock, iflags);
2016 	return ret_val;
2017 }
2018 
2019 /**
2020  * lpfc_sli_next_iocb_slot - Get next iocb slot in the ring
2021  * @phba: Pointer to HBA context object.
2022  * @pring: Pointer to driver SLI ring object.
2023  *
2024  * This function is called with hbalock held and the caller must post the
2025  * iocb without releasing the lock. If the caller releases the lock,
2026  * iocb slot returned by the function is not guaranteed to be available.
2027  * The function returns pointer to the next available iocb slot if there
2028  * is available slot in the ring, else it returns NULL.
2029  * If the get index of the ring is ahead of the put index, the function
2030  * will post an error attention event to the worker thread to take the
2031  * HBA to offline state.
2032  **/
2033 static IOCB_t *
2034 lpfc_sli_next_iocb_slot (struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
2035 {
2036 	struct lpfc_pgp *pgp = &phba->port_gp[pring->ringno];
2037 	uint32_t  max_cmd_idx = pring->sli.sli3.numCiocb;
2038 
2039 	lockdep_assert_held(&phba->hbalock);
2040 
2041 	if ((pring->sli.sli3.next_cmdidx == pring->sli.sli3.cmdidx) &&
2042 	   (++pring->sli.sli3.next_cmdidx >= max_cmd_idx))
2043 		pring->sli.sli3.next_cmdidx = 0;
2044 
2045 	if (unlikely(pring->sli.sli3.local_getidx ==
2046 		pring->sli.sli3.next_cmdidx)) {
2047 
2048 		pring->sli.sli3.local_getidx = le32_to_cpu(pgp->cmdGetInx);
2049 
2050 		if (unlikely(pring->sli.sli3.local_getidx >= max_cmd_idx)) {
2051 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
2052 					"0315 Ring %d issue: portCmdGet %d "
2053 					"is bigger than cmd ring %d\n",
2054 					pring->ringno,
2055 					pring->sli.sli3.local_getidx,
2056 					max_cmd_idx);
2057 
2058 			phba->link_state = LPFC_HBA_ERROR;
2059 			/*
2060 			 * All error attention handlers are posted to
2061 			 * worker thread
2062 			 */
2063 			phba->work_ha |= HA_ERATT;
2064 			phba->work_hs = HS_FFER3;
2065 
2066 			lpfc_worker_wake_up(phba);
2067 
2068 			return NULL;
2069 		}
2070 
2071 		if (pring->sli.sli3.local_getidx == pring->sli.sli3.next_cmdidx)
2072 			return NULL;
2073 	}
2074 
2075 	return lpfc_cmd_iocb(phba, pring);
2076 }
2077 
2078 /**
2079  * lpfc_sli_next_iotag - Get an iotag for the iocb
2080  * @phba: Pointer to HBA context object.
2081  * @iocbq: Pointer to driver iocb object.
2082  *
2083  * This function gets an iotag for the iocb. If there is no unused iotag and
2084  * the iocbq_lookup_len < 0xffff, this function allocates a bigger iotag_lookup
2085  * array and assigns a new iotag.
2086  * The function returns the allocated iotag if successful, else returns zero.
2087  * Zero is not a valid iotag.
2088  * The caller is not required to hold any lock.
2089  **/
2090 uint16_t
2091 lpfc_sli_next_iotag(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq)
2092 {
2093 	struct lpfc_iocbq **new_arr;
2094 	struct lpfc_iocbq **old_arr;
2095 	size_t new_len;
2096 	struct lpfc_sli *psli = &phba->sli;
2097 	uint16_t iotag;
2098 
2099 	spin_lock_irq(&phba->hbalock);
2100 	iotag = psli->last_iotag;
2101 	if(++iotag < psli->iocbq_lookup_len) {
2102 		psli->last_iotag = iotag;
2103 		psli->iocbq_lookup[iotag] = iocbq;
2104 		spin_unlock_irq(&phba->hbalock);
2105 		iocbq->iotag = iotag;
2106 		return iotag;
2107 	} else if (psli->iocbq_lookup_len < (0xffff
2108 					   - LPFC_IOCBQ_LOOKUP_INCREMENT)) {
2109 		new_len = psli->iocbq_lookup_len + LPFC_IOCBQ_LOOKUP_INCREMENT;
2110 		spin_unlock_irq(&phba->hbalock);
2111 		new_arr = kcalloc(new_len, sizeof(struct lpfc_iocbq *),
2112 				  GFP_KERNEL);
2113 		if (new_arr) {
2114 			spin_lock_irq(&phba->hbalock);
2115 			old_arr = psli->iocbq_lookup;
2116 			if (new_len <= psli->iocbq_lookup_len) {
2117 				/* highly unprobable case */
2118 				kfree(new_arr);
2119 				iotag = psli->last_iotag;
2120 				if(++iotag < psli->iocbq_lookup_len) {
2121 					psli->last_iotag = iotag;
2122 					psli->iocbq_lookup[iotag] = iocbq;
2123 					spin_unlock_irq(&phba->hbalock);
2124 					iocbq->iotag = iotag;
2125 					return iotag;
2126 				}
2127 				spin_unlock_irq(&phba->hbalock);
2128 				return 0;
2129 			}
2130 			if (psli->iocbq_lookup)
2131 				memcpy(new_arr, old_arr,
2132 				       ((psli->last_iotag  + 1) *
2133 					sizeof (struct lpfc_iocbq *)));
2134 			psli->iocbq_lookup = new_arr;
2135 			psli->iocbq_lookup_len = new_len;
2136 			psli->last_iotag = iotag;
2137 			psli->iocbq_lookup[iotag] = iocbq;
2138 			spin_unlock_irq(&phba->hbalock);
2139 			iocbq->iotag = iotag;
2140 			kfree(old_arr);
2141 			return iotag;
2142 		}
2143 	} else
2144 		spin_unlock_irq(&phba->hbalock);
2145 
2146 	lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
2147 			"0318 Failed to allocate IOTAG.last IOTAG is %d\n",
2148 			psli->last_iotag);
2149 
2150 	return 0;
2151 }
2152 
2153 /**
2154  * lpfc_sli_submit_iocb - Submit an iocb to the firmware
2155  * @phba: Pointer to HBA context object.
2156  * @pring: Pointer to driver SLI ring object.
2157  * @iocb: Pointer to iocb slot in the ring.
2158  * @nextiocb: Pointer to driver iocb object which need to be
2159  *            posted to firmware.
2160  *
2161  * This function is called to post a new iocb to the firmware. This
2162  * function copies the new iocb to ring iocb slot and updates the
2163  * ring pointers. It adds the new iocb to txcmplq if there is
2164  * a completion call back for this iocb else the function will free the
2165  * iocb object.  The hbalock is asserted held in the code path calling
2166  * this routine.
2167  **/
2168 static void
2169 lpfc_sli_submit_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
2170 		IOCB_t *iocb, struct lpfc_iocbq *nextiocb)
2171 {
2172 	/*
2173 	 * Set up an iotag
2174 	 */
2175 	nextiocb->iocb.ulpIoTag = (nextiocb->iocb_cmpl) ? nextiocb->iotag : 0;
2176 
2177 
2178 	if (pring->ringno == LPFC_ELS_RING) {
2179 		lpfc_debugfs_slow_ring_trc(phba,
2180 			"IOCB cmd ring:   wd4:x%08x wd6:x%08x wd7:x%08x",
2181 			*(((uint32_t *) &nextiocb->iocb) + 4),
2182 			*(((uint32_t *) &nextiocb->iocb) + 6),
2183 			*(((uint32_t *) &nextiocb->iocb) + 7));
2184 	}
2185 
2186 	/*
2187 	 * Issue iocb command to adapter
2188 	 */
2189 	lpfc_sli_pcimem_bcopy(&nextiocb->iocb, iocb, phba->iocb_cmd_size);
2190 	wmb();
2191 	pring->stats.iocb_cmd++;
2192 
2193 	/*
2194 	 * If there is no completion routine to call, we can release the
2195 	 * IOCB buffer back right now. For IOCBs, like QUE_RING_BUF,
2196 	 * that have no rsp ring completion, iocb_cmpl MUST be NULL.
2197 	 */
2198 	if (nextiocb->iocb_cmpl)
2199 		lpfc_sli_ringtxcmpl_put(phba, pring, nextiocb);
2200 	else
2201 		__lpfc_sli_release_iocbq(phba, nextiocb);
2202 
2203 	/*
2204 	 * Let the HBA know what IOCB slot will be the next one the
2205 	 * driver will put a command into.
2206 	 */
2207 	pring->sli.sli3.cmdidx = pring->sli.sli3.next_cmdidx;
2208 	writel(pring->sli.sli3.cmdidx, &phba->host_gp[pring->ringno].cmdPutInx);
2209 }
2210 
2211 /**
2212  * lpfc_sli_update_full_ring - Update the chip attention register
2213  * @phba: Pointer to HBA context object.
2214  * @pring: Pointer to driver SLI ring object.
2215  *
2216  * The caller is not required to hold any lock for calling this function.
2217  * This function updates the chip attention bits for the ring to inform firmware
2218  * that there are pending work to be done for this ring and requests an
2219  * interrupt when there is space available in the ring. This function is
2220  * called when the driver is unable to post more iocbs to the ring due
2221  * to unavailability of space in the ring.
2222  **/
2223 static void
2224 lpfc_sli_update_full_ring(struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
2225 {
2226 	int ringno = pring->ringno;
2227 
2228 	pring->flag |= LPFC_CALL_RING_AVAILABLE;
2229 
2230 	wmb();
2231 
2232 	/*
2233 	 * Set ring 'ringno' to SET R0CE_REQ in Chip Att register.
2234 	 * The HBA will tell us when an IOCB entry is available.
2235 	 */
2236 	writel((CA_R0ATT|CA_R0CE_REQ) << (ringno*4), phba->CAregaddr);
2237 	readl(phba->CAregaddr); /* flush */
2238 
2239 	pring->stats.iocb_cmd_full++;
2240 }
2241 
2242 /**
2243  * lpfc_sli_update_ring - Update chip attention register
2244  * @phba: Pointer to HBA context object.
2245  * @pring: Pointer to driver SLI ring object.
2246  *
2247  * This function updates the chip attention register bit for the
2248  * given ring to inform HBA that there is more work to be done
2249  * in this ring. The caller is not required to hold any lock.
2250  **/
2251 static void
2252 lpfc_sli_update_ring(struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
2253 {
2254 	int ringno = pring->ringno;
2255 
2256 	/*
2257 	 * Tell the HBA that there is work to do in this ring.
2258 	 */
2259 	if (!(phba->sli3_options & LPFC_SLI3_CRP_ENABLED)) {
2260 		wmb();
2261 		writel(CA_R0ATT << (ringno * 4), phba->CAregaddr);
2262 		readl(phba->CAregaddr); /* flush */
2263 	}
2264 }
2265 
2266 /**
2267  * lpfc_sli_resume_iocb - Process iocbs in the txq
2268  * @phba: Pointer to HBA context object.
2269  * @pring: Pointer to driver SLI ring object.
2270  *
2271  * This function is called with hbalock held to post pending iocbs
2272  * in the txq to the firmware. This function is called when driver
2273  * detects space available in the ring.
2274  **/
2275 static void
2276 lpfc_sli_resume_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
2277 {
2278 	IOCB_t *iocb;
2279 	struct lpfc_iocbq *nextiocb;
2280 
2281 	lockdep_assert_held(&phba->hbalock);
2282 
2283 	/*
2284 	 * Check to see if:
2285 	 *  (a) there is anything on the txq to send
2286 	 *  (b) link is up
2287 	 *  (c) link attention events can be processed (fcp ring only)
2288 	 *  (d) IOCB processing is not blocked by the outstanding mbox command.
2289 	 */
2290 
2291 	if (lpfc_is_link_up(phba) &&
2292 	    (!list_empty(&pring->txq)) &&
2293 	    (pring->ringno != LPFC_FCP_RING ||
2294 	     phba->sli.sli_flag & LPFC_PROCESS_LA)) {
2295 
2296 		while ((iocb = lpfc_sli_next_iocb_slot(phba, pring)) &&
2297 		       (nextiocb = lpfc_sli_ringtx_get(phba, pring)))
2298 			lpfc_sli_submit_iocb(phba, pring, iocb, nextiocb);
2299 
2300 		if (iocb)
2301 			lpfc_sli_update_ring(phba, pring);
2302 		else
2303 			lpfc_sli_update_full_ring(phba, pring);
2304 	}
2305 
2306 	return;
2307 }
2308 
2309 /**
2310  * lpfc_sli_next_hbq_slot - Get next hbq entry for the HBQ
2311  * @phba: Pointer to HBA context object.
2312  * @hbqno: HBQ number.
2313  *
2314  * This function is called with hbalock held to get the next
2315  * available slot for the given HBQ. If there is free slot
2316  * available for the HBQ it will return pointer to the next available
2317  * HBQ entry else it will return NULL.
2318  **/
2319 static struct lpfc_hbq_entry *
2320 lpfc_sli_next_hbq_slot(struct lpfc_hba *phba, uint32_t hbqno)
2321 {
2322 	struct hbq_s *hbqp = &phba->hbqs[hbqno];
2323 
2324 	lockdep_assert_held(&phba->hbalock);
2325 
2326 	if (hbqp->next_hbqPutIdx == hbqp->hbqPutIdx &&
2327 	    ++hbqp->next_hbqPutIdx >= hbqp->entry_count)
2328 		hbqp->next_hbqPutIdx = 0;
2329 
2330 	if (unlikely(hbqp->local_hbqGetIdx == hbqp->next_hbqPutIdx)) {
2331 		uint32_t raw_index = phba->hbq_get[hbqno];
2332 		uint32_t getidx = le32_to_cpu(raw_index);
2333 
2334 		hbqp->local_hbqGetIdx = getidx;
2335 
2336 		if (unlikely(hbqp->local_hbqGetIdx >= hbqp->entry_count)) {
2337 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
2338 					"1802 HBQ %d: local_hbqGetIdx "
2339 					"%u is > than hbqp->entry_count %u\n",
2340 					hbqno, hbqp->local_hbqGetIdx,
2341 					hbqp->entry_count);
2342 
2343 			phba->link_state = LPFC_HBA_ERROR;
2344 			return NULL;
2345 		}
2346 
2347 		if (hbqp->local_hbqGetIdx == hbqp->next_hbqPutIdx)
2348 			return NULL;
2349 	}
2350 
2351 	return (struct lpfc_hbq_entry *) phba->hbqs[hbqno].hbq_virt +
2352 			hbqp->hbqPutIdx;
2353 }
2354 
2355 /**
2356  * lpfc_sli_hbqbuf_free_all - Free all the hbq buffers
2357  * @phba: Pointer to HBA context object.
2358  *
2359  * This function is called with no lock held to free all the
2360  * hbq buffers while uninitializing the SLI interface. It also
2361  * frees the HBQ buffers returned by the firmware but not yet
2362  * processed by the upper layers.
2363  **/
2364 void
2365 lpfc_sli_hbqbuf_free_all(struct lpfc_hba *phba)
2366 {
2367 	struct lpfc_dmabuf *dmabuf, *next_dmabuf;
2368 	struct hbq_dmabuf *hbq_buf;
2369 	unsigned long flags;
2370 	int i, hbq_count;
2371 
2372 	hbq_count = lpfc_sli_hbq_count();
2373 	/* Return all memory used by all HBQs */
2374 	spin_lock_irqsave(&phba->hbalock, flags);
2375 	for (i = 0; i < hbq_count; ++i) {
2376 		list_for_each_entry_safe(dmabuf, next_dmabuf,
2377 				&phba->hbqs[i].hbq_buffer_list, list) {
2378 			hbq_buf = container_of(dmabuf, struct hbq_dmabuf, dbuf);
2379 			list_del(&hbq_buf->dbuf.list);
2380 			(phba->hbqs[i].hbq_free_buffer)(phba, hbq_buf);
2381 		}
2382 		phba->hbqs[i].buffer_count = 0;
2383 	}
2384 
2385 	/* Mark the HBQs not in use */
2386 	phba->hbq_in_use = 0;
2387 	spin_unlock_irqrestore(&phba->hbalock, flags);
2388 }
2389 
2390 /**
2391  * lpfc_sli_hbq_to_firmware - Post the hbq buffer to firmware
2392  * @phba: Pointer to HBA context object.
2393  * @hbqno: HBQ number.
2394  * @hbq_buf: Pointer to HBQ buffer.
2395  *
2396  * This function is called with the hbalock held to post a
2397  * hbq buffer to the firmware. If the function finds an empty
2398  * slot in the HBQ, it will post the buffer. The function will return
2399  * pointer to the hbq entry if it successfully post the buffer
2400  * else it will return NULL.
2401  **/
2402 static int
2403 lpfc_sli_hbq_to_firmware(struct lpfc_hba *phba, uint32_t hbqno,
2404 			 struct hbq_dmabuf *hbq_buf)
2405 {
2406 	lockdep_assert_held(&phba->hbalock);
2407 	return phba->lpfc_sli_hbq_to_firmware(phba, hbqno, hbq_buf);
2408 }
2409 
2410 /**
2411  * lpfc_sli_hbq_to_firmware_s3 - Post the hbq buffer to SLI3 firmware
2412  * @phba: Pointer to HBA context object.
2413  * @hbqno: HBQ number.
2414  * @hbq_buf: Pointer to HBQ buffer.
2415  *
2416  * This function is called with the hbalock held to post a hbq buffer to the
2417  * firmware. If the function finds an empty slot in the HBQ, it will post the
2418  * buffer and place it on the hbq_buffer_list. The function will return zero if
2419  * it successfully post the buffer else it will return an error.
2420  **/
2421 static int
2422 lpfc_sli_hbq_to_firmware_s3(struct lpfc_hba *phba, uint32_t hbqno,
2423 			    struct hbq_dmabuf *hbq_buf)
2424 {
2425 	struct lpfc_hbq_entry *hbqe;
2426 	dma_addr_t physaddr = hbq_buf->dbuf.phys;
2427 
2428 	lockdep_assert_held(&phba->hbalock);
2429 	/* Get next HBQ entry slot to use */
2430 	hbqe = lpfc_sli_next_hbq_slot(phba, hbqno);
2431 	if (hbqe) {
2432 		struct hbq_s *hbqp = &phba->hbqs[hbqno];
2433 
2434 		hbqe->bde.addrHigh = le32_to_cpu(putPaddrHigh(physaddr));
2435 		hbqe->bde.addrLow  = le32_to_cpu(putPaddrLow(physaddr));
2436 		hbqe->bde.tus.f.bdeSize = hbq_buf->total_size;
2437 		hbqe->bde.tus.f.bdeFlags = 0;
2438 		hbqe->bde.tus.w = le32_to_cpu(hbqe->bde.tus.w);
2439 		hbqe->buffer_tag = le32_to_cpu(hbq_buf->tag);
2440 				/* Sync SLIM */
2441 		hbqp->hbqPutIdx = hbqp->next_hbqPutIdx;
2442 		writel(hbqp->hbqPutIdx, phba->hbq_put + hbqno);
2443 				/* flush */
2444 		readl(phba->hbq_put + hbqno);
2445 		list_add_tail(&hbq_buf->dbuf.list, &hbqp->hbq_buffer_list);
2446 		return 0;
2447 	} else
2448 		return -ENOMEM;
2449 }
2450 
2451 /**
2452  * lpfc_sli_hbq_to_firmware_s4 - Post the hbq buffer to SLI4 firmware
2453  * @phba: Pointer to HBA context object.
2454  * @hbqno: HBQ number.
2455  * @hbq_buf: Pointer to HBQ buffer.
2456  *
2457  * This function is called with the hbalock held to post an RQE to the SLI4
2458  * firmware. If able to post the RQE to the RQ it will queue the hbq entry to
2459  * the hbq_buffer_list and return zero, otherwise it will return an error.
2460  **/
2461 static int
2462 lpfc_sli_hbq_to_firmware_s4(struct lpfc_hba *phba, uint32_t hbqno,
2463 			    struct hbq_dmabuf *hbq_buf)
2464 {
2465 	int rc;
2466 	struct lpfc_rqe hrqe;
2467 	struct lpfc_rqe drqe;
2468 	struct lpfc_queue *hrq;
2469 	struct lpfc_queue *drq;
2470 
2471 	if (hbqno != LPFC_ELS_HBQ)
2472 		return 1;
2473 	hrq = phba->sli4_hba.hdr_rq;
2474 	drq = phba->sli4_hba.dat_rq;
2475 
2476 	lockdep_assert_held(&phba->hbalock);
2477 	hrqe.address_lo = putPaddrLow(hbq_buf->hbuf.phys);
2478 	hrqe.address_hi = putPaddrHigh(hbq_buf->hbuf.phys);
2479 	drqe.address_lo = putPaddrLow(hbq_buf->dbuf.phys);
2480 	drqe.address_hi = putPaddrHigh(hbq_buf->dbuf.phys);
2481 	rc = lpfc_sli4_rq_put(hrq, drq, &hrqe, &drqe);
2482 	if (rc < 0)
2483 		return rc;
2484 	hbq_buf->tag = (rc | (hbqno << 16));
2485 	list_add_tail(&hbq_buf->dbuf.list, &phba->hbqs[hbqno].hbq_buffer_list);
2486 	return 0;
2487 }
2488 
2489 /* HBQ for ELS and CT traffic. */
2490 static struct lpfc_hbq_init lpfc_els_hbq = {
2491 	.rn = 1,
2492 	.entry_count = 256,
2493 	.mask_count = 0,
2494 	.profile = 0,
2495 	.ring_mask = (1 << LPFC_ELS_RING),
2496 	.buffer_count = 0,
2497 	.init_count = 40,
2498 	.add_count = 40,
2499 };
2500 
2501 /* Array of HBQs */
2502 struct lpfc_hbq_init *lpfc_hbq_defs[] = {
2503 	&lpfc_els_hbq,
2504 };
2505 
2506 /**
2507  * lpfc_sli_hbqbuf_fill_hbqs - Post more hbq buffers to HBQ
2508  * @phba: Pointer to HBA context object.
2509  * @hbqno: HBQ number.
2510  * @count: Number of HBQ buffers to be posted.
2511  *
2512  * This function is called with no lock held to post more hbq buffers to the
2513  * given HBQ. The function returns the number of HBQ buffers successfully
2514  * posted.
2515  **/
2516 static int
2517 lpfc_sli_hbqbuf_fill_hbqs(struct lpfc_hba *phba, uint32_t hbqno, uint32_t count)
2518 {
2519 	uint32_t i, posted = 0;
2520 	unsigned long flags;
2521 	struct hbq_dmabuf *hbq_buffer;
2522 	LIST_HEAD(hbq_buf_list);
2523 	if (!phba->hbqs[hbqno].hbq_alloc_buffer)
2524 		return 0;
2525 
2526 	if ((phba->hbqs[hbqno].buffer_count + count) >
2527 	    lpfc_hbq_defs[hbqno]->entry_count)
2528 		count = lpfc_hbq_defs[hbqno]->entry_count -
2529 					phba->hbqs[hbqno].buffer_count;
2530 	if (!count)
2531 		return 0;
2532 	/* Allocate HBQ entries */
2533 	for (i = 0; i < count; i++) {
2534 		hbq_buffer = (phba->hbqs[hbqno].hbq_alloc_buffer)(phba);
2535 		if (!hbq_buffer)
2536 			break;
2537 		list_add_tail(&hbq_buffer->dbuf.list, &hbq_buf_list);
2538 	}
2539 	/* Check whether HBQ is still in use */
2540 	spin_lock_irqsave(&phba->hbalock, flags);
2541 	if (!phba->hbq_in_use)
2542 		goto err;
2543 	while (!list_empty(&hbq_buf_list)) {
2544 		list_remove_head(&hbq_buf_list, hbq_buffer, struct hbq_dmabuf,
2545 				 dbuf.list);
2546 		hbq_buffer->tag = (phba->hbqs[hbqno].buffer_count |
2547 				      (hbqno << 16));
2548 		if (!lpfc_sli_hbq_to_firmware(phba, hbqno, hbq_buffer)) {
2549 			phba->hbqs[hbqno].buffer_count++;
2550 			posted++;
2551 		} else
2552 			(phba->hbqs[hbqno].hbq_free_buffer)(phba, hbq_buffer);
2553 	}
2554 	spin_unlock_irqrestore(&phba->hbalock, flags);
2555 	return posted;
2556 err:
2557 	spin_unlock_irqrestore(&phba->hbalock, flags);
2558 	while (!list_empty(&hbq_buf_list)) {
2559 		list_remove_head(&hbq_buf_list, hbq_buffer, struct hbq_dmabuf,
2560 				 dbuf.list);
2561 		(phba->hbqs[hbqno].hbq_free_buffer)(phba, hbq_buffer);
2562 	}
2563 	return 0;
2564 }
2565 
2566 /**
2567  * lpfc_sli_hbqbuf_add_hbqs - Post more HBQ buffers to firmware
2568  * @phba: Pointer to HBA context object.
2569  * @qno: HBQ number.
2570  *
2571  * This function posts more buffers to the HBQ. This function
2572  * is called with no lock held. The function returns the number of HBQ entries
2573  * successfully allocated.
2574  **/
2575 int
2576 lpfc_sli_hbqbuf_add_hbqs(struct lpfc_hba *phba, uint32_t qno)
2577 {
2578 	if (phba->sli_rev == LPFC_SLI_REV4)
2579 		return 0;
2580 	else
2581 		return lpfc_sli_hbqbuf_fill_hbqs(phba, qno,
2582 					 lpfc_hbq_defs[qno]->add_count);
2583 }
2584 
2585 /**
2586  * lpfc_sli_hbqbuf_init_hbqs - Post initial buffers to the HBQ
2587  * @phba: Pointer to HBA context object.
2588  * @qno:  HBQ queue number.
2589  *
2590  * This function is called from SLI initialization code path with
2591  * no lock held to post initial HBQ buffers to firmware. The
2592  * function returns the number of HBQ entries successfully allocated.
2593  **/
2594 static int
2595 lpfc_sli_hbqbuf_init_hbqs(struct lpfc_hba *phba, uint32_t qno)
2596 {
2597 	if (phba->sli_rev == LPFC_SLI_REV4)
2598 		return lpfc_sli_hbqbuf_fill_hbqs(phba, qno,
2599 					lpfc_hbq_defs[qno]->entry_count);
2600 	else
2601 		return lpfc_sli_hbqbuf_fill_hbqs(phba, qno,
2602 					 lpfc_hbq_defs[qno]->init_count);
2603 }
2604 
2605 /*
2606  * lpfc_sli_hbqbuf_get - Remove the first hbq off of an hbq list
2607  *
2608  * This function removes the first hbq buffer on an hbq list and returns a
2609  * pointer to that buffer. If it finds no buffers on the list it returns NULL.
2610  **/
2611 static struct hbq_dmabuf *
2612 lpfc_sli_hbqbuf_get(struct list_head *rb_list)
2613 {
2614 	struct lpfc_dmabuf *d_buf;
2615 
2616 	list_remove_head(rb_list, d_buf, struct lpfc_dmabuf, list);
2617 	if (!d_buf)
2618 		return NULL;
2619 	return container_of(d_buf, struct hbq_dmabuf, dbuf);
2620 }
2621 
2622 /**
2623  * lpfc_sli_rqbuf_get - Remove the first dma buffer off of an RQ list
2624  * @phba: Pointer to HBA context object.
2625  * @hrq: HBQ number.
2626  *
2627  * This function removes the first RQ buffer on an RQ buffer list and returns a
2628  * pointer to that buffer. If it finds no buffers on the list it returns NULL.
2629  **/
2630 static struct rqb_dmabuf *
2631 lpfc_sli_rqbuf_get(struct lpfc_hba *phba, struct lpfc_queue *hrq)
2632 {
2633 	struct lpfc_dmabuf *h_buf;
2634 	struct lpfc_rqb *rqbp;
2635 
2636 	rqbp = hrq->rqbp;
2637 	list_remove_head(&rqbp->rqb_buffer_list, h_buf,
2638 			 struct lpfc_dmabuf, list);
2639 	if (!h_buf)
2640 		return NULL;
2641 	rqbp->buffer_count--;
2642 	return container_of(h_buf, struct rqb_dmabuf, hbuf);
2643 }
2644 
2645 /**
2646  * lpfc_sli_hbqbuf_find - Find the hbq buffer associated with a tag
2647  * @phba: Pointer to HBA context object.
2648  * @tag: Tag of the hbq buffer.
2649  *
2650  * This function searches for the hbq buffer associated with the given tag in
2651  * the hbq buffer list. If it finds the hbq buffer, it returns the hbq_buffer
2652  * otherwise it returns NULL.
2653  **/
2654 static struct hbq_dmabuf *
2655 lpfc_sli_hbqbuf_find(struct lpfc_hba *phba, uint32_t tag)
2656 {
2657 	struct lpfc_dmabuf *d_buf;
2658 	struct hbq_dmabuf *hbq_buf;
2659 	uint32_t hbqno;
2660 
2661 	hbqno = tag >> 16;
2662 	if (hbqno >= LPFC_MAX_HBQS)
2663 		return NULL;
2664 
2665 	spin_lock_irq(&phba->hbalock);
2666 	list_for_each_entry(d_buf, &phba->hbqs[hbqno].hbq_buffer_list, list) {
2667 		hbq_buf = container_of(d_buf, struct hbq_dmabuf, dbuf);
2668 		if (hbq_buf->tag == tag) {
2669 			spin_unlock_irq(&phba->hbalock);
2670 			return hbq_buf;
2671 		}
2672 	}
2673 	spin_unlock_irq(&phba->hbalock);
2674 	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
2675 			"1803 Bad hbq tag. Data: x%x x%x\n",
2676 			tag, phba->hbqs[tag >> 16].buffer_count);
2677 	return NULL;
2678 }
2679 
2680 /**
2681  * lpfc_sli_free_hbq - Give back the hbq buffer to firmware
2682  * @phba: Pointer to HBA context object.
2683  * @hbq_buffer: Pointer to HBQ buffer.
2684  *
2685  * This function is called with hbalock. This function gives back
2686  * the hbq buffer to firmware. If the HBQ does not have space to
2687  * post the buffer, it will free the buffer.
2688  **/
2689 void
2690 lpfc_sli_free_hbq(struct lpfc_hba *phba, struct hbq_dmabuf *hbq_buffer)
2691 {
2692 	uint32_t hbqno;
2693 
2694 	if (hbq_buffer) {
2695 		hbqno = hbq_buffer->tag >> 16;
2696 		if (lpfc_sli_hbq_to_firmware(phba, hbqno, hbq_buffer))
2697 			(phba->hbqs[hbqno].hbq_free_buffer)(phba, hbq_buffer);
2698 	}
2699 }
2700 
2701 /**
2702  * lpfc_sli_chk_mbx_command - Check if the mailbox is a legitimate mailbox
2703  * @mbxCommand: mailbox command code.
2704  *
2705  * This function is called by the mailbox event handler function to verify
2706  * that the completed mailbox command is a legitimate mailbox command. If the
2707  * completed mailbox is not known to the function, it will return MBX_SHUTDOWN
2708  * and the mailbox event handler will take the HBA offline.
2709  **/
2710 static int
2711 lpfc_sli_chk_mbx_command(uint8_t mbxCommand)
2712 {
2713 	uint8_t ret;
2714 
2715 	switch (mbxCommand) {
2716 	case MBX_LOAD_SM:
2717 	case MBX_READ_NV:
2718 	case MBX_WRITE_NV:
2719 	case MBX_WRITE_VPARMS:
2720 	case MBX_RUN_BIU_DIAG:
2721 	case MBX_INIT_LINK:
2722 	case MBX_DOWN_LINK:
2723 	case MBX_CONFIG_LINK:
2724 	case MBX_CONFIG_RING:
2725 	case MBX_RESET_RING:
2726 	case MBX_READ_CONFIG:
2727 	case MBX_READ_RCONFIG:
2728 	case MBX_READ_SPARM:
2729 	case MBX_READ_STATUS:
2730 	case MBX_READ_RPI:
2731 	case MBX_READ_XRI:
2732 	case MBX_READ_REV:
2733 	case MBX_READ_LNK_STAT:
2734 	case MBX_REG_LOGIN:
2735 	case MBX_UNREG_LOGIN:
2736 	case MBX_CLEAR_LA:
2737 	case MBX_DUMP_MEMORY:
2738 	case MBX_DUMP_CONTEXT:
2739 	case MBX_RUN_DIAGS:
2740 	case MBX_RESTART:
2741 	case MBX_UPDATE_CFG:
2742 	case MBX_DOWN_LOAD:
2743 	case MBX_DEL_LD_ENTRY:
2744 	case MBX_RUN_PROGRAM:
2745 	case MBX_SET_MASK:
2746 	case MBX_SET_VARIABLE:
2747 	case MBX_UNREG_D_ID:
2748 	case MBX_KILL_BOARD:
2749 	case MBX_CONFIG_FARP:
2750 	case MBX_BEACON:
2751 	case MBX_LOAD_AREA:
2752 	case MBX_RUN_BIU_DIAG64:
2753 	case MBX_CONFIG_PORT:
2754 	case MBX_READ_SPARM64:
2755 	case MBX_READ_RPI64:
2756 	case MBX_REG_LOGIN64:
2757 	case MBX_READ_TOPOLOGY:
2758 	case MBX_WRITE_WWN:
2759 	case MBX_SET_DEBUG:
2760 	case MBX_LOAD_EXP_ROM:
2761 	case MBX_ASYNCEVT_ENABLE:
2762 	case MBX_REG_VPI:
2763 	case MBX_UNREG_VPI:
2764 	case MBX_HEARTBEAT:
2765 	case MBX_PORT_CAPABILITIES:
2766 	case MBX_PORT_IOV_CONTROL:
2767 	case MBX_SLI4_CONFIG:
2768 	case MBX_SLI4_REQ_FTRS:
2769 	case MBX_REG_FCFI:
2770 	case MBX_UNREG_FCFI:
2771 	case MBX_REG_VFI:
2772 	case MBX_UNREG_VFI:
2773 	case MBX_INIT_VPI:
2774 	case MBX_INIT_VFI:
2775 	case MBX_RESUME_RPI:
2776 	case MBX_READ_EVENT_LOG_STATUS:
2777 	case MBX_READ_EVENT_LOG:
2778 	case MBX_SECURITY_MGMT:
2779 	case MBX_AUTH_PORT:
2780 	case MBX_ACCESS_VDATA:
2781 		ret = mbxCommand;
2782 		break;
2783 	default:
2784 		ret = MBX_SHUTDOWN;
2785 		break;
2786 	}
2787 	return ret;
2788 }
2789 
2790 /**
2791  * lpfc_sli_wake_mbox_wait - lpfc_sli_issue_mbox_wait mbox completion handler
2792  * @phba: Pointer to HBA context object.
2793  * @pmboxq: Pointer to mailbox command.
2794  *
2795  * This is completion handler function for mailbox commands issued from
2796  * lpfc_sli_issue_mbox_wait function. This function is called by the
2797  * mailbox event handler function with no lock held. This function
2798  * will wake up thread waiting on the wait queue pointed by context1
2799  * of the mailbox.
2800  **/
2801 void
2802 lpfc_sli_wake_mbox_wait(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq)
2803 {
2804 	unsigned long drvr_flag;
2805 	struct completion *pmbox_done;
2806 
2807 	/*
2808 	 * If pmbox_done is empty, the driver thread gave up waiting and
2809 	 * continued running.
2810 	 */
2811 	pmboxq->mbox_flag |= LPFC_MBX_WAKE;
2812 	spin_lock_irqsave(&phba->hbalock, drvr_flag);
2813 	pmbox_done = (struct completion *)pmboxq->context3;
2814 	if (pmbox_done)
2815 		complete(pmbox_done);
2816 	spin_unlock_irqrestore(&phba->hbalock, drvr_flag);
2817 	return;
2818 }
2819 
2820 static void
2821 __lpfc_sli_rpi_release(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
2822 {
2823 	unsigned long iflags;
2824 
2825 	if (ndlp->nlp_flag & NLP_RELEASE_RPI) {
2826 		lpfc_sli4_free_rpi(vport->phba, ndlp->nlp_rpi);
2827 		spin_lock_irqsave(&ndlp->lock, iflags);
2828 		ndlp->nlp_flag &= ~NLP_RELEASE_RPI;
2829 		ndlp->nlp_rpi = LPFC_RPI_ALLOC_ERROR;
2830 		spin_unlock_irqrestore(&ndlp->lock, iflags);
2831 	}
2832 	ndlp->nlp_flag &= ~NLP_UNREG_INP;
2833 }
2834 
2835 /**
2836  * lpfc_sli_def_mbox_cmpl - Default mailbox completion handler
2837  * @phba: Pointer to HBA context object.
2838  * @pmb: Pointer to mailbox object.
2839  *
2840  * This function is the default mailbox completion handler. It
2841  * frees the memory resources associated with the completed mailbox
2842  * command. If the completed command is a REG_LOGIN mailbox command,
2843  * this function will issue a UREG_LOGIN to re-claim the RPI.
2844  **/
2845 void
2846 lpfc_sli_def_mbox_cmpl(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb)
2847 {
2848 	struct lpfc_vport  *vport = pmb->vport;
2849 	struct lpfc_dmabuf *mp;
2850 	struct lpfc_nodelist *ndlp;
2851 	struct Scsi_Host *shost;
2852 	uint16_t rpi, vpi;
2853 	int rc;
2854 
2855 	mp = (struct lpfc_dmabuf *)(pmb->ctx_buf);
2856 
2857 	if (mp) {
2858 		lpfc_mbuf_free(phba, mp->virt, mp->phys);
2859 		kfree(mp);
2860 	}
2861 
2862 	/*
2863 	 * If a REG_LOGIN succeeded  after node is destroyed or node
2864 	 * is in re-discovery driver need to cleanup the RPI.
2865 	 */
2866 	if (!(phba->pport->load_flag & FC_UNLOADING) &&
2867 	    pmb->u.mb.mbxCommand == MBX_REG_LOGIN64 &&
2868 	    !pmb->u.mb.mbxStatus) {
2869 		rpi = pmb->u.mb.un.varWords[0];
2870 		vpi = pmb->u.mb.un.varRegLogin.vpi;
2871 		if (phba->sli_rev == LPFC_SLI_REV4)
2872 			vpi -= phba->sli4_hba.max_cfg_param.vpi_base;
2873 		lpfc_unreg_login(phba, vpi, rpi, pmb);
2874 		pmb->vport = vport;
2875 		pmb->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
2876 		rc = lpfc_sli_issue_mbox(phba, pmb, MBX_NOWAIT);
2877 		if (rc != MBX_NOT_FINISHED)
2878 			return;
2879 	}
2880 
2881 	if ((pmb->u.mb.mbxCommand == MBX_REG_VPI) &&
2882 		!(phba->pport->load_flag & FC_UNLOADING) &&
2883 		!pmb->u.mb.mbxStatus) {
2884 		shost = lpfc_shost_from_vport(vport);
2885 		spin_lock_irq(shost->host_lock);
2886 		vport->vpi_state |= LPFC_VPI_REGISTERED;
2887 		vport->fc_flag &= ~FC_VPORT_NEEDS_REG_VPI;
2888 		spin_unlock_irq(shost->host_lock);
2889 	}
2890 
2891 	if (pmb->u.mb.mbxCommand == MBX_REG_LOGIN64) {
2892 		ndlp = (struct lpfc_nodelist *)pmb->ctx_ndlp;
2893 		lpfc_nlp_put(ndlp);
2894 		pmb->ctx_buf = NULL;
2895 		pmb->ctx_ndlp = NULL;
2896 	}
2897 
2898 	if (pmb->u.mb.mbxCommand == MBX_UNREG_LOGIN) {
2899 		ndlp = (struct lpfc_nodelist *)pmb->ctx_ndlp;
2900 
2901 		/* Check to see if there are any deferred events to process */
2902 		if (ndlp) {
2903 			lpfc_printf_vlog(
2904 				vport,
2905 				KERN_INFO, LOG_MBOX | LOG_DISCOVERY,
2906 				"1438 UNREG cmpl deferred mbox x%x "
2907 				"on NPort x%x Data: x%x x%x x%px x%x x%x\n",
2908 				ndlp->nlp_rpi, ndlp->nlp_DID,
2909 				ndlp->nlp_flag, ndlp->nlp_defer_did,
2910 				ndlp, vport->load_flag, kref_read(&ndlp->kref));
2911 
2912 			if ((ndlp->nlp_flag & NLP_UNREG_INP) &&
2913 			    (ndlp->nlp_defer_did != NLP_EVT_NOTHING_PENDING)) {
2914 				ndlp->nlp_flag &= ~NLP_UNREG_INP;
2915 				ndlp->nlp_defer_did = NLP_EVT_NOTHING_PENDING;
2916 				lpfc_issue_els_plogi(vport, ndlp->nlp_DID, 0);
2917 			} else {
2918 				__lpfc_sli_rpi_release(vport, ndlp);
2919 			}
2920 
2921 			/* The unreg_login mailbox is complete and had a
2922 			 * reference that has to be released.  The PLOGI
2923 			 * got its own ref.
2924 			 */
2925 			lpfc_nlp_put(ndlp);
2926 			pmb->ctx_ndlp = NULL;
2927 		}
2928 	}
2929 
2930 	/* This nlp_put pairs with lpfc_sli4_resume_rpi */
2931 	if (pmb->u.mb.mbxCommand == MBX_RESUME_RPI) {
2932 		ndlp = (struct lpfc_nodelist *)pmb->ctx_ndlp;
2933 		lpfc_nlp_put(ndlp);
2934 	}
2935 
2936 	/* Check security permission status on INIT_LINK mailbox command */
2937 	if ((pmb->u.mb.mbxCommand == MBX_INIT_LINK) &&
2938 	    (pmb->u.mb.mbxStatus == MBXERR_SEC_NO_PERMISSION))
2939 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
2940 				"2860 SLI authentication is required "
2941 				"for INIT_LINK but has not done yet\n");
2942 
2943 	if (bf_get(lpfc_mqe_command, &pmb->u.mqe) == MBX_SLI4_CONFIG)
2944 		lpfc_sli4_mbox_cmd_free(phba, pmb);
2945 	else
2946 		mempool_free(pmb, phba->mbox_mem_pool);
2947 }
2948  /**
2949  * lpfc_sli4_unreg_rpi_cmpl_clr - mailbox completion handler
2950  * @phba: Pointer to HBA context object.
2951  * @pmb: Pointer to mailbox object.
2952  *
2953  * This function is the unreg rpi mailbox completion handler. It
2954  * frees the memory resources associated with the completed mailbox
2955  * command. An additional reference is put on the ndlp to prevent
2956  * lpfc_nlp_release from freeing the rpi bit in the bitmask before
2957  * the unreg mailbox command completes, this routine puts the
2958  * reference back.
2959  *
2960  **/
2961 void
2962 lpfc_sli4_unreg_rpi_cmpl_clr(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb)
2963 {
2964 	struct lpfc_vport  *vport = pmb->vport;
2965 	struct lpfc_nodelist *ndlp;
2966 
2967 	ndlp = pmb->ctx_ndlp;
2968 	if (pmb->u.mb.mbxCommand == MBX_UNREG_LOGIN) {
2969 		if (phba->sli_rev == LPFC_SLI_REV4 &&
2970 		    (bf_get(lpfc_sli_intf_if_type,
2971 		     &phba->sli4_hba.sli_intf) >=
2972 		     LPFC_SLI_INTF_IF_TYPE_2)) {
2973 			if (ndlp) {
2974 				lpfc_printf_vlog(
2975 					 vport, KERN_INFO, LOG_MBOX | LOG_SLI,
2976 					 "0010 UNREG_LOGIN vpi:%x "
2977 					 "rpi:%x DID:%x defer x%x flg x%x "
2978 					 "x%px\n",
2979 					 vport->vpi, ndlp->nlp_rpi,
2980 					 ndlp->nlp_DID, ndlp->nlp_defer_did,
2981 					 ndlp->nlp_flag,
2982 					 ndlp);
2983 				ndlp->nlp_flag &= ~NLP_LOGO_ACC;
2984 
2985 				/* Check to see if there are any deferred
2986 				 * events to process
2987 				 */
2988 				if ((ndlp->nlp_flag & NLP_UNREG_INP) &&
2989 				    (ndlp->nlp_defer_did !=
2990 				    NLP_EVT_NOTHING_PENDING)) {
2991 					lpfc_printf_vlog(
2992 						vport, KERN_INFO, LOG_DISCOVERY,
2993 						"4111 UNREG cmpl deferred "
2994 						"clr x%x on "
2995 						"NPort x%x Data: x%x x%px\n",
2996 						ndlp->nlp_rpi, ndlp->nlp_DID,
2997 						ndlp->nlp_defer_did, ndlp);
2998 					ndlp->nlp_flag &= ~NLP_UNREG_INP;
2999 					ndlp->nlp_defer_did =
3000 						NLP_EVT_NOTHING_PENDING;
3001 					lpfc_issue_els_plogi(
3002 						vport, ndlp->nlp_DID, 0);
3003 				} else {
3004 					__lpfc_sli_rpi_release(vport, ndlp);
3005 				}
3006 				lpfc_nlp_put(ndlp);
3007 			}
3008 		}
3009 	}
3010 
3011 	mempool_free(pmb, phba->mbox_mem_pool);
3012 }
3013 
3014 /**
3015  * lpfc_sli_handle_mb_event - Handle mailbox completions from firmware
3016  * @phba: Pointer to HBA context object.
3017  *
3018  * This function is called with no lock held. This function processes all
3019  * the completed mailbox commands and gives it to upper layers. The interrupt
3020  * service routine processes mailbox completion interrupt and adds completed
3021  * mailbox commands to the mboxq_cmpl queue and signals the worker thread.
3022  * Worker thread call lpfc_sli_handle_mb_event, which will return the
3023  * completed mailbox commands in mboxq_cmpl queue to the upper layers. This
3024  * function returns the mailbox commands to the upper layer by calling the
3025  * completion handler function of each mailbox.
3026  **/
3027 int
3028 lpfc_sli_handle_mb_event(struct lpfc_hba *phba)
3029 {
3030 	MAILBOX_t *pmbox;
3031 	LPFC_MBOXQ_t *pmb;
3032 	int rc;
3033 	LIST_HEAD(cmplq);
3034 
3035 	phba->sli.slistat.mbox_event++;
3036 
3037 	/* Get all completed mailboxe buffers into the cmplq */
3038 	spin_lock_irq(&phba->hbalock);
3039 	list_splice_init(&phba->sli.mboxq_cmpl, &cmplq);
3040 	spin_unlock_irq(&phba->hbalock);
3041 
3042 	/* Get a Mailbox buffer to setup mailbox commands for callback */
3043 	do {
3044 		list_remove_head(&cmplq, pmb, LPFC_MBOXQ_t, list);
3045 		if (pmb == NULL)
3046 			break;
3047 
3048 		pmbox = &pmb->u.mb;
3049 
3050 		if (pmbox->mbxCommand != MBX_HEARTBEAT) {
3051 			if (pmb->vport) {
3052 				lpfc_debugfs_disc_trc(pmb->vport,
3053 					LPFC_DISC_TRC_MBOX_VPORT,
3054 					"MBOX cmpl vport: cmd:x%x mb:x%x x%x",
3055 					(uint32_t)pmbox->mbxCommand,
3056 					pmbox->un.varWords[0],
3057 					pmbox->un.varWords[1]);
3058 			}
3059 			else {
3060 				lpfc_debugfs_disc_trc(phba->pport,
3061 					LPFC_DISC_TRC_MBOX,
3062 					"MBOX cmpl:       cmd:x%x mb:x%x x%x",
3063 					(uint32_t)pmbox->mbxCommand,
3064 					pmbox->un.varWords[0],
3065 					pmbox->un.varWords[1]);
3066 			}
3067 		}
3068 
3069 		/*
3070 		 * It is a fatal error if unknown mbox command completion.
3071 		 */
3072 		if (lpfc_sli_chk_mbx_command(pmbox->mbxCommand) ==
3073 		    MBX_SHUTDOWN) {
3074 			/* Unknown mailbox command compl */
3075 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
3076 					"(%d):0323 Unknown Mailbox command "
3077 					"x%x (x%x/x%x) Cmpl\n",
3078 					pmb->vport ? pmb->vport->vpi :
3079 					LPFC_VPORT_UNKNOWN,
3080 					pmbox->mbxCommand,
3081 					lpfc_sli_config_mbox_subsys_get(phba,
3082 									pmb),
3083 					lpfc_sli_config_mbox_opcode_get(phba,
3084 									pmb));
3085 			phba->link_state = LPFC_HBA_ERROR;
3086 			phba->work_hs = HS_FFER3;
3087 			lpfc_handle_eratt(phba);
3088 			continue;
3089 		}
3090 
3091 		if (pmbox->mbxStatus) {
3092 			phba->sli.slistat.mbox_stat_err++;
3093 			if (pmbox->mbxStatus == MBXERR_NO_RESOURCES) {
3094 				/* Mbox cmd cmpl error - RETRYing */
3095 				lpfc_printf_log(phba, KERN_INFO,
3096 					LOG_MBOX | LOG_SLI,
3097 					"(%d):0305 Mbox cmd cmpl "
3098 					"error - RETRYing Data: x%x "
3099 					"(x%x/x%x) x%x x%x x%x\n",
3100 					pmb->vport ? pmb->vport->vpi :
3101 					LPFC_VPORT_UNKNOWN,
3102 					pmbox->mbxCommand,
3103 					lpfc_sli_config_mbox_subsys_get(phba,
3104 									pmb),
3105 					lpfc_sli_config_mbox_opcode_get(phba,
3106 									pmb),
3107 					pmbox->mbxStatus,
3108 					pmbox->un.varWords[0],
3109 					pmb->vport ? pmb->vport->port_state :
3110 					LPFC_VPORT_UNKNOWN);
3111 				pmbox->mbxStatus = 0;
3112 				pmbox->mbxOwner = OWN_HOST;
3113 				rc = lpfc_sli_issue_mbox(phba, pmb, MBX_NOWAIT);
3114 				if (rc != MBX_NOT_FINISHED)
3115 					continue;
3116 			}
3117 		}
3118 
3119 		/* Mailbox cmd <cmd> Cmpl <cmpl> */
3120 		lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI,
3121 				"(%d):0307 Mailbox cmd x%x (x%x/x%x) Cmpl %ps "
3122 				"Data: x%x x%x x%x x%x x%x x%x x%x x%x x%x "
3123 				"x%x x%x x%x\n",
3124 				pmb->vport ? pmb->vport->vpi : 0,
3125 				pmbox->mbxCommand,
3126 				lpfc_sli_config_mbox_subsys_get(phba, pmb),
3127 				lpfc_sli_config_mbox_opcode_get(phba, pmb),
3128 				pmb->mbox_cmpl,
3129 				*((uint32_t *) pmbox),
3130 				pmbox->un.varWords[0],
3131 				pmbox->un.varWords[1],
3132 				pmbox->un.varWords[2],
3133 				pmbox->un.varWords[3],
3134 				pmbox->un.varWords[4],
3135 				pmbox->un.varWords[5],
3136 				pmbox->un.varWords[6],
3137 				pmbox->un.varWords[7],
3138 				pmbox->un.varWords[8],
3139 				pmbox->un.varWords[9],
3140 				pmbox->un.varWords[10]);
3141 
3142 		if (pmb->mbox_cmpl)
3143 			pmb->mbox_cmpl(phba,pmb);
3144 	} while (1);
3145 	return 0;
3146 }
3147 
3148 /**
3149  * lpfc_sli_get_buff - Get the buffer associated with the buffer tag
3150  * @phba: Pointer to HBA context object.
3151  * @pring: Pointer to driver SLI ring object.
3152  * @tag: buffer tag.
3153  *
3154  * This function is called with no lock held. When QUE_BUFTAG_BIT bit
3155  * is set in the tag the buffer is posted for a particular exchange,
3156  * the function will return the buffer without replacing the buffer.
3157  * If the buffer is for unsolicited ELS or CT traffic, this function
3158  * returns the buffer and also posts another buffer to the firmware.
3159  **/
3160 static struct lpfc_dmabuf *
3161 lpfc_sli_get_buff(struct lpfc_hba *phba,
3162 		  struct lpfc_sli_ring *pring,
3163 		  uint32_t tag)
3164 {
3165 	struct hbq_dmabuf *hbq_entry;
3166 
3167 	if (tag & QUE_BUFTAG_BIT)
3168 		return lpfc_sli_ring_taggedbuf_get(phba, pring, tag);
3169 	hbq_entry = lpfc_sli_hbqbuf_find(phba, tag);
3170 	if (!hbq_entry)
3171 		return NULL;
3172 	return &hbq_entry->dbuf;
3173 }
3174 
3175 /**
3176  * lpfc_nvme_unsol_ls_handler - Process an unsolicited event data buffer
3177  *                              containing a NVME LS request.
3178  * @phba: pointer to lpfc hba data structure.
3179  * @piocb: pointer to the iocbq struct representing the sequence starting
3180  *        frame.
3181  *
3182  * This routine initially validates the NVME LS, validates there is a login
3183  * with the port that sent the LS, and then calls the appropriate nvme host
3184  * or target LS request handler.
3185  **/
3186 static void
3187 lpfc_nvme_unsol_ls_handler(struct lpfc_hba *phba, struct lpfc_iocbq *piocb)
3188 {
3189 	struct lpfc_nodelist *ndlp;
3190 	struct lpfc_dmabuf *d_buf;
3191 	struct hbq_dmabuf *nvmebuf;
3192 	struct fc_frame_header *fc_hdr;
3193 	struct lpfc_async_xchg_ctx *axchg = NULL;
3194 	char *failwhy = NULL;
3195 	uint32_t oxid, sid, did, fctl, size;
3196 	int ret = 1;
3197 
3198 	d_buf = piocb->context2;
3199 
3200 	nvmebuf = container_of(d_buf, struct hbq_dmabuf, dbuf);
3201 	fc_hdr = nvmebuf->hbuf.virt;
3202 	oxid = be16_to_cpu(fc_hdr->fh_ox_id);
3203 	sid = sli4_sid_from_fc_hdr(fc_hdr);
3204 	did = sli4_did_from_fc_hdr(fc_hdr);
3205 	fctl = (fc_hdr->fh_f_ctl[0] << 16 |
3206 		fc_hdr->fh_f_ctl[1] << 8 |
3207 		fc_hdr->fh_f_ctl[2]);
3208 	size = bf_get(lpfc_rcqe_length, &nvmebuf->cq_event.cqe.rcqe_cmpl);
3209 
3210 	lpfc_nvmeio_data(phba, "NVME LS    RCV: xri x%x sz %d from %06x\n",
3211 			 oxid, size, sid);
3212 
3213 	if (phba->pport->load_flag & FC_UNLOADING) {
3214 		failwhy = "Driver Unloading";
3215 	} else if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME)) {
3216 		failwhy = "NVME FC4 Disabled";
3217 	} else if (!phba->nvmet_support && !phba->pport->localport) {
3218 		failwhy = "No Localport";
3219 	} else if (phba->nvmet_support && !phba->targetport) {
3220 		failwhy = "No Targetport";
3221 	} else if (unlikely(fc_hdr->fh_r_ctl != FC_RCTL_ELS4_REQ)) {
3222 		failwhy = "Bad NVME LS R_CTL";
3223 	} else if (unlikely((fctl & 0x00FF0000) !=
3224 			(FC_FC_FIRST_SEQ | FC_FC_END_SEQ | FC_FC_SEQ_INIT))) {
3225 		failwhy = "Bad NVME LS F_CTL";
3226 	} else {
3227 		axchg = kzalloc(sizeof(*axchg), GFP_ATOMIC);
3228 		if (!axchg)
3229 			failwhy = "No CTX memory";
3230 	}
3231 
3232 	if (unlikely(failwhy)) {
3233 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
3234 				"6154 Drop NVME LS: SID %06X OXID x%X: %s\n",
3235 				sid, oxid, failwhy);
3236 		goto out_fail;
3237 	}
3238 
3239 	/* validate the source of the LS is logged in */
3240 	ndlp = lpfc_findnode_did(phba->pport, sid);
3241 	if (!ndlp ||
3242 	    ((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) &&
3243 	     (ndlp->nlp_state != NLP_STE_MAPPED_NODE))) {
3244 		lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC,
3245 				"6216 NVME Unsol rcv: No ndlp: "
3246 				"NPort_ID x%x oxid x%x\n",
3247 				sid, oxid);
3248 		goto out_fail;
3249 	}
3250 
3251 	axchg->phba = phba;
3252 	axchg->ndlp = ndlp;
3253 	axchg->size = size;
3254 	axchg->oxid = oxid;
3255 	axchg->sid = sid;
3256 	axchg->wqeq = NULL;
3257 	axchg->state = LPFC_NVME_STE_LS_RCV;
3258 	axchg->entry_cnt = 1;
3259 	axchg->rqb_buffer = (void *)nvmebuf;
3260 	axchg->hdwq = &phba->sli4_hba.hdwq[0];
3261 	axchg->payload = nvmebuf->dbuf.virt;
3262 	INIT_LIST_HEAD(&axchg->list);
3263 
3264 	if (phba->nvmet_support) {
3265 		ret = lpfc_nvmet_handle_lsreq(phba, axchg);
3266 		spin_lock_irq(&ndlp->lock);
3267 		if (!ret && !(ndlp->fc4_xpt_flags & NLP_XPT_HAS_HH)) {
3268 			ndlp->fc4_xpt_flags |= NLP_XPT_HAS_HH;
3269 			spin_unlock_irq(&ndlp->lock);
3270 
3271 			/* This reference is a single occurrence to hold the
3272 			 * node valid until the nvmet transport calls
3273 			 * host_release.
3274 			 */
3275 			if (!lpfc_nlp_get(ndlp))
3276 				goto out_fail;
3277 
3278 			lpfc_printf_log(phba, KERN_ERR, LOG_NODE,
3279 					"6206 NVMET unsol ls_req ndlp x%px "
3280 					"DID x%x xflags x%x refcnt %d\n",
3281 					ndlp, ndlp->nlp_DID,
3282 					ndlp->fc4_xpt_flags,
3283 					kref_read(&ndlp->kref));
3284 		} else {
3285 			spin_unlock_irq(&ndlp->lock);
3286 		}
3287 	} else {
3288 		ret = lpfc_nvme_handle_lsreq(phba, axchg);
3289 	}
3290 
3291 	/* if zero, LS was successfully handled. If non-zero, LS not handled */
3292 	if (!ret)
3293 		return;
3294 
3295 out_fail:
3296 	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
3297 			"6155 Drop NVME LS from DID %06X: SID %06X OXID x%X "
3298 			"NVMe%s handler failed %d\n",
3299 			did, sid, oxid,
3300 			(phba->nvmet_support) ? "T" : "I", ret);
3301 
3302 	/* recycle receive buffer */
3303 	lpfc_in_buf_free(phba, &nvmebuf->dbuf);
3304 
3305 	/* If start of new exchange, abort it */
3306 	if (axchg && (fctl & FC_FC_FIRST_SEQ && !(fctl & FC_FC_EX_CTX)))
3307 		ret = lpfc_nvme_unsol_ls_issue_abort(phba, axchg, sid, oxid);
3308 
3309 	if (ret)
3310 		kfree(axchg);
3311 }
3312 
3313 /**
3314  * lpfc_complete_unsol_iocb - Complete an unsolicited sequence
3315  * @phba: Pointer to HBA context object.
3316  * @pring: Pointer to driver SLI ring object.
3317  * @saveq: Pointer to the iocbq struct representing the sequence starting frame.
3318  * @fch_r_ctl: the r_ctl for the first frame of the sequence.
3319  * @fch_type: the type for the first frame of the sequence.
3320  *
3321  * This function is called with no lock held. This function uses the r_ctl and
3322  * type of the received sequence to find the correct callback function to call
3323  * to process the sequence.
3324  **/
3325 static int
3326 lpfc_complete_unsol_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
3327 			 struct lpfc_iocbq *saveq, uint32_t fch_r_ctl,
3328 			 uint32_t fch_type)
3329 {
3330 	int i;
3331 
3332 	switch (fch_type) {
3333 	case FC_TYPE_NVME:
3334 		lpfc_nvme_unsol_ls_handler(phba, saveq);
3335 		return 1;
3336 	default:
3337 		break;
3338 	}
3339 
3340 	/* unSolicited Responses */
3341 	if (pring->prt[0].profile) {
3342 		if (pring->prt[0].lpfc_sli_rcv_unsol_event)
3343 			(pring->prt[0].lpfc_sli_rcv_unsol_event) (phba, pring,
3344 									saveq);
3345 		return 1;
3346 	}
3347 	/* We must search, based on rctl / type
3348 	   for the right routine */
3349 	for (i = 0; i < pring->num_mask; i++) {
3350 		if ((pring->prt[i].rctl == fch_r_ctl) &&
3351 		    (pring->prt[i].type == fch_type)) {
3352 			if (pring->prt[i].lpfc_sli_rcv_unsol_event)
3353 				(pring->prt[i].lpfc_sli_rcv_unsol_event)
3354 						(phba, pring, saveq);
3355 			return 1;
3356 		}
3357 	}
3358 	return 0;
3359 }
3360 
3361 /**
3362  * lpfc_sli_process_unsol_iocb - Unsolicited iocb handler
3363  * @phba: Pointer to HBA context object.
3364  * @pring: Pointer to driver SLI ring object.
3365  * @saveq: Pointer to the unsolicited iocb.
3366  *
3367  * This function is called with no lock held by the ring event handler
3368  * when there is an unsolicited iocb posted to the response ring by the
3369  * firmware. This function gets the buffer associated with the iocbs
3370  * and calls the event handler for the ring. This function handles both
3371  * qring buffers and hbq buffers.
3372  * When the function returns 1 the caller can free the iocb object otherwise
3373  * upper layer functions will free the iocb objects.
3374  **/
3375 static int
3376 lpfc_sli_process_unsol_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
3377 			    struct lpfc_iocbq *saveq)
3378 {
3379 	IOCB_t           * irsp;
3380 	WORD5            * w5p;
3381 	uint32_t           Rctl, Type;
3382 	struct lpfc_iocbq *iocbq;
3383 	struct lpfc_dmabuf *dmzbuf;
3384 
3385 	irsp = &(saveq->iocb);
3386 
3387 	if (irsp->ulpCommand == CMD_ASYNC_STATUS) {
3388 		if (pring->lpfc_sli_rcv_async_status)
3389 			pring->lpfc_sli_rcv_async_status(phba, pring, saveq);
3390 		else
3391 			lpfc_printf_log(phba,
3392 					KERN_WARNING,
3393 					LOG_SLI,
3394 					"0316 Ring %d handler: unexpected "
3395 					"ASYNC_STATUS iocb received evt_code "
3396 					"0x%x\n",
3397 					pring->ringno,
3398 					irsp->un.asyncstat.evt_code);
3399 		return 1;
3400 	}
3401 
3402 	if ((irsp->ulpCommand == CMD_IOCB_RET_XRI64_CX) &&
3403 		(phba->sli3_options & LPFC_SLI3_HBQ_ENABLED)) {
3404 		if (irsp->ulpBdeCount > 0) {
3405 			dmzbuf = lpfc_sli_get_buff(phba, pring,
3406 					irsp->un.ulpWord[3]);
3407 			lpfc_in_buf_free(phba, dmzbuf);
3408 		}
3409 
3410 		if (irsp->ulpBdeCount > 1) {
3411 			dmzbuf = lpfc_sli_get_buff(phba, pring,
3412 					irsp->unsli3.sli3Words[3]);
3413 			lpfc_in_buf_free(phba, dmzbuf);
3414 		}
3415 
3416 		if (irsp->ulpBdeCount > 2) {
3417 			dmzbuf = lpfc_sli_get_buff(phba, pring,
3418 				irsp->unsli3.sli3Words[7]);
3419 			lpfc_in_buf_free(phba, dmzbuf);
3420 		}
3421 
3422 		return 1;
3423 	}
3424 
3425 	if (phba->sli3_options & LPFC_SLI3_HBQ_ENABLED) {
3426 		if (irsp->ulpBdeCount != 0) {
3427 			saveq->context2 = lpfc_sli_get_buff(phba, pring,
3428 						irsp->un.ulpWord[3]);
3429 			if (!saveq->context2)
3430 				lpfc_printf_log(phba,
3431 					KERN_ERR,
3432 					LOG_SLI,
3433 					"0341 Ring %d Cannot find buffer for "
3434 					"an unsolicited iocb. tag 0x%x\n",
3435 					pring->ringno,
3436 					irsp->un.ulpWord[3]);
3437 		}
3438 		if (irsp->ulpBdeCount == 2) {
3439 			saveq->context3 = lpfc_sli_get_buff(phba, pring,
3440 						irsp->unsli3.sli3Words[7]);
3441 			if (!saveq->context3)
3442 				lpfc_printf_log(phba,
3443 					KERN_ERR,
3444 					LOG_SLI,
3445 					"0342 Ring %d Cannot find buffer for an"
3446 					" unsolicited iocb. tag 0x%x\n",
3447 					pring->ringno,
3448 					irsp->unsli3.sli3Words[7]);
3449 		}
3450 		list_for_each_entry(iocbq, &saveq->list, list) {
3451 			irsp = &(iocbq->iocb);
3452 			if (irsp->ulpBdeCount != 0) {
3453 				iocbq->context2 = lpfc_sli_get_buff(phba, pring,
3454 							irsp->un.ulpWord[3]);
3455 				if (!iocbq->context2)
3456 					lpfc_printf_log(phba,
3457 						KERN_ERR,
3458 						LOG_SLI,
3459 						"0343 Ring %d Cannot find "
3460 						"buffer for an unsolicited iocb"
3461 						". tag 0x%x\n", pring->ringno,
3462 						irsp->un.ulpWord[3]);
3463 			}
3464 			if (irsp->ulpBdeCount == 2) {
3465 				iocbq->context3 = lpfc_sli_get_buff(phba, pring,
3466 						irsp->unsli3.sli3Words[7]);
3467 				if (!iocbq->context3)
3468 					lpfc_printf_log(phba,
3469 						KERN_ERR,
3470 						LOG_SLI,
3471 						"0344 Ring %d Cannot find "
3472 						"buffer for an unsolicited "
3473 						"iocb. tag 0x%x\n",
3474 						pring->ringno,
3475 						irsp->unsli3.sli3Words[7]);
3476 			}
3477 		}
3478 	}
3479 	if (irsp->ulpBdeCount != 0 &&
3480 	    (irsp->ulpCommand == CMD_IOCB_RCV_CONT64_CX ||
3481 	     irsp->ulpStatus == IOSTAT_INTERMED_RSP)) {
3482 		int found = 0;
3483 
3484 		/* search continue save q for same XRI */
3485 		list_for_each_entry(iocbq, &pring->iocb_continue_saveq, clist) {
3486 			if (iocbq->iocb.unsli3.rcvsli3.ox_id ==
3487 				saveq->iocb.unsli3.rcvsli3.ox_id) {
3488 				list_add_tail(&saveq->list, &iocbq->list);
3489 				found = 1;
3490 				break;
3491 			}
3492 		}
3493 		if (!found)
3494 			list_add_tail(&saveq->clist,
3495 				      &pring->iocb_continue_saveq);
3496 		if (saveq->iocb.ulpStatus != IOSTAT_INTERMED_RSP) {
3497 			list_del_init(&iocbq->clist);
3498 			saveq = iocbq;
3499 			irsp = &(saveq->iocb);
3500 		} else
3501 			return 0;
3502 	}
3503 	if ((irsp->ulpCommand == CMD_RCV_ELS_REQ64_CX) ||
3504 	    (irsp->ulpCommand == CMD_RCV_ELS_REQ_CX) ||
3505 	    (irsp->ulpCommand == CMD_IOCB_RCV_ELS64_CX)) {
3506 		Rctl = FC_RCTL_ELS_REQ;
3507 		Type = FC_TYPE_ELS;
3508 	} else {
3509 		w5p = (WORD5 *)&(saveq->iocb.un.ulpWord[5]);
3510 		Rctl = w5p->hcsw.Rctl;
3511 		Type = w5p->hcsw.Type;
3512 
3513 		/* Firmware Workaround */
3514 		if ((Rctl == 0) && (pring->ringno == LPFC_ELS_RING) &&
3515 			(irsp->ulpCommand == CMD_RCV_SEQUENCE64_CX ||
3516 			 irsp->ulpCommand == CMD_IOCB_RCV_SEQ64_CX)) {
3517 			Rctl = FC_RCTL_ELS_REQ;
3518 			Type = FC_TYPE_ELS;
3519 			w5p->hcsw.Rctl = Rctl;
3520 			w5p->hcsw.Type = Type;
3521 		}
3522 	}
3523 
3524 	if (!lpfc_complete_unsol_iocb(phba, pring, saveq, Rctl, Type))
3525 		lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
3526 				"0313 Ring %d handler: unexpected Rctl x%x "
3527 				"Type x%x received\n",
3528 				pring->ringno, Rctl, Type);
3529 
3530 	return 1;
3531 }
3532 
3533 /**
3534  * lpfc_sli_iocbq_lookup - Find command iocb for the given response iocb
3535  * @phba: Pointer to HBA context object.
3536  * @pring: Pointer to driver SLI ring object.
3537  * @prspiocb: Pointer to response iocb object.
3538  *
3539  * This function looks up the iocb_lookup table to get the command iocb
3540  * corresponding to the given response iocb using the iotag of the
3541  * response iocb. The driver calls this function with the hbalock held
3542  * for SLI3 ports or the ring lock held for SLI4 ports.
3543  * This function returns the command iocb object if it finds the command
3544  * iocb else returns NULL.
3545  **/
3546 static struct lpfc_iocbq *
3547 lpfc_sli_iocbq_lookup(struct lpfc_hba *phba,
3548 		      struct lpfc_sli_ring *pring,
3549 		      struct lpfc_iocbq *prspiocb)
3550 {
3551 	struct lpfc_iocbq *cmd_iocb = NULL;
3552 	uint16_t iotag;
3553 	spinlock_t *temp_lock = NULL;
3554 	unsigned long iflag = 0;
3555 
3556 	if (phba->sli_rev == LPFC_SLI_REV4)
3557 		temp_lock = &pring->ring_lock;
3558 	else
3559 		temp_lock = &phba->hbalock;
3560 
3561 	spin_lock_irqsave(temp_lock, iflag);
3562 	iotag = prspiocb->iocb.ulpIoTag;
3563 
3564 	if (iotag != 0 && iotag <= phba->sli.last_iotag) {
3565 		cmd_iocb = phba->sli.iocbq_lookup[iotag];
3566 		if (cmd_iocb->iocb_flag & LPFC_IO_ON_TXCMPLQ) {
3567 			/* remove from txcmpl queue list */
3568 			list_del_init(&cmd_iocb->list);
3569 			cmd_iocb->iocb_flag &= ~LPFC_IO_ON_TXCMPLQ;
3570 			pring->txcmplq_cnt--;
3571 			spin_unlock_irqrestore(temp_lock, iflag);
3572 			return cmd_iocb;
3573 		}
3574 	}
3575 
3576 	spin_unlock_irqrestore(temp_lock, iflag);
3577 	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
3578 			"0317 iotag x%x is out of "
3579 			"range: max iotag x%x wd0 x%x\n",
3580 			iotag, phba->sli.last_iotag,
3581 			*(((uint32_t *) &prspiocb->iocb) + 7));
3582 	return NULL;
3583 }
3584 
3585 /**
3586  * lpfc_sli_iocbq_lookup_by_tag - Find command iocb for the iotag
3587  * @phba: Pointer to HBA context object.
3588  * @pring: Pointer to driver SLI ring object.
3589  * @iotag: IOCB tag.
3590  *
3591  * This function looks up the iocb_lookup table to get the command iocb
3592  * corresponding to the given iotag. The driver calls this function with
3593  * the ring lock held because this function is an SLI4 port only helper.
3594  * This function returns the command iocb object if it finds the command
3595  * iocb else returns NULL.
3596  **/
3597 static struct lpfc_iocbq *
3598 lpfc_sli_iocbq_lookup_by_tag(struct lpfc_hba *phba,
3599 			     struct lpfc_sli_ring *pring, uint16_t iotag)
3600 {
3601 	struct lpfc_iocbq *cmd_iocb = NULL;
3602 	spinlock_t *temp_lock = NULL;
3603 	unsigned long iflag = 0;
3604 
3605 	if (phba->sli_rev == LPFC_SLI_REV4)
3606 		temp_lock = &pring->ring_lock;
3607 	else
3608 		temp_lock = &phba->hbalock;
3609 
3610 	spin_lock_irqsave(temp_lock, iflag);
3611 	if (iotag != 0 && iotag <= phba->sli.last_iotag) {
3612 		cmd_iocb = phba->sli.iocbq_lookup[iotag];
3613 		if (cmd_iocb->iocb_flag & LPFC_IO_ON_TXCMPLQ) {
3614 			/* remove from txcmpl queue list */
3615 			list_del_init(&cmd_iocb->list);
3616 			cmd_iocb->iocb_flag &= ~LPFC_IO_ON_TXCMPLQ;
3617 			pring->txcmplq_cnt--;
3618 			spin_unlock_irqrestore(temp_lock, iflag);
3619 			return cmd_iocb;
3620 		}
3621 	}
3622 
3623 	spin_unlock_irqrestore(temp_lock, iflag);
3624 	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
3625 			"0372 iotag x%x lookup error: max iotag (x%x) "
3626 			"iocb_flag x%x\n",
3627 			iotag, phba->sli.last_iotag,
3628 			cmd_iocb ? cmd_iocb->iocb_flag : 0xffff);
3629 	return NULL;
3630 }
3631 
3632 /**
3633  * lpfc_sli_process_sol_iocb - process solicited iocb completion
3634  * @phba: Pointer to HBA context object.
3635  * @pring: Pointer to driver SLI ring object.
3636  * @saveq: Pointer to the response iocb to be processed.
3637  *
3638  * This function is called by the ring event handler for non-fcp
3639  * rings when there is a new response iocb in the response ring.
3640  * The caller is not required to hold any locks. This function
3641  * gets the command iocb associated with the response iocb and
3642  * calls the completion handler for the command iocb. If there
3643  * is no completion handler, the function will free the resources
3644  * associated with command iocb. If the response iocb is for
3645  * an already aborted command iocb, the status of the completion
3646  * is changed to IOSTAT_LOCAL_REJECT/IOERR_SLI_ABORTED.
3647  * This function always returns 1.
3648  **/
3649 static int
3650 lpfc_sli_process_sol_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
3651 			  struct lpfc_iocbq *saveq)
3652 {
3653 	struct lpfc_iocbq *cmdiocbp;
3654 	int rc = 1;
3655 	unsigned long iflag;
3656 
3657 	cmdiocbp = lpfc_sli_iocbq_lookup(phba, pring, saveq);
3658 	if (cmdiocbp) {
3659 		if (cmdiocbp->iocb_cmpl) {
3660 			/*
3661 			 * If an ELS command failed send an event to mgmt
3662 			 * application.
3663 			 */
3664 			if (saveq->iocb.ulpStatus &&
3665 			     (pring->ringno == LPFC_ELS_RING) &&
3666 			     (cmdiocbp->iocb.ulpCommand ==
3667 				CMD_ELS_REQUEST64_CR))
3668 				lpfc_send_els_failure_event(phba,
3669 					cmdiocbp, saveq);
3670 
3671 			/*
3672 			 * Post all ELS completions to the worker thread.
3673 			 * All other are passed to the completion callback.
3674 			 */
3675 			if (pring->ringno == LPFC_ELS_RING) {
3676 				if ((phba->sli_rev < LPFC_SLI_REV4) &&
3677 				    (cmdiocbp->iocb_flag &
3678 							LPFC_DRIVER_ABORTED)) {
3679 					spin_lock_irqsave(&phba->hbalock,
3680 							  iflag);
3681 					cmdiocbp->iocb_flag &=
3682 						~LPFC_DRIVER_ABORTED;
3683 					spin_unlock_irqrestore(&phba->hbalock,
3684 							       iflag);
3685 					saveq->iocb.ulpStatus =
3686 						IOSTAT_LOCAL_REJECT;
3687 					saveq->iocb.un.ulpWord[4] =
3688 						IOERR_SLI_ABORTED;
3689 
3690 					/* Firmware could still be in progress
3691 					 * of DMAing payload, so don't free data
3692 					 * buffer till after a hbeat.
3693 					 */
3694 					spin_lock_irqsave(&phba->hbalock,
3695 							  iflag);
3696 					saveq->iocb_flag |= LPFC_DELAY_MEM_FREE;
3697 					spin_unlock_irqrestore(&phba->hbalock,
3698 							       iflag);
3699 				}
3700 				if (phba->sli_rev == LPFC_SLI_REV4) {
3701 					if (saveq->iocb_flag &
3702 					    LPFC_EXCHANGE_BUSY) {
3703 						/* Set cmdiocb flag for the
3704 						 * exchange busy so sgl (xri)
3705 						 * will not be released until
3706 						 * the abort xri is received
3707 						 * from hba.
3708 						 */
3709 						spin_lock_irqsave(
3710 							&phba->hbalock, iflag);
3711 						cmdiocbp->iocb_flag |=
3712 							LPFC_EXCHANGE_BUSY;
3713 						spin_unlock_irqrestore(
3714 							&phba->hbalock, iflag);
3715 					}
3716 					if (cmdiocbp->iocb_flag &
3717 					    LPFC_DRIVER_ABORTED) {
3718 						/*
3719 						 * Clear LPFC_DRIVER_ABORTED
3720 						 * bit in case it was driver
3721 						 * initiated abort.
3722 						 */
3723 						spin_lock_irqsave(
3724 							&phba->hbalock, iflag);
3725 						cmdiocbp->iocb_flag &=
3726 							~LPFC_DRIVER_ABORTED;
3727 						spin_unlock_irqrestore(
3728 							&phba->hbalock, iflag);
3729 						cmdiocbp->iocb.ulpStatus =
3730 							IOSTAT_LOCAL_REJECT;
3731 						cmdiocbp->iocb.un.ulpWord[4] =
3732 							IOERR_ABORT_REQUESTED;
3733 						/*
3734 						 * For SLI4, irsiocb contains
3735 						 * NO_XRI in sli_xritag, it
3736 						 * shall not affect releasing
3737 						 * sgl (xri) process.
3738 						 */
3739 						saveq->iocb.ulpStatus =
3740 							IOSTAT_LOCAL_REJECT;
3741 						saveq->iocb.un.ulpWord[4] =
3742 							IOERR_SLI_ABORTED;
3743 						spin_lock_irqsave(
3744 							&phba->hbalock, iflag);
3745 						saveq->iocb_flag |=
3746 							LPFC_DELAY_MEM_FREE;
3747 						spin_unlock_irqrestore(
3748 							&phba->hbalock, iflag);
3749 					}
3750 				}
3751 			}
3752 			(cmdiocbp->iocb_cmpl) (phba, cmdiocbp, saveq);
3753 		} else
3754 			lpfc_sli_release_iocbq(phba, cmdiocbp);
3755 	} else {
3756 		/*
3757 		 * Unknown initiating command based on the response iotag.
3758 		 * This could be the case on the ELS ring because of
3759 		 * lpfc_els_abort().
3760 		 */
3761 		if (pring->ringno != LPFC_ELS_RING) {
3762 			/*
3763 			 * Ring <ringno> handler: unexpected completion IoTag
3764 			 * <IoTag>
3765 			 */
3766 			lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
3767 					 "0322 Ring %d handler: "
3768 					 "unexpected completion IoTag x%x "
3769 					 "Data: x%x x%x x%x x%x\n",
3770 					 pring->ringno,
3771 					 saveq->iocb.ulpIoTag,
3772 					 saveq->iocb.ulpStatus,
3773 					 saveq->iocb.un.ulpWord[4],
3774 					 saveq->iocb.ulpCommand,
3775 					 saveq->iocb.ulpContext);
3776 		}
3777 	}
3778 
3779 	return rc;
3780 }
3781 
3782 /**
3783  * lpfc_sli_rsp_pointers_error - Response ring pointer error handler
3784  * @phba: Pointer to HBA context object.
3785  * @pring: Pointer to driver SLI ring object.
3786  *
3787  * This function is called from the iocb ring event handlers when
3788  * put pointer is ahead of the get pointer for a ring. This function signal
3789  * an error attention condition to the worker thread and the worker
3790  * thread will transition the HBA to offline state.
3791  **/
3792 static void
3793 lpfc_sli_rsp_pointers_error(struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
3794 {
3795 	struct lpfc_pgp *pgp = &phba->port_gp[pring->ringno];
3796 	/*
3797 	 * Ring <ringno> handler: portRspPut <portRspPut> is bigger than
3798 	 * rsp ring <portRspMax>
3799 	 */
3800 	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
3801 			"0312 Ring %d handler: portRspPut %d "
3802 			"is bigger than rsp ring %d\n",
3803 			pring->ringno, le32_to_cpu(pgp->rspPutInx),
3804 			pring->sli.sli3.numRiocb);
3805 
3806 	phba->link_state = LPFC_HBA_ERROR;
3807 
3808 	/*
3809 	 * All error attention handlers are posted to
3810 	 * worker thread
3811 	 */
3812 	phba->work_ha |= HA_ERATT;
3813 	phba->work_hs = HS_FFER3;
3814 
3815 	lpfc_worker_wake_up(phba);
3816 
3817 	return;
3818 }
3819 
3820 /**
3821  * lpfc_poll_eratt - Error attention polling timer timeout handler
3822  * @t: Context to fetch pointer to address of HBA context object from.
3823  *
3824  * This function is invoked by the Error Attention polling timer when the
3825  * timer times out. It will check the SLI Error Attention register for
3826  * possible attention events. If so, it will post an Error Attention event
3827  * and wake up worker thread to process it. Otherwise, it will set up the
3828  * Error Attention polling timer for the next poll.
3829  **/
3830 void lpfc_poll_eratt(struct timer_list *t)
3831 {
3832 	struct lpfc_hba *phba;
3833 	uint32_t eratt = 0;
3834 	uint64_t sli_intr, cnt;
3835 
3836 	phba = from_timer(phba, t, eratt_poll);
3837 
3838 	/* Here we will also keep track of interrupts per sec of the hba */
3839 	sli_intr = phba->sli.slistat.sli_intr;
3840 
3841 	if (phba->sli.slistat.sli_prev_intr > sli_intr)
3842 		cnt = (((uint64_t)(-1) - phba->sli.slistat.sli_prev_intr) +
3843 			sli_intr);
3844 	else
3845 		cnt = (sli_intr - phba->sli.slistat.sli_prev_intr);
3846 
3847 	/* 64-bit integer division not supported on 32-bit x86 - use do_div */
3848 	do_div(cnt, phba->eratt_poll_interval);
3849 	phba->sli.slistat.sli_ips = cnt;
3850 
3851 	phba->sli.slistat.sli_prev_intr = sli_intr;
3852 
3853 	/* Check chip HA register for error event */
3854 	eratt = lpfc_sli_check_eratt(phba);
3855 
3856 	if (eratt)
3857 		/* Tell the worker thread there is work to do */
3858 		lpfc_worker_wake_up(phba);
3859 	else
3860 		/* Restart the timer for next eratt poll */
3861 		mod_timer(&phba->eratt_poll,
3862 			  jiffies +
3863 			  msecs_to_jiffies(1000 * phba->eratt_poll_interval));
3864 	return;
3865 }
3866 
3867 
3868 /**
3869  * lpfc_sli_handle_fast_ring_event - Handle ring events on FCP ring
3870  * @phba: Pointer to HBA context object.
3871  * @pring: Pointer to driver SLI ring object.
3872  * @mask: Host attention register mask for this ring.
3873  *
3874  * This function is called from the interrupt context when there is a ring
3875  * event for the fcp ring. The caller does not hold any lock.
3876  * The function processes each response iocb in the response ring until it
3877  * finds an iocb with LE bit set and chains all the iocbs up to the iocb with
3878  * LE bit set. The function will call the completion handler of the command iocb
3879  * if the response iocb indicates a completion for a command iocb or it is
3880  * an abort completion. The function will call lpfc_sli_process_unsol_iocb
3881  * function if this is an unsolicited iocb.
3882  * This routine presumes LPFC_FCP_RING handling and doesn't bother
3883  * to check it explicitly.
3884  */
3885 int
3886 lpfc_sli_handle_fast_ring_event(struct lpfc_hba *phba,
3887 				struct lpfc_sli_ring *pring, uint32_t mask)
3888 {
3889 	struct lpfc_pgp *pgp = &phba->port_gp[pring->ringno];
3890 	IOCB_t *irsp = NULL;
3891 	IOCB_t *entry = NULL;
3892 	struct lpfc_iocbq *cmdiocbq = NULL;
3893 	struct lpfc_iocbq rspiocbq;
3894 	uint32_t status;
3895 	uint32_t portRspPut, portRspMax;
3896 	int rc = 1;
3897 	lpfc_iocb_type type;
3898 	unsigned long iflag;
3899 	uint32_t rsp_cmpl = 0;
3900 
3901 	spin_lock_irqsave(&phba->hbalock, iflag);
3902 	pring->stats.iocb_event++;
3903 
3904 	/*
3905 	 * The next available response entry should never exceed the maximum
3906 	 * entries.  If it does, treat it as an adapter hardware error.
3907 	 */
3908 	portRspMax = pring->sli.sli3.numRiocb;
3909 	portRspPut = le32_to_cpu(pgp->rspPutInx);
3910 	if (unlikely(portRspPut >= portRspMax)) {
3911 		lpfc_sli_rsp_pointers_error(phba, pring);
3912 		spin_unlock_irqrestore(&phba->hbalock, iflag);
3913 		return 1;
3914 	}
3915 	if (phba->fcp_ring_in_use) {
3916 		spin_unlock_irqrestore(&phba->hbalock, iflag);
3917 		return 1;
3918 	} else
3919 		phba->fcp_ring_in_use = 1;
3920 
3921 	rmb();
3922 	while (pring->sli.sli3.rspidx != portRspPut) {
3923 		/*
3924 		 * Fetch an entry off the ring and copy it into a local data
3925 		 * structure.  The copy involves a byte-swap since the
3926 		 * network byte order and pci byte orders are different.
3927 		 */
3928 		entry = lpfc_resp_iocb(phba, pring);
3929 		phba->last_completion_time = jiffies;
3930 
3931 		if (++pring->sli.sli3.rspidx >= portRspMax)
3932 			pring->sli.sli3.rspidx = 0;
3933 
3934 		lpfc_sli_pcimem_bcopy((uint32_t *) entry,
3935 				      (uint32_t *) &rspiocbq.iocb,
3936 				      phba->iocb_rsp_size);
3937 		INIT_LIST_HEAD(&(rspiocbq.list));
3938 		irsp = &rspiocbq.iocb;
3939 
3940 		type = lpfc_sli_iocb_cmd_type(irsp->ulpCommand & CMD_IOCB_MASK);
3941 		pring->stats.iocb_rsp++;
3942 		rsp_cmpl++;
3943 
3944 		if (unlikely(irsp->ulpStatus)) {
3945 			/*
3946 			 * If resource errors reported from HBA, reduce
3947 			 * queuedepths of the SCSI device.
3948 			 */
3949 			if ((irsp->ulpStatus == IOSTAT_LOCAL_REJECT) &&
3950 			    ((irsp->un.ulpWord[4] & IOERR_PARAM_MASK) ==
3951 			     IOERR_NO_RESOURCES)) {
3952 				spin_unlock_irqrestore(&phba->hbalock, iflag);
3953 				phba->lpfc_rampdown_queue_depth(phba);
3954 				spin_lock_irqsave(&phba->hbalock, iflag);
3955 			}
3956 
3957 			/* Rsp ring <ringno> error: IOCB */
3958 			lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
3959 					"0336 Rsp Ring %d error: IOCB Data: "
3960 					"x%x x%x x%x x%x x%x x%x x%x x%x\n",
3961 					pring->ringno,
3962 					irsp->un.ulpWord[0],
3963 					irsp->un.ulpWord[1],
3964 					irsp->un.ulpWord[2],
3965 					irsp->un.ulpWord[3],
3966 					irsp->un.ulpWord[4],
3967 					irsp->un.ulpWord[5],
3968 					*(uint32_t *)&irsp->un1,
3969 					*((uint32_t *)&irsp->un1 + 1));
3970 		}
3971 
3972 		switch (type) {
3973 		case LPFC_ABORT_IOCB:
3974 		case LPFC_SOL_IOCB:
3975 			/*
3976 			 * Idle exchange closed via ABTS from port.  No iocb
3977 			 * resources need to be recovered.
3978 			 */
3979 			if (unlikely(irsp->ulpCommand == CMD_XRI_ABORTED_CX)) {
3980 				lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
3981 						"0333 IOCB cmd 0x%x"
3982 						" processed. Skipping"
3983 						" completion\n",
3984 						irsp->ulpCommand);
3985 				break;
3986 			}
3987 
3988 			spin_unlock_irqrestore(&phba->hbalock, iflag);
3989 			cmdiocbq = lpfc_sli_iocbq_lookup(phba, pring,
3990 							 &rspiocbq);
3991 			spin_lock_irqsave(&phba->hbalock, iflag);
3992 			if (unlikely(!cmdiocbq))
3993 				break;
3994 			if (cmdiocbq->iocb_flag & LPFC_DRIVER_ABORTED)
3995 				cmdiocbq->iocb_flag &= ~LPFC_DRIVER_ABORTED;
3996 			if (cmdiocbq->iocb_cmpl) {
3997 				spin_unlock_irqrestore(&phba->hbalock, iflag);
3998 				(cmdiocbq->iocb_cmpl)(phba, cmdiocbq,
3999 						      &rspiocbq);
4000 				spin_lock_irqsave(&phba->hbalock, iflag);
4001 			}
4002 			break;
4003 		case LPFC_UNSOL_IOCB:
4004 			spin_unlock_irqrestore(&phba->hbalock, iflag);
4005 			lpfc_sli_process_unsol_iocb(phba, pring, &rspiocbq);
4006 			spin_lock_irqsave(&phba->hbalock, iflag);
4007 			break;
4008 		default:
4009 			if (irsp->ulpCommand == CMD_ADAPTER_MSG) {
4010 				char adaptermsg[LPFC_MAX_ADPTMSG];
4011 				memset(adaptermsg, 0, LPFC_MAX_ADPTMSG);
4012 				memcpy(&adaptermsg[0], (uint8_t *) irsp,
4013 				       MAX_MSG_DATA);
4014 				dev_warn(&((phba->pcidev)->dev),
4015 					 "lpfc%d: %s\n",
4016 					 phba->brd_no, adaptermsg);
4017 			} else {
4018 				/* Unknown IOCB command */
4019 				lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
4020 						"0334 Unknown IOCB command "
4021 						"Data: x%x, x%x x%x x%x x%x\n",
4022 						type, irsp->ulpCommand,
4023 						irsp->ulpStatus,
4024 						irsp->ulpIoTag,
4025 						irsp->ulpContext);
4026 			}
4027 			break;
4028 		}
4029 
4030 		/*
4031 		 * The response IOCB has been processed.  Update the ring
4032 		 * pointer in SLIM.  If the port response put pointer has not
4033 		 * been updated, sync the pgp->rspPutInx and fetch the new port
4034 		 * response put pointer.
4035 		 */
4036 		writel(pring->sli.sli3.rspidx,
4037 			&phba->host_gp[pring->ringno].rspGetInx);
4038 
4039 		if (pring->sli.sli3.rspidx == portRspPut)
4040 			portRspPut = le32_to_cpu(pgp->rspPutInx);
4041 	}
4042 
4043 	if ((rsp_cmpl > 0) && (mask & HA_R0RE_REQ)) {
4044 		pring->stats.iocb_rsp_full++;
4045 		status = ((CA_R0ATT | CA_R0RE_RSP) << (pring->ringno * 4));
4046 		writel(status, phba->CAregaddr);
4047 		readl(phba->CAregaddr);
4048 	}
4049 	if ((mask & HA_R0CE_RSP) && (pring->flag & LPFC_CALL_RING_AVAILABLE)) {
4050 		pring->flag &= ~LPFC_CALL_RING_AVAILABLE;
4051 		pring->stats.iocb_cmd_empty++;
4052 
4053 		/* Force update of the local copy of cmdGetInx */
4054 		pring->sli.sli3.local_getidx = le32_to_cpu(pgp->cmdGetInx);
4055 		lpfc_sli_resume_iocb(phba, pring);
4056 
4057 		if ((pring->lpfc_sli_cmd_available))
4058 			(pring->lpfc_sli_cmd_available) (phba, pring);
4059 
4060 	}
4061 
4062 	phba->fcp_ring_in_use = 0;
4063 	spin_unlock_irqrestore(&phba->hbalock, iflag);
4064 	return rc;
4065 }
4066 
4067 /**
4068  * lpfc_sli_sp_handle_rspiocb - Handle slow-path response iocb
4069  * @phba: Pointer to HBA context object.
4070  * @pring: Pointer to driver SLI ring object.
4071  * @rspiocbp: Pointer to driver response IOCB object.
4072  *
4073  * This function is called from the worker thread when there is a slow-path
4074  * response IOCB to process. This function chains all the response iocbs until
4075  * seeing the iocb with the LE bit set. The function will call
4076  * lpfc_sli_process_sol_iocb function if the response iocb indicates a
4077  * completion of a command iocb. The function will call the
4078  * lpfc_sli_process_unsol_iocb function if this is an unsolicited iocb.
4079  * The function frees the resources or calls the completion handler if this
4080  * iocb is an abort completion. The function returns NULL when the response
4081  * iocb has the LE bit set and all the chained iocbs are processed, otherwise
4082  * this function shall chain the iocb on to the iocb_continueq and return the
4083  * response iocb passed in.
4084  **/
4085 static struct lpfc_iocbq *
4086 lpfc_sli_sp_handle_rspiocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
4087 			struct lpfc_iocbq *rspiocbp)
4088 {
4089 	struct lpfc_iocbq *saveq;
4090 	struct lpfc_iocbq *cmdiocbp;
4091 	struct lpfc_iocbq *next_iocb;
4092 	IOCB_t *irsp = NULL;
4093 	uint32_t free_saveq;
4094 	uint8_t iocb_cmd_type;
4095 	lpfc_iocb_type type;
4096 	unsigned long iflag;
4097 	int rc;
4098 
4099 	spin_lock_irqsave(&phba->hbalock, iflag);
4100 	/* First add the response iocb to the countinueq list */
4101 	list_add_tail(&rspiocbp->list, &(pring->iocb_continueq));
4102 	pring->iocb_continueq_cnt++;
4103 
4104 	/* Now, determine whether the list is completed for processing */
4105 	irsp = &rspiocbp->iocb;
4106 	if (irsp->ulpLe) {
4107 		/*
4108 		 * By default, the driver expects to free all resources
4109 		 * associated with this iocb completion.
4110 		 */
4111 		free_saveq = 1;
4112 		saveq = list_get_first(&pring->iocb_continueq,
4113 				       struct lpfc_iocbq, list);
4114 		irsp = &(saveq->iocb);
4115 		list_del_init(&pring->iocb_continueq);
4116 		pring->iocb_continueq_cnt = 0;
4117 
4118 		pring->stats.iocb_rsp++;
4119 
4120 		/*
4121 		 * If resource errors reported from HBA, reduce
4122 		 * queuedepths of the SCSI device.
4123 		 */
4124 		if ((irsp->ulpStatus == IOSTAT_LOCAL_REJECT) &&
4125 		    ((irsp->un.ulpWord[4] & IOERR_PARAM_MASK) ==
4126 		     IOERR_NO_RESOURCES)) {
4127 			spin_unlock_irqrestore(&phba->hbalock, iflag);
4128 			phba->lpfc_rampdown_queue_depth(phba);
4129 			spin_lock_irqsave(&phba->hbalock, iflag);
4130 		}
4131 
4132 		if (irsp->ulpStatus) {
4133 			/* Rsp ring <ringno> error: IOCB */
4134 			lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
4135 					"0328 Rsp Ring %d error: "
4136 					"IOCB Data: "
4137 					"x%x x%x x%x x%x "
4138 					"x%x x%x x%x x%x "
4139 					"x%x x%x x%x x%x "
4140 					"x%x x%x x%x x%x\n",
4141 					pring->ringno,
4142 					irsp->un.ulpWord[0],
4143 					irsp->un.ulpWord[1],
4144 					irsp->un.ulpWord[2],
4145 					irsp->un.ulpWord[3],
4146 					irsp->un.ulpWord[4],
4147 					irsp->un.ulpWord[5],
4148 					*(((uint32_t *) irsp) + 6),
4149 					*(((uint32_t *) irsp) + 7),
4150 					*(((uint32_t *) irsp) + 8),
4151 					*(((uint32_t *) irsp) + 9),
4152 					*(((uint32_t *) irsp) + 10),
4153 					*(((uint32_t *) irsp) + 11),
4154 					*(((uint32_t *) irsp) + 12),
4155 					*(((uint32_t *) irsp) + 13),
4156 					*(((uint32_t *) irsp) + 14),
4157 					*(((uint32_t *) irsp) + 15));
4158 		}
4159 
4160 		/*
4161 		 * Fetch the IOCB command type and call the correct completion
4162 		 * routine. Solicited and Unsolicited IOCBs on the ELS ring
4163 		 * get freed back to the lpfc_iocb_list by the discovery
4164 		 * kernel thread.
4165 		 */
4166 		iocb_cmd_type = irsp->ulpCommand & CMD_IOCB_MASK;
4167 		type = lpfc_sli_iocb_cmd_type(iocb_cmd_type);
4168 		switch (type) {
4169 		case LPFC_SOL_IOCB:
4170 			spin_unlock_irqrestore(&phba->hbalock, iflag);
4171 			rc = lpfc_sli_process_sol_iocb(phba, pring, saveq);
4172 			spin_lock_irqsave(&phba->hbalock, iflag);
4173 			break;
4174 
4175 		case LPFC_UNSOL_IOCB:
4176 			spin_unlock_irqrestore(&phba->hbalock, iflag);
4177 			rc = lpfc_sli_process_unsol_iocb(phba, pring, saveq);
4178 			spin_lock_irqsave(&phba->hbalock, iflag);
4179 			if (!rc)
4180 				free_saveq = 0;
4181 			break;
4182 
4183 		case LPFC_ABORT_IOCB:
4184 			cmdiocbp = NULL;
4185 			if (irsp->ulpCommand != CMD_XRI_ABORTED_CX) {
4186 				spin_unlock_irqrestore(&phba->hbalock, iflag);
4187 				cmdiocbp = lpfc_sli_iocbq_lookup(phba, pring,
4188 								 saveq);
4189 				spin_lock_irqsave(&phba->hbalock, iflag);
4190 			}
4191 			if (cmdiocbp) {
4192 				/* Call the specified completion routine */
4193 				if (cmdiocbp->iocb_cmpl) {
4194 					spin_unlock_irqrestore(&phba->hbalock,
4195 							       iflag);
4196 					(cmdiocbp->iocb_cmpl)(phba, cmdiocbp,
4197 							      saveq);
4198 					spin_lock_irqsave(&phba->hbalock,
4199 							  iflag);
4200 				} else
4201 					__lpfc_sli_release_iocbq(phba,
4202 								 cmdiocbp);
4203 			}
4204 			break;
4205 
4206 		case LPFC_UNKNOWN_IOCB:
4207 			if (irsp->ulpCommand == CMD_ADAPTER_MSG) {
4208 				char adaptermsg[LPFC_MAX_ADPTMSG];
4209 				memset(adaptermsg, 0, LPFC_MAX_ADPTMSG);
4210 				memcpy(&adaptermsg[0], (uint8_t *)irsp,
4211 				       MAX_MSG_DATA);
4212 				dev_warn(&((phba->pcidev)->dev),
4213 					 "lpfc%d: %s\n",
4214 					 phba->brd_no, adaptermsg);
4215 			} else {
4216 				/* Unknown IOCB command */
4217 				lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
4218 						"0335 Unknown IOCB "
4219 						"command Data: x%x "
4220 						"x%x x%x x%x\n",
4221 						irsp->ulpCommand,
4222 						irsp->ulpStatus,
4223 						irsp->ulpIoTag,
4224 						irsp->ulpContext);
4225 			}
4226 			break;
4227 		}
4228 
4229 		if (free_saveq) {
4230 			list_for_each_entry_safe(rspiocbp, next_iocb,
4231 						 &saveq->list, list) {
4232 				list_del_init(&rspiocbp->list);
4233 				__lpfc_sli_release_iocbq(phba, rspiocbp);
4234 			}
4235 			__lpfc_sli_release_iocbq(phba, saveq);
4236 		}
4237 		rspiocbp = NULL;
4238 	}
4239 	spin_unlock_irqrestore(&phba->hbalock, iflag);
4240 	return rspiocbp;
4241 }
4242 
4243 /**
4244  * lpfc_sli_handle_slow_ring_event - Wrapper func for handling slow-path iocbs
4245  * @phba: Pointer to HBA context object.
4246  * @pring: Pointer to driver SLI ring object.
4247  * @mask: Host attention register mask for this ring.
4248  *
4249  * This routine wraps the actual slow_ring event process routine from the
4250  * API jump table function pointer from the lpfc_hba struct.
4251  **/
4252 void
4253 lpfc_sli_handle_slow_ring_event(struct lpfc_hba *phba,
4254 				struct lpfc_sli_ring *pring, uint32_t mask)
4255 {
4256 	phba->lpfc_sli_handle_slow_ring_event(phba, pring, mask);
4257 }
4258 
4259 /**
4260  * lpfc_sli_handle_slow_ring_event_s3 - Handle SLI3 ring event for non-FCP rings
4261  * @phba: Pointer to HBA context object.
4262  * @pring: Pointer to driver SLI ring object.
4263  * @mask: Host attention register mask for this ring.
4264  *
4265  * This function is called from the worker thread when there is a ring event
4266  * for non-fcp rings. The caller does not hold any lock. The function will
4267  * remove each response iocb in the response ring and calls the handle
4268  * response iocb routine (lpfc_sli_sp_handle_rspiocb) to process it.
4269  **/
4270 static void
4271 lpfc_sli_handle_slow_ring_event_s3(struct lpfc_hba *phba,
4272 				   struct lpfc_sli_ring *pring, uint32_t mask)
4273 {
4274 	struct lpfc_pgp *pgp;
4275 	IOCB_t *entry;
4276 	IOCB_t *irsp = NULL;
4277 	struct lpfc_iocbq *rspiocbp = NULL;
4278 	uint32_t portRspPut, portRspMax;
4279 	unsigned long iflag;
4280 	uint32_t status;
4281 
4282 	pgp = &phba->port_gp[pring->ringno];
4283 	spin_lock_irqsave(&phba->hbalock, iflag);
4284 	pring->stats.iocb_event++;
4285 
4286 	/*
4287 	 * The next available response entry should never exceed the maximum
4288 	 * entries.  If it does, treat it as an adapter hardware error.
4289 	 */
4290 	portRspMax = pring->sli.sli3.numRiocb;
4291 	portRspPut = le32_to_cpu(pgp->rspPutInx);
4292 	if (portRspPut >= portRspMax) {
4293 		/*
4294 		 * Ring <ringno> handler: portRspPut <portRspPut> is bigger than
4295 		 * rsp ring <portRspMax>
4296 		 */
4297 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
4298 				"0303 Ring %d handler: portRspPut %d "
4299 				"is bigger than rsp ring %d\n",
4300 				pring->ringno, portRspPut, portRspMax);
4301 
4302 		phba->link_state = LPFC_HBA_ERROR;
4303 		spin_unlock_irqrestore(&phba->hbalock, iflag);
4304 
4305 		phba->work_hs = HS_FFER3;
4306 		lpfc_handle_eratt(phba);
4307 
4308 		return;
4309 	}
4310 
4311 	rmb();
4312 	while (pring->sli.sli3.rspidx != portRspPut) {
4313 		/*
4314 		 * Build a completion list and call the appropriate handler.
4315 		 * The process is to get the next available response iocb, get
4316 		 * a free iocb from the list, copy the response data into the
4317 		 * free iocb, insert to the continuation list, and update the
4318 		 * next response index to slim.  This process makes response
4319 		 * iocb's in the ring available to DMA as fast as possible but
4320 		 * pays a penalty for a copy operation.  Since the iocb is
4321 		 * only 32 bytes, this penalty is considered small relative to
4322 		 * the PCI reads for register values and a slim write.  When
4323 		 * the ulpLe field is set, the entire Command has been
4324 		 * received.
4325 		 */
4326 		entry = lpfc_resp_iocb(phba, pring);
4327 
4328 		phba->last_completion_time = jiffies;
4329 		rspiocbp = __lpfc_sli_get_iocbq(phba);
4330 		if (rspiocbp == NULL) {
4331 			printk(KERN_ERR "%s: out of buffers! Failing "
4332 			       "completion.\n", __func__);
4333 			break;
4334 		}
4335 
4336 		lpfc_sli_pcimem_bcopy(entry, &rspiocbp->iocb,
4337 				      phba->iocb_rsp_size);
4338 		irsp = &rspiocbp->iocb;
4339 
4340 		if (++pring->sli.sli3.rspidx >= portRspMax)
4341 			pring->sli.sli3.rspidx = 0;
4342 
4343 		if (pring->ringno == LPFC_ELS_RING) {
4344 			lpfc_debugfs_slow_ring_trc(phba,
4345 			"IOCB rsp ring:   wd4:x%08x wd6:x%08x wd7:x%08x",
4346 				*(((uint32_t *) irsp) + 4),
4347 				*(((uint32_t *) irsp) + 6),
4348 				*(((uint32_t *) irsp) + 7));
4349 		}
4350 
4351 		writel(pring->sli.sli3.rspidx,
4352 			&phba->host_gp[pring->ringno].rspGetInx);
4353 
4354 		spin_unlock_irqrestore(&phba->hbalock, iflag);
4355 		/* Handle the response IOCB */
4356 		rspiocbp = lpfc_sli_sp_handle_rspiocb(phba, pring, rspiocbp);
4357 		spin_lock_irqsave(&phba->hbalock, iflag);
4358 
4359 		/*
4360 		 * If the port response put pointer has not been updated, sync
4361 		 * the pgp->rspPutInx in the MAILBOX_tand fetch the new port
4362 		 * response put pointer.
4363 		 */
4364 		if (pring->sli.sli3.rspidx == portRspPut) {
4365 			portRspPut = le32_to_cpu(pgp->rspPutInx);
4366 		}
4367 	} /* while (pring->sli.sli3.rspidx != portRspPut) */
4368 
4369 	if ((rspiocbp != NULL) && (mask & HA_R0RE_REQ)) {
4370 		/* At least one response entry has been freed */
4371 		pring->stats.iocb_rsp_full++;
4372 		/* SET RxRE_RSP in Chip Att register */
4373 		status = ((CA_R0ATT | CA_R0RE_RSP) << (pring->ringno * 4));
4374 		writel(status, phba->CAregaddr);
4375 		readl(phba->CAregaddr); /* flush */
4376 	}
4377 	if ((mask & HA_R0CE_RSP) && (pring->flag & LPFC_CALL_RING_AVAILABLE)) {
4378 		pring->flag &= ~LPFC_CALL_RING_AVAILABLE;
4379 		pring->stats.iocb_cmd_empty++;
4380 
4381 		/* Force update of the local copy of cmdGetInx */
4382 		pring->sli.sli3.local_getidx = le32_to_cpu(pgp->cmdGetInx);
4383 		lpfc_sli_resume_iocb(phba, pring);
4384 
4385 		if ((pring->lpfc_sli_cmd_available))
4386 			(pring->lpfc_sli_cmd_available) (phba, pring);
4387 
4388 	}
4389 
4390 	spin_unlock_irqrestore(&phba->hbalock, iflag);
4391 	return;
4392 }
4393 
4394 /**
4395  * lpfc_sli_handle_slow_ring_event_s4 - Handle SLI4 slow-path els events
4396  * @phba: Pointer to HBA context object.
4397  * @pring: Pointer to driver SLI ring object.
4398  * @mask: Host attention register mask for this ring.
4399  *
4400  * This function is called from the worker thread when there is a pending
4401  * ELS response iocb on the driver internal slow-path response iocb worker
4402  * queue. The caller does not hold any lock. The function will remove each
4403  * response iocb from the response worker queue and calls the handle
4404  * response iocb routine (lpfc_sli_sp_handle_rspiocb) to process it.
4405  **/
4406 static void
4407 lpfc_sli_handle_slow_ring_event_s4(struct lpfc_hba *phba,
4408 				   struct lpfc_sli_ring *pring, uint32_t mask)
4409 {
4410 	struct lpfc_iocbq *irspiocbq;
4411 	struct hbq_dmabuf *dmabuf;
4412 	struct lpfc_cq_event *cq_event;
4413 	unsigned long iflag;
4414 	int count = 0;
4415 
4416 	spin_lock_irqsave(&phba->hbalock, iflag);
4417 	phba->hba_flag &= ~HBA_SP_QUEUE_EVT;
4418 	spin_unlock_irqrestore(&phba->hbalock, iflag);
4419 	while (!list_empty(&phba->sli4_hba.sp_queue_event)) {
4420 		/* Get the response iocb from the head of work queue */
4421 		spin_lock_irqsave(&phba->hbalock, iflag);
4422 		list_remove_head(&phba->sli4_hba.sp_queue_event,
4423 				 cq_event, struct lpfc_cq_event, list);
4424 		spin_unlock_irqrestore(&phba->hbalock, iflag);
4425 
4426 		switch (bf_get(lpfc_wcqe_c_code, &cq_event->cqe.wcqe_cmpl)) {
4427 		case CQE_CODE_COMPL_WQE:
4428 			irspiocbq = container_of(cq_event, struct lpfc_iocbq,
4429 						 cq_event);
4430 			/* Translate ELS WCQE to response IOCBQ */
4431 			irspiocbq = lpfc_sli4_els_wcqe_to_rspiocbq(phba,
4432 								   irspiocbq);
4433 			if (irspiocbq)
4434 				lpfc_sli_sp_handle_rspiocb(phba, pring,
4435 							   irspiocbq);
4436 			count++;
4437 			break;
4438 		case CQE_CODE_RECEIVE:
4439 		case CQE_CODE_RECEIVE_V1:
4440 			dmabuf = container_of(cq_event, struct hbq_dmabuf,
4441 					      cq_event);
4442 			lpfc_sli4_handle_received_buffer(phba, dmabuf);
4443 			count++;
4444 			break;
4445 		default:
4446 			break;
4447 		}
4448 
4449 		/* Limit the number of events to 64 to avoid soft lockups */
4450 		if (count == 64)
4451 			break;
4452 	}
4453 }
4454 
4455 /**
4456  * lpfc_sli_abort_iocb_ring - Abort all iocbs in the ring
4457  * @phba: Pointer to HBA context object.
4458  * @pring: Pointer to driver SLI ring object.
4459  *
4460  * This function aborts all iocbs in the given ring and frees all the iocb
4461  * objects in txq. This function issues an abort iocb for all the iocb commands
4462  * in txcmplq. The iocbs in the txcmplq is not guaranteed to complete before
4463  * the return of this function. The caller is not required to hold any locks.
4464  **/
4465 void
4466 lpfc_sli_abort_iocb_ring(struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
4467 {
4468 	LIST_HEAD(completions);
4469 	struct lpfc_iocbq *iocb, *next_iocb;
4470 
4471 	if (pring->ringno == LPFC_ELS_RING) {
4472 		lpfc_fabric_abort_hba(phba);
4473 	}
4474 
4475 	/* Error everything on txq and txcmplq
4476 	 * First do the txq.
4477 	 */
4478 	if (phba->sli_rev >= LPFC_SLI_REV4) {
4479 		spin_lock_irq(&pring->ring_lock);
4480 		list_splice_init(&pring->txq, &completions);
4481 		pring->txq_cnt = 0;
4482 		spin_unlock_irq(&pring->ring_lock);
4483 
4484 		spin_lock_irq(&phba->hbalock);
4485 		/* Next issue ABTS for everything on the txcmplq */
4486 		list_for_each_entry_safe(iocb, next_iocb, &pring->txcmplq, list)
4487 			lpfc_sli_issue_abort_iotag(phba, pring, iocb, NULL);
4488 		spin_unlock_irq(&phba->hbalock);
4489 	} else {
4490 		spin_lock_irq(&phba->hbalock);
4491 		list_splice_init(&pring->txq, &completions);
4492 		pring->txq_cnt = 0;
4493 
4494 		/* Next issue ABTS for everything on the txcmplq */
4495 		list_for_each_entry_safe(iocb, next_iocb, &pring->txcmplq, list)
4496 			lpfc_sli_issue_abort_iotag(phba, pring, iocb, NULL);
4497 		spin_unlock_irq(&phba->hbalock);
4498 	}
4499 	/* Make sure HBA is alive */
4500 	lpfc_issue_hb_tmo(phba);
4501 
4502 	/* Cancel all the IOCBs from the completions list */
4503 	lpfc_sli_cancel_iocbs(phba, &completions, IOSTAT_LOCAL_REJECT,
4504 			      IOERR_SLI_ABORTED);
4505 }
4506 
4507 /**
4508  * lpfc_sli_abort_fcp_rings - Abort all iocbs in all FCP rings
4509  * @phba: Pointer to HBA context object.
4510  *
4511  * This function aborts all iocbs in FCP rings and frees all the iocb
4512  * objects in txq. This function issues an abort iocb for all the iocb commands
4513  * in txcmplq. The iocbs in the txcmplq is not guaranteed to complete before
4514  * the return of this function. The caller is not required to hold any locks.
4515  **/
4516 void
4517 lpfc_sli_abort_fcp_rings(struct lpfc_hba *phba)
4518 {
4519 	struct lpfc_sli *psli = &phba->sli;
4520 	struct lpfc_sli_ring  *pring;
4521 	uint32_t i;
4522 
4523 	/* Look on all the FCP Rings for the iotag */
4524 	if (phba->sli_rev >= LPFC_SLI_REV4) {
4525 		for (i = 0; i < phba->cfg_hdw_queue; i++) {
4526 			pring = phba->sli4_hba.hdwq[i].io_wq->pring;
4527 			lpfc_sli_abort_iocb_ring(phba, pring);
4528 		}
4529 	} else {
4530 		pring = &psli->sli3_ring[LPFC_FCP_RING];
4531 		lpfc_sli_abort_iocb_ring(phba, pring);
4532 	}
4533 }
4534 
4535 /**
4536  * lpfc_sli_flush_io_rings - flush all iocbs in the IO ring
4537  * @phba: Pointer to HBA context object.
4538  *
4539  * This function flushes all iocbs in the IO ring and frees all the iocb
4540  * objects in txq and txcmplq. This function will not issue abort iocbs
4541  * for all the iocb commands in txcmplq, they will just be returned with
4542  * IOERR_SLI_DOWN. This function is invoked with EEH when device's PCI
4543  * slot has been permanently disabled.
4544  **/
4545 void
4546 lpfc_sli_flush_io_rings(struct lpfc_hba *phba)
4547 {
4548 	LIST_HEAD(txq);
4549 	LIST_HEAD(txcmplq);
4550 	struct lpfc_sli *psli = &phba->sli;
4551 	struct lpfc_sli_ring  *pring;
4552 	uint32_t i;
4553 	struct lpfc_iocbq *piocb, *next_iocb;
4554 
4555 	spin_lock_irq(&phba->hbalock);
4556 	if (phba->hba_flag & HBA_IOQ_FLUSH ||
4557 	    !phba->sli4_hba.hdwq) {
4558 		spin_unlock_irq(&phba->hbalock);
4559 		return;
4560 	}
4561 	/* Indicate the I/O queues are flushed */
4562 	phba->hba_flag |= HBA_IOQ_FLUSH;
4563 	spin_unlock_irq(&phba->hbalock);
4564 
4565 	/* Look on all the FCP Rings for the iotag */
4566 	if (phba->sli_rev >= LPFC_SLI_REV4) {
4567 		for (i = 0; i < phba->cfg_hdw_queue; i++) {
4568 			pring = phba->sli4_hba.hdwq[i].io_wq->pring;
4569 
4570 			spin_lock_irq(&pring->ring_lock);
4571 			/* Retrieve everything on txq */
4572 			list_splice_init(&pring->txq, &txq);
4573 			list_for_each_entry_safe(piocb, next_iocb,
4574 						 &pring->txcmplq, list)
4575 				piocb->iocb_flag &= ~LPFC_IO_ON_TXCMPLQ;
4576 			/* Retrieve everything on the txcmplq */
4577 			list_splice_init(&pring->txcmplq, &txcmplq);
4578 			pring->txq_cnt = 0;
4579 			pring->txcmplq_cnt = 0;
4580 			spin_unlock_irq(&pring->ring_lock);
4581 
4582 			/* Flush the txq */
4583 			lpfc_sli_cancel_iocbs(phba, &txq,
4584 					      IOSTAT_LOCAL_REJECT,
4585 					      IOERR_SLI_DOWN);
4586 			/* Flush the txcmpq */
4587 			lpfc_sli_cancel_iocbs(phba, &txcmplq,
4588 					      IOSTAT_LOCAL_REJECT,
4589 					      IOERR_SLI_DOWN);
4590 		}
4591 	} else {
4592 		pring = &psli->sli3_ring[LPFC_FCP_RING];
4593 
4594 		spin_lock_irq(&phba->hbalock);
4595 		/* Retrieve everything on txq */
4596 		list_splice_init(&pring->txq, &txq);
4597 		list_for_each_entry_safe(piocb, next_iocb,
4598 					 &pring->txcmplq, list)
4599 			piocb->iocb_flag &= ~LPFC_IO_ON_TXCMPLQ;
4600 		/* Retrieve everything on the txcmplq */
4601 		list_splice_init(&pring->txcmplq, &txcmplq);
4602 		pring->txq_cnt = 0;
4603 		pring->txcmplq_cnt = 0;
4604 		spin_unlock_irq(&phba->hbalock);
4605 
4606 		/* Flush the txq */
4607 		lpfc_sli_cancel_iocbs(phba, &txq, IOSTAT_LOCAL_REJECT,
4608 				      IOERR_SLI_DOWN);
4609 		/* Flush the txcmpq */
4610 		lpfc_sli_cancel_iocbs(phba, &txcmplq, IOSTAT_LOCAL_REJECT,
4611 				      IOERR_SLI_DOWN);
4612 	}
4613 }
4614 
4615 /**
4616  * lpfc_sli_brdready_s3 - Check for sli3 host ready status
4617  * @phba: Pointer to HBA context object.
4618  * @mask: Bit mask to be checked.
4619  *
4620  * This function reads the host status register and compares
4621  * with the provided bit mask to check if HBA completed
4622  * the restart. This function will wait in a loop for the
4623  * HBA to complete restart. If the HBA does not restart within
4624  * 15 iterations, the function will reset the HBA again. The
4625  * function returns 1 when HBA fail to restart otherwise returns
4626  * zero.
4627  **/
4628 static int
4629 lpfc_sli_brdready_s3(struct lpfc_hba *phba, uint32_t mask)
4630 {
4631 	uint32_t status;
4632 	int i = 0;
4633 	int retval = 0;
4634 
4635 	/* Read the HBA Host Status Register */
4636 	if (lpfc_readl(phba->HSregaddr, &status))
4637 		return 1;
4638 
4639 	phba->hba_flag |= HBA_NEEDS_CFG_PORT;
4640 
4641 	/*
4642 	 * Check status register every 100ms for 5 retries, then every
4643 	 * 500ms for 5, then every 2.5 sec for 5, then reset board and
4644 	 * every 2.5 sec for 4.
4645 	 * Break our of the loop if errors occurred during init.
4646 	 */
4647 	while (((status & mask) != mask) &&
4648 	       !(status & HS_FFERM) &&
4649 	       i++ < 20) {
4650 
4651 		if (i <= 5)
4652 			msleep(10);
4653 		else if (i <= 10)
4654 			msleep(500);
4655 		else
4656 			msleep(2500);
4657 
4658 		if (i == 15) {
4659 				/* Do post */
4660 			phba->pport->port_state = LPFC_VPORT_UNKNOWN;
4661 			lpfc_sli_brdrestart(phba);
4662 		}
4663 		/* Read the HBA Host Status Register */
4664 		if (lpfc_readl(phba->HSregaddr, &status)) {
4665 			retval = 1;
4666 			break;
4667 		}
4668 	}
4669 
4670 	/* Check to see if any errors occurred during init */
4671 	if ((status & HS_FFERM) || (i >= 20)) {
4672 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
4673 				"2751 Adapter failed to restart, "
4674 				"status reg x%x, FW Data: A8 x%x AC x%x\n",
4675 				status,
4676 				readl(phba->MBslimaddr + 0xa8),
4677 				readl(phba->MBslimaddr + 0xac));
4678 		phba->link_state = LPFC_HBA_ERROR;
4679 		retval = 1;
4680 	}
4681 
4682 	return retval;
4683 }
4684 
4685 /**
4686  * lpfc_sli_brdready_s4 - Check for sli4 host ready status
4687  * @phba: Pointer to HBA context object.
4688  * @mask: Bit mask to be checked.
4689  *
4690  * This function checks the host status register to check if HBA is
4691  * ready. This function will wait in a loop for the HBA to be ready
4692  * If the HBA is not ready , the function will will reset the HBA PCI
4693  * function again. The function returns 1 when HBA fail to be ready
4694  * otherwise returns zero.
4695  **/
4696 static int
4697 lpfc_sli_brdready_s4(struct lpfc_hba *phba, uint32_t mask)
4698 {
4699 	uint32_t status;
4700 	int retval = 0;
4701 
4702 	/* Read the HBA Host Status Register */
4703 	status = lpfc_sli4_post_status_check(phba);
4704 
4705 	if (status) {
4706 		phba->pport->port_state = LPFC_VPORT_UNKNOWN;
4707 		lpfc_sli_brdrestart(phba);
4708 		status = lpfc_sli4_post_status_check(phba);
4709 	}
4710 
4711 	/* Check to see if any errors occurred during init */
4712 	if (status) {
4713 		phba->link_state = LPFC_HBA_ERROR;
4714 		retval = 1;
4715 	} else
4716 		phba->sli4_hba.intr_enable = 0;
4717 
4718 	phba->hba_flag &= ~HBA_SETUP;
4719 	return retval;
4720 }
4721 
4722 /**
4723  * lpfc_sli_brdready - Wrapper func for checking the hba readyness
4724  * @phba: Pointer to HBA context object.
4725  * @mask: Bit mask to be checked.
4726  *
4727  * This routine wraps the actual SLI3 or SLI4 hba readyness check routine
4728  * from the API jump table function pointer from the lpfc_hba struct.
4729  **/
4730 int
4731 lpfc_sli_brdready(struct lpfc_hba *phba, uint32_t mask)
4732 {
4733 	return phba->lpfc_sli_brdready(phba, mask);
4734 }
4735 
4736 #define BARRIER_TEST_PATTERN (0xdeadbeef)
4737 
4738 /**
4739  * lpfc_reset_barrier - Make HBA ready for HBA reset
4740  * @phba: Pointer to HBA context object.
4741  *
4742  * This function is called before resetting an HBA. This function is called
4743  * with hbalock held and requests HBA to quiesce DMAs before a reset.
4744  **/
4745 void lpfc_reset_barrier(struct lpfc_hba *phba)
4746 {
4747 	uint32_t __iomem *resp_buf;
4748 	uint32_t __iomem *mbox_buf;
4749 	volatile uint32_t mbox;
4750 	uint32_t hc_copy, ha_copy, resp_data;
4751 	int  i;
4752 	uint8_t hdrtype;
4753 
4754 	lockdep_assert_held(&phba->hbalock);
4755 
4756 	pci_read_config_byte(phba->pcidev, PCI_HEADER_TYPE, &hdrtype);
4757 	if (hdrtype != 0x80 ||
4758 	    (FC_JEDEC_ID(phba->vpd.rev.biuRev) != HELIOS_JEDEC_ID &&
4759 	     FC_JEDEC_ID(phba->vpd.rev.biuRev) != THOR_JEDEC_ID))
4760 		return;
4761 
4762 	/*
4763 	 * Tell the other part of the chip to suspend temporarily all
4764 	 * its DMA activity.
4765 	 */
4766 	resp_buf = phba->MBslimaddr;
4767 
4768 	/* Disable the error attention */
4769 	if (lpfc_readl(phba->HCregaddr, &hc_copy))
4770 		return;
4771 	writel((hc_copy & ~HC_ERINT_ENA), phba->HCregaddr);
4772 	readl(phba->HCregaddr); /* flush */
4773 	phba->link_flag |= LS_IGNORE_ERATT;
4774 
4775 	if (lpfc_readl(phba->HAregaddr, &ha_copy))
4776 		return;
4777 	if (ha_copy & HA_ERATT) {
4778 		/* Clear Chip error bit */
4779 		writel(HA_ERATT, phba->HAregaddr);
4780 		phba->pport->stopped = 1;
4781 	}
4782 
4783 	mbox = 0;
4784 	((MAILBOX_t *)&mbox)->mbxCommand = MBX_KILL_BOARD;
4785 	((MAILBOX_t *)&mbox)->mbxOwner = OWN_CHIP;
4786 
4787 	writel(BARRIER_TEST_PATTERN, (resp_buf + 1));
4788 	mbox_buf = phba->MBslimaddr;
4789 	writel(mbox, mbox_buf);
4790 
4791 	for (i = 0; i < 50; i++) {
4792 		if (lpfc_readl((resp_buf + 1), &resp_data))
4793 			return;
4794 		if (resp_data != ~(BARRIER_TEST_PATTERN))
4795 			mdelay(1);
4796 		else
4797 			break;
4798 	}
4799 	resp_data = 0;
4800 	if (lpfc_readl((resp_buf + 1), &resp_data))
4801 		return;
4802 	if (resp_data  != ~(BARRIER_TEST_PATTERN)) {
4803 		if (phba->sli.sli_flag & LPFC_SLI_ACTIVE ||
4804 		    phba->pport->stopped)
4805 			goto restore_hc;
4806 		else
4807 			goto clear_errat;
4808 	}
4809 
4810 	((MAILBOX_t *)&mbox)->mbxOwner = OWN_HOST;
4811 	resp_data = 0;
4812 	for (i = 0; i < 500; i++) {
4813 		if (lpfc_readl(resp_buf, &resp_data))
4814 			return;
4815 		if (resp_data != mbox)
4816 			mdelay(1);
4817 		else
4818 			break;
4819 	}
4820 
4821 clear_errat:
4822 
4823 	while (++i < 500) {
4824 		if (lpfc_readl(phba->HAregaddr, &ha_copy))
4825 			return;
4826 		if (!(ha_copy & HA_ERATT))
4827 			mdelay(1);
4828 		else
4829 			break;
4830 	}
4831 
4832 	if (readl(phba->HAregaddr) & HA_ERATT) {
4833 		writel(HA_ERATT, phba->HAregaddr);
4834 		phba->pport->stopped = 1;
4835 	}
4836 
4837 restore_hc:
4838 	phba->link_flag &= ~LS_IGNORE_ERATT;
4839 	writel(hc_copy, phba->HCregaddr);
4840 	readl(phba->HCregaddr); /* flush */
4841 }
4842 
4843 /**
4844  * lpfc_sli_brdkill - Issue a kill_board mailbox command
4845  * @phba: Pointer to HBA context object.
4846  *
4847  * This function issues a kill_board mailbox command and waits for
4848  * the error attention interrupt. This function is called for stopping
4849  * the firmware processing. The caller is not required to hold any
4850  * locks. This function calls lpfc_hba_down_post function to free
4851  * any pending commands after the kill. The function will return 1 when it
4852  * fails to kill the board else will return 0.
4853  **/
4854 int
4855 lpfc_sli_brdkill(struct lpfc_hba *phba)
4856 {
4857 	struct lpfc_sli *psli;
4858 	LPFC_MBOXQ_t *pmb;
4859 	uint32_t status;
4860 	uint32_t ha_copy;
4861 	int retval;
4862 	int i = 0;
4863 
4864 	psli = &phba->sli;
4865 
4866 	/* Kill HBA */
4867 	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
4868 			"0329 Kill HBA Data: x%x x%x\n",
4869 			phba->pport->port_state, psli->sli_flag);
4870 
4871 	pmb = (LPFC_MBOXQ_t *) mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
4872 	if (!pmb)
4873 		return 1;
4874 
4875 	/* Disable the error attention */
4876 	spin_lock_irq(&phba->hbalock);
4877 	if (lpfc_readl(phba->HCregaddr, &status)) {
4878 		spin_unlock_irq(&phba->hbalock);
4879 		mempool_free(pmb, phba->mbox_mem_pool);
4880 		return 1;
4881 	}
4882 	status &= ~HC_ERINT_ENA;
4883 	writel(status, phba->HCregaddr);
4884 	readl(phba->HCregaddr); /* flush */
4885 	phba->link_flag |= LS_IGNORE_ERATT;
4886 	spin_unlock_irq(&phba->hbalock);
4887 
4888 	lpfc_kill_board(phba, pmb);
4889 	pmb->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
4890 	retval = lpfc_sli_issue_mbox(phba, pmb, MBX_NOWAIT);
4891 
4892 	if (retval != MBX_SUCCESS) {
4893 		if (retval != MBX_BUSY)
4894 			mempool_free(pmb, phba->mbox_mem_pool);
4895 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
4896 				"2752 KILL_BOARD command failed retval %d\n",
4897 				retval);
4898 		spin_lock_irq(&phba->hbalock);
4899 		phba->link_flag &= ~LS_IGNORE_ERATT;
4900 		spin_unlock_irq(&phba->hbalock);
4901 		return 1;
4902 	}
4903 
4904 	spin_lock_irq(&phba->hbalock);
4905 	psli->sli_flag &= ~LPFC_SLI_ACTIVE;
4906 	spin_unlock_irq(&phba->hbalock);
4907 
4908 	mempool_free(pmb, phba->mbox_mem_pool);
4909 
4910 	/* There is no completion for a KILL_BOARD mbox cmd. Check for an error
4911 	 * attention every 100ms for 3 seconds. If we don't get ERATT after
4912 	 * 3 seconds we still set HBA_ERROR state because the status of the
4913 	 * board is now undefined.
4914 	 */
4915 	if (lpfc_readl(phba->HAregaddr, &ha_copy))
4916 		return 1;
4917 	while ((i++ < 30) && !(ha_copy & HA_ERATT)) {
4918 		mdelay(100);
4919 		if (lpfc_readl(phba->HAregaddr, &ha_copy))
4920 			return 1;
4921 	}
4922 
4923 	del_timer_sync(&psli->mbox_tmo);
4924 	if (ha_copy & HA_ERATT) {
4925 		writel(HA_ERATT, phba->HAregaddr);
4926 		phba->pport->stopped = 1;
4927 	}
4928 	spin_lock_irq(&phba->hbalock);
4929 	psli->sli_flag &= ~LPFC_SLI_MBOX_ACTIVE;
4930 	psli->mbox_active = NULL;
4931 	phba->link_flag &= ~LS_IGNORE_ERATT;
4932 	spin_unlock_irq(&phba->hbalock);
4933 
4934 	lpfc_hba_down_post(phba);
4935 	phba->link_state = LPFC_HBA_ERROR;
4936 
4937 	return ha_copy & HA_ERATT ? 0 : 1;
4938 }
4939 
4940 /**
4941  * lpfc_sli_brdreset - Reset a sli-2 or sli-3 HBA
4942  * @phba: Pointer to HBA context object.
4943  *
4944  * This function resets the HBA by writing HC_INITFF to the control
4945  * register. After the HBA resets, this function resets all the iocb ring
4946  * indices. This function disables PCI layer parity checking during
4947  * the reset.
4948  * This function returns 0 always.
4949  * The caller is not required to hold any locks.
4950  **/
4951 int
4952 lpfc_sli_brdreset(struct lpfc_hba *phba)
4953 {
4954 	struct lpfc_sli *psli;
4955 	struct lpfc_sli_ring *pring;
4956 	uint16_t cfg_value;
4957 	int i;
4958 
4959 	psli = &phba->sli;
4960 
4961 	/* Reset HBA */
4962 	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
4963 			"0325 Reset HBA Data: x%x x%x\n",
4964 			(phba->pport) ? phba->pport->port_state : 0,
4965 			psli->sli_flag);
4966 
4967 	/* perform board reset */
4968 	phba->fc_eventTag = 0;
4969 	phba->link_events = 0;
4970 	phba->hba_flag |= HBA_NEEDS_CFG_PORT;
4971 	if (phba->pport) {
4972 		phba->pport->fc_myDID = 0;
4973 		phba->pport->fc_prevDID = 0;
4974 	}
4975 
4976 	/* Turn off parity checking and serr during the physical reset */
4977 	if (pci_read_config_word(phba->pcidev, PCI_COMMAND, &cfg_value))
4978 		return -EIO;
4979 
4980 	pci_write_config_word(phba->pcidev, PCI_COMMAND,
4981 			      (cfg_value &
4982 			       ~(PCI_COMMAND_PARITY | PCI_COMMAND_SERR)));
4983 
4984 	psli->sli_flag &= ~(LPFC_SLI_ACTIVE | LPFC_PROCESS_LA);
4985 
4986 	/* Now toggle INITFF bit in the Host Control Register */
4987 	writel(HC_INITFF, phba->HCregaddr);
4988 	mdelay(1);
4989 	readl(phba->HCregaddr); /* flush */
4990 	writel(0, phba->HCregaddr);
4991 	readl(phba->HCregaddr); /* flush */
4992 
4993 	/* Restore PCI cmd register */
4994 	pci_write_config_word(phba->pcidev, PCI_COMMAND, cfg_value);
4995 
4996 	/* Initialize relevant SLI info */
4997 	for (i = 0; i < psli->num_rings; i++) {
4998 		pring = &psli->sli3_ring[i];
4999 		pring->flag = 0;
5000 		pring->sli.sli3.rspidx = 0;
5001 		pring->sli.sli3.next_cmdidx  = 0;
5002 		pring->sli.sli3.local_getidx = 0;
5003 		pring->sli.sli3.cmdidx = 0;
5004 		pring->missbufcnt = 0;
5005 	}
5006 
5007 	phba->link_state = LPFC_WARM_START;
5008 	return 0;
5009 }
5010 
5011 /**
5012  * lpfc_sli4_brdreset - Reset a sli-4 HBA
5013  * @phba: Pointer to HBA context object.
5014  *
5015  * This function resets a SLI4 HBA. This function disables PCI layer parity
5016  * checking during resets the device. The caller is not required to hold
5017  * any locks.
5018  *
5019  * This function returns 0 on success else returns negative error code.
5020  **/
5021 int
5022 lpfc_sli4_brdreset(struct lpfc_hba *phba)
5023 {
5024 	struct lpfc_sli *psli = &phba->sli;
5025 	uint16_t cfg_value;
5026 	int rc = 0;
5027 
5028 	/* Reset HBA */
5029 	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
5030 			"0295 Reset HBA Data: x%x x%x x%x\n",
5031 			phba->pport->port_state, psli->sli_flag,
5032 			phba->hba_flag);
5033 
5034 	/* perform board reset */
5035 	phba->fc_eventTag = 0;
5036 	phba->link_events = 0;
5037 	phba->pport->fc_myDID = 0;
5038 	phba->pport->fc_prevDID = 0;
5039 	phba->hba_flag &= ~HBA_SETUP;
5040 
5041 	spin_lock_irq(&phba->hbalock);
5042 	psli->sli_flag &= ~(LPFC_PROCESS_LA);
5043 	phba->fcf.fcf_flag = 0;
5044 	spin_unlock_irq(&phba->hbalock);
5045 
5046 	/* SLI4 INTF 2: if FW dump is being taken skip INIT_PORT */
5047 	if (phba->hba_flag & HBA_FW_DUMP_OP) {
5048 		phba->hba_flag &= ~HBA_FW_DUMP_OP;
5049 		return rc;
5050 	}
5051 
5052 	/* Now physically reset the device */
5053 	lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
5054 			"0389 Performing PCI function reset!\n");
5055 
5056 	/* Turn off parity checking and serr during the physical reset */
5057 	if (pci_read_config_word(phba->pcidev, PCI_COMMAND, &cfg_value)) {
5058 		lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
5059 				"3205 PCI read Config failed\n");
5060 		return -EIO;
5061 	}
5062 
5063 	pci_write_config_word(phba->pcidev, PCI_COMMAND, (cfg_value &
5064 			      ~(PCI_COMMAND_PARITY | PCI_COMMAND_SERR)));
5065 
5066 	/* Perform FCoE PCI function reset before freeing queue memory */
5067 	rc = lpfc_pci_function_reset(phba);
5068 
5069 	/* Restore PCI cmd register */
5070 	pci_write_config_word(phba->pcidev, PCI_COMMAND, cfg_value);
5071 
5072 	return rc;
5073 }
5074 
5075 /**
5076  * lpfc_sli_brdrestart_s3 - Restart a sli-3 hba
5077  * @phba: Pointer to HBA context object.
5078  *
5079  * This function is called in the SLI initialization code path to
5080  * restart the HBA. The caller is not required to hold any lock.
5081  * This function writes MBX_RESTART mailbox command to the SLIM and
5082  * resets the HBA. At the end of the function, it calls lpfc_hba_down_post
5083  * function to free any pending commands. The function enables
5084  * POST only during the first initialization. The function returns zero.
5085  * The function does not guarantee completion of MBX_RESTART mailbox
5086  * command before the return of this function.
5087  **/
5088 static int
5089 lpfc_sli_brdrestart_s3(struct lpfc_hba *phba)
5090 {
5091 	MAILBOX_t *mb;
5092 	struct lpfc_sli *psli;
5093 	volatile uint32_t word0;
5094 	void __iomem *to_slim;
5095 	uint32_t hba_aer_enabled;
5096 
5097 	spin_lock_irq(&phba->hbalock);
5098 
5099 	/* Take PCIe device Advanced Error Reporting (AER) state */
5100 	hba_aer_enabled = phba->hba_flag & HBA_AER_ENABLED;
5101 
5102 	psli = &phba->sli;
5103 
5104 	/* Restart HBA */
5105 	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
5106 			"0337 Restart HBA Data: x%x x%x\n",
5107 			(phba->pport) ? phba->pport->port_state : 0,
5108 			psli->sli_flag);
5109 
5110 	word0 = 0;
5111 	mb = (MAILBOX_t *) &word0;
5112 	mb->mbxCommand = MBX_RESTART;
5113 	mb->mbxHc = 1;
5114 
5115 	lpfc_reset_barrier(phba);
5116 
5117 	to_slim = phba->MBslimaddr;
5118 	writel(*(uint32_t *) mb, to_slim);
5119 	readl(to_slim); /* flush */
5120 
5121 	/* Only skip post after fc_ffinit is completed */
5122 	if (phba->pport && phba->pport->port_state)
5123 		word0 = 1;	/* This is really setting up word1 */
5124 	else
5125 		word0 = 0;	/* This is really setting up word1 */
5126 	to_slim = phba->MBslimaddr + sizeof (uint32_t);
5127 	writel(*(uint32_t *) mb, to_slim);
5128 	readl(to_slim); /* flush */
5129 
5130 	lpfc_sli_brdreset(phba);
5131 	if (phba->pport)
5132 		phba->pport->stopped = 0;
5133 	phba->link_state = LPFC_INIT_START;
5134 	phba->hba_flag = 0;
5135 	spin_unlock_irq(&phba->hbalock);
5136 
5137 	memset(&psli->lnk_stat_offsets, 0, sizeof(psli->lnk_stat_offsets));
5138 	psli->stats_start = ktime_get_seconds();
5139 
5140 	/* Give the INITFF and Post time to settle. */
5141 	mdelay(100);
5142 
5143 	/* Reset HBA AER if it was enabled, note hba_flag was reset above */
5144 	if (hba_aer_enabled)
5145 		pci_disable_pcie_error_reporting(phba->pcidev);
5146 
5147 	lpfc_hba_down_post(phba);
5148 
5149 	return 0;
5150 }
5151 
5152 /**
5153  * lpfc_sli_brdrestart_s4 - Restart the sli-4 hba
5154  * @phba: Pointer to HBA context object.
5155  *
5156  * This function is called in the SLI initialization code path to restart
5157  * a SLI4 HBA. The caller is not required to hold any lock.
5158  * At the end of the function, it calls lpfc_hba_down_post function to
5159  * free any pending commands.
5160  **/
5161 static int
5162 lpfc_sli_brdrestart_s4(struct lpfc_hba *phba)
5163 {
5164 	struct lpfc_sli *psli = &phba->sli;
5165 	uint32_t hba_aer_enabled;
5166 	int rc;
5167 
5168 	/* Restart HBA */
5169 	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
5170 			"0296 Restart HBA Data: x%x x%x\n",
5171 			phba->pport->port_state, psli->sli_flag);
5172 
5173 	/* Take PCIe device Advanced Error Reporting (AER) state */
5174 	hba_aer_enabled = phba->hba_flag & HBA_AER_ENABLED;
5175 
5176 	rc = lpfc_sli4_brdreset(phba);
5177 	if (rc) {
5178 		phba->link_state = LPFC_HBA_ERROR;
5179 		goto hba_down_queue;
5180 	}
5181 
5182 	spin_lock_irq(&phba->hbalock);
5183 	phba->pport->stopped = 0;
5184 	phba->link_state = LPFC_INIT_START;
5185 	phba->hba_flag = 0;
5186 	spin_unlock_irq(&phba->hbalock);
5187 
5188 	memset(&psli->lnk_stat_offsets, 0, sizeof(psli->lnk_stat_offsets));
5189 	psli->stats_start = ktime_get_seconds();
5190 
5191 	/* Reset HBA AER if it was enabled, note hba_flag was reset above */
5192 	if (hba_aer_enabled)
5193 		pci_disable_pcie_error_reporting(phba->pcidev);
5194 
5195 hba_down_queue:
5196 	lpfc_hba_down_post(phba);
5197 	lpfc_sli4_queue_destroy(phba);
5198 
5199 	return rc;
5200 }
5201 
5202 /**
5203  * lpfc_sli_brdrestart - Wrapper func for restarting hba
5204  * @phba: Pointer to HBA context object.
5205  *
5206  * This routine wraps the actual SLI3 or SLI4 hba restart routine from the
5207  * API jump table function pointer from the lpfc_hba struct.
5208 **/
5209 int
5210 lpfc_sli_brdrestart(struct lpfc_hba *phba)
5211 {
5212 	return phba->lpfc_sli_brdrestart(phba);
5213 }
5214 
5215 /**
5216  * lpfc_sli_chipset_init - Wait for the restart of the HBA after a restart
5217  * @phba: Pointer to HBA context object.
5218  *
5219  * This function is called after a HBA restart to wait for successful
5220  * restart of the HBA. Successful restart of the HBA is indicated by
5221  * HS_FFRDY and HS_MBRDY bits. If the HBA fails to restart even after 15
5222  * iteration, the function will restart the HBA again. The function returns
5223  * zero if HBA successfully restarted else returns negative error code.
5224  **/
5225 int
5226 lpfc_sli_chipset_init(struct lpfc_hba *phba)
5227 {
5228 	uint32_t status, i = 0;
5229 
5230 	/* Read the HBA Host Status Register */
5231 	if (lpfc_readl(phba->HSregaddr, &status))
5232 		return -EIO;
5233 
5234 	/* Check status register to see what current state is */
5235 	i = 0;
5236 	while ((status & (HS_FFRDY | HS_MBRDY)) != (HS_FFRDY | HS_MBRDY)) {
5237 
5238 		/* Check every 10ms for 10 retries, then every 100ms for 90
5239 		 * retries, then every 1 sec for 50 retires for a total of
5240 		 * ~60 seconds before reset the board again and check every
5241 		 * 1 sec for 50 retries. The up to 60 seconds before the
5242 		 * board ready is required by the Falcon FIPS zeroization
5243 		 * complete, and any reset the board in between shall cause
5244 		 * restart of zeroization, further delay the board ready.
5245 		 */
5246 		if (i++ >= 200) {
5247 			/* Adapter failed to init, timeout, status reg
5248 			   <status> */
5249 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
5250 					"0436 Adapter failed to init, "
5251 					"timeout, status reg x%x, "
5252 					"FW Data: A8 x%x AC x%x\n", status,
5253 					readl(phba->MBslimaddr + 0xa8),
5254 					readl(phba->MBslimaddr + 0xac));
5255 			phba->link_state = LPFC_HBA_ERROR;
5256 			return -ETIMEDOUT;
5257 		}
5258 
5259 		/* Check to see if any errors occurred during init */
5260 		if (status & HS_FFERM) {
5261 			/* ERROR: During chipset initialization */
5262 			/* Adapter failed to init, chipset, status reg
5263 			   <status> */
5264 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
5265 					"0437 Adapter failed to init, "
5266 					"chipset, status reg x%x, "
5267 					"FW Data: A8 x%x AC x%x\n", status,
5268 					readl(phba->MBslimaddr + 0xa8),
5269 					readl(phba->MBslimaddr + 0xac));
5270 			phba->link_state = LPFC_HBA_ERROR;
5271 			return -EIO;
5272 		}
5273 
5274 		if (i <= 10)
5275 			msleep(10);
5276 		else if (i <= 100)
5277 			msleep(100);
5278 		else
5279 			msleep(1000);
5280 
5281 		if (i == 150) {
5282 			/* Do post */
5283 			phba->pport->port_state = LPFC_VPORT_UNKNOWN;
5284 			lpfc_sli_brdrestart(phba);
5285 		}
5286 		/* Read the HBA Host Status Register */
5287 		if (lpfc_readl(phba->HSregaddr, &status))
5288 			return -EIO;
5289 	}
5290 
5291 	/* Check to see if any errors occurred during init */
5292 	if (status & HS_FFERM) {
5293 		/* ERROR: During chipset initialization */
5294 		/* Adapter failed to init, chipset, status reg <status> */
5295 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
5296 				"0438 Adapter failed to init, chipset, "
5297 				"status reg x%x, "
5298 				"FW Data: A8 x%x AC x%x\n", status,
5299 				readl(phba->MBslimaddr + 0xa8),
5300 				readl(phba->MBslimaddr + 0xac));
5301 		phba->link_state = LPFC_HBA_ERROR;
5302 		return -EIO;
5303 	}
5304 
5305 	phba->hba_flag |= HBA_NEEDS_CFG_PORT;
5306 
5307 	/* Clear all interrupt enable conditions */
5308 	writel(0, phba->HCregaddr);
5309 	readl(phba->HCregaddr); /* flush */
5310 
5311 	/* setup host attn register */
5312 	writel(0xffffffff, phba->HAregaddr);
5313 	readl(phba->HAregaddr); /* flush */
5314 	return 0;
5315 }
5316 
5317 /**
5318  * lpfc_sli_hbq_count - Get the number of HBQs to be configured
5319  *
5320  * This function calculates and returns the number of HBQs required to be
5321  * configured.
5322  **/
5323 int
5324 lpfc_sli_hbq_count(void)
5325 {
5326 	return ARRAY_SIZE(lpfc_hbq_defs);
5327 }
5328 
5329 /**
5330  * lpfc_sli_hbq_entry_count - Calculate total number of hbq entries
5331  *
5332  * This function adds the number of hbq entries in every HBQ to get
5333  * the total number of hbq entries required for the HBA and returns
5334  * the total count.
5335  **/
5336 static int
5337 lpfc_sli_hbq_entry_count(void)
5338 {
5339 	int  hbq_count = lpfc_sli_hbq_count();
5340 	int  count = 0;
5341 	int  i;
5342 
5343 	for (i = 0; i < hbq_count; ++i)
5344 		count += lpfc_hbq_defs[i]->entry_count;
5345 	return count;
5346 }
5347 
5348 /**
5349  * lpfc_sli_hbq_size - Calculate memory required for all hbq entries
5350  *
5351  * This function calculates amount of memory required for all hbq entries
5352  * to be configured and returns the total memory required.
5353  **/
5354 int
5355 lpfc_sli_hbq_size(void)
5356 {
5357 	return lpfc_sli_hbq_entry_count() * sizeof(struct lpfc_hbq_entry);
5358 }
5359 
5360 /**
5361  * lpfc_sli_hbq_setup - configure and initialize HBQs
5362  * @phba: Pointer to HBA context object.
5363  *
5364  * This function is called during the SLI initialization to configure
5365  * all the HBQs and post buffers to the HBQ. The caller is not
5366  * required to hold any locks. This function will return zero if successful
5367  * else it will return negative error code.
5368  **/
5369 static int
5370 lpfc_sli_hbq_setup(struct lpfc_hba *phba)
5371 {
5372 	int  hbq_count = lpfc_sli_hbq_count();
5373 	LPFC_MBOXQ_t *pmb;
5374 	MAILBOX_t *pmbox;
5375 	uint32_t hbqno;
5376 	uint32_t hbq_entry_index;
5377 
5378 				/* Get a Mailbox buffer to setup mailbox
5379 				 * commands for HBA initialization
5380 				 */
5381 	pmb = (LPFC_MBOXQ_t *) mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
5382 
5383 	if (!pmb)
5384 		return -ENOMEM;
5385 
5386 	pmbox = &pmb->u.mb;
5387 
5388 	/* Initialize the struct lpfc_sli_hbq structure for each hbq */
5389 	phba->link_state = LPFC_INIT_MBX_CMDS;
5390 	phba->hbq_in_use = 1;
5391 
5392 	hbq_entry_index = 0;
5393 	for (hbqno = 0; hbqno < hbq_count; ++hbqno) {
5394 		phba->hbqs[hbqno].next_hbqPutIdx = 0;
5395 		phba->hbqs[hbqno].hbqPutIdx      = 0;
5396 		phba->hbqs[hbqno].local_hbqGetIdx   = 0;
5397 		phba->hbqs[hbqno].entry_count =
5398 			lpfc_hbq_defs[hbqno]->entry_count;
5399 		lpfc_config_hbq(phba, hbqno, lpfc_hbq_defs[hbqno],
5400 			hbq_entry_index, pmb);
5401 		hbq_entry_index += phba->hbqs[hbqno].entry_count;
5402 
5403 		if (lpfc_sli_issue_mbox(phba, pmb, MBX_POLL) != MBX_SUCCESS) {
5404 			/* Adapter failed to init, mbxCmd <cmd> CFG_RING,
5405 			   mbxStatus <status>, ring <num> */
5406 
5407 			lpfc_printf_log(phba, KERN_ERR,
5408 					LOG_SLI | LOG_VPORT,
5409 					"1805 Adapter failed to init. "
5410 					"Data: x%x x%x x%x\n",
5411 					pmbox->mbxCommand,
5412 					pmbox->mbxStatus, hbqno);
5413 
5414 			phba->link_state = LPFC_HBA_ERROR;
5415 			mempool_free(pmb, phba->mbox_mem_pool);
5416 			return -ENXIO;
5417 		}
5418 	}
5419 	phba->hbq_count = hbq_count;
5420 
5421 	mempool_free(pmb, phba->mbox_mem_pool);
5422 
5423 	/* Initially populate or replenish the HBQs */
5424 	for (hbqno = 0; hbqno < hbq_count; ++hbqno)
5425 		lpfc_sli_hbqbuf_init_hbqs(phba, hbqno);
5426 	return 0;
5427 }
5428 
5429 /**
5430  * lpfc_sli4_rb_setup - Initialize and post RBs to HBA
5431  * @phba: Pointer to HBA context object.
5432  *
5433  * This function is called during the SLI initialization to configure
5434  * all the HBQs and post buffers to the HBQ. The caller is not
5435  * required to hold any locks. This function will return zero if successful
5436  * else it will return negative error code.
5437  **/
5438 static int
5439 lpfc_sli4_rb_setup(struct lpfc_hba *phba)
5440 {
5441 	phba->hbq_in_use = 1;
5442 	/**
5443 	 * Specific case when the MDS diagnostics is enabled and supported.
5444 	 * The receive buffer count is truncated to manage the incoming
5445 	 * traffic.
5446 	 **/
5447 	if (phba->cfg_enable_mds_diags && phba->mds_diags_support)
5448 		phba->hbqs[LPFC_ELS_HBQ].entry_count =
5449 			lpfc_hbq_defs[LPFC_ELS_HBQ]->entry_count >> 1;
5450 	else
5451 		phba->hbqs[LPFC_ELS_HBQ].entry_count =
5452 			lpfc_hbq_defs[LPFC_ELS_HBQ]->entry_count;
5453 	phba->hbq_count = 1;
5454 	lpfc_sli_hbqbuf_init_hbqs(phba, LPFC_ELS_HBQ);
5455 	/* Initially populate or replenish the HBQs */
5456 	return 0;
5457 }
5458 
5459 /**
5460  * lpfc_sli_config_port - Issue config port mailbox command
5461  * @phba: Pointer to HBA context object.
5462  * @sli_mode: sli mode - 2/3
5463  *
5464  * This function is called by the sli initialization code path
5465  * to issue config_port mailbox command. This function restarts the
5466  * HBA firmware and issues a config_port mailbox command to configure
5467  * the SLI interface in the sli mode specified by sli_mode
5468  * variable. The caller is not required to hold any locks.
5469  * The function returns 0 if successful, else returns negative error
5470  * code.
5471  **/
5472 int
5473 lpfc_sli_config_port(struct lpfc_hba *phba, int sli_mode)
5474 {
5475 	LPFC_MBOXQ_t *pmb;
5476 	uint32_t resetcount = 0, rc = 0, done = 0;
5477 
5478 	pmb = (LPFC_MBOXQ_t *) mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
5479 	if (!pmb) {
5480 		phba->link_state = LPFC_HBA_ERROR;
5481 		return -ENOMEM;
5482 	}
5483 
5484 	phba->sli_rev = sli_mode;
5485 	while (resetcount < 2 && !done) {
5486 		spin_lock_irq(&phba->hbalock);
5487 		phba->sli.sli_flag |= LPFC_SLI_MBOX_ACTIVE;
5488 		spin_unlock_irq(&phba->hbalock);
5489 		phba->pport->port_state = LPFC_VPORT_UNKNOWN;
5490 		lpfc_sli_brdrestart(phba);
5491 		rc = lpfc_sli_chipset_init(phba);
5492 		if (rc)
5493 			break;
5494 
5495 		spin_lock_irq(&phba->hbalock);
5496 		phba->sli.sli_flag &= ~LPFC_SLI_MBOX_ACTIVE;
5497 		spin_unlock_irq(&phba->hbalock);
5498 		resetcount++;
5499 
5500 		/* Call pre CONFIG_PORT mailbox command initialization.  A
5501 		 * value of 0 means the call was successful.  Any other
5502 		 * nonzero value is a failure, but if ERESTART is returned,
5503 		 * the driver may reset the HBA and try again.
5504 		 */
5505 		rc = lpfc_config_port_prep(phba);
5506 		if (rc == -ERESTART) {
5507 			phba->link_state = LPFC_LINK_UNKNOWN;
5508 			continue;
5509 		} else if (rc)
5510 			break;
5511 
5512 		phba->link_state = LPFC_INIT_MBX_CMDS;
5513 		lpfc_config_port(phba, pmb);
5514 		rc = lpfc_sli_issue_mbox(phba, pmb, MBX_POLL);
5515 		phba->sli3_options &= ~(LPFC_SLI3_NPIV_ENABLED |
5516 					LPFC_SLI3_HBQ_ENABLED |
5517 					LPFC_SLI3_CRP_ENABLED |
5518 					LPFC_SLI3_DSS_ENABLED);
5519 		if (rc != MBX_SUCCESS) {
5520 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
5521 				"0442 Adapter failed to init, mbxCmd x%x "
5522 				"CONFIG_PORT, mbxStatus x%x Data: x%x\n",
5523 				pmb->u.mb.mbxCommand, pmb->u.mb.mbxStatus, 0);
5524 			spin_lock_irq(&phba->hbalock);
5525 			phba->sli.sli_flag &= ~LPFC_SLI_ACTIVE;
5526 			spin_unlock_irq(&phba->hbalock);
5527 			rc = -ENXIO;
5528 		} else {
5529 			/* Allow asynchronous mailbox command to go through */
5530 			spin_lock_irq(&phba->hbalock);
5531 			phba->sli.sli_flag &= ~LPFC_SLI_ASYNC_MBX_BLK;
5532 			spin_unlock_irq(&phba->hbalock);
5533 			done = 1;
5534 
5535 			if ((pmb->u.mb.un.varCfgPort.casabt == 1) &&
5536 			    (pmb->u.mb.un.varCfgPort.gasabt == 0))
5537 				lpfc_printf_log(phba, KERN_WARNING, LOG_INIT,
5538 					"3110 Port did not grant ASABT\n");
5539 		}
5540 	}
5541 	if (!done) {
5542 		rc = -EINVAL;
5543 		goto do_prep_failed;
5544 	}
5545 	if (pmb->u.mb.un.varCfgPort.sli_mode == 3) {
5546 		if (!pmb->u.mb.un.varCfgPort.cMA) {
5547 			rc = -ENXIO;
5548 			goto do_prep_failed;
5549 		}
5550 		if (phba->max_vpi && pmb->u.mb.un.varCfgPort.gmv) {
5551 			phba->sli3_options |= LPFC_SLI3_NPIV_ENABLED;
5552 			phba->max_vpi = pmb->u.mb.un.varCfgPort.max_vpi;
5553 			phba->max_vports = (phba->max_vpi > phba->max_vports) ?
5554 				phba->max_vpi : phba->max_vports;
5555 
5556 		} else
5557 			phba->max_vpi = 0;
5558 		if (pmb->u.mb.un.varCfgPort.gerbm)
5559 			phba->sli3_options |= LPFC_SLI3_HBQ_ENABLED;
5560 		if (pmb->u.mb.un.varCfgPort.gcrp)
5561 			phba->sli3_options |= LPFC_SLI3_CRP_ENABLED;
5562 
5563 		phba->hbq_get = phba->mbox->us.s3_pgp.hbq_get;
5564 		phba->port_gp = phba->mbox->us.s3_pgp.port;
5565 
5566 		if (phba->sli3_options & LPFC_SLI3_BG_ENABLED) {
5567 			if (pmb->u.mb.un.varCfgPort.gbg == 0) {
5568 				phba->cfg_enable_bg = 0;
5569 				phba->sli3_options &= ~LPFC_SLI3_BG_ENABLED;
5570 				lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
5571 						"0443 Adapter did not grant "
5572 						"BlockGuard\n");
5573 			}
5574 		}
5575 	} else {
5576 		phba->hbq_get = NULL;
5577 		phba->port_gp = phba->mbox->us.s2.port;
5578 		phba->max_vpi = 0;
5579 	}
5580 do_prep_failed:
5581 	mempool_free(pmb, phba->mbox_mem_pool);
5582 	return rc;
5583 }
5584 
5585 
5586 /**
5587  * lpfc_sli_hba_setup - SLI initialization function
5588  * @phba: Pointer to HBA context object.
5589  *
5590  * This function is the main SLI initialization function. This function
5591  * is called by the HBA initialization code, HBA reset code and HBA
5592  * error attention handler code. Caller is not required to hold any
5593  * locks. This function issues config_port mailbox command to configure
5594  * the SLI, setup iocb rings and HBQ rings. In the end the function
5595  * calls the config_port_post function to issue init_link mailbox
5596  * command and to start the discovery. The function will return zero
5597  * if successful, else it will return negative error code.
5598  **/
5599 int
5600 lpfc_sli_hba_setup(struct lpfc_hba *phba)
5601 {
5602 	uint32_t rc;
5603 	int  i;
5604 	int longs;
5605 
5606 	/* Enable ISR already does config_port because of config_msi mbx */
5607 	if (phba->hba_flag & HBA_NEEDS_CFG_PORT) {
5608 		rc = lpfc_sli_config_port(phba, LPFC_SLI_REV3);
5609 		if (rc)
5610 			return -EIO;
5611 		phba->hba_flag &= ~HBA_NEEDS_CFG_PORT;
5612 	}
5613 	phba->fcp_embed_io = 0;	/* SLI4 FC support only */
5614 
5615 	/* Enable PCIe device Advanced Error Reporting (AER) if configured */
5616 	if (phba->cfg_aer_support == 1 && !(phba->hba_flag & HBA_AER_ENABLED)) {
5617 		rc = pci_enable_pcie_error_reporting(phba->pcidev);
5618 		if (!rc) {
5619 			lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
5620 					"2709 This device supports "
5621 					"Advanced Error Reporting (AER)\n");
5622 			spin_lock_irq(&phba->hbalock);
5623 			phba->hba_flag |= HBA_AER_ENABLED;
5624 			spin_unlock_irq(&phba->hbalock);
5625 		} else {
5626 			lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
5627 					"2708 This device does not support "
5628 					"Advanced Error Reporting (AER): %d\n",
5629 					rc);
5630 			phba->cfg_aer_support = 0;
5631 		}
5632 	}
5633 
5634 	if (phba->sli_rev == 3) {
5635 		phba->iocb_cmd_size = SLI3_IOCB_CMD_SIZE;
5636 		phba->iocb_rsp_size = SLI3_IOCB_RSP_SIZE;
5637 	} else {
5638 		phba->iocb_cmd_size = SLI2_IOCB_CMD_SIZE;
5639 		phba->iocb_rsp_size = SLI2_IOCB_RSP_SIZE;
5640 		phba->sli3_options = 0;
5641 	}
5642 
5643 	lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
5644 			"0444 Firmware in SLI %x mode. Max_vpi %d\n",
5645 			phba->sli_rev, phba->max_vpi);
5646 	rc = lpfc_sli_ring_map(phba);
5647 
5648 	if (rc)
5649 		goto lpfc_sli_hba_setup_error;
5650 
5651 	/* Initialize VPIs. */
5652 	if (phba->sli_rev == LPFC_SLI_REV3) {
5653 		/*
5654 		 * The VPI bitmask and physical ID array are allocated
5655 		 * and initialized once only - at driver load.  A port
5656 		 * reset doesn't need to reinitialize this memory.
5657 		 */
5658 		if ((phba->vpi_bmask == NULL) && (phba->vpi_ids == NULL)) {
5659 			longs = (phba->max_vpi + BITS_PER_LONG) / BITS_PER_LONG;
5660 			phba->vpi_bmask = kcalloc(longs,
5661 						  sizeof(unsigned long),
5662 						  GFP_KERNEL);
5663 			if (!phba->vpi_bmask) {
5664 				rc = -ENOMEM;
5665 				goto lpfc_sli_hba_setup_error;
5666 			}
5667 
5668 			phba->vpi_ids = kcalloc(phba->max_vpi + 1,
5669 						sizeof(uint16_t),
5670 						GFP_KERNEL);
5671 			if (!phba->vpi_ids) {
5672 				kfree(phba->vpi_bmask);
5673 				rc = -ENOMEM;
5674 				goto lpfc_sli_hba_setup_error;
5675 			}
5676 			for (i = 0; i < phba->max_vpi; i++)
5677 				phba->vpi_ids[i] = i;
5678 		}
5679 	}
5680 
5681 	/* Init HBQs */
5682 	if (phba->sli3_options & LPFC_SLI3_HBQ_ENABLED) {
5683 		rc = lpfc_sli_hbq_setup(phba);
5684 		if (rc)
5685 			goto lpfc_sli_hba_setup_error;
5686 	}
5687 	spin_lock_irq(&phba->hbalock);
5688 	phba->sli.sli_flag |= LPFC_PROCESS_LA;
5689 	spin_unlock_irq(&phba->hbalock);
5690 
5691 	rc = lpfc_config_port_post(phba);
5692 	if (rc)
5693 		goto lpfc_sli_hba_setup_error;
5694 
5695 	return rc;
5696 
5697 lpfc_sli_hba_setup_error:
5698 	phba->link_state = LPFC_HBA_ERROR;
5699 	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
5700 			"0445 Firmware initialization failed\n");
5701 	return rc;
5702 }
5703 
5704 /**
5705  * lpfc_sli4_read_fcoe_params - Read fcoe params from conf region
5706  * @phba: Pointer to HBA context object.
5707  *
5708  * This function issue a dump mailbox command to read config region
5709  * 23 and parse the records in the region and populate driver
5710  * data structure.
5711  **/
5712 static int
5713 lpfc_sli4_read_fcoe_params(struct lpfc_hba *phba)
5714 {
5715 	LPFC_MBOXQ_t *mboxq;
5716 	struct lpfc_dmabuf *mp;
5717 	struct lpfc_mqe *mqe;
5718 	uint32_t data_length;
5719 	int rc;
5720 
5721 	/* Program the default value of vlan_id and fc_map */
5722 	phba->valid_vlan = 0;
5723 	phba->fc_map[0] = LPFC_FCOE_FCF_MAP0;
5724 	phba->fc_map[1] = LPFC_FCOE_FCF_MAP1;
5725 	phba->fc_map[2] = LPFC_FCOE_FCF_MAP2;
5726 
5727 	mboxq = (LPFC_MBOXQ_t *)mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
5728 	if (!mboxq)
5729 		return -ENOMEM;
5730 
5731 	mqe = &mboxq->u.mqe;
5732 	if (lpfc_sli4_dump_cfg_rg23(phba, mboxq)) {
5733 		rc = -ENOMEM;
5734 		goto out_free_mboxq;
5735 	}
5736 
5737 	mp = (struct lpfc_dmabuf *)mboxq->ctx_buf;
5738 	rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
5739 
5740 	lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI,
5741 			"(%d):2571 Mailbox cmd x%x Status x%x "
5742 			"Data: x%x x%x x%x x%x x%x x%x x%x x%x x%x "
5743 			"x%x x%x x%x x%x x%x x%x x%x x%x x%x "
5744 			"CQ: x%x x%x x%x x%x\n",
5745 			mboxq->vport ? mboxq->vport->vpi : 0,
5746 			bf_get(lpfc_mqe_command, mqe),
5747 			bf_get(lpfc_mqe_status, mqe),
5748 			mqe->un.mb_words[0], mqe->un.mb_words[1],
5749 			mqe->un.mb_words[2], mqe->un.mb_words[3],
5750 			mqe->un.mb_words[4], mqe->un.mb_words[5],
5751 			mqe->un.mb_words[6], mqe->un.mb_words[7],
5752 			mqe->un.mb_words[8], mqe->un.mb_words[9],
5753 			mqe->un.mb_words[10], mqe->un.mb_words[11],
5754 			mqe->un.mb_words[12], mqe->un.mb_words[13],
5755 			mqe->un.mb_words[14], mqe->un.mb_words[15],
5756 			mqe->un.mb_words[16], mqe->un.mb_words[50],
5757 			mboxq->mcqe.word0,
5758 			mboxq->mcqe.mcqe_tag0, 	mboxq->mcqe.mcqe_tag1,
5759 			mboxq->mcqe.trailer);
5760 
5761 	if (rc) {
5762 		lpfc_mbuf_free(phba, mp->virt, mp->phys);
5763 		kfree(mp);
5764 		rc = -EIO;
5765 		goto out_free_mboxq;
5766 	}
5767 	data_length = mqe->un.mb_words[5];
5768 	if (data_length > DMP_RGN23_SIZE) {
5769 		lpfc_mbuf_free(phba, mp->virt, mp->phys);
5770 		kfree(mp);
5771 		rc = -EIO;
5772 		goto out_free_mboxq;
5773 	}
5774 
5775 	lpfc_parse_fcoe_conf(phba, mp->virt, data_length);
5776 	lpfc_mbuf_free(phba, mp->virt, mp->phys);
5777 	kfree(mp);
5778 	rc = 0;
5779 
5780 out_free_mboxq:
5781 	mempool_free(mboxq, phba->mbox_mem_pool);
5782 	return rc;
5783 }
5784 
5785 /**
5786  * lpfc_sli4_read_rev - Issue READ_REV and collect vpd data
5787  * @phba: pointer to lpfc hba data structure.
5788  * @mboxq: pointer to the LPFC_MBOXQ_t structure.
5789  * @vpd: pointer to the memory to hold resulting port vpd data.
5790  * @vpd_size: On input, the number of bytes allocated to @vpd.
5791  *	      On output, the number of data bytes in @vpd.
5792  *
5793  * This routine executes a READ_REV SLI4 mailbox command.  In
5794  * addition, this routine gets the port vpd data.
5795  *
5796  * Return codes
5797  * 	0 - successful
5798  * 	-ENOMEM - could not allocated memory.
5799  **/
5800 static int
5801 lpfc_sli4_read_rev(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq,
5802 		    uint8_t *vpd, uint32_t *vpd_size)
5803 {
5804 	int rc = 0;
5805 	uint32_t dma_size;
5806 	struct lpfc_dmabuf *dmabuf;
5807 	struct lpfc_mqe *mqe;
5808 
5809 	dmabuf = kzalloc(sizeof(struct lpfc_dmabuf), GFP_KERNEL);
5810 	if (!dmabuf)
5811 		return -ENOMEM;
5812 
5813 	/*
5814 	 * Get a DMA buffer for the vpd data resulting from the READ_REV
5815 	 * mailbox command.
5816 	 */
5817 	dma_size = *vpd_size;
5818 	dmabuf->virt = dma_alloc_coherent(&phba->pcidev->dev, dma_size,
5819 					  &dmabuf->phys, GFP_KERNEL);
5820 	if (!dmabuf->virt) {
5821 		kfree(dmabuf);
5822 		return -ENOMEM;
5823 	}
5824 
5825 	/*
5826 	 * The SLI4 implementation of READ_REV conflicts at word1,
5827 	 * bits 31:16 and SLI4 adds vpd functionality not present
5828 	 * in SLI3.  This code corrects the conflicts.
5829 	 */
5830 	lpfc_read_rev(phba, mboxq);
5831 	mqe = &mboxq->u.mqe;
5832 	mqe->un.read_rev.vpd_paddr_high = putPaddrHigh(dmabuf->phys);
5833 	mqe->un.read_rev.vpd_paddr_low = putPaddrLow(dmabuf->phys);
5834 	mqe->un.read_rev.word1 &= 0x0000FFFF;
5835 	bf_set(lpfc_mbx_rd_rev_vpd, &mqe->un.read_rev, 1);
5836 	bf_set(lpfc_mbx_rd_rev_avail_len, &mqe->un.read_rev, dma_size);
5837 
5838 	rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
5839 	if (rc) {
5840 		dma_free_coherent(&phba->pcidev->dev, dma_size,
5841 				  dmabuf->virt, dmabuf->phys);
5842 		kfree(dmabuf);
5843 		return -EIO;
5844 	}
5845 
5846 	/*
5847 	 * The available vpd length cannot be bigger than the
5848 	 * DMA buffer passed to the port.  Catch the less than
5849 	 * case and update the caller's size.
5850 	 */
5851 	if (mqe->un.read_rev.avail_vpd_len < *vpd_size)
5852 		*vpd_size = mqe->un.read_rev.avail_vpd_len;
5853 
5854 	memcpy(vpd, dmabuf->virt, *vpd_size);
5855 
5856 	dma_free_coherent(&phba->pcidev->dev, dma_size,
5857 			  dmabuf->virt, dmabuf->phys);
5858 	kfree(dmabuf);
5859 	return 0;
5860 }
5861 
5862 /**
5863  * lpfc_sli4_get_ctl_attr - Retrieve SLI4 device controller attributes
5864  * @phba: pointer to lpfc hba data structure.
5865  *
5866  * This routine retrieves SLI4 device physical port name this PCI function
5867  * is attached to.
5868  *
5869  * Return codes
5870  *      0 - successful
5871  *      otherwise - failed to retrieve controller attributes
5872  **/
5873 static int
5874 lpfc_sli4_get_ctl_attr(struct lpfc_hba *phba)
5875 {
5876 	LPFC_MBOXQ_t *mboxq;
5877 	struct lpfc_mbx_get_cntl_attributes *mbx_cntl_attr;
5878 	struct lpfc_controller_attribute *cntl_attr;
5879 	void *virtaddr = NULL;
5880 	uint32_t alloclen, reqlen;
5881 	uint32_t shdr_status, shdr_add_status;
5882 	union lpfc_sli4_cfg_shdr *shdr;
5883 	int rc;
5884 
5885 	mboxq = (LPFC_MBOXQ_t *)mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
5886 	if (!mboxq)
5887 		return -ENOMEM;
5888 
5889 	/* Send COMMON_GET_CNTL_ATTRIBUTES mbox cmd */
5890 	reqlen = sizeof(struct lpfc_mbx_get_cntl_attributes);
5891 	alloclen = lpfc_sli4_config(phba, mboxq, LPFC_MBOX_SUBSYSTEM_COMMON,
5892 			LPFC_MBOX_OPCODE_GET_CNTL_ATTRIBUTES, reqlen,
5893 			LPFC_SLI4_MBX_NEMBED);
5894 
5895 	if (alloclen < reqlen) {
5896 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
5897 				"3084 Allocated DMA memory size (%d) is "
5898 				"less than the requested DMA memory size "
5899 				"(%d)\n", alloclen, reqlen);
5900 		rc = -ENOMEM;
5901 		goto out_free_mboxq;
5902 	}
5903 	rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
5904 	virtaddr = mboxq->sge_array->addr[0];
5905 	mbx_cntl_attr = (struct lpfc_mbx_get_cntl_attributes *)virtaddr;
5906 	shdr = &mbx_cntl_attr->cfg_shdr;
5907 	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
5908 	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
5909 	if (shdr_status || shdr_add_status || rc) {
5910 		lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
5911 				"3085 Mailbox x%x (x%x/x%x) failed, "
5912 				"rc:x%x, status:x%x, add_status:x%x\n",
5913 				bf_get(lpfc_mqe_command, &mboxq->u.mqe),
5914 				lpfc_sli_config_mbox_subsys_get(phba, mboxq),
5915 				lpfc_sli_config_mbox_opcode_get(phba, mboxq),
5916 				rc, shdr_status, shdr_add_status);
5917 		rc = -ENXIO;
5918 		goto out_free_mboxq;
5919 	}
5920 
5921 	cntl_attr = &mbx_cntl_attr->cntl_attr;
5922 	phba->sli4_hba.lnk_info.lnk_dv = LPFC_LNK_DAT_VAL;
5923 	phba->sli4_hba.lnk_info.lnk_tp =
5924 		bf_get(lpfc_cntl_attr_lnk_type, cntl_attr);
5925 	phba->sli4_hba.lnk_info.lnk_no =
5926 		bf_get(lpfc_cntl_attr_lnk_numb, cntl_attr);
5927 	phba->sli4_hba.flash_id = bf_get(lpfc_cntl_attr_flash_id, cntl_attr);
5928 	phba->sli4_hba.asic_rev = bf_get(lpfc_cntl_attr_asic_rev, cntl_attr);
5929 
5930 	memset(phba->BIOSVersion, 0, sizeof(phba->BIOSVersion));
5931 	strlcat(phba->BIOSVersion, (char *)cntl_attr->bios_ver_str,
5932 		sizeof(phba->BIOSVersion));
5933 
5934 	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
5935 			"3086 lnk_type:%d, lnk_numb:%d, bios_ver:%s, "
5936 			"flash_id: x%02x, asic_rev: x%02x\n",
5937 			phba->sli4_hba.lnk_info.lnk_tp,
5938 			phba->sli4_hba.lnk_info.lnk_no,
5939 			phba->BIOSVersion, phba->sli4_hba.flash_id,
5940 			phba->sli4_hba.asic_rev);
5941 out_free_mboxq:
5942 	if (bf_get(lpfc_mqe_command, &mboxq->u.mqe) == MBX_SLI4_CONFIG)
5943 		lpfc_sli4_mbox_cmd_free(phba, mboxq);
5944 	else
5945 		mempool_free(mboxq, phba->mbox_mem_pool);
5946 	return rc;
5947 }
5948 
5949 /**
5950  * lpfc_sli4_retrieve_pport_name - Retrieve SLI4 device physical port name
5951  * @phba: pointer to lpfc hba data structure.
5952  *
5953  * This routine retrieves SLI4 device physical port name this PCI function
5954  * is attached to.
5955  *
5956  * Return codes
5957  *      0 - successful
5958  *      otherwise - failed to retrieve physical port name
5959  **/
5960 static int
5961 lpfc_sli4_retrieve_pport_name(struct lpfc_hba *phba)
5962 {
5963 	LPFC_MBOXQ_t *mboxq;
5964 	struct lpfc_mbx_get_port_name *get_port_name;
5965 	uint32_t shdr_status, shdr_add_status;
5966 	union lpfc_sli4_cfg_shdr *shdr;
5967 	char cport_name = 0;
5968 	int rc;
5969 
5970 	/* We assume nothing at this point */
5971 	phba->sli4_hba.lnk_info.lnk_dv = LPFC_LNK_DAT_INVAL;
5972 	phba->sli4_hba.pport_name_sta = LPFC_SLI4_PPNAME_NON;
5973 
5974 	mboxq = (LPFC_MBOXQ_t *)mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
5975 	if (!mboxq)
5976 		return -ENOMEM;
5977 	/* obtain link type and link number via READ_CONFIG */
5978 	phba->sli4_hba.lnk_info.lnk_dv = LPFC_LNK_DAT_INVAL;
5979 	lpfc_sli4_read_config(phba);
5980 	if (phba->sli4_hba.lnk_info.lnk_dv == LPFC_LNK_DAT_VAL)
5981 		goto retrieve_ppname;
5982 
5983 	/* obtain link type and link number via COMMON_GET_CNTL_ATTRIBUTES */
5984 	rc = lpfc_sli4_get_ctl_attr(phba);
5985 	if (rc)
5986 		goto out_free_mboxq;
5987 
5988 retrieve_ppname:
5989 	lpfc_sli4_config(phba, mboxq, LPFC_MBOX_SUBSYSTEM_COMMON,
5990 		LPFC_MBOX_OPCODE_GET_PORT_NAME,
5991 		sizeof(struct lpfc_mbx_get_port_name) -
5992 		sizeof(struct lpfc_sli4_cfg_mhdr),
5993 		LPFC_SLI4_MBX_EMBED);
5994 	get_port_name = &mboxq->u.mqe.un.get_port_name;
5995 	shdr = (union lpfc_sli4_cfg_shdr *)&get_port_name->header.cfg_shdr;
5996 	bf_set(lpfc_mbox_hdr_version, &shdr->request, LPFC_OPCODE_VERSION_1);
5997 	bf_set(lpfc_mbx_get_port_name_lnk_type, &get_port_name->u.request,
5998 		phba->sli4_hba.lnk_info.lnk_tp);
5999 	rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
6000 	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
6001 	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
6002 	if (shdr_status || shdr_add_status || rc) {
6003 		lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
6004 				"3087 Mailbox x%x (x%x/x%x) failed: "
6005 				"rc:x%x, status:x%x, add_status:x%x\n",
6006 				bf_get(lpfc_mqe_command, &mboxq->u.mqe),
6007 				lpfc_sli_config_mbox_subsys_get(phba, mboxq),
6008 				lpfc_sli_config_mbox_opcode_get(phba, mboxq),
6009 				rc, shdr_status, shdr_add_status);
6010 		rc = -ENXIO;
6011 		goto out_free_mboxq;
6012 	}
6013 	switch (phba->sli4_hba.lnk_info.lnk_no) {
6014 	case LPFC_LINK_NUMBER_0:
6015 		cport_name = bf_get(lpfc_mbx_get_port_name_name0,
6016 				&get_port_name->u.response);
6017 		phba->sli4_hba.pport_name_sta = LPFC_SLI4_PPNAME_GET;
6018 		break;
6019 	case LPFC_LINK_NUMBER_1:
6020 		cport_name = bf_get(lpfc_mbx_get_port_name_name1,
6021 				&get_port_name->u.response);
6022 		phba->sli4_hba.pport_name_sta = LPFC_SLI4_PPNAME_GET;
6023 		break;
6024 	case LPFC_LINK_NUMBER_2:
6025 		cport_name = bf_get(lpfc_mbx_get_port_name_name2,
6026 				&get_port_name->u.response);
6027 		phba->sli4_hba.pport_name_sta = LPFC_SLI4_PPNAME_GET;
6028 		break;
6029 	case LPFC_LINK_NUMBER_3:
6030 		cport_name = bf_get(lpfc_mbx_get_port_name_name3,
6031 				&get_port_name->u.response);
6032 		phba->sli4_hba.pport_name_sta = LPFC_SLI4_PPNAME_GET;
6033 		break;
6034 	default:
6035 		break;
6036 	}
6037 
6038 	if (phba->sli4_hba.pport_name_sta == LPFC_SLI4_PPNAME_GET) {
6039 		phba->Port[0] = cport_name;
6040 		phba->Port[1] = '\0';
6041 		lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
6042 				"3091 SLI get port name: %s\n", phba->Port);
6043 	}
6044 
6045 out_free_mboxq:
6046 	if (bf_get(lpfc_mqe_command, &mboxq->u.mqe) == MBX_SLI4_CONFIG)
6047 		lpfc_sli4_mbox_cmd_free(phba, mboxq);
6048 	else
6049 		mempool_free(mboxq, phba->mbox_mem_pool);
6050 	return rc;
6051 }
6052 
6053 /**
6054  * lpfc_sli4_arm_cqeq_intr - Arm sli-4 device completion and event queues
6055  * @phba: pointer to lpfc hba data structure.
6056  *
6057  * This routine is called to explicitly arm the SLI4 device's completion and
6058  * event queues
6059  **/
6060 static void
6061 lpfc_sli4_arm_cqeq_intr(struct lpfc_hba *phba)
6062 {
6063 	int qidx;
6064 	struct lpfc_sli4_hba *sli4_hba = &phba->sli4_hba;
6065 	struct lpfc_sli4_hdw_queue *qp;
6066 	struct lpfc_queue *eq;
6067 
6068 	sli4_hba->sli4_write_cq_db(phba, sli4_hba->mbx_cq, 0, LPFC_QUEUE_REARM);
6069 	sli4_hba->sli4_write_cq_db(phba, sli4_hba->els_cq, 0, LPFC_QUEUE_REARM);
6070 	if (sli4_hba->nvmels_cq)
6071 		sli4_hba->sli4_write_cq_db(phba, sli4_hba->nvmels_cq, 0,
6072 					   LPFC_QUEUE_REARM);
6073 
6074 	if (sli4_hba->hdwq) {
6075 		/* Loop thru all Hardware Queues */
6076 		for (qidx = 0; qidx < phba->cfg_hdw_queue; qidx++) {
6077 			qp = &sli4_hba->hdwq[qidx];
6078 			/* ARM the corresponding CQ */
6079 			sli4_hba->sli4_write_cq_db(phba, qp->io_cq, 0,
6080 						LPFC_QUEUE_REARM);
6081 		}
6082 
6083 		/* Loop thru all IRQ vectors */
6084 		for (qidx = 0; qidx < phba->cfg_irq_chann; qidx++) {
6085 			eq = sli4_hba->hba_eq_hdl[qidx].eq;
6086 			/* ARM the corresponding EQ */
6087 			sli4_hba->sli4_write_eq_db(phba, eq,
6088 						   0, LPFC_QUEUE_REARM);
6089 		}
6090 	}
6091 
6092 	if (phba->nvmet_support) {
6093 		for (qidx = 0; qidx < phba->cfg_nvmet_mrq; qidx++) {
6094 			sli4_hba->sli4_write_cq_db(phba,
6095 				sli4_hba->nvmet_cqset[qidx], 0,
6096 				LPFC_QUEUE_REARM);
6097 		}
6098 	}
6099 }
6100 
6101 /**
6102  * lpfc_sli4_get_avail_extnt_rsrc - Get available resource extent count.
6103  * @phba: Pointer to HBA context object.
6104  * @type: The resource extent type.
6105  * @extnt_count: buffer to hold port available extent count.
6106  * @extnt_size: buffer to hold element count per extent.
6107  *
6108  * This function calls the port and retrievs the number of available
6109  * extents and their size for a particular extent type.
6110  *
6111  * Returns: 0 if successful.  Nonzero otherwise.
6112  **/
6113 int
6114 lpfc_sli4_get_avail_extnt_rsrc(struct lpfc_hba *phba, uint16_t type,
6115 			       uint16_t *extnt_count, uint16_t *extnt_size)
6116 {
6117 	int rc = 0;
6118 	uint32_t length;
6119 	uint32_t mbox_tmo;
6120 	struct lpfc_mbx_get_rsrc_extent_info *rsrc_info;
6121 	LPFC_MBOXQ_t *mbox;
6122 
6123 	mbox = (LPFC_MBOXQ_t *) mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
6124 	if (!mbox)
6125 		return -ENOMEM;
6126 
6127 	/* Find out how many extents are available for this resource type */
6128 	length = (sizeof(struct lpfc_mbx_get_rsrc_extent_info) -
6129 		  sizeof(struct lpfc_sli4_cfg_mhdr));
6130 	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
6131 			 LPFC_MBOX_OPCODE_GET_RSRC_EXTENT_INFO,
6132 			 length, LPFC_SLI4_MBX_EMBED);
6133 
6134 	/* Send an extents count of 0 - the GET doesn't use it. */
6135 	rc = lpfc_sli4_mbox_rsrc_extent(phba, mbox, 0, type,
6136 					LPFC_SLI4_MBX_EMBED);
6137 	if (unlikely(rc)) {
6138 		rc = -EIO;
6139 		goto err_exit;
6140 	}
6141 
6142 	if (!phba->sli4_hba.intr_enable)
6143 		rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL);
6144 	else {
6145 		mbox_tmo = lpfc_mbox_tmo_val(phba, mbox);
6146 		rc = lpfc_sli_issue_mbox_wait(phba, mbox, mbox_tmo);
6147 	}
6148 	if (unlikely(rc)) {
6149 		rc = -EIO;
6150 		goto err_exit;
6151 	}
6152 
6153 	rsrc_info = &mbox->u.mqe.un.rsrc_extent_info;
6154 	if (bf_get(lpfc_mbox_hdr_status,
6155 		   &rsrc_info->header.cfg_shdr.response)) {
6156 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
6157 				"2930 Failed to get resource extents "
6158 				"Status 0x%x Add'l Status 0x%x\n",
6159 				bf_get(lpfc_mbox_hdr_status,
6160 				       &rsrc_info->header.cfg_shdr.response),
6161 				bf_get(lpfc_mbox_hdr_add_status,
6162 				       &rsrc_info->header.cfg_shdr.response));
6163 		rc = -EIO;
6164 		goto err_exit;
6165 	}
6166 
6167 	*extnt_count = bf_get(lpfc_mbx_get_rsrc_extent_info_cnt,
6168 			      &rsrc_info->u.rsp);
6169 	*extnt_size = bf_get(lpfc_mbx_get_rsrc_extent_info_size,
6170 			     &rsrc_info->u.rsp);
6171 
6172 	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
6173 			"3162 Retrieved extents type-%d from port: count:%d, "
6174 			"size:%d\n", type, *extnt_count, *extnt_size);
6175 
6176 err_exit:
6177 	mempool_free(mbox, phba->mbox_mem_pool);
6178 	return rc;
6179 }
6180 
6181 /**
6182  * lpfc_sli4_chk_avail_extnt_rsrc - Check for available SLI4 resource extents.
6183  * @phba: Pointer to HBA context object.
6184  * @type: The extent type to check.
6185  *
6186  * This function reads the current available extents from the port and checks
6187  * if the extent count or extent size has changed since the last access.
6188  * Callers use this routine post port reset to understand if there is a
6189  * extent reprovisioning requirement.
6190  *
6191  * Returns:
6192  *   -Error: error indicates problem.
6193  *   1: Extent count or size has changed.
6194  *   0: No changes.
6195  **/
6196 static int
6197 lpfc_sli4_chk_avail_extnt_rsrc(struct lpfc_hba *phba, uint16_t type)
6198 {
6199 	uint16_t curr_ext_cnt, rsrc_ext_cnt;
6200 	uint16_t size_diff, rsrc_ext_size;
6201 	int rc = 0;
6202 	struct lpfc_rsrc_blks *rsrc_entry;
6203 	struct list_head *rsrc_blk_list = NULL;
6204 
6205 	size_diff = 0;
6206 	curr_ext_cnt = 0;
6207 	rc = lpfc_sli4_get_avail_extnt_rsrc(phba, type,
6208 					    &rsrc_ext_cnt,
6209 					    &rsrc_ext_size);
6210 	if (unlikely(rc))
6211 		return -EIO;
6212 
6213 	switch (type) {
6214 	case LPFC_RSC_TYPE_FCOE_RPI:
6215 		rsrc_blk_list = &phba->sli4_hba.lpfc_rpi_blk_list;
6216 		break;
6217 	case LPFC_RSC_TYPE_FCOE_VPI:
6218 		rsrc_blk_list = &phba->lpfc_vpi_blk_list;
6219 		break;
6220 	case LPFC_RSC_TYPE_FCOE_XRI:
6221 		rsrc_blk_list = &phba->sli4_hba.lpfc_xri_blk_list;
6222 		break;
6223 	case LPFC_RSC_TYPE_FCOE_VFI:
6224 		rsrc_blk_list = &phba->sli4_hba.lpfc_vfi_blk_list;
6225 		break;
6226 	default:
6227 		break;
6228 	}
6229 
6230 	list_for_each_entry(rsrc_entry, rsrc_blk_list, list) {
6231 		curr_ext_cnt++;
6232 		if (rsrc_entry->rsrc_size != rsrc_ext_size)
6233 			size_diff++;
6234 	}
6235 
6236 	if (curr_ext_cnt != rsrc_ext_cnt || size_diff != 0)
6237 		rc = 1;
6238 
6239 	return rc;
6240 }
6241 
6242 /**
6243  * lpfc_sli4_cfg_post_extnts -
6244  * @phba: Pointer to HBA context object.
6245  * @extnt_cnt: number of available extents.
6246  * @type: the extent type (rpi, xri, vfi, vpi).
6247  * @emb: buffer to hold either MBX_EMBED or MBX_NEMBED operation.
6248  * @mbox: pointer to the caller's allocated mailbox structure.
6249  *
6250  * This function executes the extents allocation request.  It also
6251  * takes care of the amount of memory needed to allocate or get the
6252  * allocated extents. It is the caller's responsibility to evaluate
6253  * the response.
6254  *
6255  * Returns:
6256  *   -Error:  Error value describes the condition found.
6257  *   0: if successful
6258  **/
6259 static int
6260 lpfc_sli4_cfg_post_extnts(struct lpfc_hba *phba, uint16_t extnt_cnt,
6261 			  uint16_t type, bool *emb, LPFC_MBOXQ_t *mbox)
6262 {
6263 	int rc = 0;
6264 	uint32_t req_len;
6265 	uint32_t emb_len;
6266 	uint32_t alloc_len, mbox_tmo;
6267 
6268 	/* Calculate the total requested length of the dma memory */
6269 	req_len = extnt_cnt * sizeof(uint16_t);
6270 
6271 	/*
6272 	 * Calculate the size of an embedded mailbox.  The uint32_t
6273 	 * accounts for extents-specific word.
6274 	 */
6275 	emb_len = sizeof(MAILBOX_t) - sizeof(struct mbox_header) -
6276 		sizeof(uint32_t);
6277 
6278 	/*
6279 	 * Presume the allocation and response will fit into an embedded
6280 	 * mailbox.  If not true, reconfigure to a non-embedded mailbox.
6281 	 */
6282 	*emb = LPFC_SLI4_MBX_EMBED;
6283 	if (req_len > emb_len) {
6284 		req_len = extnt_cnt * sizeof(uint16_t) +
6285 			sizeof(union lpfc_sli4_cfg_shdr) +
6286 			sizeof(uint32_t);
6287 		*emb = LPFC_SLI4_MBX_NEMBED;
6288 	}
6289 
6290 	alloc_len = lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
6291 				     LPFC_MBOX_OPCODE_ALLOC_RSRC_EXTENT,
6292 				     req_len, *emb);
6293 	if (alloc_len < req_len) {
6294 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
6295 			"2982 Allocated DMA memory size (x%x) is "
6296 			"less than the requested DMA memory "
6297 			"size (x%x)\n", alloc_len, req_len);
6298 		return -ENOMEM;
6299 	}
6300 	rc = lpfc_sli4_mbox_rsrc_extent(phba, mbox, extnt_cnt, type, *emb);
6301 	if (unlikely(rc))
6302 		return -EIO;
6303 
6304 	if (!phba->sli4_hba.intr_enable)
6305 		rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL);
6306 	else {
6307 		mbox_tmo = lpfc_mbox_tmo_val(phba, mbox);
6308 		rc = lpfc_sli_issue_mbox_wait(phba, mbox, mbox_tmo);
6309 	}
6310 
6311 	if (unlikely(rc))
6312 		rc = -EIO;
6313 	return rc;
6314 }
6315 
6316 /**
6317  * lpfc_sli4_alloc_extent - Allocate an SLI4 resource extent.
6318  * @phba: Pointer to HBA context object.
6319  * @type:  The resource extent type to allocate.
6320  *
6321  * This function allocates the number of elements for the specified
6322  * resource type.
6323  **/
6324 static int
6325 lpfc_sli4_alloc_extent(struct lpfc_hba *phba, uint16_t type)
6326 {
6327 	bool emb = false;
6328 	uint16_t rsrc_id_cnt, rsrc_cnt, rsrc_size;
6329 	uint16_t rsrc_id, rsrc_start, j, k;
6330 	uint16_t *ids;
6331 	int i, rc;
6332 	unsigned long longs;
6333 	unsigned long *bmask;
6334 	struct lpfc_rsrc_blks *rsrc_blks;
6335 	LPFC_MBOXQ_t *mbox;
6336 	uint32_t length;
6337 	struct lpfc_id_range *id_array = NULL;
6338 	void *virtaddr = NULL;
6339 	struct lpfc_mbx_nembed_rsrc_extent *n_rsrc;
6340 	struct lpfc_mbx_alloc_rsrc_extents *rsrc_ext;
6341 	struct list_head *ext_blk_list;
6342 
6343 	rc = lpfc_sli4_get_avail_extnt_rsrc(phba, type,
6344 					    &rsrc_cnt,
6345 					    &rsrc_size);
6346 	if (unlikely(rc))
6347 		return -EIO;
6348 
6349 	if ((rsrc_cnt == 0) || (rsrc_size == 0)) {
6350 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
6351 			"3009 No available Resource Extents "
6352 			"for resource type 0x%x: Count: 0x%x, "
6353 			"Size 0x%x\n", type, rsrc_cnt,
6354 			rsrc_size);
6355 		return -ENOMEM;
6356 	}
6357 
6358 	lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_INIT | LOG_SLI,
6359 			"2903 Post resource extents type-0x%x: "
6360 			"count:%d, size %d\n", type, rsrc_cnt, rsrc_size);
6361 
6362 	mbox = (LPFC_MBOXQ_t *) mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
6363 	if (!mbox)
6364 		return -ENOMEM;
6365 
6366 	rc = lpfc_sli4_cfg_post_extnts(phba, rsrc_cnt, type, &emb, mbox);
6367 	if (unlikely(rc)) {
6368 		rc = -EIO;
6369 		goto err_exit;
6370 	}
6371 
6372 	/*
6373 	 * Figure out where the response is located.  Then get local pointers
6374 	 * to the response data.  The port does not guarantee to respond to
6375 	 * all extents counts request so update the local variable with the
6376 	 * allocated count from the port.
6377 	 */
6378 	if (emb == LPFC_SLI4_MBX_EMBED) {
6379 		rsrc_ext = &mbox->u.mqe.un.alloc_rsrc_extents;
6380 		id_array = &rsrc_ext->u.rsp.id[0];
6381 		rsrc_cnt = bf_get(lpfc_mbx_rsrc_cnt, &rsrc_ext->u.rsp);
6382 	} else {
6383 		virtaddr = mbox->sge_array->addr[0];
6384 		n_rsrc = (struct lpfc_mbx_nembed_rsrc_extent *) virtaddr;
6385 		rsrc_cnt = bf_get(lpfc_mbx_rsrc_cnt, n_rsrc);
6386 		id_array = &n_rsrc->id;
6387 	}
6388 
6389 	longs = ((rsrc_cnt * rsrc_size) + BITS_PER_LONG - 1) / BITS_PER_LONG;
6390 	rsrc_id_cnt = rsrc_cnt * rsrc_size;
6391 
6392 	/*
6393 	 * Based on the resource size and count, correct the base and max
6394 	 * resource values.
6395 	 */
6396 	length = sizeof(struct lpfc_rsrc_blks);
6397 	switch (type) {
6398 	case LPFC_RSC_TYPE_FCOE_RPI:
6399 		phba->sli4_hba.rpi_bmask = kcalloc(longs,
6400 						   sizeof(unsigned long),
6401 						   GFP_KERNEL);
6402 		if (unlikely(!phba->sli4_hba.rpi_bmask)) {
6403 			rc = -ENOMEM;
6404 			goto err_exit;
6405 		}
6406 		phba->sli4_hba.rpi_ids = kcalloc(rsrc_id_cnt,
6407 						 sizeof(uint16_t),
6408 						 GFP_KERNEL);
6409 		if (unlikely(!phba->sli4_hba.rpi_ids)) {
6410 			kfree(phba->sli4_hba.rpi_bmask);
6411 			rc = -ENOMEM;
6412 			goto err_exit;
6413 		}
6414 
6415 		/*
6416 		 * The next_rpi was initialized with the maximum available
6417 		 * count but the port may allocate a smaller number.  Catch
6418 		 * that case and update the next_rpi.
6419 		 */
6420 		phba->sli4_hba.next_rpi = rsrc_id_cnt;
6421 
6422 		/* Initialize local ptrs for common extent processing later. */
6423 		bmask = phba->sli4_hba.rpi_bmask;
6424 		ids = phba->sli4_hba.rpi_ids;
6425 		ext_blk_list = &phba->sli4_hba.lpfc_rpi_blk_list;
6426 		break;
6427 	case LPFC_RSC_TYPE_FCOE_VPI:
6428 		phba->vpi_bmask = kcalloc(longs, sizeof(unsigned long),
6429 					  GFP_KERNEL);
6430 		if (unlikely(!phba->vpi_bmask)) {
6431 			rc = -ENOMEM;
6432 			goto err_exit;
6433 		}
6434 		phba->vpi_ids = kcalloc(rsrc_id_cnt, sizeof(uint16_t),
6435 					 GFP_KERNEL);
6436 		if (unlikely(!phba->vpi_ids)) {
6437 			kfree(phba->vpi_bmask);
6438 			rc = -ENOMEM;
6439 			goto err_exit;
6440 		}
6441 
6442 		/* Initialize local ptrs for common extent processing later. */
6443 		bmask = phba->vpi_bmask;
6444 		ids = phba->vpi_ids;
6445 		ext_blk_list = &phba->lpfc_vpi_blk_list;
6446 		break;
6447 	case LPFC_RSC_TYPE_FCOE_XRI:
6448 		phba->sli4_hba.xri_bmask = kcalloc(longs,
6449 						   sizeof(unsigned long),
6450 						   GFP_KERNEL);
6451 		if (unlikely(!phba->sli4_hba.xri_bmask)) {
6452 			rc = -ENOMEM;
6453 			goto err_exit;
6454 		}
6455 		phba->sli4_hba.max_cfg_param.xri_used = 0;
6456 		phba->sli4_hba.xri_ids = kcalloc(rsrc_id_cnt,
6457 						 sizeof(uint16_t),
6458 						 GFP_KERNEL);
6459 		if (unlikely(!phba->sli4_hba.xri_ids)) {
6460 			kfree(phba->sli4_hba.xri_bmask);
6461 			rc = -ENOMEM;
6462 			goto err_exit;
6463 		}
6464 
6465 		/* Initialize local ptrs for common extent processing later. */
6466 		bmask = phba->sli4_hba.xri_bmask;
6467 		ids = phba->sli4_hba.xri_ids;
6468 		ext_blk_list = &phba->sli4_hba.lpfc_xri_blk_list;
6469 		break;
6470 	case LPFC_RSC_TYPE_FCOE_VFI:
6471 		phba->sli4_hba.vfi_bmask = kcalloc(longs,
6472 						   sizeof(unsigned long),
6473 						   GFP_KERNEL);
6474 		if (unlikely(!phba->sli4_hba.vfi_bmask)) {
6475 			rc = -ENOMEM;
6476 			goto err_exit;
6477 		}
6478 		phba->sli4_hba.vfi_ids = kcalloc(rsrc_id_cnt,
6479 						 sizeof(uint16_t),
6480 						 GFP_KERNEL);
6481 		if (unlikely(!phba->sli4_hba.vfi_ids)) {
6482 			kfree(phba->sli4_hba.vfi_bmask);
6483 			rc = -ENOMEM;
6484 			goto err_exit;
6485 		}
6486 
6487 		/* Initialize local ptrs for common extent processing later. */
6488 		bmask = phba->sli4_hba.vfi_bmask;
6489 		ids = phba->sli4_hba.vfi_ids;
6490 		ext_blk_list = &phba->sli4_hba.lpfc_vfi_blk_list;
6491 		break;
6492 	default:
6493 		/* Unsupported Opcode.  Fail call. */
6494 		id_array = NULL;
6495 		bmask = NULL;
6496 		ids = NULL;
6497 		ext_blk_list = NULL;
6498 		goto err_exit;
6499 	}
6500 
6501 	/*
6502 	 * Complete initializing the extent configuration with the
6503 	 * allocated ids assigned to this function.  The bitmask serves
6504 	 * as an index into the array and manages the available ids.  The
6505 	 * array just stores the ids communicated to the port via the wqes.
6506 	 */
6507 	for (i = 0, j = 0, k = 0; i < rsrc_cnt; i++) {
6508 		if ((i % 2) == 0)
6509 			rsrc_id = bf_get(lpfc_mbx_rsrc_id_word4_0,
6510 					 &id_array[k]);
6511 		else
6512 			rsrc_id = bf_get(lpfc_mbx_rsrc_id_word4_1,
6513 					 &id_array[k]);
6514 
6515 		rsrc_blks = kzalloc(length, GFP_KERNEL);
6516 		if (unlikely(!rsrc_blks)) {
6517 			rc = -ENOMEM;
6518 			kfree(bmask);
6519 			kfree(ids);
6520 			goto err_exit;
6521 		}
6522 		rsrc_blks->rsrc_start = rsrc_id;
6523 		rsrc_blks->rsrc_size = rsrc_size;
6524 		list_add_tail(&rsrc_blks->list, ext_blk_list);
6525 		rsrc_start = rsrc_id;
6526 		if ((type == LPFC_RSC_TYPE_FCOE_XRI) && (j == 0)) {
6527 			phba->sli4_hba.io_xri_start = rsrc_start +
6528 				lpfc_sli4_get_iocb_cnt(phba);
6529 		}
6530 
6531 		while (rsrc_id < (rsrc_start + rsrc_size)) {
6532 			ids[j] = rsrc_id;
6533 			rsrc_id++;
6534 			j++;
6535 		}
6536 		/* Entire word processed.  Get next word.*/
6537 		if ((i % 2) == 1)
6538 			k++;
6539 	}
6540  err_exit:
6541 	lpfc_sli4_mbox_cmd_free(phba, mbox);
6542 	return rc;
6543 }
6544 
6545 
6546 
6547 /**
6548  * lpfc_sli4_dealloc_extent - Deallocate an SLI4 resource extent.
6549  * @phba: Pointer to HBA context object.
6550  * @type: the extent's type.
6551  *
6552  * This function deallocates all extents of a particular resource type.
6553  * SLI4 does not allow for deallocating a particular extent range.  It
6554  * is the caller's responsibility to release all kernel memory resources.
6555  **/
6556 static int
6557 lpfc_sli4_dealloc_extent(struct lpfc_hba *phba, uint16_t type)
6558 {
6559 	int rc;
6560 	uint32_t length, mbox_tmo = 0;
6561 	LPFC_MBOXQ_t *mbox;
6562 	struct lpfc_mbx_dealloc_rsrc_extents *dealloc_rsrc;
6563 	struct lpfc_rsrc_blks *rsrc_blk, *rsrc_blk_next;
6564 
6565 	mbox = (LPFC_MBOXQ_t *) mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
6566 	if (!mbox)
6567 		return -ENOMEM;
6568 
6569 	/*
6570 	 * This function sends an embedded mailbox because it only sends the
6571 	 * the resource type.  All extents of this type are released by the
6572 	 * port.
6573 	 */
6574 	length = (sizeof(struct lpfc_mbx_dealloc_rsrc_extents) -
6575 		  sizeof(struct lpfc_sli4_cfg_mhdr));
6576 	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
6577 			 LPFC_MBOX_OPCODE_DEALLOC_RSRC_EXTENT,
6578 			 length, LPFC_SLI4_MBX_EMBED);
6579 
6580 	/* Send an extents count of 0 - the dealloc doesn't use it. */
6581 	rc = lpfc_sli4_mbox_rsrc_extent(phba, mbox, 0, type,
6582 					LPFC_SLI4_MBX_EMBED);
6583 	if (unlikely(rc)) {
6584 		rc = -EIO;
6585 		goto out_free_mbox;
6586 	}
6587 	if (!phba->sli4_hba.intr_enable)
6588 		rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL);
6589 	else {
6590 		mbox_tmo = lpfc_mbox_tmo_val(phba, mbox);
6591 		rc = lpfc_sli_issue_mbox_wait(phba, mbox, mbox_tmo);
6592 	}
6593 	if (unlikely(rc)) {
6594 		rc = -EIO;
6595 		goto out_free_mbox;
6596 	}
6597 
6598 	dealloc_rsrc = &mbox->u.mqe.un.dealloc_rsrc_extents;
6599 	if (bf_get(lpfc_mbox_hdr_status,
6600 		   &dealloc_rsrc->header.cfg_shdr.response)) {
6601 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
6602 				"2919 Failed to release resource extents "
6603 				"for type %d - Status 0x%x Add'l Status 0x%x. "
6604 				"Resource memory not released.\n",
6605 				type,
6606 				bf_get(lpfc_mbox_hdr_status,
6607 				    &dealloc_rsrc->header.cfg_shdr.response),
6608 				bf_get(lpfc_mbox_hdr_add_status,
6609 				    &dealloc_rsrc->header.cfg_shdr.response));
6610 		rc = -EIO;
6611 		goto out_free_mbox;
6612 	}
6613 
6614 	/* Release kernel memory resources for the specific type. */
6615 	switch (type) {
6616 	case LPFC_RSC_TYPE_FCOE_VPI:
6617 		kfree(phba->vpi_bmask);
6618 		kfree(phba->vpi_ids);
6619 		bf_set(lpfc_vpi_rsrc_rdy, &phba->sli4_hba.sli4_flags, 0);
6620 		list_for_each_entry_safe(rsrc_blk, rsrc_blk_next,
6621 				    &phba->lpfc_vpi_blk_list, list) {
6622 			list_del_init(&rsrc_blk->list);
6623 			kfree(rsrc_blk);
6624 		}
6625 		phba->sli4_hba.max_cfg_param.vpi_used = 0;
6626 		break;
6627 	case LPFC_RSC_TYPE_FCOE_XRI:
6628 		kfree(phba->sli4_hba.xri_bmask);
6629 		kfree(phba->sli4_hba.xri_ids);
6630 		list_for_each_entry_safe(rsrc_blk, rsrc_blk_next,
6631 				    &phba->sli4_hba.lpfc_xri_blk_list, list) {
6632 			list_del_init(&rsrc_blk->list);
6633 			kfree(rsrc_blk);
6634 		}
6635 		break;
6636 	case LPFC_RSC_TYPE_FCOE_VFI:
6637 		kfree(phba->sli4_hba.vfi_bmask);
6638 		kfree(phba->sli4_hba.vfi_ids);
6639 		bf_set(lpfc_vfi_rsrc_rdy, &phba->sli4_hba.sli4_flags, 0);
6640 		list_for_each_entry_safe(rsrc_blk, rsrc_blk_next,
6641 				    &phba->sli4_hba.lpfc_vfi_blk_list, list) {
6642 			list_del_init(&rsrc_blk->list);
6643 			kfree(rsrc_blk);
6644 		}
6645 		break;
6646 	case LPFC_RSC_TYPE_FCOE_RPI:
6647 		/* RPI bitmask and physical id array are cleaned up earlier. */
6648 		list_for_each_entry_safe(rsrc_blk, rsrc_blk_next,
6649 				    &phba->sli4_hba.lpfc_rpi_blk_list, list) {
6650 			list_del_init(&rsrc_blk->list);
6651 			kfree(rsrc_blk);
6652 		}
6653 		break;
6654 	default:
6655 		break;
6656 	}
6657 
6658 	bf_set(lpfc_idx_rsrc_rdy, &phba->sli4_hba.sli4_flags, 0);
6659 
6660  out_free_mbox:
6661 	mempool_free(mbox, phba->mbox_mem_pool);
6662 	return rc;
6663 }
6664 
6665 static void
6666 lpfc_set_features(struct lpfc_hba *phba, LPFC_MBOXQ_t *mbox,
6667 		  uint32_t feature)
6668 {
6669 	uint32_t len;
6670 	u32 sig_freq = 0;
6671 
6672 	len = sizeof(struct lpfc_mbx_set_feature) -
6673 		sizeof(struct lpfc_sli4_cfg_mhdr);
6674 	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
6675 			 LPFC_MBOX_OPCODE_SET_FEATURES, len,
6676 			 LPFC_SLI4_MBX_EMBED);
6677 
6678 	switch (feature) {
6679 	case LPFC_SET_UE_RECOVERY:
6680 		bf_set(lpfc_mbx_set_feature_UER,
6681 		       &mbox->u.mqe.un.set_feature, 1);
6682 		mbox->u.mqe.un.set_feature.feature = LPFC_SET_UE_RECOVERY;
6683 		mbox->u.mqe.un.set_feature.param_len = 8;
6684 		break;
6685 	case LPFC_SET_MDS_DIAGS:
6686 		bf_set(lpfc_mbx_set_feature_mds,
6687 		       &mbox->u.mqe.un.set_feature, 1);
6688 		bf_set(lpfc_mbx_set_feature_mds_deep_loopbk,
6689 		       &mbox->u.mqe.un.set_feature, 1);
6690 		mbox->u.mqe.un.set_feature.feature = LPFC_SET_MDS_DIAGS;
6691 		mbox->u.mqe.un.set_feature.param_len = 8;
6692 		break;
6693 	case LPFC_SET_CGN_SIGNAL:
6694 		if (phba->cmf_active_mode == LPFC_CFG_OFF)
6695 			sig_freq = 0;
6696 		else
6697 			sig_freq = phba->cgn_sig_freq;
6698 
6699 		if (phba->cgn_reg_signal == EDC_CG_SIG_WARN_ALARM) {
6700 			bf_set(lpfc_mbx_set_feature_CGN_alarm_freq,
6701 			       &mbox->u.mqe.un.set_feature, sig_freq);
6702 			bf_set(lpfc_mbx_set_feature_CGN_warn_freq,
6703 			       &mbox->u.mqe.un.set_feature, sig_freq);
6704 		}
6705 
6706 		if (phba->cgn_reg_signal == EDC_CG_SIG_WARN_ONLY)
6707 			bf_set(lpfc_mbx_set_feature_CGN_warn_freq,
6708 			       &mbox->u.mqe.un.set_feature, sig_freq);
6709 
6710 		if (phba->cmf_active_mode == LPFC_CFG_OFF ||
6711 		    phba->cgn_reg_signal == EDC_CG_SIG_NOTSUPPORTED)
6712 			sig_freq = 0;
6713 		else
6714 			sig_freq = lpfc_acqe_cgn_frequency;
6715 
6716 		bf_set(lpfc_mbx_set_feature_CGN_acqe_freq,
6717 		       &mbox->u.mqe.un.set_feature, sig_freq);
6718 
6719 		mbox->u.mqe.un.set_feature.feature = LPFC_SET_CGN_SIGNAL;
6720 		mbox->u.mqe.un.set_feature.param_len = 12;
6721 		break;
6722 	case LPFC_SET_DUAL_DUMP:
6723 		bf_set(lpfc_mbx_set_feature_dd,
6724 		       &mbox->u.mqe.un.set_feature, LPFC_ENABLE_DUAL_DUMP);
6725 		bf_set(lpfc_mbx_set_feature_ddquery,
6726 		       &mbox->u.mqe.un.set_feature, 0);
6727 		mbox->u.mqe.un.set_feature.feature = LPFC_SET_DUAL_DUMP;
6728 		mbox->u.mqe.un.set_feature.param_len = 4;
6729 		break;
6730 	case LPFC_SET_ENABLE_MI:
6731 		mbox->u.mqe.un.set_feature.feature = LPFC_SET_ENABLE_MI;
6732 		mbox->u.mqe.un.set_feature.param_len = 4;
6733 		bf_set(lpfc_mbx_set_feature_milunq, &mbox->u.mqe.un.set_feature,
6734 		       phba->pport->cfg_lun_queue_depth);
6735 		bf_set(lpfc_mbx_set_feature_mi, &mbox->u.mqe.un.set_feature,
6736 		       phba->sli4_hba.pc_sli4_params.mi_ver);
6737 		break;
6738 	case LPFC_SET_ENABLE_CMF:
6739 		bf_set(lpfc_mbx_set_feature_dd, &mbox->u.mqe.un.set_feature, 1);
6740 		mbox->u.mqe.un.set_feature.feature = LPFC_SET_ENABLE_CMF;
6741 		mbox->u.mqe.un.set_feature.param_len = 4;
6742 		bf_set(lpfc_mbx_set_feature_cmf,
6743 		       &mbox->u.mqe.un.set_feature, 1);
6744 		break;
6745 	}
6746 	return;
6747 }
6748 
6749 /**
6750  * lpfc_ras_stop_fwlog: Disable FW logging by the adapter
6751  * @phba: Pointer to HBA context object.
6752  *
6753  * Disable FW logging into host memory on the adapter. To
6754  * be done before reading logs from the host memory.
6755  **/
6756 void
6757 lpfc_ras_stop_fwlog(struct lpfc_hba *phba)
6758 {
6759 	struct lpfc_ras_fwlog *ras_fwlog = &phba->ras_fwlog;
6760 
6761 	spin_lock_irq(&phba->hbalock);
6762 	ras_fwlog->state = INACTIVE;
6763 	spin_unlock_irq(&phba->hbalock);
6764 
6765 	/* Disable FW logging to host memory */
6766 	writel(LPFC_CTL_PDEV_CTL_DDL_RAS,
6767 	       phba->sli4_hba.conf_regs_memmap_p + LPFC_CTL_PDEV_CTL_OFFSET);
6768 
6769 	/* Wait 10ms for firmware to stop using DMA buffer */
6770 	usleep_range(10 * 1000, 20 * 1000);
6771 }
6772 
6773 /**
6774  * lpfc_sli4_ras_dma_free - Free memory allocated for FW logging.
6775  * @phba: Pointer to HBA context object.
6776  *
6777  * This function is called to free memory allocated for RAS FW logging
6778  * support in the driver.
6779  **/
6780 void
6781 lpfc_sli4_ras_dma_free(struct lpfc_hba *phba)
6782 {
6783 	struct lpfc_ras_fwlog *ras_fwlog = &phba->ras_fwlog;
6784 	struct lpfc_dmabuf *dmabuf, *next;
6785 
6786 	if (!list_empty(&ras_fwlog->fwlog_buff_list)) {
6787 		list_for_each_entry_safe(dmabuf, next,
6788 				    &ras_fwlog->fwlog_buff_list,
6789 				    list) {
6790 			list_del(&dmabuf->list);
6791 			dma_free_coherent(&phba->pcidev->dev,
6792 					  LPFC_RAS_MAX_ENTRY_SIZE,
6793 					  dmabuf->virt, dmabuf->phys);
6794 			kfree(dmabuf);
6795 		}
6796 	}
6797 
6798 	if (ras_fwlog->lwpd.virt) {
6799 		dma_free_coherent(&phba->pcidev->dev,
6800 				  sizeof(uint32_t) * 2,
6801 				  ras_fwlog->lwpd.virt,
6802 				  ras_fwlog->lwpd.phys);
6803 		ras_fwlog->lwpd.virt = NULL;
6804 	}
6805 
6806 	spin_lock_irq(&phba->hbalock);
6807 	ras_fwlog->state = INACTIVE;
6808 	spin_unlock_irq(&phba->hbalock);
6809 }
6810 
6811 /**
6812  * lpfc_sli4_ras_dma_alloc: Allocate memory for FW support
6813  * @phba: Pointer to HBA context object.
6814  * @fwlog_buff_count: Count of buffers to be created.
6815  *
6816  * This routine DMA memory for Log Write Position Data[LPWD] and buffer
6817  * to update FW log is posted to the adapter.
6818  * Buffer count is calculated based on module param ras_fwlog_buffsize
6819  * Size of each buffer posted to FW is 64K.
6820  **/
6821 
6822 static int
6823 lpfc_sli4_ras_dma_alloc(struct lpfc_hba *phba,
6824 			uint32_t fwlog_buff_count)
6825 {
6826 	struct lpfc_ras_fwlog *ras_fwlog = &phba->ras_fwlog;
6827 	struct lpfc_dmabuf *dmabuf;
6828 	int rc = 0, i = 0;
6829 
6830 	/* Initialize List */
6831 	INIT_LIST_HEAD(&ras_fwlog->fwlog_buff_list);
6832 
6833 	/* Allocate memory for the LWPD */
6834 	ras_fwlog->lwpd.virt = dma_alloc_coherent(&phba->pcidev->dev,
6835 					    sizeof(uint32_t) * 2,
6836 					    &ras_fwlog->lwpd.phys,
6837 					    GFP_KERNEL);
6838 	if (!ras_fwlog->lwpd.virt) {
6839 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
6840 				"6185 LWPD Memory Alloc Failed\n");
6841 
6842 		return -ENOMEM;
6843 	}
6844 
6845 	ras_fwlog->fw_buffcount = fwlog_buff_count;
6846 	for (i = 0; i < ras_fwlog->fw_buffcount; i++) {
6847 		dmabuf = kzalloc(sizeof(struct lpfc_dmabuf),
6848 				 GFP_KERNEL);
6849 		if (!dmabuf) {
6850 			rc = -ENOMEM;
6851 			lpfc_printf_log(phba, KERN_WARNING, LOG_INIT,
6852 					"6186 Memory Alloc failed FW logging");
6853 			goto free_mem;
6854 		}
6855 
6856 		dmabuf->virt = dma_alloc_coherent(&phba->pcidev->dev,
6857 						  LPFC_RAS_MAX_ENTRY_SIZE,
6858 						  &dmabuf->phys, GFP_KERNEL);
6859 		if (!dmabuf->virt) {
6860 			kfree(dmabuf);
6861 			rc = -ENOMEM;
6862 			lpfc_printf_log(phba, KERN_WARNING, LOG_INIT,
6863 					"6187 DMA Alloc Failed FW logging");
6864 			goto free_mem;
6865 		}
6866 		dmabuf->buffer_tag = i;
6867 		list_add_tail(&dmabuf->list, &ras_fwlog->fwlog_buff_list);
6868 	}
6869 
6870 free_mem:
6871 	if (rc)
6872 		lpfc_sli4_ras_dma_free(phba);
6873 
6874 	return rc;
6875 }
6876 
6877 /**
6878  * lpfc_sli4_ras_mbox_cmpl: Completion handler for RAS MBX command
6879  * @phba: pointer to lpfc hba data structure.
6880  * @pmb: pointer to the driver internal queue element for mailbox command.
6881  *
6882  * Completion handler for driver's RAS MBX command to the device.
6883  **/
6884 static void
6885 lpfc_sli4_ras_mbox_cmpl(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb)
6886 {
6887 	MAILBOX_t *mb;
6888 	union lpfc_sli4_cfg_shdr *shdr;
6889 	uint32_t shdr_status, shdr_add_status;
6890 	struct lpfc_ras_fwlog *ras_fwlog = &phba->ras_fwlog;
6891 
6892 	mb = &pmb->u.mb;
6893 
6894 	shdr = (union lpfc_sli4_cfg_shdr *)
6895 		&pmb->u.mqe.un.ras_fwlog.header.cfg_shdr;
6896 	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
6897 	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
6898 
6899 	if (mb->mbxStatus != MBX_SUCCESS || shdr_status) {
6900 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
6901 				"6188 FW LOG mailbox "
6902 				"completed with status x%x add_status x%x,"
6903 				" mbx status x%x\n",
6904 				shdr_status, shdr_add_status, mb->mbxStatus);
6905 
6906 		ras_fwlog->ras_hwsupport = false;
6907 		goto disable_ras;
6908 	}
6909 
6910 	spin_lock_irq(&phba->hbalock);
6911 	ras_fwlog->state = ACTIVE;
6912 	spin_unlock_irq(&phba->hbalock);
6913 	mempool_free(pmb, phba->mbox_mem_pool);
6914 
6915 	return;
6916 
6917 disable_ras:
6918 	/* Free RAS DMA memory */
6919 	lpfc_sli4_ras_dma_free(phba);
6920 	mempool_free(pmb, phba->mbox_mem_pool);
6921 }
6922 
6923 /**
6924  * lpfc_sli4_ras_fwlog_init: Initialize memory and post RAS MBX command
6925  * @phba: pointer to lpfc hba data structure.
6926  * @fwlog_level: Logging verbosity level.
6927  * @fwlog_enable: Enable/Disable logging.
6928  *
6929  * Initialize memory and post mailbox command to enable FW logging in host
6930  * memory.
6931  **/
6932 int
6933 lpfc_sli4_ras_fwlog_init(struct lpfc_hba *phba,
6934 			 uint32_t fwlog_level,
6935 			 uint32_t fwlog_enable)
6936 {
6937 	struct lpfc_ras_fwlog *ras_fwlog = &phba->ras_fwlog;
6938 	struct lpfc_mbx_set_ras_fwlog *mbx_fwlog = NULL;
6939 	struct lpfc_dmabuf *dmabuf;
6940 	LPFC_MBOXQ_t *mbox;
6941 	uint32_t len = 0, fwlog_buffsize, fwlog_entry_count;
6942 	int rc = 0;
6943 
6944 	spin_lock_irq(&phba->hbalock);
6945 	ras_fwlog->state = INACTIVE;
6946 	spin_unlock_irq(&phba->hbalock);
6947 
6948 	fwlog_buffsize = (LPFC_RAS_MIN_BUFF_POST_SIZE *
6949 			  phba->cfg_ras_fwlog_buffsize);
6950 	fwlog_entry_count = (fwlog_buffsize/LPFC_RAS_MAX_ENTRY_SIZE);
6951 
6952 	/*
6953 	 * If re-enabling FW logging support use earlier allocated
6954 	 * DMA buffers while posting MBX command.
6955 	 **/
6956 	if (!ras_fwlog->lwpd.virt) {
6957 		rc = lpfc_sli4_ras_dma_alloc(phba, fwlog_entry_count);
6958 		if (rc) {
6959 			lpfc_printf_log(phba, KERN_WARNING, LOG_INIT,
6960 					"6189 FW Log Memory Allocation Failed");
6961 			return rc;
6962 		}
6963 	}
6964 
6965 	/* Setup Mailbox command */
6966 	mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
6967 	if (!mbox) {
6968 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
6969 				"6190 RAS MBX Alloc Failed");
6970 		rc = -ENOMEM;
6971 		goto mem_free;
6972 	}
6973 
6974 	ras_fwlog->fw_loglevel = fwlog_level;
6975 	len = (sizeof(struct lpfc_mbx_set_ras_fwlog) -
6976 		sizeof(struct lpfc_sli4_cfg_mhdr));
6977 
6978 	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_LOWLEVEL,
6979 			 LPFC_MBOX_OPCODE_SET_DIAG_LOG_OPTION,
6980 			 len, LPFC_SLI4_MBX_EMBED);
6981 
6982 	mbx_fwlog = (struct lpfc_mbx_set_ras_fwlog *)&mbox->u.mqe.un.ras_fwlog;
6983 	bf_set(lpfc_fwlog_enable, &mbx_fwlog->u.request,
6984 	       fwlog_enable);
6985 	bf_set(lpfc_fwlog_loglvl, &mbx_fwlog->u.request,
6986 	       ras_fwlog->fw_loglevel);
6987 	bf_set(lpfc_fwlog_buffcnt, &mbx_fwlog->u.request,
6988 	       ras_fwlog->fw_buffcount);
6989 	bf_set(lpfc_fwlog_buffsz, &mbx_fwlog->u.request,
6990 	       LPFC_RAS_MAX_ENTRY_SIZE/SLI4_PAGE_SIZE);
6991 
6992 	/* Update DMA buffer address */
6993 	list_for_each_entry(dmabuf, &ras_fwlog->fwlog_buff_list, list) {
6994 		memset(dmabuf->virt, 0, LPFC_RAS_MAX_ENTRY_SIZE);
6995 
6996 		mbx_fwlog->u.request.buff_fwlog[dmabuf->buffer_tag].addr_lo =
6997 			putPaddrLow(dmabuf->phys);
6998 
6999 		mbx_fwlog->u.request.buff_fwlog[dmabuf->buffer_tag].addr_hi =
7000 			putPaddrHigh(dmabuf->phys);
7001 	}
7002 
7003 	/* Update LPWD address */
7004 	mbx_fwlog->u.request.lwpd.addr_lo = putPaddrLow(ras_fwlog->lwpd.phys);
7005 	mbx_fwlog->u.request.lwpd.addr_hi = putPaddrHigh(ras_fwlog->lwpd.phys);
7006 
7007 	spin_lock_irq(&phba->hbalock);
7008 	ras_fwlog->state = REG_INPROGRESS;
7009 	spin_unlock_irq(&phba->hbalock);
7010 	mbox->vport = phba->pport;
7011 	mbox->mbox_cmpl = lpfc_sli4_ras_mbox_cmpl;
7012 
7013 	rc = lpfc_sli_issue_mbox(phba, mbox, MBX_NOWAIT);
7014 
7015 	if (rc == MBX_NOT_FINISHED) {
7016 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
7017 				"6191 FW-Log Mailbox failed. "
7018 				"status %d mbxStatus : x%x", rc,
7019 				bf_get(lpfc_mqe_status, &mbox->u.mqe));
7020 		mempool_free(mbox, phba->mbox_mem_pool);
7021 		rc = -EIO;
7022 		goto mem_free;
7023 	} else
7024 		rc = 0;
7025 mem_free:
7026 	if (rc)
7027 		lpfc_sli4_ras_dma_free(phba);
7028 
7029 	return rc;
7030 }
7031 
7032 /**
7033  * lpfc_sli4_ras_setup - Check if RAS supported on the adapter
7034  * @phba: Pointer to HBA context object.
7035  *
7036  * Check if RAS is supported on the adapter and initialize it.
7037  **/
7038 void
7039 lpfc_sli4_ras_setup(struct lpfc_hba *phba)
7040 {
7041 	/* Check RAS FW Log needs to be enabled or not */
7042 	if (lpfc_check_fwlog_support(phba))
7043 		return;
7044 
7045 	lpfc_sli4_ras_fwlog_init(phba, phba->cfg_ras_fwlog_level,
7046 				 LPFC_RAS_ENABLE_LOGGING);
7047 }
7048 
7049 /**
7050  * lpfc_sli4_alloc_resource_identifiers - Allocate all SLI4 resource extents.
7051  * @phba: Pointer to HBA context object.
7052  *
7053  * This function allocates all SLI4 resource identifiers.
7054  **/
7055 int
7056 lpfc_sli4_alloc_resource_identifiers(struct lpfc_hba *phba)
7057 {
7058 	int i, rc, error = 0;
7059 	uint16_t count, base;
7060 	unsigned long longs;
7061 
7062 	if (!phba->sli4_hba.rpi_hdrs_in_use)
7063 		phba->sli4_hba.next_rpi = phba->sli4_hba.max_cfg_param.max_rpi;
7064 	if (phba->sli4_hba.extents_in_use) {
7065 		/*
7066 		 * The port supports resource extents. The XRI, VPI, VFI, RPI
7067 		 * resource extent count must be read and allocated before
7068 		 * provisioning the resource id arrays.
7069 		 */
7070 		if (bf_get(lpfc_idx_rsrc_rdy, &phba->sli4_hba.sli4_flags) ==
7071 		    LPFC_IDX_RSRC_RDY) {
7072 			/*
7073 			 * Extent-based resources are set - the driver could
7074 			 * be in a port reset. Figure out if any corrective
7075 			 * actions need to be taken.
7076 			 */
7077 			rc = lpfc_sli4_chk_avail_extnt_rsrc(phba,
7078 						 LPFC_RSC_TYPE_FCOE_VFI);
7079 			if (rc != 0)
7080 				error++;
7081 			rc = lpfc_sli4_chk_avail_extnt_rsrc(phba,
7082 						 LPFC_RSC_TYPE_FCOE_VPI);
7083 			if (rc != 0)
7084 				error++;
7085 			rc = lpfc_sli4_chk_avail_extnt_rsrc(phba,
7086 						 LPFC_RSC_TYPE_FCOE_XRI);
7087 			if (rc != 0)
7088 				error++;
7089 			rc = lpfc_sli4_chk_avail_extnt_rsrc(phba,
7090 						 LPFC_RSC_TYPE_FCOE_RPI);
7091 			if (rc != 0)
7092 				error++;
7093 
7094 			/*
7095 			 * It's possible that the number of resources
7096 			 * provided to this port instance changed between
7097 			 * resets.  Detect this condition and reallocate
7098 			 * resources.  Otherwise, there is no action.
7099 			 */
7100 			if (error) {
7101 				lpfc_printf_log(phba, KERN_INFO,
7102 						LOG_MBOX | LOG_INIT,
7103 						"2931 Detected extent resource "
7104 						"change.  Reallocating all "
7105 						"extents.\n");
7106 				rc = lpfc_sli4_dealloc_extent(phba,
7107 						 LPFC_RSC_TYPE_FCOE_VFI);
7108 				rc = lpfc_sli4_dealloc_extent(phba,
7109 						 LPFC_RSC_TYPE_FCOE_VPI);
7110 				rc = lpfc_sli4_dealloc_extent(phba,
7111 						 LPFC_RSC_TYPE_FCOE_XRI);
7112 				rc = lpfc_sli4_dealloc_extent(phba,
7113 						 LPFC_RSC_TYPE_FCOE_RPI);
7114 			} else
7115 				return 0;
7116 		}
7117 
7118 		rc = lpfc_sli4_alloc_extent(phba, LPFC_RSC_TYPE_FCOE_VFI);
7119 		if (unlikely(rc))
7120 			goto err_exit;
7121 
7122 		rc = lpfc_sli4_alloc_extent(phba, LPFC_RSC_TYPE_FCOE_VPI);
7123 		if (unlikely(rc))
7124 			goto err_exit;
7125 
7126 		rc = lpfc_sli4_alloc_extent(phba, LPFC_RSC_TYPE_FCOE_RPI);
7127 		if (unlikely(rc))
7128 			goto err_exit;
7129 
7130 		rc = lpfc_sli4_alloc_extent(phba, LPFC_RSC_TYPE_FCOE_XRI);
7131 		if (unlikely(rc))
7132 			goto err_exit;
7133 		bf_set(lpfc_idx_rsrc_rdy, &phba->sli4_hba.sli4_flags,
7134 		       LPFC_IDX_RSRC_RDY);
7135 		return rc;
7136 	} else {
7137 		/*
7138 		 * The port does not support resource extents.  The XRI, VPI,
7139 		 * VFI, RPI resource ids were determined from READ_CONFIG.
7140 		 * Just allocate the bitmasks and provision the resource id
7141 		 * arrays.  If a port reset is active, the resources don't
7142 		 * need any action - just exit.
7143 		 */
7144 		if (bf_get(lpfc_idx_rsrc_rdy, &phba->sli4_hba.sli4_flags) ==
7145 		    LPFC_IDX_RSRC_RDY) {
7146 			lpfc_sli4_dealloc_resource_identifiers(phba);
7147 			lpfc_sli4_remove_rpis(phba);
7148 		}
7149 		/* RPIs. */
7150 		count = phba->sli4_hba.max_cfg_param.max_rpi;
7151 		if (count <= 0) {
7152 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
7153 					"3279 Invalid provisioning of "
7154 					"rpi:%d\n", count);
7155 			rc = -EINVAL;
7156 			goto err_exit;
7157 		}
7158 		base = phba->sli4_hba.max_cfg_param.rpi_base;
7159 		longs = (count + BITS_PER_LONG - 1) / BITS_PER_LONG;
7160 		phba->sli4_hba.rpi_bmask = kcalloc(longs,
7161 						   sizeof(unsigned long),
7162 						   GFP_KERNEL);
7163 		if (unlikely(!phba->sli4_hba.rpi_bmask)) {
7164 			rc = -ENOMEM;
7165 			goto err_exit;
7166 		}
7167 		phba->sli4_hba.rpi_ids = kcalloc(count, sizeof(uint16_t),
7168 						 GFP_KERNEL);
7169 		if (unlikely(!phba->sli4_hba.rpi_ids)) {
7170 			rc = -ENOMEM;
7171 			goto free_rpi_bmask;
7172 		}
7173 
7174 		for (i = 0; i < count; i++)
7175 			phba->sli4_hba.rpi_ids[i] = base + i;
7176 
7177 		/* VPIs. */
7178 		count = phba->sli4_hba.max_cfg_param.max_vpi;
7179 		if (count <= 0) {
7180 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
7181 					"3280 Invalid provisioning of "
7182 					"vpi:%d\n", count);
7183 			rc = -EINVAL;
7184 			goto free_rpi_ids;
7185 		}
7186 		base = phba->sli4_hba.max_cfg_param.vpi_base;
7187 		longs = (count + BITS_PER_LONG - 1) / BITS_PER_LONG;
7188 		phba->vpi_bmask = kcalloc(longs, sizeof(unsigned long),
7189 					  GFP_KERNEL);
7190 		if (unlikely(!phba->vpi_bmask)) {
7191 			rc = -ENOMEM;
7192 			goto free_rpi_ids;
7193 		}
7194 		phba->vpi_ids = kcalloc(count, sizeof(uint16_t),
7195 					GFP_KERNEL);
7196 		if (unlikely(!phba->vpi_ids)) {
7197 			rc = -ENOMEM;
7198 			goto free_vpi_bmask;
7199 		}
7200 
7201 		for (i = 0; i < count; i++)
7202 			phba->vpi_ids[i] = base + i;
7203 
7204 		/* XRIs. */
7205 		count = phba->sli4_hba.max_cfg_param.max_xri;
7206 		if (count <= 0) {
7207 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
7208 					"3281 Invalid provisioning of "
7209 					"xri:%d\n", count);
7210 			rc = -EINVAL;
7211 			goto free_vpi_ids;
7212 		}
7213 		base = phba->sli4_hba.max_cfg_param.xri_base;
7214 		longs = (count + BITS_PER_LONG - 1) / BITS_PER_LONG;
7215 		phba->sli4_hba.xri_bmask = kcalloc(longs,
7216 						   sizeof(unsigned long),
7217 						   GFP_KERNEL);
7218 		if (unlikely(!phba->sli4_hba.xri_bmask)) {
7219 			rc = -ENOMEM;
7220 			goto free_vpi_ids;
7221 		}
7222 		phba->sli4_hba.max_cfg_param.xri_used = 0;
7223 		phba->sli4_hba.xri_ids = kcalloc(count, sizeof(uint16_t),
7224 						 GFP_KERNEL);
7225 		if (unlikely(!phba->sli4_hba.xri_ids)) {
7226 			rc = -ENOMEM;
7227 			goto free_xri_bmask;
7228 		}
7229 
7230 		for (i = 0; i < count; i++)
7231 			phba->sli4_hba.xri_ids[i] = base + i;
7232 
7233 		/* VFIs. */
7234 		count = phba->sli4_hba.max_cfg_param.max_vfi;
7235 		if (count <= 0) {
7236 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
7237 					"3282 Invalid provisioning of "
7238 					"vfi:%d\n", count);
7239 			rc = -EINVAL;
7240 			goto free_xri_ids;
7241 		}
7242 		base = phba->sli4_hba.max_cfg_param.vfi_base;
7243 		longs = (count + BITS_PER_LONG - 1) / BITS_PER_LONG;
7244 		phba->sli4_hba.vfi_bmask = kcalloc(longs,
7245 						   sizeof(unsigned long),
7246 						   GFP_KERNEL);
7247 		if (unlikely(!phba->sli4_hba.vfi_bmask)) {
7248 			rc = -ENOMEM;
7249 			goto free_xri_ids;
7250 		}
7251 		phba->sli4_hba.vfi_ids = kcalloc(count, sizeof(uint16_t),
7252 						 GFP_KERNEL);
7253 		if (unlikely(!phba->sli4_hba.vfi_ids)) {
7254 			rc = -ENOMEM;
7255 			goto free_vfi_bmask;
7256 		}
7257 
7258 		for (i = 0; i < count; i++)
7259 			phba->sli4_hba.vfi_ids[i] = base + i;
7260 
7261 		/*
7262 		 * Mark all resources ready.  An HBA reset doesn't need
7263 		 * to reset the initialization.
7264 		 */
7265 		bf_set(lpfc_idx_rsrc_rdy, &phba->sli4_hba.sli4_flags,
7266 		       LPFC_IDX_RSRC_RDY);
7267 		return 0;
7268 	}
7269 
7270  free_vfi_bmask:
7271 	kfree(phba->sli4_hba.vfi_bmask);
7272 	phba->sli4_hba.vfi_bmask = NULL;
7273  free_xri_ids:
7274 	kfree(phba->sli4_hba.xri_ids);
7275 	phba->sli4_hba.xri_ids = NULL;
7276  free_xri_bmask:
7277 	kfree(phba->sli4_hba.xri_bmask);
7278 	phba->sli4_hba.xri_bmask = NULL;
7279  free_vpi_ids:
7280 	kfree(phba->vpi_ids);
7281 	phba->vpi_ids = NULL;
7282  free_vpi_bmask:
7283 	kfree(phba->vpi_bmask);
7284 	phba->vpi_bmask = NULL;
7285  free_rpi_ids:
7286 	kfree(phba->sli4_hba.rpi_ids);
7287 	phba->sli4_hba.rpi_ids = NULL;
7288  free_rpi_bmask:
7289 	kfree(phba->sli4_hba.rpi_bmask);
7290 	phba->sli4_hba.rpi_bmask = NULL;
7291  err_exit:
7292 	return rc;
7293 }
7294 
7295 /**
7296  * lpfc_sli4_dealloc_resource_identifiers - Deallocate all SLI4 resource extents.
7297  * @phba: Pointer to HBA context object.
7298  *
7299  * This function allocates the number of elements for the specified
7300  * resource type.
7301  **/
7302 int
7303 lpfc_sli4_dealloc_resource_identifiers(struct lpfc_hba *phba)
7304 {
7305 	if (phba->sli4_hba.extents_in_use) {
7306 		lpfc_sli4_dealloc_extent(phba, LPFC_RSC_TYPE_FCOE_VPI);
7307 		lpfc_sli4_dealloc_extent(phba, LPFC_RSC_TYPE_FCOE_RPI);
7308 		lpfc_sli4_dealloc_extent(phba, LPFC_RSC_TYPE_FCOE_XRI);
7309 		lpfc_sli4_dealloc_extent(phba, LPFC_RSC_TYPE_FCOE_VFI);
7310 	} else {
7311 		kfree(phba->vpi_bmask);
7312 		phba->sli4_hba.max_cfg_param.vpi_used = 0;
7313 		kfree(phba->vpi_ids);
7314 		bf_set(lpfc_vpi_rsrc_rdy, &phba->sli4_hba.sli4_flags, 0);
7315 		kfree(phba->sli4_hba.xri_bmask);
7316 		kfree(phba->sli4_hba.xri_ids);
7317 		kfree(phba->sli4_hba.vfi_bmask);
7318 		kfree(phba->sli4_hba.vfi_ids);
7319 		bf_set(lpfc_vfi_rsrc_rdy, &phba->sli4_hba.sli4_flags, 0);
7320 		bf_set(lpfc_idx_rsrc_rdy, &phba->sli4_hba.sli4_flags, 0);
7321 	}
7322 
7323 	return 0;
7324 }
7325 
7326 /**
7327  * lpfc_sli4_get_allocated_extnts - Get the port's allocated extents.
7328  * @phba: Pointer to HBA context object.
7329  * @type: The resource extent type.
7330  * @extnt_cnt: buffer to hold port extent count response
7331  * @extnt_size: buffer to hold port extent size response.
7332  *
7333  * This function calls the port to read the host allocated extents
7334  * for a particular type.
7335  **/
7336 int
7337 lpfc_sli4_get_allocated_extnts(struct lpfc_hba *phba, uint16_t type,
7338 			       uint16_t *extnt_cnt, uint16_t *extnt_size)
7339 {
7340 	bool emb;
7341 	int rc = 0;
7342 	uint16_t curr_blks = 0;
7343 	uint32_t req_len, emb_len;
7344 	uint32_t alloc_len, mbox_tmo;
7345 	struct list_head *blk_list_head;
7346 	struct lpfc_rsrc_blks *rsrc_blk;
7347 	LPFC_MBOXQ_t *mbox;
7348 	void *virtaddr = NULL;
7349 	struct lpfc_mbx_nembed_rsrc_extent *n_rsrc;
7350 	struct lpfc_mbx_alloc_rsrc_extents *rsrc_ext;
7351 	union  lpfc_sli4_cfg_shdr *shdr;
7352 
7353 	switch (type) {
7354 	case LPFC_RSC_TYPE_FCOE_VPI:
7355 		blk_list_head = &phba->lpfc_vpi_blk_list;
7356 		break;
7357 	case LPFC_RSC_TYPE_FCOE_XRI:
7358 		blk_list_head = &phba->sli4_hba.lpfc_xri_blk_list;
7359 		break;
7360 	case LPFC_RSC_TYPE_FCOE_VFI:
7361 		blk_list_head = &phba->sli4_hba.lpfc_vfi_blk_list;
7362 		break;
7363 	case LPFC_RSC_TYPE_FCOE_RPI:
7364 		blk_list_head = &phba->sli4_hba.lpfc_rpi_blk_list;
7365 		break;
7366 	default:
7367 		return -EIO;
7368 	}
7369 
7370 	/* Count the number of extents currently allocatd for this type. */
7371 	list_for_each_entry(rsrc_blk, blk_list_head, list) {
7372 		if (curr_blks == 0) {
7373 			/*
7374 			 * The GET_ALLOCATED mailbox does not return the size,
7375 			 * just the count.  The size should be just the size
7376 			 * stored in the current allocated block and all sizes
7377 			 * for an extent type are the same so set the return
7378 			 * value now.
7379 			 */
7380 			*extnt_size = rsrc_blk->rsrc_size;
7381 		}
7382 		curr_blks++;
7383 	}
7384 
7385 	/*
7386 	 * Calculate the size of an embedded mailbox.  The uint32_t
7387 	 * accounts for extents-specific word.
7388 	 */
7389 	emb_len = sizeof(MAILBOX_t) - sizeof(struct mbox_header) -
7390 		sizeof(uint32_t);
7391 
7392 	/*
7393 	 * Presume the allocation and response will fit into an embedded
7394 	 * mailbox.  If not true, reconfigure to a non-embedded mailbox.
7395 	 */
7396 	emb = LPFC_SLI4_MBX_EMBED;
7397 	req_len = emb_len;
7398 	if (req_len > emb_len) {
7399 		req_len = curr_blks * sizeof(uint16_t) +
7400 			sizeof(union lpfc_sli4_cfg_shdr) +
7401 			sizeof(uint32_t);
7402 		emb = LPFC_SLI4_MBX_NEMBED;
7403 	}
7404 
7405 	mbox = (LPFC_MBOXQ_t *) mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
7406 	if (!mbox)
7407 		return -ENOMEM;
7408 	memset(mbox, 0, sizeof(LPFC_MBOXQ_t));
7409 
7410 	alloc_len = lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
7411 				     LPFC_MBOX_OPCODE_GET_ALLOC_RSRC_EXTENT,
7412 				     req_len, emb);
7413 	if (alloc_len < req_len) {
7414 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
7415 			"2983 Allocated DMA memory size (x%x) is "
7416 			"less than the requested DMA memory "
7417 			"size (x%x)\n", alloc_len, req_len);
7418 		rc = -ENOMEM;
7419 		goto err_exit;
7420 	}
7421 	rc = lpfc_sli4_mbox_rsrc_extent(phba, mbox, curr_blks, type, emb);
7422 	if (unlikely(rc)) {
7423 		rc = -EIO;
7424 		goto err_exit;
7425 	}
7426 
7427 	if (!phba->sli4_hba.intr_enable)
7428 		rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL);
7429 	else {
7430 		mbox_tmo = lpfc_mbox_tmo_val(phba, mbox);
7431 		rc = lpfc_sli_issue_mbox_wait(phba, mbox, mbox_tmo);
7432 	}
7433 
7434 	if (unlikely(rc)) {
7435 		rc = -EIO;
7436 		goto err_exit;
7437 	}
7438 
7439 	/*
7440 	 * Figure out where the response is located.  Then get local pointers
7441 	 * to the response data.  The port does not guarantee to respond to
7442 	 * all extents counts request so update the local variable with the
7443 	 * allocated count from the port.
7444 	 */
7445 	if (emb == LPFC_SLI4_MBX_EMBED) {
7446 		rsrc_ext = &mbox->u.mqe.un.alloc_rsrc_extents;
7447 		shdr = &rsrc_ext->header.cfg_shdr;
7448 		*extnt_cnt = bf_get(lpfc_mbx_rsrc_cnt, &rsrc_ext->u.rsp);
7449 	} else {
7450 		virtaddr = mbox->sge_array->addr[0];
7451 		n_rsrc = (struct lpfc_mbx_nembed_rsrc_extent *) virtaddr;
7452 		shdr = &n_rsrc->cfg_shdr;
7453 		*extnt_cnt = bf_get(lpfc_mbx_rsrc_cnt, n_rsrc);
7454 	}
7455 
7456 	if (bf_get(lpfc_mbox_hdr_status, &shdr->response)) {
7457 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
7458 			"2984 Failed to read allocated resources "
7459 			"for type %d - Status 0x%x Add'l Status 0x%x.\n",
7460 			type,
7461 			bf_get(lpfc_mbox_hdr_status, &shdr->response),
7462 			bf_get(lpfc_mbox_hdr_add_status, &shdr->response));
7463 		rc = -EIO;
7464 		goto err_exit;
7465 	}
7466  err_exit:
7467 	lpfc_sli4_mbox_cmd_free(phba, mbox);
7468 	return rc;
7469 }
7470 
7471 /**
7472  * lpfc_sli4_repost_sgl_list - Repost the buffers sgl pages as block
7473  * @phba: pointer to lpfc hba data structure.
7474  * @sgl_list: linked link of sgl buffers to post
7475  * @cnt: number of linked list buffers
7476  *
7477  * This routine walks the list of buffers that have been allocated and
7478  * repost them to the port by using SGL block post. This is needed after a
7479  * pci_function_reset/warm_start or start. It attempts to construct blocks
7480  * of buffer sgls which contains contiguous xris and uses the non-embedded
7481  * SGL block post mailbox commands to post them to the port. For single
7482  * buffer sgl with non-contiguous xri, if any, it shall use embedded SGL post
7483  * mailbox command for posting.
7484  *
7485  * Returns: 0 = success, non-zero failure.
7486  **/
7487 static int
7488 lpfc_sli4_repost_sgl_list(struct lpfc_hba *phba,
7489 			  struct list_head *sgl_list, int cnt)
7490 {
7491 	struct lpfc_sglq *sglq_entry = NULL;
7492 	struct lpfc_sglq *sglq_entry_next = NULL;
7493 	struct lpfc_sglq *sglq_entry_first = NULL;
7494 	int status, total_cnt;
7495 	int post_cnt = 0, num_posted = 0, block_cnt = 0;
7496 	int last_xritag = NO_XRI;
7497 	LIST_HEAD(prep_sgl_list);
7498 	LIST_HEAD(blck_sgl_list);
7499 	LIST_HEAD(allc_sgl_list);
7500 	LIST_HEAD(post_sgl_list);
7501 	LIST_HEAD(free_sgl_list);
7502 
7503 	spin_lock_irq(&phba->hbalock);
7504 	spin_lock(&phba->sli4_hba.sgl_list_lock);
7505 	list_splice_init(sgl_list, &allc_sgl_list);
7506 	spin_unlock(&phba->sli4_hba.sgl_list_lock);
7507 	spin_unlock_irq(&phba->hbalock);
7508 
7509 	total_cnt = cnt;
7510 	list_for_each_entry_safe(sglq_entry, sglq_entry_next,
7511 				 &allc_sgl_list, list) {
7512 		list_del_init(&sglq_entry->list);
7513 		block_cnt++;
7514 		if ((last_xritag != NO_XRI) &&
7515 		    (sglq_entry->sli4_xritag != last_xritag + 1)) {
7516 			/* a hole in xri block, form a sgl posting block */
7517 			list_splice_init(&prep_sgl_list, &blck_sgl_list);
7518 			post_cnt = block_cnt - 1;
7519 			/* prepare list for next posting block */
7520 			list_add_tail(&sglq_entry->list, &prep_sgl_list);
7521 			block_cnt = 1;
7522 		} else {
7523 			/* prepare list for next posting block */
7524 			list_add_tail(&sglq_entry->list, &prep_sgl_list);
7525 			/* enough sgls for non-embed sgl mbox command */
7526 			if (block_cnt == LPFC_NEMBED_MBOX_SGL_CNT) {
7527 				list_splice_init(&prep_sgl_list,
7528 						 &blck_sgl_list);
7529 				post_cnt = block_cnt;
7530 				block_cnt = 0;
7531 			}
7532 		}
7533 		num_posted++;
7534 
7535 		/* keep track of last sgl's xritag */
7536 		last_xritag = sglq_entry->sli4_xritag;
7537 
7538 		/* end of repost sgl list condition for buffers */
7539 		if (num_posted == total_cnt) {
7540 			if (post_cnt == 0) {
7541 				list_splice_init(&prep_sgl_list,
7542 						 &blck_sgl_list);
7543 				post_cnt = block_cnt;
7544 			} else if (block_cnt == 1) {
7545 				status = lpfc_sli4_post_sgl(phba,
7546 						sglq_entry->phys, 0,
7547 						sglq_entry->sli4_xritag);
7548 				if (!status) {
7549 					/* successful, put sgl to posted list */
7550 					list_add_tail(&sglq_entry->list,
7551 						      &post_sgl_list);
7552 				} else {
7553 					/* Failure, put sgl to free list */
7554 					lpfc_printf_log(phba, KERN_WARNING,
7555 						LOG_SLI,
7556 						"3159 Failed to post "
7557 						"sgl, xritag:x%x\n",
7558 						sglq_entry->sli4_xritag);
7559 					list_add_tail(&sglq_entry->list,
7560 						      &free_sgl_list);
7561 					total_cnt--;
7562 				}
7563 			}
7564 		}
7565 
7566 		/* continue until a nembed page worth of sgls */
7567 		if (post_cnt == 0)
7568 			continue;
7569 
7570 		/* post the buffer list sgls as a block */
7571 		status = lpfc_sli4_post_sgl_list(phba, &blck_sgl_list,
7572 						 post_cnt);
7573 
7574 		if (!status) {
7575 			/* success, put sgl list to posted sgl list */
7576 			list_splice_init(&blck_sgl_list, &post_sgl_list);
7577 		} else {
7578 			/* Failure, put sgl list to free sgl list */
7579 			sglq_entry_first = list_first_entry(&blck_sgl_list,
7580 							    struct lpfc_sglq,
7581 							    list);
7582 			lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
7583 					"3160 Failed to post sgl-list, "
7584 					"xritag:x%x-x%x\n",
7585 					sglq_entry_first->sli4_xritag,
7586 					(sglq_entry_first->sli4_xritag +
7587 					 post_cnt - 1));
7588 			list_splice_init(&blck_sgl_list, &free_sgl_list);
7589 			total_cnt -= post_cnt;
7590 		}
7591 
7592 		/* don't reset xirtag due to hole in xri block */
7593 		if (block_cnt == 0)
7594 			last_xritag = NO_XRI;
7595 
7596 		/* reset sgl post count for next round of posting */
7597 		post_cnt = 0;
7598 	}
7599 
7600 	/* free the sgls failed to post */
7601 	lpfc_free_sgl_list(phba, &free_sgl_list);
7602 
7603 	/* push sgls posted to the available list */
7604 	if (!list_empty(&post_sgl_list)) {
7605 		spin_lock_irq(&phba->hbalock);
7606 		spin_lock(&phba->sli4_hba.sgl_list_lock);
7607 		list_splice_init(&post_sgl_list, sgl_list);
7608 		spin_unlock(&phba->sli4_hba.sgl_list_lock);
7609 		spin_unlock_irq(&phba->hbalock);
7610 	} else {
7611 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
7612 				"3161 Failure to post sgl to port.\n");
7613 		return -EIO;
7614 	}
7615 
7616 	/* return the number of XRIs actually posted */
7617 	return total_cnt;
7618 }
7619 
7620 /**
7621  * lpfc_sli4_repost_io_sgl_list - Repost all the allocated nvme buffer sgls
7622  * @phba: pointer to lpfc hba data structure.
7623  *
7624  * This routine walks the list of nvme buffers that have been allocated and
7625  * repost them to the port by using SGL block post. This is needed after a
7626  * pci_function_reset/warm_start or start. The lpfc_hba_down_post_s4 routine
7627  * is responsible for moving all nvme buffers on the lpfc_abts_nvme_sgl_list
7628  * to the lpfc_io_buf_list. If the repost fails, reject all nvme buffers.
7629  *
7630  * Returns: 0 = success, non-zero failure.
7631  **/
7632 static int
7633 lpfc_sli4_repost_io_sgl_list(struct lpfc_hba *phba)
7634 {
7635 	LIST_HEAD(post_nblist);
7636 	int num_posted, rc = 0;
7637 
7638 	/* get all NVME buffers need to repost to a local list */
7639 	lpfc_io_buf_flush(phba, &post_nblist);
7640 
7641 	/* post the list of nvme buffer sgls to port if available */
7642 	if (!list_empty(&post_nblist)) {
7643 		num_posted = lpfc_sli4_post_io_sgl_list(
7644 			phba, &post_nblist, phba->sli4_hba.io_xri_cnt);
7645 		/* failed to post any nvme buffer, return error */
7646 		if (num_posted == 0)
7647 			rc = -EIO;
7648 	}
7649 	return rc;
7650 }
7651 
7652 static void
7653 lpfc_set_host_data(struct lpfc_hba *phba, LPFC_MBOXQ_t *mbox)
7654 {
7655 	uint32_t len;
7656 
7657 	len = sizeof(struct lpfc_mbx_set_host_data) -
7658 		sizeof(struct lpfc_sli4_cfg_mhdr);
7659 	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
7660 			 LPFC_MBOX_OPCODE_SET_HOST_DATA, len,
7661 			 LPFC_SLI4_MBX_EMBED);
7662 
7663 	mbox->u.mqe.un.set_host_data.param_id = LPFC_SET_HOST_OS_DRIVER_VERSION;
7664 	mbox->u.mqe.un.set_host_data.param_len =
7665 					LPFC_HOST_OS_DRIVER_VERSION_SIZE;
7666 	snprintf(mbox->u.mqe.un.set_host_data.un.data,
7667 		 LPFC_HOST_OS_DRIVER_VERSION_SIZE,
7668 		 "Linux %s v"LPFC_DRIVER_VERSION,
7669 		 (phba->hba_flag & HBA_FCOE_MODE) ? "FCoE" : "FC");
7670 }
7671 
7672 int
7673 lpfc_post_rq_buffer(struct lpfc_hba *phba, struct lpfc_queue *hrq,
7674 		    struct lpfc_queue *drq, int count, int idx)
7675 {
7676 	int rc, i;
7677 	struct lpfc_rqe hrqe;
7678 	struct lpfc_rqe drqe;
7679 	struct lpfc_rqb *rqbp;
7680 	unsigned long flags;
7681 	struct rqb_dmabuf *rqb_buffer;
7682 	LIST_HEAD(rqb_buf_list);
7683 
7684 	rqbp = hrq->rqbp;
7685 	for (i = 0; i < count; i++) {
7686 		spin_lock_irqsave(&phba->hbalock, flags);
7687 		/* IF RQ is already full, don't bother */
7688 		if (rqbp->buffer_count + i >= rqbp->entry_count - 1) {
7689 			spin_unlock_irqrestore(&phba->hbalock, flags);
7690 			break;
7691 		}
7692 		spin_unlock_irqrestore(&phba->hbalock, flags);
7693 
7694 		rqb_buffer = rqbp->rqb_alloc_buffer(phba);
7695 		if (!rqb_buffer)
7696 			break;
7697 		rqb_buffer->hrq = hrq;
7698 		rqb_buffer->drq = drq;
7699 		rqb_buffer->idx = idx;
7700 		list_add_tail(&rqb_buffer->hbuf.list, &rqb_buf_list);
7701 	}
7702 
7703 	spin_lock_irqsave(&phba->hbalock, flags);
7704 	while (!list_empty(&rqb_buf_list)) {
7705 		list_remove_head(&rqb_buf_list, rqb_buffer, struct rqb_dmabuf,
7706 				 hbuf.list);
7707 
7708 		hrqe.address_lo = putPaddrLow(rqb_buffer->hbuf.phys);
7709 		hrqe.address_hi = putPaddrHigh(rqb_buffer->hbuf.phys);
7710 		drqe.address_lo = putPaddrLow(rqb_buffer->dbuf.phys);
7711 		drqe.address_hi = putPaddrHigh(rqb_buffer->dbuf.phys);
7712 		rc = lpfc_sli4_rq_put(hrq, drq, &hrqe, &drqe);
7713 		if (rc < 0) {
7714 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
7715 					"6421 Cannot post to HRQ %d: %x %x %x "
7716 					"DRQ %x %x\n",
7717 					hrq->queue_id,
7718 					hrq->host_index,
7719 					hrq->hba_index,
7720 					hrq->entry_count,
7721 					drq->host_index,
7722 					drq->hba_index);
7723 			rqbp->rqb_free_buffer(phba, rqb_buffer);
7724 		} else {
7725 			list_add_tail(&rqb_buffer->hbuf.list,
7726 				      &rqbp->rqb_buffer_list);
7727 			rqbp->buffer_count++;
7728 		}
7729 	}
7730 	spin_unlock_irqrestore(&phba->hbalock, flags);
7731 	return 1;
7732 }
7733 
7734 static void
7735 lpfc_mbx_cmpl_cgn_set_ftrs(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb)
7736 {
7737 	struct lpfc_vport *vport = pmb->vport;
7738 	union lpfc_sli4_cfg_shdr *shdr;
7739 	u32 shdr_status, shdr_add_status;
7740 	u32 sig, acqe;
7741 
7742 	/* Two outcomes. (1) Set featurs was successul and EDC negotiation
7743 	 * is done. (2) Mailbox failed and send FPIN support only.
7744 	 */
7745 	shdr = (union lpfc_sli4_cfg_shdr *)
7746 		&pmb->u.mqe.un.sli4_config.header.cfg_shdr;
7747 	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
7748 	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
7749 	if (shdr_status || shdr_add_status || pmb->u.mb.mbxStatus) {
7750 		lpfc_printf_log(phba, KERN_ERR, LOG_INIT | LOG_CGN_MGMT,
7751 				"2516 CGN SET_FEATURE mbox failed with "
7752 				"status x%x add_status x%x, mbx status x%x "
7753 				"Reset Congestion to FPINs only\n",
7754 				shdr_status, shdr_add_status,
7755 				pmb->u.mb.mbxStatus);
7756 		/* If there is a mbox error, move on to RDF */
7757 		phba->cgn_reg_signal = EDC_CG_SIG_NOTSUPPORTED;
7758 		phba->cgn_reg_fpin = LPFC_CGN_FPIN_WARN | LPFC_CGN_FPIN_ALARM;
7759 		goto out;
7760 	}
7761 
7762 	/* Zero out Congestion Signal ACQE counter */
7763 	phba->cgn_acqe_cnt = 0;
7764 	atomic64_set(&phba->cgn_acqe_stat.warn, 0);
7765 	atomic64_set(&phba->cgn_acqe_stat.alarm, 0);
7766 
7767 	acqe = bf_get(lpfc_mbx_set_feature_CGN_acqe_freq,
7768 		      &pmb->u.mqe.un.set_feature);
7769 	sig = bf_get(lpfc_mbx_set_feature_CGN_warn_freq,
7770 		     &pmb->u.mqe.un.set_feature);
7771 	lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT,
7772 			"4620 SET_FEATURES Success: Freq: %ds %dms "
7773 			" Reg: x%x x%x\n", acqe, sig,
7774 			phba->cgn_reg_signal, phba->cgn_reg_fpin);
7775 out:
7776 	mempool_free(pmb, phba->mbox_mem_pool);
7777 
7778 	/* Register for FPIN events from the fabric now that the
7779 	 * EDC common_set_features has completed.
7780 	 */
7781 	lpfc_issue_els_rdf(vport, 0);
7782 }
7783 
7784 int
7785 lpfc_config_cgn_signal(struct lpfc_hba *phba)
7786 {
7787 	LPFC_MBOXQ_t *mboxq;
7788 	u32 rc;
7789 
7790 	mboxq = (LPFC_MBOXQ_t *)mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
7791 	if (!mboxq)
7792 		goto out_rdf;
7793 
7794 	lpfc_set_features(phba, mboxq, LPFC_SET_CGN_SIGNAL);
7795 	mboxq->vport = phba->pport;
7796 	mboxq->mbox_cmpl = lpfc_mbx_cmpl_cgn_set_ftrs;
7797 
7798 	lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT,
7799 			"4621 SET_FEATURES: FREQ sig x%x acqe x%x: "
7800 			"Reg: x%x x%x\n",
7801 			phba->cgn_sig_freq, lpfc_acqe_cgn_frequency,
7802 			phba->cgn_reg_signal, phba->cgn_reg_fpin);
7803 
7804 	rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_NOWAIT);
7805 	if (rc == MBX_NOT_FINISHED)
7806 		goto out;
7807 	return 0;
7808 
7809 out:
7810 	mempool_free(mboxq, phba->mbox_mem_pool);
7811 out_rdf:
7812 	/* If there is a mbox error, move on to RDF */
7813 	phba->cgn_reg_fpin = LPFC_CGN_FPIN_WARN | LPFC_CGN_FPIN_ALARM;
7814 	phba->cgn_reg_signal = EDC_CG_SIG_NOTSUPPORTED;
7815 	lpfc_issue_els_rdf(phba->pport, 0);
7816 	return -EIO;
7817 }
7818 
7819 /**
7820  * lpfc_init_idle_stat_hb - Initialize idle_stat tracking
7821  * @phba: pointer to lpfc hba data structure.
7822  *
7823  * This routine initializes the per-cq idle_stat to dynamically dictate
7824  * polling decisions.
7825  *
7826  * Return codes:
7827  *   None
7828  **/
7829 static void lpfc_init_idle_stat_hb(struct lpfc_hba *phba)
7830 {
7831 	int i;
7832 	struct lpfc_sli4_hdw_queue *hdwq;
7833 	struct lpfc_queue *cq;
7834 	struct lpfc_idle_stat *idle_stat;
7835 	u64 wall;
7836 
7837 	for_each_present_cpu(i) {
7838 		hdwq = &phba->sli4_hba.hdwq[phba->sli4_hba.cpu_map[i].hdwq];
7839 		cq = hdwq->io_cq;
7840 
7841 		/* Skip if we've already handled this cq's primary CPU */
7842 		if (cq->chann != i)
7843 			continue;
7844 
7845 		idle_stat = &phba->sli4_hba.idle_stat[i];
7846 
7847 		idle_stat->prev_idle = get_cpu_idle_time(i, &wall, 1);
7848 		idle_stat->prev_wall = wall;
7849 
7850 		if (phba->nvmet_support ||
7851 		    phba->cmf_active_mode != LPFC_CFG_OFF)
7852 			cq->poll_mode = LPFC_QUEUE_WORK;
7853 		else
7854 			cq->poll_mode = LPFC_IRQ_POLL;
7855 	}
7856 
7857 	if (!phba->nvmet_support)
7858 		schedule_delayed_work(&phba->idle_stat_delay_work,
7859 				      msecs_to_jiffies(LPFC_IDLE_STAT_DELAY));
7860 }
7861 
7862 static void lpfc_sli4_dip(struct lpfc_hba *phba)
7863 {
7864 	uint32_t if_type;
7865 
7866 	if_type = bf_get(lpfc_sli_intf_if_type, &phba->sli4_hba.sli_intf);
7867 	if (if_type == LPFC_SLI_INTF_IF_TYPE_2 ||
7868 	    if_type == LPFC_SLI_INTF_IF_TYPE_6) {
7869 		struct lpfc_register reg_data;
7870 
7871 		if (lpfc_readl(phba->sli4_hba.u.if_type2.STATUSregaddr,
7872 			       &reg_data.word0))
7873 			return;
7874 
7875 		if (bf_get(lpfc_sliport_status_dip, &reg_data))
7876 			lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
7877 					"2904 Firmware Dump Image Present"
7878 					" on Adapter");
7879 	}
7880 }
7881 
7882 /**
7883  * lpfc_cmf_setup - Initialize idle_stat tracking
7884  * @phba: Pointer to HBA context object.
7885  *
7886  * This is called from HBA setup during driver load or when the HBA
7887  * comes online. this does all the initialization to support CMF and MI.
7888  **/
7889 static int
7890 lpfc_cmf_setup(struct lpfc_hba *phba)
7891 {
7892 	LPFC_MBOXQ_t *mboxq;
7893 	struct lpfc_mqe *mqe;
7894 	struct lpfc_dmabuf *mp;
7895 	struct lpfc_pc_sli4_params *sli4_params;
7896 	struct lpfc_sli4_parameters *mbx_sli4_parameters;
7897 	int length;
7898 	int rc, cmf, mi_ver;
7899 
7900 	mboxq = (LPFC_MBOXQ_t *)mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
7901 	if (!mboxq)
7902 		return -ENOMEM;
7903 	mqe = &mboxq->u.mqe;
7904 
7905 	/* Read the port's SLI4 Config Parameters */
7906 	length = (sizeof(struct lpfc_mbx_get_sli4_parameters) -
7907 		  sizeof(struct lpfc_sli4_cfg_mhdr));
7908 	lpfc_sli4_config(phba, mboxq, LPFC_MBOX_SUBSYSTEM_COMMON,
7909 			 LPFC_MBOX_OPCODE_GET_SLI4_PARAMETERS,
7910 			 length, LPFC_SLI4_MBX_EMBED);
7911 
7912 	rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
7913 	if (unlikely(rc)) {
7914 		mempool_free(mboxq, phba->mbox_mem_pool);
7915 		return rc;
7916 	}
7917 
7918 	/* Gather info on CMF and MI support */
7919 	sli4_params = &phba->sli4_hba.pc_sli4_params;
7920 	mbx_sli4_parameters = &mqe->un.get_sli4_parameters.sli4_parameters;
7921 	sli4_params->mi_ver = bf_get(cfg_mi_ver, mbx_sli4_parameters);
7922 	sli4_params->cmf = bf_get(cfg_cmf, mbx_sli4_parameters);
7923 
7924 	/* Are we forcing MI off via module parameter? */
7925 	if (!phba->cfg_enable_mi)
7926 		sli4_params->mi_ver = 0;
7927 
7928 	/* Always try to enable MI feature if we can */
7929 	if (sli4_params->mi_ver) {
7930 		lpfc_set_features(phba, mboxq, LPFC_SET_ENABLE_MI);
7931 		rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
7932 		mi_ver = bf_get(lpfc_mbx_set_feature_mi,
7933 				 &mboxq->u.mqe.un.set_feature);
7934 
7935 		if (rc == MBX_SUCCESS) {
7936 			if (mi_ver) {
7937 				lpfc_printf_log(phba,
7938 						KERN_WARNING, LOG_CGN_MGMT,
7939 						"6215 MI is enabled\n");
7940 				sli4_params->mi_ver = mi_ver;
7941 			} else {
7942 				lpfc_printf_log(phba,
7943 						KERN_WARNING, LOG_CGN_MGMT,
7944 						"6338 MI is disabled\n");
7945 				sli4_params->mi_ver = 0;
7946 			}
7947 		} else {
7948 			/* mi_ver is already set from GET_SLI4_PARAMETERS */
7949 			lpfc_printf_log(phba, KERN_INFO,
7950 					LOG_CGN_MGMT | LOG_INIT,
7951 					"6245 Enable MI Mailbox x%x (x%x/x%x) "
7952 					"failed, rc:x%x mi:x%x\n",
7953 					bf_get(lpfc_mqe_command, &mboxq->u.mqe),
7954 					lpfc_sli_config_mbox_subsys_get
7955 						(phba, mboxq),
7956 					lpfc_sli_config_mbox_opcode_get
7957 						(phba, mboxq),
7958 					rc, sli4_params->mi_ver);
7959 		}
7960 	} else {
7961 		lpfc_printf_log(phba, KERN_WARNING, LOG_CGN_MGMT,
7962 				"6217 MI is disabled\n");
7963 	}
7964 
7965 	/* Ensure FDMI is enabled for MI if enable_mi is set */
7966 	if (sli4_params->mi_ver)
7967 		phba->cfg_fdmi_on = LPFC_FDMI_SUPPORT;
7968 
7969 	/* Always try to enable CMF feature if we can */
7970 	if (sli4_params->cmf) {
7971 		lpfc_set_features(phba, mboxq, LPFC_SET_ENABLE_CMF);
7972 		rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
7973 		cmf = bf_get(lpfc_mbx_set_feature_cmf,
7974 			     &mboxq->u.mqe.un.set_feature);
7975 		if (rc == MBX_SUCCESS && cmf) {
7976 			lpfc_printf_log(phba, KERN_WARNING, LOG_CGN_MGMT,
7977 					"6218 CMF is enabled: mode %d\n",
7978 					phba->cmf_active_mode);
7979 		} else {
7980 			lpfc_printf_log(phba, KERN_WARNING,
7981 					LOG_CGN_MGMT | LOG_INIT,
7982 					"6219 Enable CMF Mailbox x%x (x%x/x%x) "
7983 					"failed, rc:x%x dd:x%x\n",
7984 					bf_get(lpfc_mqe_command, &mboxq->u.mqe),
7985 					lpfc_sli_config_mbox_subsys_get
7986 						(phba, mboxq),
7987 					lpfc_sli_config_mbox_opcode_get
7988 						(phba, mboxq),
7989 					rc, cmf);
7990 			sli4_params->cmf = 0;
7991 			phba->cmf_active_mode = LPFC_CFG_OFF;
7992 			goto no_cmf;
7993 		}
7994 
7995 		/* Allocate Congestion Information Buffer */
7996 		if (!phba->cgn_i) {
7997 			mp = kmalloc(sizeof(*mp), GFP_KERNEL);
7998 			if (mp)
7999 				mp->virt = dma_alloc_coherent
8000 						(&phba->pcidev->dev,
8001 						sizeof(struct lpfc_cgn_info),
8002 						&mp->phys, GFP_KERNEL);
8003 			if (!mp || !mp->virt) {
8004 				lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
8005 						"2640 Failed to alloc memory "
8006 						"for Congestion Info\n");
8007 				kfree(mp);
8008 				sli4_params->cmf = 0;
8009 				phba->cmf_active_mode = LPFC_CFG_OFF;
8010 				goto no_cmf;
8011 			}
8012 			phba->cgn_i = mp;
8013 
8014 			/* initialize congestion buffer info */
8015 			lpfc_init_congestion_buf(phba);
8016 			lpfc_init_congestion_stat(phba);
8017 		}
8018 
8019 		rc = lpfc_sli4_cgn_params_read(phba);
8020 		if (rc < 0) {
8021 			lpfc_printf_log(phba, KERN_ERR, LOG_CGN_MGMT | LOG_INIT,
8022 					"6242 Error reading Cgn Params (%d)\n",
8023 					rc);
8024 			/* Ensure CGN Mode is off */
8025 			sli4_params->cmf = 0;
8026 		} else if (!rc) {
8027 			lpfc_printf_log(phba, KERN_ERR, LOG_CGN_MGMT | LOG_INIT,
8028 					"6243 CGN Event empty object.\n");
8029 			/* Ensure CGN Mode is off */
8030 			sli4_params->cmf = 0;
8031 		}
8032 	} else {
8033 no_cmf:
8034 		lpfc_printf_log(phba, KERN_WARNING, LOG_CGN_MGMT,
8035 				"6220 CMF is disabled\n");
8036 	}
8037 
8038 	/* Only register congestion buffer with firmware if BOTH
8039 	 * CMF and E2E are enabled.
8040 	 */
8041 	if (sli4_params->cmf && sli4_params->mi_ver) {
8042 		rc = lpfc_reg_congestion_buf(phba);
8043 		if (rc) {
8044 			dma_free_coherent(&phba->pcidev->dev,
8045 					  sizeof(struct lpfc_cgn_info),
8046 					  phba->cgn_i->virt, phba->cgn_i->phys);
8047 			kfree(phba->cgn_i);
8048 			phba->cgn_i = NULL;
8049 			/* Ensure CGN Mode is off */
8050 			phba->cmf_active_mode = LPFC_CFG_OFF;
8051 			return 0;
8052 		}
8053 	}
8054 	lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
8055 			"6470 Setup MI version %d CMF %d mode %d\n",
8056 			sli4_params->mi_ver, sli4_params->cmf,
8057 			phba->cmf_active_mode);
8058 
8059 	mempool_free(mboxq, phba->mbox_mem_pool);
8060 
8061 	/* Initialize atomic counters */
8062 	atomic_set(&phba->cgn_fabric_warn_cnt, 0);
8063 	atomic_set(&phba->cgn_fabric_alarm_cnt, 0);
8064 	atomic_set(&phba->cgn_sync_alarm_cnt, 0);
8065 	atomic_set(&phba->cgn_sync_warn_cnt, 0);
8066 	atomic_set(&phba->cgn_driver_evt_cnt, 0);
8067 	atomic_set(&phba->cgn_latency_evt_cnt, 0);
8068 	atomic64_set(&phba->cgn_latency_evt, 0);
8069 
8070 	phba->cmf_interval_rate = LPFC_CMF_INTERVAL;
8071 
8072 	/* Allocate RX Monitor Buffer */
8073 	if (!phba->rxtable) {
8074 		phba->rxtable = kmalloc_array(LPFC_MAX_RXMONITOR_ENTRY,
8075 					      sizeof(struct rxtable_entry),
8076 					      GFP_KERNEL);
8077 		if (!phba->rxtable) {
8078 			lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
8079 					"2644 Failed to alloc memory "
8080 					"for RX Monitor Buffer\n");
8081 			return -ENOMEM;
8082 		}
8083 	}
8084 	atomic_set(&phba->rxtable_idx_head, 0);
8085 	atomic_set(&phba->rxtable_idx_tail, 0);
8086 	return 0;
8087 }
8088 
8089 static int
8090 lpfc_set_host_tm(struct lpfc_hba *phba)
8091 {
8092 	LPFC_MBOXQ_t *mboxq;
8093 	uint32_t len, rc;
8094 	struct timespec64 cur_time;
8095 	struct tm broken;
8096 	uint32_t month, day, year;
8097 	uint32_t hour, minute, second;
8098 	struct lpfc_mbx_set_host_date_time *tm;
8099 
8100 	mboxq = (LPFC_MBOXQ_t *)mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
8101 	if (!mboxq)
8102 		return -ENOMEM;
8103 
8104 	len = sizeof(struct lpfc_mbx_set_host_data) -
8105 		sizeof(struct lpfc_sli4_cfg_mhdr);
8106 	lpfc_sli4_config(phba, mboxq, LPFC_MBOX_SUBSYSTEM_COMMON,
8107 			 LPFC_MBOX_OPCODE_SET_HOST_DATA, len,
8108 			 LPFC_SLI4_MBX_EMBED);
8109 
8110 	mboxq->u.mqe.un.set_host_data.param_id = LPFC_SET_HOST_DATE_TIME;
8111 	mboxq->u.mqe.un.set_host_data.param_len =
8112 			sizeof(struct lpfc_mbx_set_host_date_time);
8113 	tm = &mboxq->u.mqe.un.set_host_data.un.tm;
8114 	ktime_get_real_ts64(&cur_time);
8115 	time64_to_tm(cur_time.tv_sec, 0, &broken);
8116 	month = broken.tm_mon + 1;
8117 	day = broken.tm_mday;
8118 	year = broken.tm_year - 100;
8119 	hour = broken.tm_hour;
8120 	minute = broken.tm_min;
8121 	second = broken.tm_sec;
8122 	bf_set(lpfc_mbx_set_host_month, tm, month);
8123 	bf_set(lpfc_mbx_set_host_day, tm, day);
8124 	bf_set(lpfc_mbx_set_host_year, tm, year);
8125 	bf_set(lpfc_mbx_set_host_hour, tm, hour);
8126 	bf_set(lpfc_mbx_set_host_min, tm, minute);
8127 	bf_set(lpfc_mbx_set_host_sec, tm, second);
8128 
8129 	rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
8130 	mempool_free(mboxq, phba->mbox_mem_pool);
8131 	return rc;
8132 }
8133 
8134 /**
8135  * lpfc_sli4_hba_setup - SLI4 device initialization PCI function
8136  * @phba: Pointer to HBA context object.
8137  *
8138  * This function is the main SLI4 device initialization PCI function. This
8139  * function is called by the HBA initialization code, HBA reset code and
8140  * HBA error attention handler code. Caller is not required to hold any
8141  * locks.
8142  **/
8143 int
8144 lpfc_sli4_hba_setup(struct lpfc_hba *phba)
8145 {
8146 	int rc, i, cnt, len, dd;
8147 	LPFC_MBOXQ_t *mboxq;
8148 	struct lpfc_mqe *mqe;
8149 	uint8_t *vpd;
8150 	uint32_t vpd_size;
8151 	uint32_t ftr_rsp = 0;
8152 	struct Scsi_Host *shost = lpfc_shost_from_vport(phba->pport);
8153 	struct lpfc_vport *vport = phba->pport;
8154 	struct lpfc_dmabuf *mp;
8155 	struct lpfc_rqb *rqbp;
8156 
8157 	/* Perform a PCI function reset to start from clean */
8158 	rc = lpfc_pci_function_reset(phba);
8159 	if (unlikely(rc))
8160 		return -ENODEV;
8161 
8162 	/* Check the HBA Host Status Register for readyness */
8163 	rc = lpfc_sli4_post_status_check(phba);
8164 	if (unlikely(rc))
8165 		return -ENODEV;
8166 	else {
8167 		spin_lock_irq(&phba->hbalock);
8168 		phba->sli.sli_flag |= LPFC_SLI_ACTIVE;
8169 		spin_unlock_irq(&phba->hbalock);
8170 	}
8171 
8172 	lpfc_sli4_dip(phba);
8173 
8174 	/*
8175 	 * Allocate a single mailbox container for initializing the
8176 	 * port.
8177 	 */
8178 	mboxq = (LPFC_MBOXQ_t *) mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
8179 	if (!mboxq)
8180 		return -ENOMEM;
8181 
8182 	/* Issue READ_REV to collect vpd and FW information. */
8183 	vpd_size = SLI4_PAGE_SIZE;
8184 	vpd = kzalloc(vpd_size, GFP_KERNEL);
8185 	if (!vpd) {
8186 		rc = -ENOMEM;
8187 		goto out_free_mbox;
8188 	}
8189 
8190 	rc = lpfc_sli4_read_rev(phba, mboxq, vpd, &vpd_size);
8191 	if (unlikely(rc)) {
8192 		kfree(vpd);
8193 		goto out_free_mbox;
8194 	}
8195 
8196 	mqe = &mboxq->u.mqe;
8197 	phba->sli_rev = bf_get(lpfc_mbx_rd_rev_sli_lvl, &mqe->un.read_rev);
8198 	if (bf_get(lpfc_mbx_rd_rev_fcoe, &mqe->un.read_rev)) {
8199 		phba->hba_flag |= HBA_FCOE_MODE;
8200 		phba->fcp_embed_io = 0;	/* SLI4 FC support only */
8201 	} else {
8202 		phba->hba_flag &= ~HBA_FCOE_MODE;
8203 	}
8204 
8205 	if (bf_get(lpfc_mbx_rd_rev_cee_ver, &mqe->un.read_rev) ==
8206 		LPFC_DCBX_CEE_MODE)
8207 		phba->hba_flag |= HBA_FIP_SUPPORT;
8208 	else
8209 		phba->hba_flag &= ~HBA_FIP_SUPPORT;
8210 
8211 	phba->hba_flag &= ~HBA_IOQ_FLUSH;
8212 
8213 	if (phba->sli_rev != LPFC_SLI_REV4) {
8214 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
8215 			"0376 READ_REV Error. SLI Level %d "
8216 			"FCoE enabled %d\n",
8217 			phba->sli_rev, phba->hba_flag & HBA_FCOE_MODE);
8218 		rc = -EIO;
8219 		kfree(vpd);
8220 		goto out_free_mbox;
8221 	}
8222 
8223 	rc = lpfc_set_host_tm(phba);
8224 	lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_INIT,
8225 			"6468 Set host date / time: Status x%x:\n", rc);
8226 
8227 	/*
8228 	 * Continue initialization with default values even if driver failed
8229 	 * to read FCoE param config regions, only read parameters if the
8230 	 * board is FCoE
8231 	 */
8232 	if (phba->hba_flag & HBA_FCOE_MODE &&
8233 	    lpfc_sli4_read_fcoe_params(phba))
8234 		lpfc_printf_log(phba, KERN_WARNING, LOG_MBOX | LOG_INIT,
8235 			"2570 Failed to read FCoE parameters\n");
8236 
8237 	/*
8238 	 * Retrieve sli4 device physical port name, failure of doing it
8239 	 * is considered as non-fatal.
8240 	 */
8241 	rc = lpfc_sli4_retrieve_pport_name(phba);
8242 	if (!rc)
8243 		lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI,
8244 				"3080 Successful retrieving SLI4 device "
8245 				"physical port name: %s.\n", phba->Port);
8246 
8247 	rc = lpfc_sli4_get_ctl_attr(phba);
8248 	if (!rc)
8249 		lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI,
8250 				"8351 Successful retrieving SLI4 device "
8251 				"CTL ATTR\n");
8252 
8253 	/*
8254 	 * Evaluate the read rev and vpd data. Populate the driver
8255 	 * state with the results. If this routine fails, the failure
8256 	 * is not fatal as the driver will use generic values.
8257 	 */
8258 	rc = lpfc_parse_vpd(phba, vpd, vpd_size);
8259 	if (unlikely(!rc)) {
8260 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
8261 				"0377 Error %d parsing vpd. "
8262 				"Using defaults.\n", rc);
8263 		rc = 0;
8264 	}
8265 	kfree(vpd);
8266 
8267 	/* Save information as VPD data */
8268 	phba->vpd.rev.biuRev = mqe->un.read_rev.first_hw_rev;
8269 	phba->vpd.rev.smRev = mqe->un.read_rev.second_hw_rev;
8270 
8271 	/*
8272 	 * This is because first G7 ASIC doesn't support the standard
8273 	 * 0x5a NVME cmd descriptor type/subtype
8274 	 */
8275 	if ((bf_get(lpfc_sli_intf_if_type, &phba->sli4_hba.sli_intf) ==
8276 			LPFC_SLI_INTF_IF_TYPE_6) &&
8277 	    (phba->vpd.rev.biuRev == LPFC_G7_ASIC_1) &&
8278 	    (phba->vpd.rev.smRev == 0) &&
8279 	    (phba->cfg_nvme_embed_cmd == 1))
8280 		phba->cfg_nvme_embed_cmd = 0;
8281 
8282 	phba->vpd.rev.endecRev = mqe->un.read_rev.third_hw_rev;
8283 	phba->vpd.rev.fcphHigh = bf_get(lpfc_mbx_rd_rev_fcph_high,
8284 					 &mqe->un.read_rev);
8285 	phba->vpd.rev.fcphLow = bf_get(lpfc_mbx_rd_rev_fcph_low,
8286 				       &mqe->un.read_rev);
8287 	phba->vpd.rev.feaLevelHigh = bf_get(lpfc_mbx_rd_rev_ftr_lvl_high,
8288 					    &mqe->un.read_rev);
8289 	phba->vpd.rev.feaLevelLow = bf_get(lpfc_mbx_rd_rev_ftr_lvl_low,
8290 					   &mqe->un.read_rev);
8291 	phba->vpd.rev.sli1FwRev = mqe->un.read_rev.fw_id_rev;
8292 	memcpy(phba->vpd.rev.sli1FwName, mqe->un.read_rev.fw_name, 16);
8293 	phba->vpd.rev.sli2FwRev = mqe->un.read_rev.ulp_fw_id_rev;
8294 	memcpy(phba->vpd.rev.sli2FwName, mqe->un.read_rev.ulp_fw_name, 16);
8295 	phba->vpd.rev.opFwRev = mqe->un.read_rev.fw_id_rev;
8296 	memcpy(phba->vpd.rev.opFwName, mqe->un.read_rev.fw_name, 16);
8297 	lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI,
8298 			"(%d):0380 READ_REV Status x%x "
8299 			"fw_rev:%s fcphHi:%x fcphLo:%x flHi:%x flLo:%x\n",
8300 			mboxq->vport ? mboxq->vport->vpi : 0,
8301 			bf_get(lpfc_mqe_status, mqe),
8302 			phba->vpd.rev.opFwName,
8303 			phba->vpd.rev.fcphHigh, phba->vpd.rev.fcphLow,
8304 			phba->vpd.rev.feaLevelHigh, phba->vpd.rev.feaLevelLow);
8305 
8306 	if (bf_get(lpfc_sli_intf_if_type, &phba->sli4_hba.sli_intf) ==
8307 	    LPFC_SLI_INTF_IF_TYPE_0) {
8308 		lpfc_set_features(phba, mboxq, LPFC_SET_UE_RECOVERY);
8309 		rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
8310 		if (rc == MBX_SUCCESS) {
8311 			phba->hba_flag |= HBA_RECOVERABLE_UE;
8312 			/* Set 1Sec interval to detect UE */
8313 			phba->eratt_poll_interval = 1;
8314 			phba->sli4_hba.ue_to_sr = bf_get(
8315 					lpfc_mbx_set_feature_UESR,
8316 					&mboxq->u.mqe.un.set_feature);
8317 			phba->sli4_hba.ue_to_rp = bf_get(
8318 					lpfc_mbx_set_feature_UERP,
8319 					&mboxq->u.mqe.un.set_feature);
8320 		}
8321 	}
8322 
8323 	if (phba->cfg_enable_mds_diags && phba->mds_diags_support) {
8324 		/* Enable MDS Diagnostics only if the SLI Port supports it */
8325 		lpfc_set_features(phba, mboxq, LPFC_SET_MDS_DIAGS);
8326 		rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
8327 		if (rc != MBX_SUCCESS)
8328 			phba->mds_diags_support = 0;
8329 	}
8330 
8331 	/*
8332 	 * Discover the port's supported feature set and match it against the
8333 	 * hosts requests.
8334 	 */
8335 	lpfc_request_features(phba, mboxq);
8336 	rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
8337 	if (unlikely(rc)) {
8338 		rc = -EIO;
8339 		goto out_free_mbox;
8340 	}
8341 
8342 	/* Disable VMID if app header is not supported */
8343 	if (phba->cfg_vmid_app_header && !(bf_get(lpfc_mbx_rq_ftr_rsp_ashdr,
8344 						  &mqe->un.req_ftrs))) {
8345 		bf_set(lpfc_ftr_ashdr, &phba->sli4_hba.sli4_flags, 0);
8346 		phba->cfg_vmid_app_header = 0;
8347 		lpfc_printf_log(phba, KERN_DEBUG, LOG_SLI,
8348 				"1242 vmid feature not supported\n");
8349 	}
8350 
8351 	/*
8352 	 * The port must support FCP initiator mode as this is the
8353 	 * only mode running in the host.
8354 	 */
8355 	if (!(bf_get(lpfc_mbx_rq_ftr_rsp_fcpi, &mqe->un.req_ftrs))) {
8356 		lpfc_printf_log(phba, KERN_WARNING, LOG_MBOX | LOG_SLI,
8357 				"0378 No support for fcpi mode.\n");
8358 		ftr_rsp++;
8359 	}
8360 
8361 	/* Performance Hints are ONLY for FCoE */
8362 	if (phba->hba_flag & HBA_FCOE_MODE) {
8363 		if (bf_get(lpfc_mbx_rq_ftr_rsp_perfh, &mqe->un.req_ftrs))
8364 			phba->sli3_options |= LPFC_SLI4_PERFH_ENABLED;
8365 		else
8366 			phba->sli3_options &= ~LPFC_SLI4_PERFH_ENABLED;
8367 	}
8368 
8369 	/*
8370 	 * If the port cannot support the host's requested features
8371 	 * then turn off the global config parameters to disable the
8372 	 * feature in the driver.  This is not a fatal error.
8373 	 */
8374 	if (phba->sli3_options & LPFC_SLI3_BG_ENABLED) {
8375 		if (!(bf_get(lpfc_mbx_rq_ftr_rsp_dif, &mqe->un.req_ftrs))) {
8376 			phba->cfg_enable_bg = 0;
8377 			phba->sli3_options &= ~LPFC_SLI3_BG_ENABLED;
8378 			ftr_rsp++;
8379 		}
8380 	}
8381 
8382 	if (phba->max_vpi && phba->cfg_enable_npiv &&
8383 	    !(bf_get(lpfc_mbx_rq_ftr_rsp_npiv, &mqe->un.req_ftrs)))
8384 		ftr_rsp++;
8385 
8386 	if (ftr_rsp) {
8387 		lpfc_printf_log(phba, KERN_WARNING, LOG_MBOX | LOG_SLI,
8388 				"0379 Feature Mismatch Data: x%08x %08x "
8389 				"x%x x%x x%x\n", mqe->un.req_ftrs.word2,
8390 				mqe->un.req_ftrs.word3, phba->cfg_enable_bg,
8391 				phba->cfg_enable_npiv, phba->max_vpi);
8392 		if (!(bf_get(lpfc_mbx_rq_ftr_rsp_dif, &mqe->un.req_ftrs)))
8393 			phba->cfg_enable_bg = 0;
8394 		if (!(bf_get(lpfc_mbx_rq_ftr_rsp_npiv, &mqe->un.req_ftrs)))
8395 			phba->cfg_enable_npiv = 0;
8396 	}
8397 
8398 	/* These SLI3 features are assumed in SLI4 */
8399 	spin_lock_irq(&phba->hbalock);
8400 	phba->sli3_options |= (LPFC_SLI3_NPIV_ENABLED | LPFC_SLI3_HBQ_ENABLED);
8401 	spin_unlock_irq(&phba->hbalock);
8402 
8403 	/* Always try to enable dual dump feature if we can */
8404 	lpfc_set_features(phba, mboxq, LPFC_SET_DUAL_DUMP);
8405 	rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
8406 	dd = bf_get(lpfc_mbx_set_feature_dd, &mboxq->u.mqe.un.set_feature);
8407 	if ((rc == MBX_SUCCESS) && (dd == LPFC_ENABLE_DUAL_DUMP))
8408 		lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
8409 				"6448 Dual Dump is enabled\n");
8410 	else
8411 		lpfc_printf_log(phba, KERN_INFO, LOG_SLI | LOG_INIT,
8412 				"6447 Dual Dump Mailbox x%x (x%x/x%x) failed, "
8413 				"rc:x%x dd:x%x\n",
8414 				bf_get(lpfc_mqe_command, &mboxq->u.mqe),
8415 				lpfc_sli_config_mbox_subsys_get(
8416 					phba, mboxq),
8417 				lpfc_sli_config_mbox_opcode_get(
8418 					phba, mboxq),
8419 				rc, dd);
8420 	/*
8421 	 * Allocate all resources (xri,rpi,vpi,vfi) now.  Subsequent
8422 	 * calls depends on these resources to complete port setup.
8423 	 */
8424 	rc = lpfc_sli4_alloc_resource_identifiers(phba);
8425 	if (rc) {
8426 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
8427 				"2920 Failed to alloc Resource IDs "
8428 				"rc = x%x\n", rc);
8429 		goto out_free_mbox;
8430 	}
8431 
8432 	lpfc_set_host_data(phba, mboxq);
8433 
8434 	rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
8435 	if (rc) {
8436 		lpfc_printf_log(phba, KERN_WARNING, LOG_MBOX | LOG_SLI,
8437 				"2134 Failed to set host os driver version %x",
8438 				rc);
8439 	}
8440 
8441 	/* Read the port's service parameters. */
8442 	rc = lpfc_read_sparam(phba, mboxq, vport->vpi);
8443 	if (rc) {
8444 		phba->link_state = LPFC_HBA_ERROR;
8445 		rc = -ENOMEM;
8446 		goto out_free_mbox;
8447 	}
8448 
8449 	mboxq->vport = vport;
8450 	rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
8451 	mp = (struct lpfc_dmabuf *)mboxq->ctx_buf;
8452 	if (rc == MBX_SUCCESS) {
8453 		memcpy(&vport->fc_sparam, mp->virt, sizeof(struct serv_parm));
8454 		rc = 0;
8455 	}
8456 
8457 	/*
8458 	 * This memory was allocated by the lpfc_read_sparam routine. Release
8459 	 * it to the mbuf pool.
8460 	 */
8461 	lpfc_mbuf_free(phba, mp->virt, mp->phys);
8462 	kfree(mp);
8463 	mboxq->ctx_buf = NULL;
8464 	if (unlikely(rc)) {
8465 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
8466 				"0382 READ_SPARAM command failed "
8467 				"status %d, mbxStatus x%x\n",
8468 				rc, bf_get(lpfc_mqe_status, mqe));
8469 		phba->link_state = LPFC_HBA_ERROR;
8470 		rc = -EIO;
8471 		goto out_free_mbox;
8472 	}
8473 
8474 	lpfc_update_vport_wwn(vport);
8475 
8476 	/* Update the fc_host data structures with new wwn. */
8477 	fc_host_node_name(shost) = wwn_to_u64(vport->fc_nodename.u.wwn);
8478 	fc_host_port_name(shost) = wwn_to_u64(vport->fc_portname.u.wwn);
8479 
8480 	/* Create all the SLI4 queues */
8481 	rc = lpfc_sli4_queue_create(phba);
8482 	if (rc) {
8483 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
8484 				"3089 Failed to allocate queues\n");
8485 		rc = -ENODEV;
8486 		goto out_free_mbox;
8487 	}
8488 	/* Set up all the queues to the device */
8489 	rc = lpfc_sli4_queue_setup(phba);
8490 	if (unlikely(rc)) {
8491 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
8492 				"0381 Error %d during queue setup.\n ", rc);
8493 		goto out_stop_timers;
8494 	}
8495 	/* Initialize the driver internal SLI layer lists. */
8496 	lpfc_sli4_setup(phba);
8497 	lpfc_sli4_queue_init(phba);
8498 
8499 	/* update host els xri-sgl sizes and mappings */
8500 	rc = lpfc_sli4_els_sgl_update(phba);
8501 	if (unlikely(rc)) {
8502 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
8503 				"1400 Failed to update xri-sgl size and "
8504 				"mapping: %d\n", rc);
8505 		goto out_destroy_queue;
8506 	}
8507 
8508 	/* register the els sgl pool to the port */
8509 	rc = lpfc_sli4_repost_sgl_list(phba, &phba->sli4_hba.lpfc_els_sgl_list,
8510 				       phba->sli4_hba.els_xri_cnt);
8511 	if (unlikely(rc < 0)) {
8512 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
8513 				"0582 Error %d during els sgl post "
8514 				"operation\n", rc);
8515 		rc = -ENODEV;
8516 		goto out_destroy_queue;
8517 	}
8518 	phba->sli4_hba.els_xri_cnt = rc;
8519 
8520 	if (phba->nvmet_support) {
8521 		/* update host nvmet xri-sgl sizes and mappings */
8522 		rc = lpfc_sli4_nvmet_sgl_update(phba);
8523 		if (unlikely(rc)) {
8524 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
8525 					"6308 Failed to update nvmet-sgl size "
8526 					"and mapping: %d\n", rc);
8527 			goto out_destroy_queue;
8528 		}
8529 
8530 		/* register the nvmet sgl pool to the port */
8531 		rc = lpfc_sli4_repost_sgl_list(
8532 			phba,
8533 			&phba->sli4_hba.lpfc_nvmet_sgl_list,
8534 			phba->sli4_hba.nvmet_xri_cnt);
8535 		if (unlikely(rc < 0)) {
8536 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
8537 					"3117 Error %d during nvmet "
8538 					"sgl post\n", rc);
8539 			rc = -ENODEV;
8540 			goto out_destroy_queue;
8541 		}
8542 		phba->sli4_hba.nvmet_xri_cnt = rc;
8543 
8544 		/* We allocate an iocbq for every receive context SGL.
8545 		 * The additional allocation is for abort and ls handling.
8546 		 */
8547 		cnt = phba->sli4_hba.nvmet_xri_cnt +
8548 			phba->sli4_hba.max_cfg_param.max_xri;
8549 	} else {
8550 		/* update host common xri-sgl sizes and mappings */
8551 		rc = lpfc_sli4_io_sgl_update(phba);
8552 		if (unlikely(rc)) {
8553 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
8554 					"6082 Failed to update nvme-sgl size "
8555 					"and mapping: %d\n", rc);
8556 			goto out_destroy_queue;
8557 		}
8558 
8559 		/* register the allocated common sgl pool to the port */
8560 		rc = lpfc_sli4_repost_io_sgl_list(phba);
8561 		if (unlikely(rc)) {
8562 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
8563 					"6116 Error %d during nvme sgl post "
8564 					"operation\n", rc);
8565 			/* Some NVME buffers were moved to abort nvme list */
8566 			/* A pci function reset will repost them */
8567 			rc = -ENODEV;
8568 			goto out_destroy_queue;
8569 		}
8570 		/* Each lpfc_io_buf job structure has an iocbq element.
8571 		 * This cnt provides for abort, els, ct and ls requests.
8572 		 */
8573 		cnt = phba->sli4_hba.max_cfg_param.max_xri;
8574 	}
8575 
8576 	if (!phba->sli.iocbq_lookup) {
8577 		/* Initialize and populate the iocb list per host */
8578 		lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
8579 				"2821 initialize iocb list with %d entries\n",
8580 				cnt);
8581 		rc = lpfc_init_iocb_list(phba, cnt);
8582 		if (rc) {
8583 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
8584 					"1413 Failed to init iocb list.\n");
8585 			goto out_destroy_queue;
8586 		}
8587 	}
8588 
8589 	if (phba->nvmet_support)
8590 		lpfc_nvmet_create_targetport(phba);
8591 
8592 	if (phba->nvmet_support && phba->cfg_nvmet_mrq) {
8593 		/* Post initial buffers to all RQs created */
8594 		for (i = 0; i < phba->cfg_nvmet_mrq; i++) {
8595 			rqbp = phba->sli4_hba.nvmet_mrq_hdr[i]->rqbp;
8596 			INIT_LIST_HEAD(&rqbp->rqb_buffer_list);
8597 			rqbp->rqb_alloc_buffer = lpfc_sli4_nvmet_alloc;
8598 			rqbp->rqb_free_buffer = lpfc_sli4_nvmet_free;
8599 			rqbp->entry_count = LPFC_NVMET_RQE_DEF_COUNT;
8600 			rqbp->buffer_count = 0;
8601 
8602 			lpfc_post_rq_buffer(
8603 				phba, phba->sli4_hba.nvmet_mrq_hdr[i],
8604 				phba->sli4_hba.nvmet_mrq_data[i],
8605 				phba->cfg_nvmet_mrq_post, i);
8606 		}
8607 	}
8608 
8609 	/* Post the rpi header region to the device. */
8610 	rc = lpfc_sli4_post_all_rpi_hdrs(phba);
8611 	if (unlikely(rc)) {
8612 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
8613 				"0393 Error %d during rpi post operation\n",
8614 				rc);
8615 		rc = -ENODEV;
8616 		goto out_free_iocblist;
8617 	}
8618 	lpfc_sli4_node_prep(phba);
8619 
8620 	if (!(phba->hba_flag & HBA_FCOE_MODE)) {
8621 		if ((phba->nvmet_support == 0) || (phba->cfg_nvmet_mrq == 1)) {
8622 			/*
8623 			 * The FC Port needs to register FCFI (index 0)
8624 			 */
8625 			lpfc_reg_fcfi(phba, mboxq);
8626 			mboxq->vport = phba->pport;
8627 			rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
8628 			if (rc != MBX_SUCCESS)
8629 				goto out_unset_queue;
8630 			rc = 0;
8631 			phba->fcf.fcfi = bf_get(lpfc_reg_fcfi_fcfi,
8632 						&mboxq->u.mqe.un.reg_fcfi);
8633 		} else {
8634 			/* We are a NVME Target mode with MRQ > 1 */
8635 
8636 			/* First register the FCFI */
8637 			lpfc_reg_fcfi_mrq(phba, mboxq, 0);
8638 			mboxq->vport = phba->pport;
8639 			rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
8640 			if (rc != MBX_SUCCESS)
8641 				goto out_unset_queue;
8642 			rc = 0;
8643 			phba->fcf.fcfi = bf_get(lpfc_reg_fcfi_mrq_fcfi,
8644 						&mboxq->u.mqe.un.reg_fcfi_mrq);
8645 
8646 			/* Next register the MRQs */
8647 			lpfc_reg_fcfi_mrq(phba, mboxq, 1);
8648 			mboxq->vport = phba->pport;
8649 			rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
8650 			if (rc != MBX_SUCCESS)
8651 				goto out_unset_queue;
8652 			rc = 0;
8653 		}
8654 		/* Check if the port is configured to be disabled */
8655 		lpfc_sli_read_link_ste(phba);
8656 	}
8657 
8658 	/* Don't post more new bufs if repost already recovered
8659 	 * the nvme sgls.
8660 	 */
8661 	if (phba->nvmet_support == 0) {
8662 		if (phba->sli4_hba.io_xri_cnt == 0) {
8663 			len = lpfc_new_io_buf(
8664 					      phba, phba->sli4_hba.io_xri_max);
8665 			if (len == 0) {
8666 				rc = -ENOMEM;
8667 				goto out_unset_queue;
8668 			}
8669 
8670 			if (phba->cfg_xri_rebalancing)
8671 				lpfc_create_multixri_pools(phba);
8672 		}
8673 	} else {
8674 		phba->cfg_xri_rebalancing = 0;
8675 	}
8676 
8677 	/* Allow asynchronous mailbox command to go through */
8678 	spin_lock_irq(&phba->hbalock);
8679 	phba->sli.sli_flag &= ~LPFC_SLI_ASYNC_MBX_BLK;
8680 	spin_unlock_irq(&phba->hbalock);
8681 
8682 	/* Post receive buffers to the device */
8683 	lpfc_sli4_rb_setup(phba);
8684 
8685 	/* Reset HBA FCF states after HBA reset */
8686 	phba->fcf.fcf_flag = 0;
8687 	phba->fcf.current_rec.flag = 0;
8688 
8689 	/* Start the ELS watchdog timer */
8690 	mod_timer(&vport->els_tmofunc,
8691 		  jiffies + msecs_to_jiffies(1000 * (phba->fc_ratov * 2)));
8692 
8693 	/* Start heart beat timer */
8694 	mod_timer(&phba->hb_tmofunc,
8695 		  jiffies + msecs_to_jiffies(1000 * LPFC_HB_MBOX_INTERVAL));
8696 	phba->hba_flag &= ~(HBA_HBEAT_INP | HBA_HBEAT_TMO);
8697 	phba->last_completion_time = jiffies;
8698 
8699 	/* start eq_delay heartbeat */
8700 	if (phba->cfg_auto_imax)
8701 		queue_delayed_work(phba->wq, &phba->eq_delay_work,
8702 				   msecs_to_jiffies(LPFC_EQ_DELAY_MSECS));
8703 
8704 	/* start per phba idle_stat_delay heartbeat */
8705 	lpfc_init_idle_stat_hb(phba);
8706 
8707 	/* Start error attention (ERATT) polling timer */
8708 	mod_timer(&phba->eratt_poll,
8709 		  jiffies + msecs_to_jiffies(1000 * phba->eratt_poll_interval));
8710 
8711 	/* Enable PCIe device Advanced Error Reporting (AER) if configured */
8712 	if (phba->cfg_aer_support == 1 && !(phba->hba_flag & HBA_AER_ENABLED)) {
8713 		rc = pci_enable_pcie_error_reporting(phba->pcidev);
8714 		if (!rc) {
8715 			lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
8716 					"2829 This device supports "
8717 					"Advanced Error Reporting (AER)\n");
8718 			spin_lock_irq(&phba->hbalock);
8719 			phba->hba_flag |= HBA_AER_ENABLED;
8720 			spin_unlock_irq(&phba->hbalock);
8721 		} else {
8722 			lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
8723 					"2830 This device does not support "
8724 					"Advanced Error Reporting (AER)\n");
8725 			phba->cfg_aer_support = 0;
8726 		}
8727 		rc = 0;
8728 	}
8729 
8730 	/*
8731 	 * The port is ready, set the host's link state to LINK_DOWN
8732 	 * in preparation for link interrupts.
8733 	 */
8734 	spin_lock_irq(&phba->hbalock);
8735 	phba->link_state = LPFC_LINK_DOWN;
8736 
8737 	/* Check if physical ports are trunked */
8738 	if (bf_get(lpfc_conf_trunk_port0, &phba->sli4_hba))
8739 		phba->trunk_link.link0.state = LPFC_LINK_DOWN;
8740 	if (bf_get(lpfc_conf_trunk_port1, &phba->sli4_hba))
8741 		phba->trunk_link.link1.state = LPFC_LINK_DOWN;
8742 	if (bf_get(lpfc_conf_trunk_port2, &phba->sli4_hba))
8743 		phba->trunk_link.link2.state = LPFC_LINK_DOWN;
8744 	if (bf_get(lpfc_conf_trunk_port3, &phba->sli4_hba))
8745 		phba->trunk_link.link3.state = LPFC_LINK_DOWN;
8746 	spin_unlock_irq(&phba->hbalock);
8747 
8748 	/* Arm the CQs and then EQs on device */
8749 	lpfc_sli4_arm_cqeq_intr(phba);
8750 
8751 	/* Indicate device interrupt mode */
8752 	phba->sli4_hba.intr_enable = 1;
8753 
8754 	/* Setup CMF after HBA is initialized */
8755 	lpfc_cmf_setup(phba);
8756 
8757 	if (!(phba->hba_flag & HBA_FCOE_MODE) &&
8758 	    (phba->hba_flag & LINK_DISABLED)) {
8759 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
8760 				"3103 Adapter Link is disabled.\n");
8761 		lpfc_down_link(phba, mboxq);
8762 		rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
8763 		if (rc != MBX_SUCCESS) {
8764 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
8765 					"3104 Adapter failed to issue "
8766 					"DOWN_LINK mbox cmd, rc:x%x\n", rc);
8767 			goto out_io_buff_free;
8768 		}
8769 	} else if (phba->cfg_suppress_link_up == LPFC_INITIALIZE_LINK) {
8770 		/* don't perform init_link on SLI4 FC port loopback test */
8771 		if (!(phba->link_flag & LS_LOOPBACK_MODE)) {
8772 			rc = phba->lpfc_hba_init_link(phba, MBX_NOWAIT);
8773 			if (rc)
8774 				goto out_io_buff_free;
8775 		}
8776 	}
8777 	mempool_free(mboxq, phba->mbox_mem_pool);
8778 
8779 	phba->hba_flag |= HBA_SETUP;
8780 	return rc;
8781 
8782 out_io_buff_free:
8783 	/* Free allocated IO Buffers */
8784 	lpfc_io_free(phba);
8785 out_unset_queue:
8786 	/* Unset all the queues set up in this routine when error out */
8787 	lpfc_sli4_queue_unset(phba);
8788 out_free_iocblist:
8789 	lpfc_free_iocb_list(phba);
8790 out_destroy_queue:
8791 	lpfc_sli4_queue_destroy(phba);
8792 out_stop_timers:
8793 	lpfc_stop_hba_timers(phba);
8794 out_free_mbox:
8795 	mempool_free(mboxq, phba->mbox_mem_pool);
8796 	return rc;
8797 }
8798 
8799 /**
8800  * lpfc_mbox_timeout - Timeout call back function for mbox timer
8801  * @t: Context to fetch pointer to hba structure from.
8802  *
8803  * This is the callback function for mailbox timer. The mailbox
8804  * timer is armed when a new mailbox command is issued and the timer
8805  * is deleted when the mailbox complete. The function is called by
8806  * the kernel timer code when a mailbox does not complete within
8807  * expected time. This function wakes up the worker thread to
8808  * process the mailbox timeout and returns. All the processing is
8809  * done by the worker thread function lpfc_mbox_timeout_handler.
8810  **/
8811 void
8812 lpfc_mbox_timeout(struct timer_list *t)
8813 {
8814 	struct lpfc_hba  *phba = from_timer(phba, t, sli.mbox_tmo);
8815 	unsigned long iflag;
8816 	uint32_t tmo_posted;
8817 
8818 	spin_lock_irqsave(&phba->pport->work_port_lock, iflag);
8819 	tmo_posted = phba->pport->work_port_events & WORKER_MBOX_TMO;
8820 	if (!tmo_posted)
8821 		phba->pport->work_port_events |= WORKER_MBOX_TMO;
8822 	spin_unlock_irqrestore(&phba->pport->work_port_lock, iflag);
8823 
8824 	if (!tmo_posted)
8825 		lpfc_worker_wake_up(phba);
8826 	return;
8827 }
8828 
8829 /**
8830  * lpfc_sli4_mbox_completions_pending - check to see if any mailbox completions
8831  *                                    are pending
8832  * @phba: Pointer to HBA context object.
8833  *
8834  * This function checks if any mailbox completions are present on the mailbox
8835  * completion queue.
8836  **/
8837 static bool
8838 lpfc_sli4_mbox_completions_pending(struct lpfc_hba *phba)
8839 {
8840 
8841 	uint32_t idx;
8842 	struct lpfc_queue *mcq;
8843 	struct lpfc_mcqe *mcqe;
8844 	bool pending_completions = false;
8845 	uint8_t	qe_valid;
8846 
8847 	if (unlikely(!phba) || (phba->sli_rev != LPFC_SLI_REV4))
8848 		return false;
8849 
8850 	/* Check for completions on mailbox completion queue */
8851 
8852 	mcq = phba->sli4_hba.mbx_cq;
8853 	idx = mcq->hba_index;
8854 	qe_valid = mcq->qe_valid;
8855 	while (bf_get_le32(lpfc_cqe_valid,
8856 	       (struct lpfc_cqe *)lpfc_sli4_qe(mcq, idx)) == qe_valid) {
8857 		mcqe = (struct lpfc_mcqe *)(lpfc_sli4_qe(mcq, idx));
8858 		if (bf_get_le32(lpfc_trailer_completed, mcqe) &&
8859 		    (!bf_get_le32(lpfc_trailer_async, mcqe))) {
8860 			pending_completions = true;
8861 			break;
8862 		}
8863 		idx = (idx + 1) % mcq->entry_count;
8864 		if (mcq->hba_index == idx)
8865 			break;
8866 
8867 		/* if the index wrapped around, toggle the valid bit */
8868 		if (phba->sli4_hba.pc_sli4_params.cqav && !idx)
8869 			qe_valid = (qe_valid) ? 0 : 1;
8870 	}
8871 	return pending_completions;
8872 
8873 }
8874 
8875 /**
8876  * lpfc_sli4_process_missed_mbox_completions - process mbox completions
8877  *					      that were missed.
8878  * @phba: Pointer to HBA context object.
8879  *
8880  * For sli4, it is possible to miss an interrupt. As such mbox completions
8881  * maybe missed causing erroneous mailbox timeouts to occur. This function
8882  * checks to see if mbox completions are on the mailbox completion queue
8883  * and will process all the completions associated with the eq for the
8884  * mailbox completion queue.
8885  **/
8886 static bool
8887 lpfc_sli4_process_missed_mbox_completions(struct lpfc_hba *phba)
8888 {
8889 	struct lpfc_sli4_hba *sli4_hba = &phba->sli4_hba;
8890 	uint32_t eqidx;
8891 	struct lpfc_queue *fpeq = NULL;
8892 	struct lpfc_queue *eq;
8893 	bool mbox_pending;
8894 
8895 	if (unlikely(!phba) || (phba->sli_rev != LPFC_SLI_REV4))
8896 		return false;
8897 
8898 	/* Find the EQ associated with the mbox CQ */
8899 	if (sli4_hba->hdwq) {
8900 		for (eqidx = 0; eqidx < phba->cfg_irq_chann; eqidx++) {
8901 			eq = phba->sli4_hba.hba_eq_hdl[eqidx].eq;
8902 			if (eq && eq->queue_id == sli4_hba->mbx_cq->assoc_qid) {
8903 				fpeq = eq;
8904 				break;
8905 			}
8906 		}
8907 	}
8908 	if (!fpeq)
8909 		return false;
8910 
8911 	/* Turn off interrupts from this EQ */
8912 
8913 	sli4_hba->sli4_eq_clr_intr(fpeq);
8914 
8915 	/* Check to see if a mbox completion is pending */
8916 
8917 	mbox_pending = lpfc_sli4_mbox_completions_pending(phba);
8918 
8919 	/*
8920 	 * If a mbox completion is pending, process all the events on EQ
8921 	 * associated with the mbox completion queue (this could include
8922 	 * mailbox commands, async events, els commands, receive queue data
8923 	 * and fcp commands)
8924 	 */
8925 
8926 	if (mbox_pending)
8927 		/* process and rearm the EQ */
8928 		lpfc_sli4_process_eq(phba, fpeq, LPFC_QUEUE_REARM);
8929 	else
8930 		/* Always clear and re-arm the EQ */
8931 		sli4_hba->sli4_write_eq_db(phba, fpeq, 0, LPFC_QUEUE_REARM);
8932 
8933 	return mbox_pending;
8934 
8935 }
8936 
8937 /**
8938  * lpfc_mbox_timeout_handler - Worker thread function to handle mailbox timeout
8939  * @phba: Pointer to HBA context object.
8940  *
8941  * This function is called from worker thread when a mailbox command times out.
8942  * The caller is not required to hold any locks. This function will reset the
8943  * HBA and recover all the pending commands.
8944  **/
8945 void
8946 lpfc_mbox_timeout_handler(struct lpfc_hba *phba)
8947 {
8948 	LPFC_MBOXQ_t *pmbox = phba->sli.mbox_active;
8949 	MAILBOX_t *mb = NULL;
8950 
8951 	struct lpfc_sli *psli = &phba->sli;
8952 
8953 	/* If the mailbox completed, process the completion */
8954 	lpfc_sli4_process_missed_mbox_completions(phba);
8955 
8956 	if (!(psli->sli_flag & LPFC_SLI_ACTIVE))
8957 		return;
8958 
8959 	if (pmbox != NULL)
8960 		mb = &pmbox->u.mb;
8961 	/* Check the pmbox pointer first.  There is a race condition
8962 	 * between the mbox timeout handler getting executed in the
8963 	 * worklist and the mailbox actually completing. When this
8964 	 * race condition occurs, the mbox_active will be NULL.
8965 	 */
8966 	spin_lock_irq(&phba->hbalock);
8967 	if (pmbox == NULL) {
8968 		lpfc_printf_log(phba, KERN_WARNING,
8969 				LOG_MBOX | LOG_SLI,
8970 				"0353 Active Mailbox cleared - mailbox timeout "
8971 				"exiting\n");
8972 		spin_unlock_irq(&phba->hbalock);
8973 		return;
8974 	}
8975 
8976 	/* Mbox cmd <mbxCommand> timeout */
8977 	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
8978 			"0310 Mailbox command x%x timeout Data: x%x x%x x%px\n",
8979 			mb->mbxCommand,
8980 			phba->pport->port_state,
8981 			phba->sli.sli_flag,
8982 			phba->sli.mbox_active);
8983 	spin_unlock_irq(&phba->hbalock);
8984 
8985 	/* Setting state unknown so lpfc_sli_abort_iocb_ring
8986 	 * would get IOCB_ERROR from lpfc_sli_issue_iocb, allowing
8987 	 * it to fail all outstanding SCSI IO.
8988 	 */
8989 	spin_lock_irq(&phba->pport->work_port_lock);
8990 	phba->pport->work_port_events &= ~WORKER_MBOX_TMO;
8991 	spin_unlock_irq(&phba->pport->work_port_lock);
8992 	spin_lock_irq(&phba->hbalock);
8993 	phba->link_state = LPFC_LINK_UNKNOWN;
8994 	psli->sli_flag &= ~LPFC_SLI_ACTIVE;
8995 	spin_unlock_irq(&phba->hbalock);
8996 
8997 	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
8998 			"0345 Resetting board due to mailbox timeout\n");
8999 
9000 	/* Reset the HBA device */
9001 	lpfc_reset_hba(phba);
9002 }
9003 
9004 /**
9005  * lpfc_sli_issue_mbox_s3 - Issue an SLI3 mailbox command to firmware
9006  * @phba: Pointer to HBA context object.
9007  * @pmbox: Pointer to mailbox object.
9008  * @flag: Flag indicating how the mailbox need to be processed.
9009  *
9010  * This function is called by discovery code and HBA management code
9011  * to submit a mailbox command to firmware with SLI-3 interface spec. This
9012  * function gets the hbalock to protect the data structures.
9013  * The mailbox command can be submitted in polling mode, in which case
9014  * this function will wait in a polling loop for the completion of the
9015  * mailbox.
9016  * If the mailbox is submitted in no_wait mode (not polling) the
9017  * function will submit the command and returns immediately without waiting
9018  * for the mailbox completion. The no_wait is supported only when HBA
9019  * is in SLI2/SLI3 mode - interrupts are enabled.
9020  * The SLI interface allows only one mailbox pending at a time. If the
9021  * mailbox is issued in polling mode and there is already a mailbox
9022  * pending, then the function will return an error. If the mailbox is issued
9023  * in NO_WAIT mode and there is a mailbox pending already, the function
9024  * will return MBX_BUSY after queuing the mailbox into mailbox queue.
9025  * The sli layer owns the mailbox object until the completion of mailbox
9026  * command if this function return MBX_BUSY or MBX_SUCCESS. For all other
9027  * return codes the caller owns the mailbox command after the return of
9028  * the function.
9029  **/
9030 static int
9031 lpfc_sli_issue_mbox_s3(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmbox,
9032 		       uint32_t flag)
9033 {
9034 	MAILBOX_t *mbx;
9035 	struct lpfc_sli *psli = &phba->sli;
9036 	uint32_t status, evtctr;
9037 	uint32_t ha_copy, hc_copy;
9038 	int i;
9039 	unsigned long timeout;
9040 	unsigned long drvr_flag = 0;
9041 	uint32_t word0, ldata;
9042 	void __iomem *to_slim;
9043 	int processing_queue = 0;
9044 
9045 	spin_lock_irqsave(&phba->hbalock, drvr_flag);
9046 	if (!pmbox) {
9047 		phba->sli.sli_flag &= ~LPFC_SLI_MBOX_ACTIVE;
9048 		/* processing mbox queue from intr_handler */
9049 		if (unlikely(psli->sli_flag & LPFC_SLI_ASYNC_MBX_BLK)) {
9050 			spin_unlock_irqrestore(&phba->hbalock, drvr_flag);
9051 			return MBX_SUCCESS;
9052 		}
9053 		processing_queue = 1;
9054 		pmbox = lpfc_mbox_get(phba);
9055 		if (!pmbox) {
9056 			spin_unlock_irqrestore(&phba->hbalock, drvr_flag);
9057 			return MBX_SUCCESS;
9058 		}
9059 	}
9060 
9061 	if (pmbox->mbox_cmpl && pmbox->mbox_cmpl != lpfc_sli_def_mbox_cmpl &&
9062 		pmbox->mbox_cmpl != lpfc_sli_wake_mbox_wait) {
9063 		if(!pmbox->vport) {
9064 			spin_unlock_irqrestore(&phba->hbalock, drvr_flag);
9065 			lpfc_printf_log(phba, KERN_ERR,
9066 					LOG_MBOX | LOG_VPORT,
9067 					"1806 Mbox x%x failed. No vport\n",
9068 					pmbox->u.mb.mbxCommand);
9069 			dump_stack();
9070 			goto out_not_finished;
9071 		}
9072 	}
9073 
9074 	/* If the PCI channel is in offline state, do not post mbox. */
9075 	if (unlikely(pci_channel_offline(phba->pcidev))) {
9076 		spin_unlock_irqrestore(&phba->hbalock, drvr_flag);
9077 		goto out_not_finished;
9078 	}
9079 
9080 	/* If HBA has a deferred error attention, fail the iocb. */
9081 	if (unlikely(phba->hba_flag & DEFER_ERATT)) {
9082 		spin_unlock_irqrestore(&phba->hbalock, drvr_flag);
9083 		goto out_not_finished;
9084 	}
9085 
9086 	psli = &phba->sli;
9087 
9088 	mbx = &pmbox->u.mb;
9089 	status = MBX_SUCCESS;
9090 
9091 	if (phba->link_state == LPFC_HBA_ERROR) {
9092 		spin_unlock_irqrestore(&phba->hbalock, drvr_flag);
9093 
9094 		/* Mbox command <mbxCommand> cannot issue */
9095 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
9096 				"(%d):0311 Mailbox command x%x cannot "
9097 				"issue Data: x%x x%x\n",
9098 				pmbox->vport ? pmbox->vport->vpi : 0,
9099 				pmbox->u.mb.mbxCommand, psli->sli_flag, flag);
9100 		goto out_not_finished;
9101 	}
9102 
9103 	if (mbx->mbxCommand != MBX_KILL_BOARD && flag & MBX_NOWAIT) {
9104 		if (lpfc_readl(phba->HCregaddr, &hc_copy) ||
9105 			!(hc_copy & HC_MBINT_ENA)) {
9106 			spin_unlock_irqrestore(&phba->hbalock, drvr_flag);
9107 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
9108 				"(%d):2528 Mailbox command x%x cannot "
9109 				"issue Data: x%x x%x\n",
9110 				pmbox->vport ? pmbox->vport->vpi : 0,
9111 				pmbox->u.mb.mbxCommand, psli->sli_flag, flag);
9112 			goto out_not_finished;
9113 		}
9114 	}
9115 
9116 	if (psli->sli_flag & LPFC_SLI_MBOX_ACTIVE) {
9117 		/* Polling for a mbox command when another one is already active
9118 		 * is not allowed in SLI. Also, the driver must have established
9119 		 * SLI2 mode to queue and process multiple mbox commands.
9120 		 */
9121 
9122 		if (flag & MBX_POLL) {
9123 			spin_unlock_irqrestore(&phba->hbalock, drvr_flag);
9124 
9125 			/* Mbox command <mbxCommand> cannot issue */
9126 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
9127 					"(%d):2529 Mailbox command x%x "
9128 					"cannot issue Data: x%x x%x\n",
9129 					pmbox->vport ? pmbox->vport->vpi : 0,
9130 					pmbox->u.mb.mbxCommand,
9131 					psli->sli_flag, flag);
9132 			goto out_not_finished;
9133 		}
9134 
9135 		if (!(psli->sli_flag & LPFC_SLI_ACTIVE)) {
9136 			spin_unlock_irqrestore(&phba->hbalock, drvr_flag);
9137 			/* Mbox command <mbxCommand> cannot issue */
9138 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
9139 					"(%d):2530 Mailbox command x%x "
9140 					"cannot issue Data: x%x x%x\n",
9141 					pmbox->vport ? pmbox->vport->vpi : 0,
9142 					pmbox->u.mb.mbxCommand,
9143 					psli->sli_flag, flag);
9144 			goto out_not_finished;
9145 		}
9146 
9147 		/* Another mailbox command is still being processed, queue this
9148 		 * command to be processed later.
9149 		 */
9150 		lpfc_mbox_put(phba, pmbox);
9151 
9152 		/* Mbox cmd issue - BUSY */
9153 		lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI,
9154 				"(%d):0308 Mbox cmd issue - BUSY Data: "
9155 				"x%x x%x x%x x%x\n",
9156 				pmbox->vport ? pmbox->vport->vpi : 0xffffff,
9157 				mbx->mbxCommand,
9158 				phba->pport ? phba->pport->port_state : 0xff,
9159 				psli->sli_flag, flag);
9160 
9161 		psli->slistat.mbox_busy++;
9162 		spin_unlock_irqrestore(&phba->hbalock, drvr_flag);
9163 
9164 		if (pmbox->vport) {
9165 			lpfc_debugfs_disc_trc(pmbox->vport,
9166 				LPFC_DISC_TRC_MBOX_VPORT,
9167 				"MBOX Bsy vport:  cmd:x%x mb:x%x x%x",
9168 				(uint32_t)mbx->mbxCommand,
9169 				mbx->un.varWords[0], mbx->un.varWords[1]);
9170 		}
9171 		else {
9172 			lpfc_debugfs_disc_trc(phba->pport,
9173 				LPFC_DISC_TRC_MBOX,
9174 				"MBOX Bsy:        cmd:x%x mb:x%x x%x",
9175 				(uint32_t)mbx->mbxCommand,
9176 				mbx->un.varWords[0], mbx->un.varWords[1]);
9177 		}
9178 
9179 		return MBX_BUSY;
9180 	}
9181 
9182 	psli->sli_flag |= LPFC_SLI_MBOX_ACTIVE;
9183 
9184 	/* If we are not polling, we MUST be in SLI2 mode */
9185 	if (flag != MBX_POLL) {
9186 		if (!(psli->sli_flag & LPFC_SLI_ACTIVE) &&
9187 		    (mbx->mbxCommand != MBX_KILL_BOARD)) {
9188 			psli->sli_flag &= ~LPFC_SLI_MBOX_ACTIVE;
9189 			spin_unlock_irqrestore(&phba->hbalock, drvr_flag);
9190 			/* Mbox command <mbxCommand> cannot issue */
9191 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
9192 					"(%d):2531 Mailbox command x%x "
9193 					"cannot issue Data: x%x x%x\n",
9194 					pmbox->vport ? pmbox->vport->vpi : 0,
9195 					pmbox->u.mb.mbxCommand,
9196 					psli->sli_flag, flag);
9197 			goto out_not_finished;
9198 		}
9199 		/* timeout active mbox command */
9200 		timeout = msecs_to_jiffies(lpfc_mbox_tmo_val(phba, pmbox) *
9201 					   1000);
9202 		mod_timer(&psli->mbox_tmo, jiffies + timeout);
9203 	}
9204 
9205 	/* Mailbox cmd <cmd> issue */
9206 	lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI,
9207 			"(%d):0309 Mailbox cmd x%x issue Data: x%x x%x "
9208 			"x%x\n",
9209 			pmbox->vport ? pmbox->vport->vpi : 0,
9210 			mbx->mbxCommand,
9211 			phba->pport ? phba->pport->port_state : 0xff,
9212 			psli->sli_flag, flag);
9213 
9214 	if (mbx->mbxCommand != MBX_HEARTBEAT) {
9215 		if (pmbox->vport) {
9216 			lpfc_debugfs_disc_trc(pmbox->vport,
9217 				LPFC_DISC_TRC_MBOX_VPORT,
9218 				"MBOX Send vport: cmd:x%x mb:x%x x%x",
9219 				(uint32_t)mbx->mbxCommand,
9220 				mbx->un.varWords[0], mbx->un.varWords[1]);
9221 		}
9222 		else {
9223 			lpfc_debugfs_disc_trc(phba->pport,
9224 				LPFC_DISC_TRC_MBOX,
9225 				"MBOX Send:       cmd:x%x mb:x%x x%x",
9226 				(uint32_t)mbx->mbxCommand,
9227 				mbx->un.varWords[0], mbx->un.varWords[1]);
9228 		}
9229 	}
9230 
9231 	psli->slistat.mbox_cmd++;
9232 	evtctr = psli->slistat.mbox_event;
9233 
9234 	/* next set own bit for the adapter and copy over command word */
9235 	mbx->mbxOwner = OWN_CHIP;
9236 
9237 	if (psli->sli_flag & LPFC_SLI_ACTIVE) {
9238 		/* Populate mbox extension offset word. */
9239 		if (pmbox->in_ext_byte_len || pmbox->out_ext_byte_len) {
9240 			*(((uint32_t *)mbx) + pmbox->mbox_offset_word)
9241 				= (uint8_t *)phba->mbox_ext
9242 				  - (uint8_t *)phba->mbox;
9243 		}
9244 
9245 		/* Copy the mailbox extension data */
9246 		if (pmbox->in_ext_byte_len && pmbox->ctx_buf) {
9247 			lpfc_sli_pcimem_bcopy(pmbox->ctx_buf,
9248 					      (uint8_t *)phba->mbox_ext,
9249 					      pmbox->in_ext_byte_len);
9250 		}
9251 		/* Copy command data to host SLIM area */
9252 		lpfc_sli_pcimem_bcopy(mbx, phba->mbox, MAILBOX_CMD_SIZE);
9253 	} else {
9254 		/* Populate mbox extension offset word. */
9255 		if (pmbox->in_ext_byte_len || pmbox->out_ext_byte_len)
9256 			*(((uint32_t *)mbx) + pmbox->mbox_offset_word)
9257 				= MAILBOX_HBA_EXT_OFFSET;
9258 
9259 		/* Copy the mailbox extension data */
9260 		if (pmbox->in_ext_byte_len && pmbox->ctx_buf)
9261 			lpfc_memcpy_to_slim(phba->MBslimaddr +
9262 				MAILBOX_HBA_EXT_OFFSET,
9263 				pmbox->ctx_buf, pmbox->in_ext_byte_len);
9264 
9265 		if (mbx->mbxCommand == MBX_CONFIG_PORT)
9266 			/* copy command data into host mbox for cmpl */
9267 			lpfc_sli_pcimem_bcopy(mbx, phba->mbox,
9268 					      MAILBOX_CMD_SIZE);
9269 
9270 		/* First copy mbox command data to HBA SLIM, skip past first
9271 		   word */
9272 		to_slim = phba->MBslimaddr + sizeof (uint32_t);
9273 		lpfc_memcpy_to_slim(to_slim, &mbx->un.varWords[0],
9274 			    MAILBOX_CMD_SIZE - sizeof (uint32_t));
9275 
9276 		/* Next copy over first word, with mbxOwner set */
9277 		ldata = *((uint32_t *)mbx);
9278 		to_slim = phba->MBslimaddr;
9279 		writel(ldata, to_slim);
9280 		readl(to_slim); /* flush */
9281 
9282 		if (mbx->mbxCommand == MBX_CONFIG_PORT)
9283 			/* switch over to host mailbox */
9284 			psli->sli_flag |= LPFC_SLI_ACTIVE;
9285 	}
9286 
9287 	wmb();
9288 
9289 	switch (flag) {
9290 	case MBX_NOWAIT:
9291 		/* Set up reference to mailbox command */
9292 		psli->mbox_active = pmbox;
9293 		/* Interrupt board to do it */
9294 		writel(CA_MBATT, phba->CAregaddr);
9295 		readl(phba->CAregaddr); /* flush */
9296 		/* Don't wait for it to finish, just return */
9297 		break;
9298 
9299 	case MBX_POLL:
9300 		/* Set up null reference to mailbox command */
9301 		psli->mbox_active = NULL;
9302 		/* Interrupt board to do it */
9303 		writel(CA_MBATT, phba->CAregaddr);
9304 		readl(phba->CAregaddr); /* flush */
9305 
9306 		if (psli->sli_flag & LPFC_SLI_ACTIVE) {
9307 			/* First read mbox status word */
9308 			word0 = *((uint32_t *)phba->mbox);
9309 			word0 = le32_to_cpu(word0);
9310 		} else {
9311 			/* First read mbox status word */
9312 			if (lpfc_readl(phba->MBslimaddr, &word0)) {
9313 				spin_unlock_irqrestore(&phba->hbalock,
9314 						       drvr_flag);
9315 				goto out_not_finished;
9316 			}
9317 		}
9318 
9319 		/* Read the HBA Host Attention Register */
9320 		if (lpfc_readl(phba->HAregaddr, &ha_copy)) {
9321 			spin_unlock_irqrestore(&phba->hbalock,
9322 						       drvr_flag);
9323 			goto out_not_finished;
9324 		}
9325 		timeout = msecs_to_jiffies(lpfc_mbox_tmo_val(phba, pmbox) *
9326 							1000) + jiffies;
9327 		i = 0;
9328 		/* Wait for command to complete */
9329 		while (((word0 & OWN_CHIP) == OWN_CHIP) ||
9330 		       (!(ha_copy & HA_MBATT) &&
9331 			(phba->link_state > LPFC_WARM_START))) {
9332 			if (time_after(jiffies, timeout)) {
9333 				psli->sli_flag &= ~LPFC_SLI_MBOX_ACTIVE;
9334 				spin_unlock_irqrestore(&phba->hbalock,
9335 						       drvr_flag);
9336 				goto out_not_finished;
9337 			}
9338 
9339 			/* Check if we took a mbox interrupt while we were
9340 			   polling */
9341 			if (((word0 & OWN_CHIP) != OWN_CHIP)
9342 			    && (evtctr != psli->slistat.mbox_event))
9343 				break;
9344 
9345 			if (i++ > 10) {
9346 				spin_unlock_irqrestore(&phba->hbalock,
9347 						       drvr_flag);
9348 				msleep(1);
9349 				spin_lock_irqsave(&phba->hbalock, drvr_flag);
9350 			}
9351 
9352 			if (psli->sli_flag & LPFC_SLI_ACTIVE) {
9353 				/* First copy command data */
9354 				word0 = *((uint32_t *)phba->mbox);
9355 				word0 = le32_to_cpu(word0);
9356 				if (mbx->mbxCommand == MBX_CONFIG_PORT) {
9357 					MAILBOX_t *slimmb;
9358 					uint32_t slimword0;
9359 					/* Check real SLIM for any errors */
9360 					slimword0 = readl(phba->MBslimaddr);
9361 					slimmb = (MAILBOX_t *) & slimword0;
9362 					if (((slimword0 & OWN_CHIP) != OWN_CHIP)
9363 					    && slimmb->mbxStatus) {
9364 						psli->sli_flag &=
9365 						    ~LPFC_SLI_ACTIVE;
9366 						word0 = slimword0;
9367 					}
9368 				}
9369 			} else {
9370 				/* First copy command data */
9371 				word0 = readl(phba->MBslimaddr);
9372 			}
9373 			/* Read the HBA Host Attention Register */
9374 			if (lpfc_readl(phba->HAregaddr, &ha_copy)) {
9375 				spin_unlock_irqrestore(&phba->hbalock,
9376 						       drvr_flag);
9377 				goto out_not_finished;
9378 			}
9379 		}
9380 
9381 		if (psli->sli_flag & LPFC_SLI_ACTIVE) {
9382 			/* copy results back to user */
9383 			lpfc_sli_pcimem_bcopy(phba->mbox, mbx,
9384 						MAILBOX_CMD_SIZE);
9385 			/* Copy the mailbox extension data */
9386 			if (pmbox->out_ext_byte_len && pmbox->ctx_buf) {
9387 				lpfc_sli_pcimem_bcopy(phba->mbox_ext,
9388 						      pmbox->ctx_buf,
9389 						      pmbox->out_ext_byte_len);
9390 			}
9391 		} else {
9392 			/* First copy command data */
9393 			lpfc_memcpy_from_slim(mbx, phba->MBslimaddr,
9394 						MAILBOX_CMD_SIZE);
9395 			/* Copy the mailbox extension data */
9396 			if (pmbox->out_ext_byte_len && pmbox->ctx_buf) {
9397 				lpfc_memcpy_from_slim(
9398 					pmbox->ctx_buf,
9399 					phba->MBslimaddr +
9400 					MAILBOX_HBA_EXT_OFFSET,
9401 					pmbox->out_ext_byte_len);
9402 			}
9403 		}
9404 
9405 		writel(HA_MBATT, phba->HAregaddr);
9406 		readl(phba->HAregaddr); /* flush */
9407 
9408 		psli->sli_flag &= ~LPFC_SLI_MBOX_ACTIVE;
9409 		status = mbx->mbxStatus;
9410 	}
9411 
9412 	spin_unlock_irqrestore(&phba->hbalock, drvr_flag);
9413 	return status;
9414 
9415 out_not_finished:
9416 	if (processing_queue) {
9417 		pmbox->u.mb.mbxStatus = MBX_NOT_FINISHED;
9418 		lpfc_mbox_cmpl_put(phba, pmbox);
9419 	}
9420 	return MBX_NOT_FINISHED;
9421 }
9422 
9423 /**
9424  * lpfc_sli4_async_mbox_block - Block posting SLI4 asynchronous mailbox command
9425  * @phba: Pointer to HBA context object.
9426  *
9427  * The function blocks the posting of SLI4 asynchronous mailbox commands from
9428  * the driver internal pending mailbox queue. It will then try to wait out the
9429  * possible outstanding mailbox command before return.
9430  *
9431  * Returns:
9432  * 	0 - the outstanding mailbox command completed; otherwise, the wait for
9433  * 	the outstanding mailbox command timed out.
9434  **/
9435 static int
9436 lpfc_sli4_async_mbox_block(struct lpfc_hba *phba)
9437 {
9438 	struct lpfc_sli *psli = &phba->sli;
9439 	LPFC_MBOXQ_t *mboxq;
9440 	int rc = 0;
9441 	unsigned long timeout = 0;
9442 	u32 sli_flag;
9443 	u8 cmd, subsys, opcode;
9444 
9445 	/* Mark the asynchronous mailbox command posting as blocked */
9446 	spin_lock_irq(&phba->hbalock);
9447 	psli->sli_flag |= LPFC_SLI_ASYNC_MBX_BLK;
9448 	/* Determine how long we might wait for the active mailbox
9449 	 * command to be gracefully completed by firmware.
9450 	 */
9451 	if (phba->sli.mbox_active)
9452 		timeout = msecs_to_jiffies(lpfc_mbox_tmo_val(phba,
9453 						phba->sli.mbox_active) *
9454 						1000) + jiffies;
9455 	spin_unlock_irq(&phba->hbalock);
9456 
9457 	/* Make sure the mailbox is really active */
9458 	if (timeout)
9459 		lpfc_sli4_process_missed_mbox_completions(phba);
9460 
9461 	/* Wait for the outstanding mailbox command to complete */
9462 	while (phba->sli.mbox_active) {
9463 		/* Check active mailbox complete status every 2ms */
9464 		msleep(2);
9465 		if (time_after(jiffies, timeout)) {
9466 			/* Timeout, mark the outstanding cmd not complete */
9467 
9468 			/* Sanity check sli.mbox_active has not completed or
9469 			 * cancelled from another context during last 2ms sleep,
9470 			 * so take hbalock to be sure before logging.
9471 			 */
9472 			spin_lock_irq(&phba->hbalock);
9473 			if (phba->sli.mbox_active) {
9474 				mboxq = phba->sli.mbox_active;
9475 				cmd = mboxq->u.mb.mbxCommand;
9476 				subsys = lpfc_sli_config_mbox_subsys_get(phba,
9477 									 mboxq);
9478 				opcode = lpfc_sli_config_mbox_opcode_get(phba,
9479 									 mboxq);
9480 				sli_flag = psli->sli_flag;
9481 				spin_unlock_irq(&phba->hbalock);
9482 				lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
9483 						"2352 Mailbox command x%x "
9484 						"(x%x/x%x) sli_flag x%x could "
9485 						"not complete\n",
9486 						cmd, subsys, opcode,
9487 						sli_flag);
9488 			} else {
9489 				spin_unlock_irq(&phba->hbalock);
9490 			}
9491 
9492 			rc = 1;
9493 			break;
9494 		}
9495 	}
9496 
9497 	/* Can not cleanly block async mailbox command, fails it */
9498 	if (rc) {
9499 		spin_lock_irq(&phba->hbalock);
9500 		psli->sli_flag &= ~LPFC_SLI_ASYNC_MBX_BLK;
9501 		spin_unlock_irq(&phba->hbalock);
9502 	}
9503 	return rc;
9504 }
9505 
9506 /**
9507  * lpfc_sli4_async_mbox_unblock - Block posting SLI4 async mailbox command
9508  * @phba: Pointer to HBA context object.
9509  *
9510  * The function unblocks and resume posting of SLI4 asynchronous mailbox
9511  * commands from the driver internal pending mailbox queue. It makes sure
9512  * that there is no outstanding mailbox command before resuming posting
9513  * asynchronous mailbox commands. If, for any reason, there is outstanding
9514  * mailbox command, it will try to wait it out before resuming asynchronous
9515  * mailbox command posting.
9516  **/
9517 static void
9518 lpfc_sli4_async_mbox_unblock(struct lpfc_hba *phba)
9519 {
9520 	struct lpfc_sli *psli = &phba->sli;
9521 
9522 	spin_lock_irq(&phba->hbalock);
9523 	if (!(psli->sli_flag & LPFC_SLI_ASYNC_MBX_BLK)) {
9524 		/* Asynchronous mailbox posting is not blocked, do nothing */
9525 		spin_unlock_irq(&phba->hbalock);
9526 		return;
9527 	}
9528 
9529 	/* Outstanding synchronous mailbox command is guaranteed to be done,
9530 	 * successful or timeout, after timing-out the outstanding mailbox
9531 	 * command shall always be removed, so just unblock posting async
9532 	 * mailbox command and resume
9533 	 */
9534 	psli->sli_flag &= ~LPFC_SLI_ASYNC_MBX_BLK;
9535 	spin_unlock_irq(&phba->hbalock);
9536 
9537 	/* wake up worker thread to post asynchronous mailbox command */
9538 	lpfc_worker_wake_up(phba);
9539 }
9540 
9541 /**
9542  * lpfc_sli4_wait_bmbx_ready - Wait for bootstrap mailbox register ready
9543  * @phba: Pointer to HBA context object.
9544  * @mboxq: Pointer to mailbox object.
9545  *
9546  * The function waits for the bootstrap mailbox register ready bit from
9547  * port for twice the regular mailbox command timeout value.
9548  *
9549  *      0 - no timeout on waiting for bootstrap mailbox register ready.
9550  *      MBXERR_ERROR - wait for bootstrap mailbox register timed out.
9551  **/
9552 static int
9553 lpfc_sli4_wait_bmbx_ready(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq)
9554 {
9555 	uint32_t db_ready;
9556 	unsigned long timeout;
9557 	struct lpfc_register bmbx_reg;
9558 
9559 	timeout = msecs_to_jiffies(lpfc_mbox_tmo_val(phba, mboxq)
9560 				   * 1000) + jiffies;
9561 
9562 	do {
9563 		bmbx_reg.word0 = readl(phba->sli4_hba.BMBXregaddr);
9564 		db_ready = bf_get(lpfc_bmbx_rdy, &bmbx_reg);
9565 		if (!db_ready)
9566 			mdelay(2);
9567 
9568 		if (time_after(jiffies, timeout))
9569 			return MBXERR_ERROR;
9570 	} while (!db_ready);
9571 
9572 	return 0;
9573 }
9574 
9575 /**
9576  * lpfc_sli4_post_sync_mbox - Post an SLI4 mailbox to the bootstrap mailbox
9577  * @phba: Pointer to HBA context object.
9578  * @mboxq: Pointer to mailbox object.
9579  *
9580  * The function posts a mailbox to the port.  The mailbox is expected
9581  * to be comletely filled in and ready for the port to operate on it.
9582  * This routine executes a synchronous completion operation on the
9583  * mailbox by polling for its completion.
9584  *
9585  * The caller must not be holding any locks when calling this routine.
9586  *
9587  * Returns:
9588  *	MBX_SUCCESS - mailbox posted successfully
9589  *	Any of the MBX error values.
9590  **/
9591 static int
9592 lpfc_sli4_post_sync_mbox(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq)
9593 {
9594 	int rc = MBX_SUCCESS;
9595 	unsigned long iflag;
9596 	uint32_t mcqe_status;
9597 	uint32_t mbx_cmnd;
9598 	struct lpfc_sli *psli = &phba->sli;
9599 	struct lpfc_mqe *mb = &mboxq->u.mqe;
9600 	struct lpfc_bmbx_create *mbox_rgn;
9601 	struct dma_address *dma_address;
9602 
9603 	/*
9604 	 * Only one mailbox can be active to the bootstrap mailbox region
9605 	 * at a time and there is no queueing provided.
9606 	 */
9607 	spin_lock_irqsave(&phba->hbalock, iflag);
9608 	if (psli->sli_flag & LPFC_SLI_MBOX_ACTIVE) {
9609 		spin_unlock_irqrestore(&phba->hbalock, iflag);
9610 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
9611 				"(%d):2532 Mailbox command x%x (x%x/x%x) "
9612 				"cannot issue Data: x%x x%x\n",
9613 				mboxq->vport ? mboxq->vport->vpi : 0,
9614 				mboxq->u.mb.mbxCommand,
9615 				lpfc_sli_config_mbox_subsys_get(phba, mboxq),
9616 				lpfc_sli_config_mbox_opcode_get(phba, mboxq),
9617 				psli->sli_flag, MBX_POLL);
9618 		return MBXERR_ERROR;
9619 	}
9620 	/* The server grabs the token and owns it until release */
9621 	psli->sli_flag |= LPFC_SLI_MBOX_ACTIVE;
9622 	phba->sli.mbox_active = mboxq;
9623 	spin_unlock_irqrestore(&phba->hbalock, iflag);
9624 
9625 	/* wait for bootstrap mbox register for readyness */
9626 	rc = lpfc_sli4_wait_bmbx_ready(phba, mboxq);
9627 	if (rc)
9628 		goto exit;
9629 	/*
9630 	 * Initialize the bootstrap memory region to avoid stale data areas
9631 	 * in the mailbox post.  Then copy the caller's mailbox contents to
9632 	 * the bmbx mailbox region.
9633 	 */
9634 	mbx_cmnd = bf_get(lpfc_mqe_command, mb);
9635 	memset(phba->sli4_hba.bmbx.avirt, 0, sizeof(struct lpfc_bmbx_create));
9636 	lpfc_sli4_pcimem_bcopy(mb, phba->sli4_hba.bmbx.avirt,
9637 			       sizeof(struct lpfc_mqe));
9638 
9639 	/* Post the high mailbox dma address to the port and wait for ready. */
9640 	dma_address = &phba->sli4_hba.bmbx.dma_address;
9641 	writel(dma_address->addr_hi, phba->sli4_hba.BMBXregaddr);
9642 
9643 	/* wait for bootstrap mbox register for hi-address write done */
9644 	rc = lpfc_sli4_wait_bmbx_ready(phba, mboxq);
9645 	if (rc)
9646 		goto exit;
9647 
9648 	/* Post the low mailbox dma address to the port. */
9649 	writel(dma_address->addr_lo, phba->sli4_hba.BMBXregaddr);
9650 
9651 	/* wait for bootstrap mbox register for low address write done */
9652 	rc = lpfc_sli4_wait_bmbx_ready(phba, mboxq);
9653 	if (rc)
9654 		goto exit;
9655 
9656 	/*
9657 	 * Read the CQ to ensure the mailbox has completed.
9658 	 * If so, update the mailbox status so that the upper layers
9659 	 * can complete the request normally.
9660 	 */
9661 	lpfc_sli4_pcimem_bcopy(phba->sli4_hba.bmbx.avirt, mb,
9662 			       sizeof(struct lpfc_mqe));
9663 	mbox_rgn = (struct lpfc_bmbx_create *) phba->sli4_hba.bmbx.avirt;
9664 	lpfc_sli4_pcimem_bcopy(&mbox_rgn->mcqe, &mboxq->mcqe,
9665 			       sizeof(struct lpfc_mcqe));
9666 	mcqe_status = bf_get(lpfc_mcqe_status, &mbox_rgn->mcqe);
9667 	/*
9668 	 * When the CQE status indicates a failure and the mailbox status
9669 	 * indicates success then copy the CQE status into the mailbox status
9670 	 * (and prefix it with x4000).
9671 	 */
9672 	if (mcqe_status != MB_CQE_STATUS_SUCCESS) {
9673 		if (bf_get(lpfc_mqe_status, mb) == MBX_SUCCESS)
9674 			bf_set(lpfc_mqe_status, mb,
9675 			       (LPFC_MBX_ERROR_RANGE | mcqe_status));
9676 		rc = MBXERR_ERROR;
9677 	} else
9678 		lpfc_sli4_swap_str(phba, mboxq);
9679 
9680 	lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI,
9681 			"(%d):0356 Mailbox cmd x%x (x%x/x%x) Status x%x "
9682 			"Data: x%x x%x x%x x%x x%x x%x x%x x%x x%x x%x x%x"
9683 			" x%x x%x CQ: x%x x%x x%x x%x\n",
9684 			mboxq->vport ? mboxq->vport->vpi : 0, mbx_cmnd,
9685 			lpfc_sli_config_mbox_subsys_get(phba, mboxq),
9686 			lpfc_sli_config_mbox_opcode_get(phba, mboxq),
9687 			bf_get(lpfc_mqe_status, mb),
9688 			mb->un.mb_words[0], mb->un.mb_words[1],
9689 			mb->un.mb_words[2], mb->un.mb_words[3],
9690 			mb->un.mb_words[4], mb->un.mb_words[5],
9691 			mb->un.mb_words[6], mb->un.mb_words[7],
9692 			mb->un.mb_words[8], mb->un.mb_words[9],
9693 			mb->un.mb_words[10], mb->un.mb_words[11],
9694 			mb->un.mb_words[12], mboxq->mcqe.word0,
9695 			mboxq->mcqe.mcqe_tag0, 	mboxq->mcqe.mcqe_tag1,
9696 			mboxq->mcqe.trailer);
9697 exit:
9698 	/* We are holding the token, no needed for lock when release */
9699 	spin_lock_irqsave(&phba->hbalock, iflag);
9700 	psli->sli_flag &= ~LPFC_SLI_MBOX_ACTIVE;
9701 	phba->sli.mbox_active = NULL;
9702 	spin_unlock_irqrestore(&phba->hbalock, iflag);
9703 	return rc;
9704 }
9705 
9706 /**
9707  * lpfc_sli_issue_mbox_s4 - Issue an SLI4 mailbox command to firmware
9708  * @phba: Pointer to HBA context object.
9709  * @mboxq: Pointer to mailbox object.
9710  * @flag: Flag indicating how the mailbox need to be processed.
9711  *
9712  * This function is called by discovery code and HBA management code to submit
9713  * a mailbox command to firmware with SLI-4 interface spec.
9714  *
9715  * Return codes the caller owns the mailbox command after the return of the
9716  * function.
9717  **/
9718 static int
9719 lpfc_sli_issue_mbox_s4(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq,
9720 		       uint32_t flag)
9721 {
9722 	struct lpfc_sli *psli = &phba->sli;
9723 	unsigned long iflags;
9724 	int rc;
9725 
9726 	/* dump from issue mailbox command if setup */
9727 	lpfc_idiag_mbxacc_dump_issue_mbox(phba, &mboxq->u.mb);
9728 
9729 	rc = lpfc_mbox_dev_check(phba);
9730 	if (unlikely(rc)) {
9731 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
9732 				"(%d):2544 Mailbox command x%x (x%x/x%x) "
9733 				"cannot issue Data: x%x x%x\n",
9734 				mboxq->vport ? mboxq->vport->vpi : 0,
9735 				mboxq->u.mb.mbxCommand,
9736 				lpfc_sli_config_mbox_subsys_get(phba, mboxq),
9737 				lpfc_sli_config_mbox_opcode_get(phba, mboxq),
9738 				psli->sli_flag, flag);
9739 		goto out_not_finished;
9740 	}
9741 
9742 	/* Detect polling mode and jump to a handler */
9743 	if (!phba->sli4_hba.intr_enable) {
9744 		if (flag == MBX_POLL)
9745 			rc = lpfc_sli4_post_sync_mbox(phba, mboxq);
9746 		else
9747 			rc = -EIO;
9748 		if (rc != MBX_SUCCESS)
9749 			lpfc_printf_log(phba, KERN_WARNING, LOG_MBOX | LOG_SLI,
9750 					"(%d):2541 Mailbox command x%x "
9751 					"(x%x/x%x) failure: "
9752 					"mqe_sta: x%x mcqe_sta: x%x/x%x "
9753 					"Data: x%x x%x\n,",
9754 					mboxq->vport ? mboxq->vport->vpi : 0,
9755 					mboxq->u.mb.mbxCommand,
9756 					lpfc_sli_config_mbox_subsys_get(phba,
9757 									mboxq),
9758 					lpfc_sli_config_mbox_opcode_get(phba,
9759 									mboxq),
9760 					bf_get(lpfc_mqe_status, &mboxq->u.mqe),
9761 					bf_get(lpfc_mcqe_status, &mboxq->mcqe),
9762 					bf_get(lpfc_mcqe_ext_status,
9763 					       &mboxq->mcqe),
9764 					psli->sli_flag, flag);
9765 		return rc;
9766 	} else if (flag == MBX_POLL) {
9767 		lpfc_printf_log(phba, KERN_WARNING, LOG_MBOX | LOG_SLI,
9768 				"(%d):2542 Try to issue mailbox command "
9769 				"x%x (x%x/x%x) synchronously ahead of async "
9770 				"mailbox command queue: x%x x%x\n",
9771 				mboxq->vport ? mboxq->vport->vpi : 0,
9772 				mboxq->u.mb.mbxCommand,
9773 				lpfc_sli_config_mbox_subsys_get(phba, mboxq),
9774 				lpfc_sli_config_mbox_opcode_get(phba, mboxq),
9775 				psli->sli_flag, flag);
9776 		/* Try to block the asynchronous mailbox posting */
9777 		rc = lpfc_sli4_async_mbox_block(phba);
9778 		if (!rc) {
9779 			/* Successfully blocked, now issue sync mbox cmd */
9780 			rc = lpfc_sli4_post_sync_mbox(phba, mboxq);
9781 			if (rc != MBX_SUCCESS)
9782 				lpfc_printf_log(phba, KERN_WARNING,
9783 					LOG_MBOX | LOG_SLI,
9784 					"(%d):2597 Sync Mailbox command "
9785 					"x%x (x%x/x%x) failure: "
9786 					"mqe_sta: x%x mcqe_sta: x%x/x%x "
9787 					"Data: x%x x%x\n,",
9788 					mboxq->vport ? mboxq->vport->vpi : 0,
9789 					mboxq->u.mb.mbxCommand,
9790 					lpfc_sli_config_mbox_subsys_get(phba,
9791 									mboxq),
9792 					lpfc_sli_config_mbox_opcode_get(phba,
9793 									mboxq),
9794 					bf_get(lpfc_mqe_status, &mboxq->u.mqe),
9795 					bf_get(lpfc_mcqe_status, &mboxq->mcqe),
9796 					bf_get(lpfc_mcqe_ext_status,
9797 					       &mboxq->mcqe),
9798 					psli->sli_flag, flag);
9799 			/* Unblock the async mailbox posting afterward */
9800 			lpfc_sli4_async_mbox_unblock(phba);
9801 		}
9802 		return rc;
9803 	}
9804 
9805 	/* Now, interrupt mode asynchronous mailbox command */
9806 	rc = lpfc_mbox_cmd_check(phba, mboxq);
9807 	if (rc) {
9808 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
9809 				"(%d):2543 Mailbox command x%x (x%x/x%x) "
9810 				"cannot issue Data: x%x x%x\n",
9811 				mboxq->vport ? mboxq->vport->vpi : 0,
9812 				mboxq->u.mb.mbxCommand,
9813 				lpfc_sli_config_mbox_subsys_get(phba, mboxq),
9814 				lpfc_sli_config_mbox_opcode_get(phba, mboxq),
9815 				psli->sli_flag, flag);
9816 		goto out_not_finished;
9817 	}
9818 
9819 	/* Put the mailbox command to the driver internal FIFO */
9820 	psli->slistat.mbox_busy++;
9821 	spin_lock_irqsave(&phba->hbalock, iflags);
9822 	lpfc_mbox_put(phba, mboxq);
9823 	spin_unlock_irqrestore(&phba->hbalock, iflags);
9824 	lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI,
9825 			"(%d):0354 Mbox cmd issue - Enqueue Data: "
9826 			"x%x (x%x/x%x) x%x x%x x%x\n",
9827 			mboxq->vport ? mboxq->vport->vpi : 0xffffff,
9828 			bf_get(lpfc_mqe_command, &mboxq->u.mqe),
9829 			lpfc_sli_config_mbox_subsys_get(phba, mboxq),
9830 			lpfc_sli_config_mbox_opcode_get(phba, mboxq),
9831 			phba->pport->port_state,
9832 			psli->sli_flag, MBX_NOWAIT);
9833 	/* Wake up worker thread to transport mailbox command from head */
9834 	lpfc_worker_wake_up(phba);
9835 
9836 	return MBX_BUSY;
9837 
9838 out_not_finished:
9839 	return MBX_NOT_FINISHED;
9840 }
9841 
9842 /**
9843  * lpfc_sli4_post_async_mbox - Post an SLI4 mailbox command to device
9844  * @phba: Pointer to HBA context object.
9845  *
9846  * This function is called by worker thread to send a mailbox command to
9847  * SLI4 HBA firmware.
9848  *
9849  **/
9850 int
9851 lpfc_sli4_post_async_mbox(struct lpfc_hba *phba)
9852 {
9853 	struct lpfc_sli *psli = &phba->sli;
9854 	LPFC_MBOXQ_t *mboxq;
9855 	int rc = MBX_SUCCESS;
9856 	unsigned long iflags;
9857 	struct lpfc_mqe *mqe;
9858 	uint32_t mbx_cmnd;
9859 
9860 	/* Check interrupt mode before post async mailbox command */
9861 	if (unlikely(!phba->sli4_hba.intr_enable))
9862 		return MBX_NOT_FINISHED;
9863 
9864 	/* Check for mailbox command service token */
9865 	spin_lock_irqsave(&phba->hbalock, iflags);
9866 	if (unlikely(psli->sli_flag & LPFC_SLI_ASYNC_MBX_BLK)) {
9867 		spin_unlock_irqrestore(&phba->hbalock, iflags);
9868 		return MBX_NOT_FINISHED;
9869 	}
9870 	if (psli->sli_flag & LPFC_SLI_MBOX_ACTIVE) {
9871 		spin_unlock_irqrestore(&phba->hbalock, iflags);
9872 		return MBX_NOT_FINISHED;
9873 	}
9874 	if (unlikely(phba->sli.mbox_active)) {
9875 		spin_unlock_irqrestore(&phba->hbalock, iflags);
9876 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
9877 				"0384 There is pending active mailbox cmd\n");
9878 		return MBX_NOT_FINISHED;
9879 	}
9880 	/* Take the mailbox command service token */
9881 	psli->sli_flag |= LPFC_SLI_MBOX_ACTIVE;
9882 
9883 	/* Get the next mailbox command from head of queue */
9884 	mboxq = lpfc_mbox_get(phba);
9885 
9886 	/* If no more mailbox command waiting for post, we're done */
9887 	if (!mboxq) {
9888 		psli->sli_flag &= ~LPFC_SLI_MBOX_ACTIVE;
9889 		spin_unlock_irqrestore(&phba->hbalock, iflags);
9890 		return MBX_SUCCESS;
9891 	}
9892 	phba->sli.mbox_active = mboxq;
9893 	spin_unlock_irqrestore(&phba->hbalock, iflags);
9894 
9895 	/* Check device readiness for posting mailbox command */
9896 	rc = lpfc_mbox_dev_check(phba);
9897 	if (unlikely(rc))
9898 		/* Driver clean routine will clean up pending mailbox */
9899 		goto out_not_finished;
9900 
9901 	/* Prepare the mbox command to be posted */
9902 	mqe = &mboxq->u.mqe;
9903 	mbx_cmnd = bf_get(lpfc_mqe_command, mqe);
9904 
9905 	/* Start timer for the mbox_tmo and log some mailbox post messages */
9906 	mod_timer(&psli->mbox_tmo, (jiffies +
9907 		  msecs_to_jiffies(1000 * lpfc_mbox_tmo_val(phba, mboxq))));
9908 
9909 	lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI,
9910 			"(%d):0355 Mailbox cmd x%x (x%x/x%x) issue Data: "
9911 			"x%x x%x\n",
9912 			mboxq->vport ? mboxq->vport->vpi : 0, mbx_cmnd,
9913 			lpfc_sli_config_mbox_subsys_get(phba, mboxq),
9914 			lpfc_sli_config_mbox_opcode_get(phba, mboxq),
9915 			phba->pport->port_state, psli->sli_flag);
9916 
9917 	if (mbx_cmnd != MBX_HEARTBEAT) {
9918 		if (mboxq->vport) {
9919 			lpfc_debugfs_disc_trc(mboxq->vport,
9920 				LPFC_DISC_TRC_MBOX_VPORT,
9921 				"MBOX Send vport: cmd:x%x mb:x%x x%x",
9922 				mbx_cmnd, mqe->un.mb_words[0],
9923 				mqe->un.mb_words[1]);
9924 		} else {
9925 			lpfc_debugfs_disc_trc(phba->pport,
9926 				LPFC_DISC_TRC_MBOX,
9927 				"MBOX Send: cmd:x%x mb:x%x x%x",
9928 				mbx_cmnd, mqe->un.mb_words[0],
9929 				mqe->un.mb_words[1]);
9930 		}
9931 	}
9932 	psli->slistat.mbox_cmd++;
9933 
9934 	/* Post the mailbox command to the port */
9935 	rc = lpfc_sli4_mq_put(phba->sli4_hba.mbx_wq, mqe);
9936 	if (rc != MBX_SUCCESS) {
9937 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
9938 				"(%d):2533 Mailbox command x%x (x%x/x%x) "
9939 				"cannot issue Data: x%x x%x\n",
9940 				mboxq->vport ? mboxq->vport->vpi : 0,
9941 				mboxq->u.mb.mbxCommand,
9942 				lpfc_sli_config_mbox_subsys_get(phba, mboxq),
9943 				lpfc_sli_config_mbox_opcode_get(phba, mboxq),
9944 				psli->sli_flag, MBX_NOWAIT);
9945 		goto out_not_finished;
9946 	}
9947 
9948 	return rc;
9949 
9950 out_not_finished:
9951 	spin_lock_irqsave(&phba->hbalock, iflags);
9952 	if (phba->sli.mbox_active) {
9953 		mboxq->u.mb.mbxStatus = MBX_NOT_FINISHED;
9954 		__lpfc_mbox_cmpl_put(phba, mboxq);
9955 		/* Release the token */
9956 		psli->sli_flag &= ~LPFC_SLI_MBOX_ACTIVE;
9957 		phba->sli.mbox_active = NULL;
9958 	}
9959 	spin_unlock_irqrestore(&phba->hbalock, iflags);
9960 
9961 	return MBX_NOT_FINISHED;
9962 }
9963 
9964 /**
9965  * lpfc_sli_issue_mbox - Wrapper func for issuing mailbox command
9966  * @phba: Pointer to HBA context object.
9967  * @pmbox: Pointer to mailbox object.
9968  * @flag: Flag indicating how the mailbox need to be processed.
9969  *
9970  * This routine wraps the actual SLI3 or SLI4 mailbox issuing routine from
9971  * the API jump table function pointer from the lpfc_hba struct.
9972  *
9973  * Return codes the caller owns the mailbox command after the return of the
9974  * function.
9975  **/
9976 int
9977 lpfc_sli_issue_mbox(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmbox, uint32_t flag)
9978 {
9979 	return phba->lpfc_sli_issue_mbox(phba, pmbox, flag);
9980 }
9981 
9982 /**
9983  * lpfc_mbox_api_table_setup - Set up mbox api function jump table
9984  * @phba: The hba struct for which this call is being executed.
9985  * @dev_grp: The HBA PCI-Device group number.
9986  *
9987  * This routine sets up the mbox interface API function jump table in @phba
9988  * struct.
9989  * Returns: 0 - success, -ENODEV - failure.
9990  **/
9991 int
9992 lpfc_mbox_api_table_setup(struct lpfc_hba *phba, uint8_t dev_grp)
9993 {
9994 
9995 	switch (dev_grp) {
9996 	case LPFC_PCI_DEV_LP:
9997 		phba->lpfc_sli_issue_mbox = lpfc_sli_issue_mbox_s3;
9998 		phba->lpfc_sli_handle_slow_ring_event =
9999 				lpfc_sli_handle_slow_ring_event_s3;
10000 		phba->lpfc_sli_hbq_to_firmware = lpfc_sli_hbq_to_firmware_s3;
10001 		phba->lpfc_sli_brdrestart = lpfc_sli_brdrestart_s3;
10002 		phba->lpfc_sli_brdready = lpfc_sli_brdready_s3;
10003 		break;
10004 	case LPFC_PCI_DEV_OC:
10005 		phba->lpfc_sli_issue_mbox = lpfc_sli_issue_mbox_s4;
10006 		phba->lpfc_sli_handle_slow_ring_event =
10007 				lpfc_sli_handle_slow_ring_event_s4;
10008 		phba->lpfc_sli_hbq_to_firmware = lpfc_sli_hbq_to_firmware_s4;
10009 		phba->lpfc_sli_brdrestart = lpfc_sli_brdrestart_s4;
10010 		phba->lpfc_sli_brdready = lpfc_sli_brdready_s4;
10011 		break;
10012 	default:
10013 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
10014 				"1420 Invalid HBA PCI-device group: 0x%x\n",
10015 				dev_grp);
10016 		return -ENODEV;
10017 	}
10018 	return 0;
10019 }
10020 
10021 /**
10022  * __lpfc_sli_ringtx_put - Add an iocb to the txq
10023  * @phba: Pointer to HBA context object.
10024  * @pring: Pointer to driver SLI ring object.
10025  * @piocb: Pointer to address of newly added command iocb.
10026  *
10027  * This function is called with hbalock held for SLI3 ports or
10028  * the ring lock held for SLI4 ports to add a command
10029  * iocb to the txq when SLI layer cannot submit the command iocb
10030  * to the ring.
10031  **/
10032 void
10033 __lpfc_sli_ringtx_put(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
10034 		    struct lpfc_iocbq *piocb)
10035 {
10036 	if (phba->sli_rev == LPFC_SLI_REV4)
10037 		lockdep_assert_held(&pring->ring_lock);
10038 	else
10039 		lockdep_assert_held(&phba->hbalock);
10040 	/* Insert the caller's iocb in the txq tail for later processing. */
10041 	list_add_tail(&piocb->list, &pring->txq);
10042 }
10043 
10044 /**
10045  * lpfc_sli_next_iocb - Get the next iocb in the txq
10046  * @phba: Pointer to HBA context object.
10047  * @pring: Pointer to driver SLI ring object.
10048  * @piocb: Pointer to address of newly added command iocb.
10049  *
10050  * This function is called with hbalock held before a new
10051  * iocb is submitted to the firmware. This function checks
10052  * txq to flush the iocbs in txq to Firmware before
10053  * submitting new iocbs to the Firmware.
10054  * If there are iocbs in the txq which need to be submitted
10055  * to firmware, lpfc_sli_next_iocb returns the first element
10056  * of the txq after dequeuing it from txq.
10057  * If there is no iocb in the txq then the function will return
10058  * *piocb and *piocb is set to NULL. Caller needs to check
10059  * *piocb to find if there are more commands in the txq.
10060  **/
10061 static struct lpfc_iocbq *
10062 lpfc_sli_next_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
10063 		   struct lpfc_iocbq **piocb)
10064 {
10065 	struct lpfc_iocbq * nextiocb;
10066 
10067 	lockdep_assert_held(&phba->hbalock);
10068 
10069 	nextiocb = lpfc_sli_ringtx_get(phba, pring);
10070 	if (!nextiocb) {
10071 		nextiocb = *piocb;
10072 		*piocb = NULL;
10073 	}
10074 
10075 	return nextiocb;
10076 }
10077 
10078 /**
10079  * __lpfc_sli_issue_iocb_s3 - SLI3 device lockless ver of lpfc_sli_issue_iocb
10080  * @phba: Pointer to HBA context object.
10081  * @ring_number: SLI ring number to issue iocb on.
10082  * @piocb: Pointer to command iocb.
10083  * @flag: Flag indicating if this command can be put into txq.
10084  *
10085  * __lpfc_sli_issue_iocb_s3 is used by other functions in the driver to issue
10086  * an iocb command to an HBA with SLI-3 interface spec. If the PCI slot is
10087  * recovering from error state, if HBA is resetting or if LPFC_STOP_IOCB_EVENT
10088  * flag is turned on, the function returns IOCB_ERROR. When the link is down,
10089  * this function allows only iocbs for posting buffers. This function finds
10090  * next available slot in the command ring and posts the command to the
10091  * available slot and writes the port attention register to request HBA start
10092  * processing new iocb. If there is no slot available in the ring and
10093  * flag & SLI_IOCB_RET_IOCB is set, the new iocb is added to the txq, otherwise
10094  * the function returns IOCB_BUSY.
10095  *
10096  * This function is called with hbalock held. The function will return success
10097  * after it successfully submit the iocb to firmware or after adding to the
10098  * txq.
10099  **/
10100 static int
10101 __lpfc_sli_issue_iocb_s3(struct lpfc_hba *phba, uint32_t ring_number,
10102 		    struct lpfc_iocbq *piocb, uint32_t flag)
10103 {
10104 	struct lpfc_iocbq *nextiocb;
10105 	IOCB_t *iocb;
10106 	struct lpfc_sli_ring *pring = &phba->sli.sli3_ring[ring_number];
10107 
10108 	lockdep_assert_held(&phba->hbalock);
10109 
10110 	if (piocb->iocb_cmpl && (!piocb->vport) &&
10111 	   (piocb->iocb.ulpCommand != CMD_ABORT_XRI_CN) &&
10112 	   (piocb->iocb.ulpCommand != CMD_CLOSE_XRI_CN)) {
10113 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
10114 				"1807 IOCB x%x failed. No vport\n",
10115 				piocb->iocb.ulpCommand);
10116 		dump_stack();
10117 		return IOCB_ERROR;
10118 	}
10119 
10120 
10121 	/* If the PCI channel is in offline state, do not post iocbs. */
10122 	if (unlikely(pci_channel_offline(phba->pcidev)))
10123 		return IOCB_ERROR;
10124 
10125 	/* If HBA has a deferred error attention, fail the iocb. */
10126 	if (unlikely(phba->hba_flag & DEFER_ERATT))
10127 		return IOCB_ERROR;
10128 
10129 	/*
10130 	 * We should never get an IOCB if we are in a < LINK_DOWN state
10131 	 */
10132 	if (unlikely(phba->link_state < LPFC_LINK_DOWN))
10133 		return IOCB_ERROR;
10134 
10135 	/*
10136 	 * Check to see if we are blocking IOCB processing because of a
10137 	 * outstanding event.
10138 	 */
10139 	if (unlikely(pring->flag & LPFC_STOP_IOCB_EVENT))
10140 		goto iocb_busy;
10141 
10142 	if (unlikely(phba->link_state == LPFC_LINK_DOWN)) {
10143 		/*
10144 		 * Only CREATE_XRI, CLOSE_XRI, and QUE_RING_BUF
10145 		 * can be issued if the link is not up.
10146 		 */
10147 		switch (piocb->iocb.ulpCommand) {
10148 		case CMD_GEN_REQUEST64_CR:
10149 		case CMD_GEN_REQUEST64_CX:
10150 			if (!(phba->sli.sli_flag & LPFC_MENLO_MAINT) ||
10151 				(piocb->iocb.un.genreq64.w5.hcsw.Rctl !=
10152 					FC_RCTL_DD_UNSOL_CMD) ||
10153 				(piocb->iocb.un.genreq64.w5.hcsw.Type !=
10154 					MENLO_TRANSPORT_TYPE))
10155 
10156 				goto iocb_busy;
10157 			break;
10158 		case CMD_QUE_RING_BUF_CN:
10159 		case CMD_QUE_RING_BUF64_CN:
10160 			/*
10161 			 * For IOCBs, like QUE_RING_BUF, that have no rsp ring
10162 			 * completion, iocb_cmpl MUST be 0.
10163 			 */
10164 			if (piocb->iocb_cmpl)
10165 				piocb->iocb_cmpl = NULL;
10166 			fallthrough;
10167 		case CMD_CREATE_XRI_CR:
10168 		case CMD_CLOSE_XRI_CN:
10169 		case CMD_CLOSE_XRI_CX:
10170 			break;
10171 		default:
10172 			goto iocb_busy;
10173 		}
10174 
10175 	/*
10176 	 * For FCP commands, we must be in a state where we can process link
10177 	 * attention events.
10178 	 */
10179 	} else if (unlikely(pring->ringno == LPFC_FCP_RING &&
10180 			    !(phba->sli.sli_flag & LPFC_PROCESS_LA))) {
10181 		goto iocb_busy;
10182 	}
10183 
10184 	while ((iocb = lpfc_sli_next_iocb_slot(phba, pring)) &&
10185 	       (nextiocb = lpfc_sli_next_iocb(phba, pring, &piocb)))
10186 		lpfc_sli_submit_iocb(phba, pring, iocb, nextiocb);
10187 
10188 	if (iocb)
10189 		lpfc_sli_update_ring(phba, pring);
10190 	else
10191 		lpfc_sli_update_full_ring(phba, pring);
10192 
10193 	if (!piocb)
10194 		return IOCB_SUCCESS;
10195 
10196 	goto out_busy;
10197 
10198  iocb_busy:
10199 	pring->stats.iocb_cmd_delay++;
10200 
10201  out_busy:
10202 
10203 	if (!(flag & SLI_IOCB_RET_IOCB)) {
10204 		__lpfc_sli_ringtx_put(phba, pring, piocb);
10205 		return IOCB_SUCCESS;
10206 	}
10207 
10208 	return IOCB_BUSY;
10209 }
10210 
10211 /**
10212  * lpfc_sli4_bpl2sgl - Convert the bpl/bde to a sgl.
10213  * @phba: Pointer to HBA context object.
10214  * @piocbq: Pointer to command iocb.
10215  * @sglq: Pointer to the scatter gather queue object.
10216  *
10217  * This routine converts the bpl or bde that is in the IOCB
10218  * to a sgl list for the sli4 hardware. The physical address
10219  * of the bpl/bde is converted back to a virtual address.
10220  * If the IOCB contains a BPL then the list of BDE's is
10221  * converted to sli4_sge's. If the IOCB contains a single
10222  * BDE then it is converted to a single sli_sge.
10223  * The IOCB is still in cpu endianess so the contents of
10224  * the bpl can be used without byte swapping.
10225  *
10226  * Returns valid XRI = Success, NO_XRI = Failure.
10227 **/
10228 static uint16_t
10229 lpfc_sli4_bpl2sgl(struct lpfc_hba *phba, struct lpfc_iocbq *piocbq,
10230 		struct lpfc_sglq *sglq)
10231 {
10232 	uint16_t xritag = NO_XRI;
10233 	struct ulp_bde64 *bpl = NULL;
10234 	struct ulp_bde64 bde;
10235 	struct sli4_sge *sgl  = NULL;
10236 	struct lpfc_dmabuf *dmabuf;
10237 	IOCB_t *icmd;
10238 	int numBdes = 0;
10239 	int i = 0;
10240 	uint32_t offset = 0; /* accumulated offset in the sg request list */
10241 	int inbound = 0; /* number of sg reply entries inbound from firmware */
10242 
10243 	if (!piocbq || !sglq)
10244 		return xritag;
10245 
10246 	sgl  = (struct sli4_sge *)sglq->sgl;
10247 	icmd = &piocbq->iocb;
10248 	if (icmd->ulpCommand == CMD_XMIT_BLS_RSP64_CX)
10249 		return sglq->sli4_xritag;
10250 	if (icmd->un.genreq64.bdl.bdeFlags == BUFF_TYPE_BLP_64) {
10251 		numBdes = icmd->un.genreq64.bdl.bdeSize /
10252 				sizeof(struct ulp_bde64);
10253 		/* The addrHigh and addrLow fields within the IOCB
10254 		 * have not been byteswapped yet so there is no
10255 		 * need to swap them back.
10256 		 */
10257 		if (piocbq->context3)
10258 			dmabuf = (struct lpfc_dmabuf *)piocbq->context3;
10259 		else
10260 			return xritag;
10261 
10262 		bpl  = (struct ulp_bde64 *)dmabuf->virt;
10263 		if (!bpl)
10264 			return xritag;
10265 
10266 		for (i = 0; i < numBdes; i++) {
10267 			/* Should already be byte swapped. */
10268 			sgl->addr_hi = bpl->addrHigh;
10269 			sgl->addr_lo = bpl->addrLow;
10270 
10271 			sgl->word2 = le32_to_cpu(sgl->word2);
10272 			if ((i+1) == numBdes)
10273 				bf_set(lpfc_sli4_sge_last, sgl, 1);
10274 			else
10275 				bf_set(lpfc_sli4_sge_last, sgl, 0);
10276 			/* swap the size field back to the cpu so we
10277 			 * can assign it to the sgl.
10278 			 */
10279 			bde.tus.w = le32_to_cpu(bpl->tus.w);
10280 			sgl->sge_len = cpu_to_le32(bde.tus.f.bdeSize);
10281 			/* The offsets in the sgl need to be accumulated
10282 			 * separately for the request and reply lists.
10283 			 * The request is always first, the reply follows.
10284 			 */
10285 			if (piocbq->iocb.ulpCommand == CMD_GEN_REQUEST64_CR) {
10286 				/* add up the reply sg entries */
10287 				if (bpl->tus.f.bdeFlags == BUFF_TYPE_BDE_64I)
10288 					inbound++;
10289 				/* first inbound? reset the offset */
10290 				if (inbound == 1)
10291 					offset = 0;
10292 				bf_set(lpfc_sli4_sge_offset, sgl, offset);
10293 				bf_set(lpfc_sli4_sge_type, sgl,
10294 					LPFC_SGE_TYPE_DATA);
10295 				offset += bde.tus.f.bdeSize;
10296 			}
10297 			sgl->word2 = cpu_to_le32(sgl->word2);
10298 			bpl++;
10299 			sgl++;
10300 		}
10301 	} else if (icmd->un.genreq64.bdl.bdeFlags == BUFF_TYPE_BDE_64) {
10302 			/* The addrHigh and addrLow fields of the BDE have not
10303 			 * been byteswapped yet so they need to be swapped
10304 			 * before putting them in the sgl.
10305 			 */
10306 			sgl->addr_hi =
10307 				cpu_to_le32(icmd->un.genreq64.bdl.addrHigh);
10308 			sgl->addr_lo =
10309 				cpu_to_le32(icmd->un.genreq64.bdl.addrLow);
10310 			sgl->word2 = le32_to_cpu(sgl->word2);
10311 			bf_set(lpfc_sli4_sge_last, sgl, 1);
10312 			sgl->word2 = cpu_to_le32(sgl->word2);
10313 			sgl->sge_len =
10314 				cpu_to_le32(icmd->un.genreq64.bdl.bdeSize);
10315 	}
10316 	return sglq->sli4_xritag;
10317 }
10318 
10319 /**
10320  * lpfc_sli4_iocb2wqe - Convert the IOCB to a work queue entry.
10321  * @phba: Pointer to HBA context object.
10322  * @iocbq: Pointer to command iocb.
10323  * @wqe: Pointer to the work queue entry.
10324  *
10325  * This routine converts the iocb command to its Work Queue Entry
10326  * equivalent. The wqe pointer should not have any fields set when
10327  * this routine is called because it will memcpy over them.
10328  * This routine does not set the CQ_ID or the WQEC bits in the
10329  * wqe.
10330  *
10331  * Returns: 0 = Success, IOCB_ERROR = Failure.
10332  **/
10333 static int
10334 lpfc_sli4_iocb2wqe(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq,
10335 		union lpfc_wqe128 *wqe)
10336 {
10337 	uint32_t xmit_len = 0, total_len = 0;
10338 	uint8_t ct = 0;
10339 	uint32_t fip;
10340 	uint32_t abort_tag;
10341 	uint8_t command_type = ELS_COMMAND_NON_FIP;
10342 	uint8_t cmnd;
10343 	uint16_t xritag;
10344 	uint16_t abrt_iotag;
10345 	struct lpfc_iocbq *abrtiocbq;
10346 	struct ulp_bde64 *bpl = NULL;
10347 	uint32_t els_id = LPFC_ELS_ID_DEFAULT;
10348 	int numBdes, i;
10349 	struct ulp_bde64 bde;
10350 	struct lpfc_nodelist *ndlp;
10351 	uint32_t *pcmd;
10352 	uint32_t if_type;
10353 
10354 	fip = phba->hba_flag & HBA_FIP_SUPPORT;
10355 	/* The fcp commands will set command type */
10356 	if (iocbq->iocb_flag &  LPFC_IO_FCP)
10357 		command_type = FCP_COMMAND;
10358 	else if (fip && (iocbq->iocb_flag & LPFC_FIP_ELS_ID_MASK))
10359 		command_type = ELS_COMMAND_FIP;
10360 	else
10361 		command_type = ELS_COMMAND_NON_FIP;
10362 
10363 	if (phba->fcp_embed_io)
10364 		memset(wqe, 0, sizeof(union lpfc_wqe128));
10365 	/* Some of the fields are in the right position already */
10366 	memcpy(wqe, &iocbq->iocb, sizeof(union lpfc_wqe));
10367 	/* The ct field has moved so reset */
10368 	wqe->generic.wqe_com.word7 = 0;
10369 	wqe->generic.wqe_com.word10 = 0;
10370 
10371 	abort_tag = (uint32_t) iocbq->iotag;
10372 	xritag = iocbq->sli4_xritag;
10373 	/* words0-2 bpl convert bde */
10374 	if (iocbq->iocb.un.genreq64.bdl.bdeFlags == BUFF_TYPE_BLP_64) {
10375 		numBdes = iocbq->iocb.un.genreq64.bdl.bdeSize /
10376 				sizeof(struct ulp_bde64);
10377 		bpl  = (struct ulp_bde64 *)
10378 			((struct lpfc_dmabuf *)iocbq->context3)->virt;
10379 		if (!bpl)
10380 			return IOCB_ERROR;
10381 
10382 		/* Should already be byte swapped. */
10383 		wqe->generic.bde.addrHigh =  le32_to_cpu(bpl->addrHigh);
10384 		wqe->generic.bde.addrLow =  le32_to_cpu(bpl->addrLow);
10385 		/* swap the size field back to the cpu so we
10386 		 * can assign it to the sgl.
10387 		 */
10388 		wqe->generic.bde.tus.w  = le32_to_cpu(bpl->tus.w);
10389 		xmit_len = wqe->generic.bde.tus.f.bdeSize;
10390 		total_len = 0;
10391 		for (i = 0; i < numBdes; i++) {
10392 			bde.tus.w  = le32_to_cpu(bpl[i].tus.w);
10393 			total_len += bde.tus.f.bdeSize;
10394 		}
10395 	} else
10396 		xmit_len = iocbq->iocb.un.fcpi64.bdl.bdeSize;
10397 
10398 	iocbq->iocb.ulpIoTag = iocbq->iotag;
10399 	cmnd = iocbq->iocb.ulpCommand;
10400 
10401 	switch (iocbq->iocb.ulpCommand) {
10402 	case CMD_ELS_REQUEST64_CR:
10403 		if (iocbq->iocb_flag & LPFC_IO_LIBDFC)
10404 			ndlp = iocbq->context_un.ndlp;
10405 		else
10406 			ndlp = (struct lpfc_nodelist *)iocbq->context1;
10407 		if (!iocbq->iocb.ulpLe) {
10408 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
10409 				"2007 Only Limited Edition cmd Format"
10410 				" supported 0x%x\n",
10411 				iocbq->iocb.ulpCommand);
10412 			return IOCB_ERROR;
10413 		}
10414 
10415 		wqe->els_req.payload_len = xmit_len;
10416 		/* Els_reguest64 has a TMO */
10417 		bf_set(wqe_tmo, &wqe->els_req.wqe_com,
10418 			iocbq->iocb.ulpTimeout);
10419 		/* Need a VF for word 4 set the vf bit*/
10420 		bf_set(els_req64_vf, &wqe->els_req, 0);
10421 		/* And a VFID for word 12 */
10422 		bf_set(els_req64_vfid, &wqe->els_req, 0);
10423 		ct = ((iocbq->iocb.ulpCt_h << 1) | iocbq->iocb.ulpCt_l);
10424 		bf_set(wqe_ctxt_tag, &wqe->els_req.wqe_com,
10425 		       iocbq->iocb.ulpContext);
10426 		bf_set(wqe_ct, &wqe->els_req.wqe_com, ct);
10427 		bf_set(wqe_pu, &wqe->els_req.wqe_com, 0);
10428 		/* CCP CCPE PV PRI in word10 were set in the memcpy */
10429 		if (command_type == ELS_COMMAND_FIP)
10430 			els_id = ((iocbq->iocb_flag & LPFC_FIP_ELS_ID_MASK)
10431 					>> LPFC_FIP_ELS_ID_SHIFT);
10432 		pcmd = (uint32_t *) (((struct lpfc_dmabuf *)
10433 					iocbq->context2)->virt);
10434 		if_type = bf_get(lpfc_sli_intf_if_type,
10435 					&phba->sli4_hba.sli_intf);
10436 		if (if_type >= LPFC_SLI_INTF_IF_TYPE_2) {
10437 			if (pcmd && (*pcmd == ELS_CMD_FLOGI ||
10438 				*pcmd == ELS_CMD_SCR ||
10439 				*pcmd == ELS_CMD_RDF ||
10440 				*pcmd == ELS_CMD_EDC ||
10441 				*pcmd == ELS_CMD_RSCN_XMT ||
10442 				*pcmd == ELS_CMD_FDISC ||
10443 				*pcmd == ELS_CMD_LOGO ||
10444 				*pcmd == ELS_CMD_QFPA ||
10445 				*pcmd == ELS_CMD_UVEM ||
10446 				*pcmd == ELS_CMD_PLOGI)) {
10447 				bf_set(els_req64_sp, &wqe->els_req, 1);
10448 				bf_set(els_req64_sid, &wqe->els_req,
10449 					iocbq->vport->fc_myDID);
10450 				if ((*pcmd == ELS_CMD_FLOGI) &&
10451 					!(phba->fc_topology ==
10452 						LPFC_TOPOLOGY_LOOP))
10453 					bf_set(els_req64_sid, &wqe->els_req, 0);
10454 				bf_set(wqe_ct, &wqe->els_req.wqe_com, 1);
10455 				bf_set(wqe_ctxt_tag, &wqe->els_req.wqe_com,
10456 					phba->vpi_ids[iocbq->vport->vpi]);
10457 			} else if (pcmd && iocbq->context1) {
10458 				bf_set(wqe_ct, &wqe->els_req.wqe_com, 0);
10459 				bf_set(wqe_ctxt_tag, &wqe->els_req.wqe_com,
10460 					phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
10461 			}
10462 		}
10463 		bf_set(wqe_temp_rpi, &wqe->els_req.wqe_com,
10464 		       phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
10465 		bf_set(wqe_els_id, &wqe->els_req.wqe_com, els_id);
10466 		bf_set(wqe_dbde, &wqe->els_req.wqe_com, 1);
10467 		bf_set(wqe_iod, &wqe->els_req.wqe_com, LPFC_WQE_IOD_READ);
10468 		bf_set(wqe_qosd, &wqe->els_req.wqe_com, 1);
10469 		bf_set(wqe_lenloc, &wqe->els_req.wqe_com, LPFC_WQE_LENLOC_NONE);
10470 		bf_set(wqe_ebde_cnt, &wqe->els_req.wqe_com, 0);
10471 		wqe->els_req.max_response_payload_len = total_len - xmit_len;
10472 		break;
10473 	case CMD_XMIT_SEQUENCE64_CX:
10474 		bf_set(wqe_ctxt_tag, &wqe->xmit_sequence.wqe_com,
10475 		       iocbq->iocb.un.ulpWord[3]);
10476 		bf_set(wqe_rcvoxid, &wqe->xmit_sequence.wqe_com,
10477 		       iocbq->iocb.unsli3.rcvsli3.ox_id);
10478 		/* The entire sequence is transmitted for this IOCB */
10479 		xmit_len = total_len;
10480 		cmnd = CMD_XMIT_SEQUENCE64_CR;
10481 		if (phba->link_flag & LS_LOOPBACK_MODE)
10482 			bf_set(wqe_xo, &wqe->xmit_sequence.wge_ctl, 1);
10483 		fallthrough;
10484 	case CMD_XMIT_SEQUENCE64_CR:
10485 		/* word3 iocb=io_tag32 wqe=reserved */
10486 		wqe->xmit_sequence.rsvd3 = 0;
10487 		/* word4 relative_offset memcpy */
10488 		/* word5 r_ctl/df_ctl memcpy */
10489 		bf_set(wqe_pu, &wqe->xmit_sequence.wqe_com, 0);
10490 		bf_set(wqe_dbde, &wqe->xmit_sequence.wqe_com, 1);
10491 		bf_set(wqe_iod, &wqe->xmit_sequence.wqe_com,
10492 		       LPFC_WQE_IOD_WRITE);
10493 		bf_set(wqe_lenloc, &wqe->xmit_sequence.wqe_com,
10494 		       LPFC_WQE_LENLOC_WORD12);
10495 		bf_set(wqe_ebde_cnt, &wqe->xmit_sequence.wqe_com, 0);
10496 		wqe->xmit_sequence.xmit_len = xmit_len;
10497 		command_type = OTHER_COMMAND;
10498 		break;
10499 	case CMD_XMIT_BCAST64_CN:
10500 		/* word3 iocb=iotag32 wqe=seq_payload_len */
10501 		wqe->xmit_bcast64.seq_payload_len = xmit_len;
10502 		/* word4 iocb=rsvd wqe=rsvd */
10503 		/* word5 iocb=rctl/type/df_ctl wqe=rctl/type/df_ctl memcpy */
10504 		/* word6 iocb=ctxt_tag/io_tag wqe=ctxt_tag/xri */
10505 		bf_set(wqe_ct, &wqe->xmit_bcast64.wqe_com,
10506 			((iocbq->iocb.ulpCt_h << 1) | iocbq->iocb.ulpCt_l));
10507 		bf_set(wqe_dbde, &wqe->xmit_bcast64.wqe_com, 1);
10508 		bf_set(wqe_iod, &wqe->xmit_bcast64.wqe_com, LPFC_WQE_IOD_WRITE);
10509 		bf_set(wqe_lenloc, &wqe->xmit_bcast64.wqe_com,
10510 		       LPFC_WQE_LENLOC_WORD3);
10511 		bf_set(wqe_ebde_cnt, &wqe->xmit_bcast64.wqe_com, 0);
10512 		break;
10513 	case CMD_FCP_IWRITE64_CR:
10514 		command_type = FCP_COMMAND_DATA_OUT;
10515 		/* word3 iocb=iotag wqe=payload_offset_len */
10516 		/* Add the FCP_CMD and FCP_RSP sizes to get the offset */
10517 		bf_set(payload_offset_len, &wqe->fcp_iwrite,
10518 		       xmit_len + sizeof(struct fcp_rsp));
10519 		bf_set(cmd_buff_len, &wqe->fcp_iwrite,
10520 		       0);
10521 		/* word4 iocb=parameter wqe=total_xfer_length memcpy */
10522 		/* word5 iocb=initial_xfer_len wqe=initial_xfer_len memcpy */
10523 		bf_set(wqe_erp, &wqe->fcp_iwrite.wqe_com,
10524 		       iocbq->iocb.ulpFCP2Rcvy);
10525 		bf_set(wqe_lnk, &wqe->fcp_iwrite.wqe_com, iocbq->iocb.ulpXS);
10526 		/* Always open the exchange */
10527 		bf_set(wqe_iod, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_IOD_WRITE);
10528 		bf_set(wqe_lenloc, &wqe->fcp_iwrite.wqe_com,
10529 		       LPFC_WQE_LENLOC_WORD4);
10530 		bf_set(wqe_pu, &wqe->fcp_iwrite.wqe_com, iocbq->iocb.ulpPU);
10531 		bf_set(wqe_dbde, &wqe->fcp_iwrite.wqe_com, 1);
10532 		if (iocbq->iocb_flag & LPFC_IO_OAS) {
10533 			bf_set(wqe_oas, &wqe->fcp_iwrite.wqe_com, 1);
10534 			bf_set(wqe_ccpe, &wqe->fcp_iwrite.wqe_com, 1);
10535 			if (iocbq->priority) {
10536 				bf_set(wqe_ccp, &wqe->fcp_iwrite.wqe_com,
10537 				       (iocbq->priority << 1));
10538 			} else {
10539 				bf_set(wqe_ccp, &wqe->fcp_iwrite.wqe_com,
10540 				       (phba->cfg_XLanePriority << 1));
10541 			}
10542 		}
10543 		/* Note, word 10 is already initialized to 0 */
10544 
10545 		/* Don't set PBDE for Perf hints, just lpfc_enable_pbde */
10546 		if (phba->cfg_enable_pbde)
10547 			bf_set(wqe_pbde, &wqe->fcp_iwrite.wqe_com, 1);
10548 		else
10549 			bf_set(wqe_pbde, &wqe->fcp_iwrite.wqe_com, 0);
10550 
10551 		if (phba->fcp_embed_io) {
10552 			struct lpfc_io_buf *lpfc_cmd;
10553 			struct sli4_sge *sgl;
10554 			struct fcp_cmnd *fcp_cmnd;
10555 			uint32_t *ptr;
10556 
10557 			/* 128 byte wqe support here */
10558 
10559 			lpfc_cmd = iocbq->context1;
10560 			sgl = (struct sli4_sge *)lpfc_cmd->dma_sgl;
10561 			fcp_cmnd = lpfc_cmd->fcp_cmnd;
10562 
10563 			/* Word 0-2 - FCP_CMND */
10564 			wqe->generic.bde.tus.f.bdeFlags =
10565 				BUFF_TYPE_BDE_IMMED;
10566 			wqe->generic.bde.tus.f.bdeSize = sgl->sge_len;
10567 			wqe->generic.bde.addrHigh = 0;
10568 			wqe->generic.bde.addrLow =  88;  /* Word 22 */
10569 
10570 			bf_set(wqe_wqes, &wqe->fcp_iwrite.wqe_com, 1);
10571 			bf_set(wqe_dbde, &wqe->fcp_iwrite.wqe_com, 0);
10572 
10573 			/* Word 22-29  FCP CMND Payload */
10574 			ptr = &wqe->words[22];
10575 			memcpy(ptr, fcp_cmnd, sizeof(struct fcp_cmnd));
10576 		}
10577 		break;
10578 	case CMD_FCP_IREAD64_CR:
10579 		/* word3 iocb=iotag wqe=payload_offset_len */
10580 		/* Add the FCP_CMD and FCP_RSP sizes to get the offset */
10581 		bf_set(payload_offset_len, &wqe->fcp_iread,
10582 		       xmit_len + sizeof(struct fcp_rsp));
10583 		bf_set(cmd_buff_len, &wqe->fcp_iread,
10584 		       0);
10585 		/* word4 iocb=parameter wqe=total_xfer_length memcpy */
10586 		/* word5 iocb=initial_xfer_len wqe=initial_xfer_len memcpy */
10587 		bf_set(wqe_erp, &wqe->fcp_iread.wqe_com,
10588 		       iocbq->iocb.ulpFCP2Rcvy);
10589 		bf_set(wqe_lnk, &wqe->fcp_iread.wqe_com, iocbq->iocb.ulpXS);
10590 		/* Always open the exchange */
10591 		bf_set(wqe_iod, &wqe->fcp_iread.wqe_com, LPFC_WQE_IOD_READ);
10592 		bf_set(wqe_lenloc, &wqe->fcp_iread.wqe_com,
10593 		       LPFC_WQE_LENLOC_WORD4);
10594 		bf_set(wqe_pu, &wqe->fcp_iread.wqe_com, iocbq->iocb.ulpPU);
10595 		bf_set(wqe_dbde, &wqe->fcp_iread.wqe_com, 1);
10596 		if (iocbq->iocb_flag & LPFC_IO_OAS) {
10597 			bf_set(wqe_oas, &wqe->fcp_iread.wqe_com, 1);
10598 			bf_set(wqe_ccpe, &wqe->fcp_iread.wqe_com, 1);
10599 			if (iocbq->priority) {
10600 				bf_set(wqe_ccp, &wqe->fcp_iread.wqe_com,
10601 				       (iocbq->priority << 1));
10602 			} else {
10603 				bf_set(wqe_ccp, &wqe->fcp_iread.wqe_com,
10604 				       (phba->cfg_XLanePriority << 1));
10605 			}
10606 		}
10607 		/* Note, word 10 is already initialized to 0 */
10608 
10609 		/* Don't set PBDE for Perf hints, just lpfc_enable_pbde */
10610 		if (phba->cfg_enable_pbde)
10611 			bf_set(wqe_pbde, &wqe->fcp_iread.wqe_com, 1);
10612 		else
10613 			bf_set(wqe_pbde, &wqe->fcp_iread.wqe_com, 0);
10614 
10615 		if (phba->fcp_embed_io) {
10616 			struct lpfc_io_buf *lpfc_cmd;
10617 			struct sli4_sge *sgl;
10618 			struct fcp_cmnd *fcp_cmnd;
10619 			uint32_t *ptr;
10620 
10621 			/* 128 byte wqe support here */
10622 
10623 			lpfc_cmd = iocbq->context1;
10624 			sgl = (struct sli4_sge *)lpfc_cmd->dma_sgl;
10625 			fcp_cmnd = lpfc_cmd->fcp_cmnd;
10626 
10627 			/* Word 0-2 - FCP_CMND */
10628 			wqe->generic.bde.tus.f.bdeFlags =
10629 				BUFF_TYPE_BDE_IMMED;
10630 			wqe->generic.bde.tus.f.bdeSize = sgl->sge_len;
10631 			wqe->generic.bde.addrHigh = 0;
10632 			wqe->generic.bde.addrLow =  88;  /* Word 22 */
10633 
10634 			bf_set(wqe_wqes, &wqe->fcp_iread.wqe_com, 1);
10635 			bf_set(wqe_dbde, &wqe->fcp_iread.wqe_com, 0);
10636 
10637 			/* Word 22-29  FCP CMND Payload */
10638 			ptr = &wqe->words[22];
10639 			memcpy(ptr, fcp_cmnd, sizeof(struct fcp_cmnd));
10640 		}
10641 		break;
10642 	case CMD_FCP_ICMND64_CR:
10643 		/* word3 iocb=iotag wqe=payload_offset_len */
10644 		/* Add the FCP_CMD and FCP_RSP sizes to get the offset */
10645 		bf_set(payload_offset_len, &wqe->fcp_icmd,
10646 		       xmit_len + sizeof(struct fcp_rsp));
10647 		bf_set(cmd_buff_len, &wqe->fcp_icmd,
10648 		       0);
10649 		/* word3 iocb=IO_TAG wqe=reserved */
10650 		bf_set(wqe_pu, &wqe->fcp_icmd.wqe_com, 0);
10651 		/* Always open the exchange */
10652 		bf_set(wqe_dbde, &wqe->fcp_icmd.wqe_com, 1);
10653 		bf_set(wqe_iod, &wqe->fcp_icmd.wqe_com, LPFC_WQE_IOD_WRITE);
10654 		bf_set(wqe_qosd, &wqe->fcp_icmd.wqe_com, 1);
10655 		bf_set(wqe_lenloc, &wqe->fcp_icmd.wqe_com,
10656 		       LPFC_WQE_LENLOC_NONE);
10657 		bf_set(wqe_erp, &wqe->fcp_icmd.wqe_com,
10658 		       iocbq->iocb.ulpFCP2Rcvy);
10659 		if (iocbq->iocb_flag & LPFC_IO_OAS) {
10660 			bf_set(wqe_oas, &wqe->fcp_icmd.wqe_com, 1);
10661 			bf_set(wqe_ccpe, &wqe->fcp_icmd.wqe_com, 1);
10662 			if (iocbq->priority) {
10663 				bf_set(wqe_ccp, &wqe->fcp_icmd.wqe_com,
10664 				       (iocbq->priority << 1));
10665 			} else {
10666 				bf_set(wqe_ccp, &wqe->fcp_icmd.wqe_com,
10667 				       (phba->cfg_XLanePriority << 1));
10668 			}
10669 		}
10670 		/* Note, word 10 is already initialized to 0 */
10671 
10672 		if (phba->fcp_embed_io) {
10673 			struct lpfc_io_buf *lpfc_cmd;
10674 			struct sli4_sge *sgl;
10675 			struct fcp_cmnd *fcp_cmnd;
10676 			uint32_t *ptr;
10677 
10678 			/* 128 byte wqe support here */
10679 
10680 			lpfc_cmd = iocbq->context1;
10681 			sgl = (struct sli4_sge *)lpfc_cmd->dma_sgl;
10682 			fcp_cmnd = lpfc_cmd->fcp_cmnd;
10683 
10684 			/* Word 0-2 - FCP_CMND */
10685 			wqe->generic.bde.tus.f.bdeFlags =
10686 				BUFF_TYPE_BDE_IMMED;
10687 			wqe->generic.bde.tus.f.bdeSize = sgl->sge_len;
10688 			wqe->generic.bde.addrHigh = 0;
10689 			wqe->generic.bde.addrLow =  88;  /* Word 22 */
10690 
10691 			bf_set(wqe_wqes, &wqe->fcp_icmd.wqe_com, 1);
10692 			bf_set(wqe_dbde, &wqe->fcp_icmd.wqe_com, 0);
10693 
10694 			/* Word 22-29  FCP CMND Payload */
10695 			ptr = &wqe->words[22];
10696 			memcpy(ptr, fcp_cmnd, sizeof(struct fcp_cmnd));
10697 		}
10698 		break;
10699 	case CMD_GEN_REQUEST64_CR:
10700 		/* For this command calculate the xmit length of the
10701 		 * request bde.
10702 		 */
10703 		xmit_len = 0;
10704 		numBdes = iocbq->iocb.un.genreq64.bdl.bdeSize /
10705 			sizeof(struct ulp_bde64);
10706 		for (i = 0; i < numBdes; i++) {
10707 			bde.tus.w = le32_to_cpu(bpl[i].tus.w);
10708 			if (bde.tus.f.bdeFlags != BUFF_TYPE_BDE_64)
10709 				break;
10710 			xmit_len += bde.tus.f.bdeSize;
10711 		}
10712 		/* word3 iocb=IO_TAG wqe=request_payload_len */
10713 		wqe->gen_req.request_payload_len = xmit_len;
10714 		/* word4 iocb=parameter wqe=relative_offset memcpy */
10715 		/* word5 [rctl, type, df_ctl, la] copied in memcpy */
10716 		/* word6 context tag copied in memcpy */
10717 		if (iocbq->iocb.ulpCt_h  || iocbq->iocb.ulpCt_l) {
10718 			ct = ((iocbq->iocb.ulpCt_h << 1) | iocbq->iocb.ulpCt_l);
10719 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
10720 				"2015 Invalid CT %x command 0x%x\n",
10721 				ct, iocbq->iocb.ulpCommand);
10722 			return IOCB_ERROR;
10723 		}
10724 		bf_set(wqe_ct, &wqe->gen_req.wqe_com, 0);
10725 		bf_set(wqe_tmo, &wqe->gen_req.wqe_com, iocbq->iocb.ulpTimeout);
10726 		bf_set(wqe_pu, &wqe->gen_req.wqe_com, iocbq->iocb.ulpPU);
10727 		bf_set(wqe_dbde, &wqe->gen_req.wqe_com, 1);
10728 		bf_set(wqe_iod, &wqe->gen_req.wqe_com, LPFC_WQE_IOD_READ);
10729 		bf_set(wqe_qosd, &wqe->gen_req.wqe_com, 1);
10730 		bf_set(wqe_lenloc, &wqe->gen_req.wqe_com, LPFC_WQE_LENLOC_NONE);
10731 		bf_set(wqe_ebde_cnt, &wqe->gen_req.wqe_com, 0);
10732 		wqe->gen_req.max_response_payload_len = total_len - xmit_len;
10733 		command_type = OTHER_COMMAND;
10734 		break;
10735 	case CMD_XMIT_ELS_RSP64_CX:
10736 		ndlp = (struct lpfc_nodelist *)iocbq->context1;
10737 		/* words0-2 BDE memcpy */
10738 		/* word3 iocb=iotag32 wqe=response_payload_len */
10739 		wqe->xmit_els_rsp.response_payload_len = xmit_len;
10740 		/* word4 */
10741 		wqe->xmit_els_rsp.word4 = 0;
10742 		/* word5 iocb=rsvd wge=did */
10743 		bf_set(wqe_els_did, &wqe->xmit_els_rsp.wqe_dest,
10744 			 iocbq->iocb.un.xseq64.xmit_els_remoteID);
10745 
10746 		if_type = bf_get(lpfc_sli_intf_if_type,
10747 					&phba->sli4_hba.sli_intf);
10748 		if (if_type >= LPFC_SLI_INTF_IF_TYPE_2) {
10749 			if (iocbq->vport->fc_flag & FC_PT2PT) {
10750 				bf_set(els_rsp64_sp, &wqe->xmit_els_rsp, 1);
10751 				bf_set(els_rsp64_sid, &wqe->xmit_els_rsp,
10752 					iocbq->vport->fc_myDID);
10753 				if (iocbq->vport->fc_myDID == Fabric_DID) {
10754 					bf_set(wqe_els_did,
10755 						&wqe->xmit_els_rsp.wqe_dest, 0);
10756 				}
10757 			}
10758 		}
10759 		bf_set(wqe_ct, &wqe->xmit_els_rsp.wqe_com,
10760 		       ((iocbq->iocb.ulpCt_h << 1) | iocbq->iocb.ulpCt_l));
10761 		bf_set(wqe_pu, &wqe->xmit_els_rsp.wqe_com, iocbq->iocb.ulpPU);
10762 		bf_set(wqe_rcvoxid, &wqe->xmit_els_rsp.wqe_com,
10763 		       iocbq->iocb.unsli3.rcvsli3.ox_id);
10764 		if (!iocbq->iocb.ulpCt_h && iocbq->iocb.ulpCt_l)
10765 			bf_set(wqe_ctxt_tag, &wqe->xmit_els_rsp.wqe_com,
10766 			       phba->vpi_ids[iocbq->vport->vpi]);
10767 		bf_set(wqe_dbde, &wqe->xmit_els_rsp.wqe_com, 1);
10768 		bf_set(wqe_iod, &wqe->xmit_els_rsp.wqe_com, LPFC_WQE_IOD_WRITE);
10769 		bf_set(wqe_qosd, &wqe->xmit_els_rsp.wqe_com, 1);
10770 		bf_set(wqe_lenloc, &wqe->xmit_els_rsp.wqe_com,
10771 		       LPFC_WQE_LENLOC_WORD3);
10772 		bf_set(wqe_ebde_cnt, &wqe->xmit_els_rsp.wqe_com, 0);
10773 		bf_set(wqe_rsp_temp_rpi, &wqe->xmit_els_rsp,
10774 		       phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
10775 		if (phba->fc_topology == LPFC_TOPOLOGY_LOOP) {
10776 				bf_set(els_rsp64_sp, &wqe->xmit_els_rsp, 1);
10777 				bf_set(els_rsp64_sid, &wqe->xmit_els_rsp,
10778 					iocbq->vport->fc_myDID);
10779 				bf_set(wqe_ct, &wqe->xmit_els_rsp.wqe_com, 1);
10780 				bf_set(wqe_ctxt_tag, &wqe->xmit_els_rsp.wqe_com,
10781 					phba->vpi_ids[phba->pport->vpi]);
10782 		}
10783 		command_type = OTHER_COMMAND;
10784 		break;
10785 	case CMD_CLOSE_XRI_CN:
10786 	case CMD_ABORT_XRI_CN:
10787 	case CMD_ABORT_XRI_CX:
10788 		/* words 0-2 memcpy should be 0 rserved */
10789 		/* port will send abts */
10790 		abrt_iotag = iocbq->iocb.un.acxri.abortContextTag;
10791 		if (abrt_iotag != 0 && abrt_iotag <= phba->sli.last_iotag) {
10792 			abrtiocbq = phba->sli.iocbq_lookup[abrt_iotag];
10793 			fip = abrtiocbq->iocb_flag & LPFC_FIP_ELS_ID_MASK;
10794 		} else
10795 			fip = 0;
10796 
10797 		if ((iocbq->iocb.ulpCommand == CMD_CLOSE_XRI_CN) || fip)
10798 			/*
10799 			 * The link is down, or the command was ELS_FIP
10800 			 * so the fw does not need to send abts
10801 			 * on the wire.
10802 			 */
10803 			bf_set(abort_cmd_ia, &wqe->abort_cmd, 1);
10804 		else
10805 			bf_set(abort_cmd_ia, &wqe->abort_cmd, 0);
10806 		bf_set(abort_cmd_criteria, &wqe->abort_cmd, T_XRI_TAG);
10807 		/* word5 iocb=CONTEXT_TAG|IO_TAG wqe=reserved */
10808 		wqe->abort_cmd.rsrvd5 = 0;
10809 		bf_set(wqe_ct, &wqe->abort_cmd.wqe_com,
10810 			((iocbq->iocb.ulpCt_h << 1) | iocbq->iocb.ulpCt_l));
10811 		abort_tag = iocbq->iocb.un.acxri.abortIoTag;
10812 		/*
10813 		 * The abort handler will send us CMD_ABORT_XRI_CN or
10814 		 * CMD_CLOSE_XRI_CN and the fw only accepts CMD_ABORT_XRI_CX
10815 		 */
10816 		bf_set(wqe_cmnd, &wqe->abort_cmd.wqe_com, CMD_ABORT_XRI_CX);
10817 		bf_set(wqe_qosd, &wqe->abort_cmd.wqe_com, 1);
10818 		bf_set(wqe_lenloc, &wqe->abort_cmd.wqe_com,
10819 		       LPFC_WQE_LENLOC_NONE);
10820 		cmnd = CMD_ABORT_XRI_CX;
10821 		command_type = OTHER_COMMAND;
10822 		xritag = 0;
10823 		break;
10824 	case CMD_XMIT_BLS_RSP64_CX:
10825 		ndlp = (struct lpfc_nodelist *)iocbq->context1;
10826 		/* As BLS ABTS RSP WQE is very different from other WQEs,
10827 		 * we re-construct this WQE here based on information in
10828 		 * iocbq from scratch.
10829 		 */
10830 		memset(wqe, 0, sizeof(*wqe));
10831 		/* OX_ID is invariable to who sent ABTS to CT exchange */
10832 		bf_set(xmit_bls_rsp64_oxid, &wqe->xmit_bls_rsp,
10833 		       bf_get(lpfc_abts_oxid, &iocbq->iocb.un.bls_rsp));
10834 		if (bf_get(lpfc_abts_orig, &iocbq->iocb.un.bls_rsp) ==
10835 		    LPFC_ABTS_UNSOL_INT) {
10836 			/* ABTS sent by initiator to CT exchange, the
10837 			 * RX_ID field will be filled with the newly
10838 			 * allocated responder XRI.
10839 			 */
10840 			bf_set(xmit_bls_rsp64_rxid, &wqe->xmit_bls_rsp,
10841 			       iocbq->sli4_xritag);
10842 		} else {
10843 			/* ABTS sent by responder to CT exchange, the
10844 			 * RX_ID field will be filled with the responder
10845 			 * RX_ID from ABTS.
10846 			 */
10847 			bf_set(xmit_bls_rsp64_rxid, &wqe->xmit_bls_rsp,
10848 			       bf_get(lpfc_abts_rxid, &iocbq->iocb.un.bls_rsp));
10849 		}
10850 		bf_set(xmit_bls_rsp64_seqcnthi, &wqe->xmit_bls_rsp, 0xffff);
10851 		bf_set(wqe_xmit_bls_pt, &wqe->xmit_bls_rsp.wqe_dest, 0x1);
10852 
10853 		/* Use CT=VPI */
10854 		bf_set(wqe_els_did, &wqe->xmit_bls_rsp.wqe_dest,
10855 			ndlp->nlp_DID);
10856 		bf_set(xmit_bls_rsp64_temprpi, &wqe->xmit_bls_rsp,
10857 			iocbq->iocb.ulpContext);
10858 		bf_set(wqe_ct, &wqe->xmit_bls_rsp.wqe_com, 1);
10859 		bf_set(wqe_ctxt_tag, &wqe->xmit_bls_rsp.wqe_com,
10860 			phba->vpi_ids[phba->pport->vpi]);
10861 		bf_set(wqe_qosd, &wqe->xmit_bls_rsp.wqe_com, 1);
10862 		bf_set(wqe_lenloc, &wqe->xmit_bls_rsp.wqe_com,
10863 		       LPFC_WQE_LENLOC_NONE);
10864 		/* Overwrite the pre-set comnd type with OTHER_COMMAND */
10865 		command_type = OTHER_COMMAND;
10866 		if (iocbq->iocb.un.xseq64.w5.hcsw.Rctl == FC_RCTL_BA_RJT) {
10867 			bf_set(xmit_bls_rsp64_rjt_vspec, &wqe->xmit_bls_rsp,
10868 			       bf_get(lpfc_vndr_code, &iocbq->iocb.un.bls_rsp));
10869 			bf_set(xmit_bls_rsp64_rjt_expc, &wqe->xmit_bls_rsp,
10870 			       bf_get(lpfc_rsn_expln, &iocbq->iocb.un.bls_rsp));
10871 			bf_set(xmit_bls_rsp64_rjt_rsnc, &wqe->xmit_bls_rsp,
10872 			       bf_get(lpfc_rsn_code, &iocbq->iocb.un.bls_rsp));
10873 		}
10874 
10875 		break;
10876 	case CMD_SEND_FRAME:
10877 		bf_set(wqe_cmnd, &wqe->generic.wqe_com, CMD_SEND_FRAME);
10878 		bf_set(wqe_sof, &wqe->generic.wqe_com, 0x2E); /* SOF byte */
10879 		bf_set(wqe_eof, &wqe->generic.wqe_com, 0x41); /* EOF byte */
10880 		bf_set(wqe_lenloc, &wqe->generic.wqe_com, 1);
10881 		bf_set(wqe_xbl, &wqe->generic.wqe_com, 1);
10882 		bf_set(wqe_dbde, &wqe->generic.wqe_com, 1);
10883 		bf_set(wqe_xc, &wqe->generic.wqe_com, 1);
10884 		bf_set(wqe_cmd_type, &wqe->generic.wqe_com, 0xA);
10885 		bf_set(wqe_cqid, &wqe->generic.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
10886 		bf_set(wqe_xri_tag, &wqe->generic.wqe_com, xritag);
10887 		bf_set(wqe_reqtag, &wqe->generic.wqe_com, iocbq->iotag);
10888 		return 0;
10889 	case CMD_XRI_ABORTED_CX:
10890 	case CMD_CREATE_XRI_CR: /* Do we expect to use this? */
10891 	case CMD_IOCB_FCP_IBIDIR64_CR: /* bidirectional xfer */
10892 	case CMD_FCP_TSEND64_CX: /* Target mode send xfer-ready */
10893 	case CMD_FCP_TRSP64_CX: /* Target mode rcv */
10894 	case CMD_FCP_AUTO_TRSP_CX: /* Auto target rsp */
10895 	default:
10896 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
10897 				"2014 Invalid command 0x%x\n",
10898 				iocbq->iocb.ulpCommand);
10899 		return IOCB_ERROR;
10900 	}
10901 
10902 	if (iocbq->iocb_flag & LPFC_IO_DIF_PASS)
10903 		bf_set(wqe_dif, &wqe->generic.wqe_com, LPFC_WQE_DIF_PASSTHRU);
10904 	else if (iocbq->iocb_flag & LPFC_IO_DIF_STRIP)
10905 		bf_set(wqe_dif, &wqe->generic.wqe_com, LPFC_WQE_DIF_STRIP);
10906 	else if (iocbq->iocb_flag & LPFC_IO_DIF_INSERT)
10907 		bf_set(wqe_dif, &wqe->generic.wqe_com, LPFC_WQE_DIF_INSERT);
10908 	iocbq->iocb_flag &= ~(LPFC_IO_DIF_PASS | LPFC_IO_DIF_STRIP |
10909 			      LPFC_IO_DIF_INSERT);
10910 	bf_set(wqe_xri_tag, &wqe->generic.wqe_com, xritag);
10911 	bf_set(wqe_reqtag, &wqe->generic.wqe_com, iocbq->iotag);
10912 	wqe->generic.wqe_com.abort_tag = abort_tag;
10913 	bf_set(wqe_cmd_type, &wqe->generic.wqe_com, command_type);
10914 	bf_set(wqe_cmnd, &wqe->generic.wqe_com, cmnd);
10915 	bf_set(wqe_class, &wqe->generic.wqe_com, iocbq->iocb.ulpClass);
10916 	bf_set(wqe_cqid, &wqe->generic.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
10917 	return 0;
10918 }
10919 
10920 /**
10921  * __lpfc_sli_issue_fcp_io_s3 - SLI3 device for sending fcp io iocb
10922  * @phba: Pointer to HBA context object.
10923  * @ring_number: SLI ring number to issue wqe on.
10924  * @piocb: Pointer to command iocb.
10925  * @flag: Flag indicating if this command can be put into txq.
10926  *
10927  * __lpfc_sli_issue_fcp_io_s3 is wrapper function to invoke lockless func to
10928  * send  an iocb command to an HBA with SLI-4 interface spec.
10929  *
10930  * This function takes the hbalock before invoking the lockless version.
10931  * The function will return success after it successfully submit the wqe to
10932  * firmware or after adding to the txq.
10933  **/
10934 static int
10935 __lpfc_sli_issue_fcp_io_s3(struct lpfc_hba *phba, uint32_t ring_number,
10936 			   struct lpfc_iocbq *piocb, uint32_t flag)
10937 {
10938 	unsigned long iflags;
10939 	int rc;
10940 
10941 	spin_lock_irqsave(&phba->hbalock, iflags);
10942 	rc = __lpfc_sli_issue_iocb_s3(phba, ring_number, piocb, flag);
10943 	spin_unlock_irqrestore(&phba->hbalock, iflags);
10944 
10945 	return rc;
10946 }
10947 
10948 /**
10949  * __lpfc_sli_issue_fcp_io_s4 - SLI4 device for sending fcp io wqe
10950  * @phba: Pointer to HBA context object.
10951  * @ring_number: SLI ring number to issue wqe on.
10952  * @piocb: Pointer to command iocb.
10953  * @flag: Flag indicating if this command can be put into txq.
10954  *
10955  * __lpfc_sli_issue_fcp_io_s4 is used by other functions in the driver to issue
10956  * an wqe command to an HBA with SLI-4 interface spec.
10957  *
10958  * This function is a lockless version. The function will return success
10959  * after it successfully submit the wqe to firmware or after adding to the
10960  * txq.
10961  **/
10962 static int
10963 __lpfc_sli_issue_fcp_io_s4(struct lpfc_hba *phba, uint32_t ring_number,
10964 			   struct lpfc_iocbq *piocb, uint32_t flag)
10965 {
10966 	int rc;
10967 	struct lpfc_io_buf *lpfc_cmd =
10968 		(struct lpfc_io_buf *)piocb->context1;
10969 	union lpfc_wqe128 *wqe = &piocb->wqe;
10970 	struct sli4_sge *sgl;
10971 
10972 	/* 128 byte wqe support here */
10973 	sgl = (struct sli4_sge *)lpfc_cmd->dma_sgl;
10974 
10975 	if (phba->fcp_embed_io) {
10976 		struct fcp_cmnd *fcp_cmnd;
10977 		u32 *ptr;
10978 
10979 		fcp_cmnd = lpfc_cmd->fcp_cmnd;
10980 
10981 		/* Word 0-2 - FCP_CMND */
10982 		wqe->generic.bde.tus.f.bdeFlags =
10983 			BUFF_TYPE_BDE_IMMED;
10984 		wqe->generic.bde.tus.f.bdeSize = sgl->sge_len;
10985 		wqe->generic.bde.addrHigh = 0;
10986 		wqe->generic.bde.addrLow =  88;  /* Word 22 */
10987 
10988 		bf_set(wqe_wqes, &wqe->fcp_iwrite.wqe_com, 1);
10989 		bf_set(wqe_dbde, &wqe->fcp_iwrite.wqe_com, 0);
10990 
10991 		/* Word 22-29  FCP CMND Payload */
10992 		ptr = &wqe->words[22];
10993 		memcpy(ptr, fcp_cmnd, sizeof(struct fcp_cmnd));
10994 	} else {
10995 		/* Word 0-2 - Inline BDE */
10996 		wqe->generic.bde.tus.f.bdeFlags =  BUFF_TYPE_BDE_64;
10997 		wqe->generic.bde.tus.f.bdeSize = sizeof(struct fcp_cmnd);
10998 		wqe->generic.bde.addrHigh = sgl->addr_hi;
10999 		wqe->generic.bde.addrLow =  sgl->addr_lo;
11000 
11001 		/* Word 10 */
11002 		bf_set(wqe_dbde, &wqe->generic.wqe_com, 1);
11003 		bf_set(wqe_wqes, &wqe->generic.wqe_com, 0);
11004 	}
11005 
11006 	/* add the VMID tags as per switch response */
11007 	if (unlikely(piocb->iocb_flag & LPFC_IO_VMID)) {
11008 		if (phba->pport->vmid_priority_tagging) {
11009 			bf_set(wqe_ccpe, &wqe->fcp_iwrite.wqe_com, 1);
11010 			bf_set(wqe_ccp, &wqe->fcp_iwrite.wqe_com,
11011 					(piocb->vmid_tag.cs_ctl_vmid));
11012 		} else {
11013 			bf_set(wqe_appid, &wqe->fcp_iwrite.wqe_com, 1);
11014 			bf_set(wqe_wqes, &wqe->fcp_iwrite.wqe_com, 1);
11015 			wqe->words[31] = piocb->vmid_tag.app_id;
11016 		}
11017 	}
11018 	rc = lpfc_sli4_issue_wqe(phba, lpfc_cmd->hdwq, piocb);
11019 	return rc;
11020 }
11021 
11022 /**
11023  * __lpfc_sli_issue_iocb_s4 - SLI4 device lockless ver of lpfc_sli_issue_iocb
11024  * @phba: Pointer to HBA context object.
11025  * @ring_number: SLI ring number to issue iocb on.
11026  * @piocb: Pointer to command iocb.
11027  * @flag: Flag indicating if this command can be put into txq.
11028  *
11029  * __lpfc_sli_issue_iocb_s4 is used by other functions in the driver to issue
11030  * an iocb command to an HBA with SLI-4 interface spec.
11031  *
11032  * This function is called with ringlock held. The function will return success
11033  * after it successfully submit the iocb to firmware or after adding to the
11034  * txq.
11035  **/
11036 static int
11037 __lpfc_sli_issue_iocb_s4(struct lpfc_hba *phba, uint32_t ring_number,
11038 			 struct lpfc_iocbq *piocb, uint32_t flag)
11039 {
11040 	struct lpfc_sglq *sglq;
11041 	union lpfc_wqe128 wqe;
11042 	struct lpfc_queue *wq;
11043 	struct lpfc_sli_ring *pring;
11044 
11045 	/* Get the WQ */
11046 	if ((piocb->iocb_flag & LPFC_IO_FCP) ||
11047 	    (piocb->iocb_flag & LPFC_USE_FCPWQIDX)) {
11048 		wq = phba->sli4_hba.hdwq[piocb->hba_wqidx].io_wq;
11049 	} else {
11050 		wq = phba->sli4_hba.els_wq;
11051 	}
11052 
11053 	/* Get corresponding ring */
11054 	pring = wq->pring;
11055 
11056 	/*
11057 	 * The WQE can be either 64 or 128 bytes,
11058 	 */
11059 
11060 	lockdep_assert_held(&pring->ring_lock);
11061 
11062 	if (piocb->sli4_xritag == NO_XRI) {
11063 		if (piocb->iocb.ulpCommand == CMD_ABORT_XRI_CN ||
11064 		    piocb->iocb.ulpCommand == CMD_CLOSE_XRI_CN)
11065 			sglq = NULL;
11066 		else {
11067 			if (!list_empty(&pring->txq)) {
11068 				if (!(flag & SLI_IOCB_RET_IOCB)) {
11069 					__lpfc_sli_ringtx_put(phba,
11070 						pring, piocb);
11071 					return IOCB_SUCCESS;
11072 				} else {
11073 					return IOCB_BUSY;
11074 				}
11075 			} else {
11076 				sglq = __lpfc_sli_get_els_sglq(phba, piocb);
11077 				if (!sglq) {
11078 					if (!(flag & SLI_IOCB_RET_IOCB)) {
11079 						__lpfc_sli_ringtx_put(phba,
11080 								pring,
11081 								piocb);
11082 						return IOCB_SUCCESS;
11083 					} else
11084 						return IOCB_BUSY;
11085 				}
11086 			}
11087 		}
11088 	} else if (piocb->iocb_flag &  LPFC_IO_FCP) {
11089 		/* These IO's already have an XRI and a mapped sgl. */
11090 		sglq = NULL;
11091 	}
11092 	else {
11093 		/*
11094 		 * This is a continuation of a commandi,(CX) so this
11095 		 * sglq is on the active list
11096 		 */
11097 		sglq = __lpfc_get_active_sglq(phba, piocb->sli4_lxritag);
11098 		if (!sglq)
11099 			return IOCB_ERROR;
11100 	}
11101 
11102 	if (sglq) {
11103 		piocb->sli4_lxritag = sglq->sli4_lxritag;
11104 		piocb->sli4_xritag = sglq->sli4_xritag;
11105 		if (NO_XRI == lpfc_sli4_bpl2sgl(phba, piocb, sglq))
11106 			return IOCB_ERROR;
11107 	}
11108 
11109 	if (lpfc_sli4_iocb2wqe(phba, piocb, &wqe))
11110 		return IOCB_ERROR;
11111 
11112 	if (lpfc_sli4_wq_put(wq, &wqe))
11113 		return IOCB_ERROR;
11114 	lpfc_sli_ringtxcmpl_put(phba, pring, piocb);
11115 
11116 	return 0;
11117 }
11118 
11119 /*
11120  * lpfc_sli_issue_fcp_io - Wrapper func for issuing fcp i/o
11121  *
11122  * This routine wraps the actual fcp i/o function for issusing WQE for sli-4
11123  * or IOCB for sli-3  function.
11124  * pointer from the lpfc_hba struct.
11125  *
11126  * Return codes:
11127  * IOCB_ERROR - Error
11128  * IOCB_SUCCESS - Success
11129  * IOCB_BUSY - Busy
11130  **/
11131 int
11132 lpfc_sli_issue_fcp_io(struct lpfc_hba *phba, uint32_t ring_number,
11133 		      struct lpfc_iocbq *piocb, uint32_t flag)
11134 {
11135 	return phba->__lpfc_sli_issue_fcp_io(phba, ring_number, piocb, flag);
11136 }
11137 
11138 /*
11139  * __lpfc_sli_issue_iocb - Wrapper func of lockless version for issuing iocb
11140  *
11141  * This routine wraps the actual lockless version for issusing IOCB function
11142  * pointer from the lpfc_hba struct.
11143  *
11144  * Return codes:
11145  * IOCB_ERROR - Error
11146  * IOCB_SUCCESS - Success
11147  * IOCB_BUSY - Busy
11148  **/
11149 int
11150 __lpfc_sli_issue_iocb(struct lpfc_hba *phba, uint32_t ring_number,
11151 		struct lpfc_iocbq *piocb, uint32_t flag)
11152 {
11153 	return phba->__lpfc_sli_issue_iocb(phba, ring_number, piocb, flag);
11154 }
11155 
11156 /**
11157  * lpfc_sli_api_table_setup - Set up sli api function jump table
11158  * @phba: The hba struct for which this call is being executed.
11159  * @dev_grp: The HBA PCI-Device group number.
11160  *
11161  * This routine sets up the SLI interface API function jump table in @phba
11162  * struct.
11163  * Returns: 0 - success, -ENODEV - failure.
11164  **/
11165 int
11166 lpfc_sli_api_table_setup(struct lpfc_hba *phba, uint8_t dev_grp)
11167 {
11168 
11169 	switch (dev_grp) {
11170 	case LPFC_PCI_DEV_LP:
11171 		phba->__lpfc_sli_issue_iocb = __lpfc_sli_issue_iocb_s3;
11172 		phba->__lpfc_sli_release_iocbq = __lpfc_sli_release_iocbq_s3;
11173 		phba->__lpfc_sli_issue_fcp_io = __lpfc_sli_issue_fcp_io_s3;
11174 		break;
11175 	case LPFC_PCI_DEV_OC:
11176 		phba->__lpfc_sli_issue_iocb = __lpfc_sli_issue_iocb_s4;
11177 		phba->__lpfc_sli_release_iocbq = __lpfc_sli_release_iocbq_s4;
11178 		phba->__lpfc_sli_issue_fcp_io = __lpfc_sli_issue_fcp_io_s4;
11179 		break;
11180 	default:
11181 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
11182 				"1419 Invalid HBA PCI-device group: 0x%x\n",
11183 				dev_grp);
11184 		return -ENODEV;
11185 	}
11186 	phba->lpfc_get_iocb_from_iocbq = lpfc_get_iocb_from_iocbq;
11187 	return 0;
11188 }
11189 
11190 /**
11191  * lpfc_sli4_calc_ring - Calculates which ring to use
11192  * @phba: Pointer to HBA context object.
11193  * @piocb: Pointer to command iocb.
11194  *
11195  * For SLI4 only, FCP IO can deferred to one fo many WQs, based on
11196  * hba_wqidx, thus we need to calculate the corresponding ring.
11197  * Since ABORTS must go on the same WQ of the command they are
11198  * aborting, we use command's hba_wqidx.
11199  */
11200 struct lpfc_sli_ring *
11201 lpfc_sli4_calc_ring(struct lpfc_hba *phba, struct lpfc_iocbq *piocb)
11202 {
11203 	struct lpfc_io_buf *lpfc_cmd;
11204 
11205 	if (piocb->iocb_flag & (LPFC_IO_FCP | LPFC_USE_FCPWQIDX)) {
11206 		if (unlikely(!phba->sli4_hba.hdwq))
11207 			return NULL;
11208 		/*
11209 		 * for abort iocb hba_wqidx should already
11210 		 * be setup based on what work queue we used.
11211 		 */
11212 		if (!(piocb->iocb_flag & LPFC_USE_FCPWQIDX)) {
11213 			lpfc_cmd = (struct lpfc_io_buf *)piocb->context1;
11214 			piocb->hba_wqidx = lpfc_cmd->hdwq_no;
11215 		}
11216 		return phba->sli4_hba.hdwq[piocb->hba_wqidx].io_wq->pring;
11217 	} else {
11218 		if (unlikely(!phba->sli4_hba.els_wq))
11219 			return NULL;
11220 		piocb->hba_wqidx = 0;
11221 		return phba->sli4_hba.els_wq->pring;
11222 	}
11223 }
11224 
11225 /**
11226  * lpfc_sli_issue_iocb - Wrapper function for __lpfc_sli_issue_iocb
11227  * @phba: Pointer to HBA context object.
11228  * @ring_number: Ring number
11229  * @piocb: Pointer to command iocb.
11230  * @flag: Flag indicating if this command can be put into txq.
11231  *
11232  * lpfc_sli_issue_iocb is a wrapper around __lpfc_sli_issue_iocb
11233  * function. This function gets the hbalock and calls
11234  * __lpfc_sli_issue_iocb function and will return the error returned
11235  * by __lpfc_sli_issue_iocb function. This wrapper is used by
11236  * functions which do not hold hbalock.
11237  **/
11238 int
11239 lpfc_sli_issue_iocb(struct lpfc_hba *phba, uint32_t ring_number,
11240 		    struct lpfc_iocbq *piocb, uint32_t flag)
11241 {
11242 	struct lpfc_sli_ring *pring;
11243 	struct lpfc_queue *eq;
11244 	unsigned long iflags;
11245 	int rc;
11246 
11247 	if (phba->sli_rev == LPFC_SLI_REV4) {
11248 		eq = phba->sli4_hba.hdwq[piocb->hba_wqidx].hba_eq;
11249 
11250 		pring = lpfc_sli4_calc_ring(phba, piocb);
11251 		if (unlikely(pring == NULL))
11252 			return IOCB_ERROR;
11253 
11254 		spin_lock_irqsave(&pring->ring_lock, iflags);
11255 		rc = __lpfc_sli_issue_iocb(phba, ring_number, piocb, flag);
11256 		spin_unlock_irqrestore(&pring->ring_lock, iflags);
11257 
11258 		lpfc_sli4_poll_eq(eq, LPFC_POLL_FASTPATH);
11259 	} else {
11260 		/* For now, SLI2/3 will still use hbalock */
11261 		spin_lock_irqsave(&phba->hbalock, iflags);
11262 		rc = __lpfc_sli_issue_iocb(phba, ring_number, piocb, flag);
11263 		spin_unlock_irqrestore(&phba->hbalock, iflags);
11264 	}
11265 	return rc;
11266 }
11267 
11268 /**
11269  * lpfc_extra_ring_setup - Extra ring setup function
11270  * @phba: Pointer to HBA context object.
11271  *
11272  * This function is called while driver attaches with the
11273  * HBA to setup the extra ring. The extra ring is used
11274  * only when driver needs to support target mode functionality
11275  * or IP over FC functionalities.
11276  *
11277  * This function is called with no lock held. SLI3 only.
11278  **/
11279 static int
11280 lpfc_extra_ring_setup( struct lpfc_hba *phba)
11281 {
11282 	struct lpfc_sli *psli;
11283 	struct lpfc_sli_ring *pring;
11284 
11285 	psli = &phba->sli;
11286 
11287 	/* Adjust cmd/rsp ring iocb entries more evenly */
11288 
11289 	/* Take some away from the FCP ring */
11290 	pring = &psli->sli3_ring[LPFC_FCP_RING];
11291 	pring->sli.sli3.numCiocb -= SLI2_IOCB_CMD_R1XTRA_ENTRIES;
11292 	pring->sli.sli3.numRiocb -= SLI2_IOCB_RSP_R1XTRA_ENTRIES;
11293 	pring->sli.sli3.numCiocb -= SLI2_IOCB_CMD_R3XTRA_ENTRIES;
11294 	pring->sli.sli3.numRiocb -= SLI2_IOCB_RSP_R3XTRA_ENTRIES;
11295 
11296 	/* and give them to the extra ring */
11297 	pring = &psli->sli3_ring[LPFC_EXTRA_RING];
11298 
11299 	pring->sli.sli3.numCiocb += SLI2_IOCB_CMD_R1XTRA_ENTRIES;
11300 	pring->sli.sli3.numRiocb += SLI2_IOCB_RSP_R1XTRA_ENTRIES;
11301 	pring->sli.sli3.numCiocb += SLI2_IOCB_CMD_R3XTRA_ENTRIES;
11302 	pring->sli.sli3.numRiocb += SLI2_IOCB_RSP_R3XTRA_ENTRIES;
11303 
11304 	/* Setup default profile for this ring */
11305 	pring->iotag_max = 4096;
11306 	pring->num_mask = 1;
11307 	pring->prt[0].profile = 0;      /* Mask 0 */
11308 	pring->prt[0].rctl = phba->cfg_multi_ring_rctl;
11309 	pring->prt[0].type = phba->cfg_multi_ring_type;
11310 	pring->prt[0].lpfc_sli_rcv_unsol_event = NULL;
11311 	return 0;
11312 }
11313 
11314 static void
11315 lpfc_sli_post_recovery_event(struct lpfc_hba *phba,
11316 			     struct lpfc_nodelist *ndlp)
11317 {
11318 	unsigned long iflags;
11319 	struct lpfc_work_evt  *evtp = &ndlp->recovery_evt;
11320 
11321 	spin_lock_irqsave(&phba->hbalock, iflags);
11322 	if (!list_empty(&evtp->evt_listp)) {
11323 		spin_unlock_irqrestore(&phba->hbalock, iflags);
11324 		return;
11325 	}
11326 
11327 	/* Incrementing the reference count until the queued work is done. */
11328 	evtp->evt_arg1  = lpfc_nlp_get(ndlp);
11329 	if (!evtp->evt_arg1) {
11330 		spin_unlock_irqrestore(&phba->hbalock, iflags);
11331 		return;
11332 	}
11333 	evtp->evt = LPFC_EVT_RECOVER_PORT;
11334 	list_add_tail(&evtp->evt_listp, &phba->work_list);
11335 	spin_unlock_irqrestore(&phba->hbalock, iflags);
11336 
11337 	lpfc_worker_wake_up(phba);
11338 }
11339 
11340 /* lpfc_sli_abts_err_handler - handle a failed ABTS request from an SLI3 port.
11341  * @phba: Pointer to HBA context object.
11342  * @iocbq: Pointer to iocb object.
11343  *
11344  * The async_event handler calls this routine when it receives
11345  * an ASYNC_STATUS_CN event from the port.  The port generates
11346  * this event when an Abort Sequence request to an rport fails
11347  * twice in succession.  The abort could be originated by the
11348  * driver or by the port.  The ABTS could have been for an ELS
11349  * or FCP IO.  The port only generates this event when an ABTS
11350  * fails to complete after one retry.
11351  */
11352 static void
11353 lpfc_sli_abts_err_handler(struct lpfc_hba *phba,
11354 			  struct lpfc_iocbq *iocbq)
11355 {
11356 	struct lpfc_nodelist *ndlp = NULL;
11357 	uint16_t rpi = 0, vpi = 0;
11358 	struct lpfc_vport *vport = NULL;
11359 
11360 	/* The rpi in the ulpContext is vport-sensitive. */
11361 	vpi = iocbq->iocb.un.asyncstat.sub_ctxt_tag;
11362 	rpi = iocbq->iocb.ulpContext;
11363 
11364 	lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
11365 			"3092 Port generated ABTS async event "
11366 			"on vpi %d rpi %d status 0x%x\n",
11367 			vpi, rpi, iocbq->iocb.ulpStatus);
11368 
11369 	vport = lpfc_find_vport_by_vpid(phba, vpi);
11370 	if (!vport)
11371 		goto err_exit;
11372 	ndlp = lpfc_findnode_rpi(vport, rpi);
11373 	if (!ndlp)
11374 		goto err_exit;
11375 
11376 	if (iocbq->iocb.ulpStatus == IOSTAT_LOCAL_REJECT)
11377 		lpfc_sli_abts_recover_port(vport, ndlp);
11378 	return;
11379 
11380  err_exit:
11381 	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
11382 			"3095 Event Context not found, no "
11383 			"action on vpi %d rpi %d status 0x%x, reason 0x%x\n",
11384 			iocbq->iocb.ulpContext, iocbq->iocb.ulpStatus,
11385 			vpi, rpi);
11386 }
11387 
11388 /* lpfc_sli4_abts_err_handler - handle a failed ABTS request from an SLI4 port.
11389  * @phba: pointer to HBA context object.
11390  * @ndlp: nodelist pointer for the impacted rport.
11391  * @axri: pointer to the wcqe containing the failed exchange.
11392  *
11393  * The driver calls this routine when it receives an ABORT_XRI_FCP CQE from the
11394  * port.  The port generates this event when an abort exchange request to an
11395  * rport fails twice in succession with no reply.  The abort could be originated
11396  * by the driver or by the port.  The ABTS could have been for an ELS or FCP IO.
11397  */
11398 void
11399 lpfc_sli4_abts_err_handler(struct lpfc_hba *phba,
11400 			   struct lpfc_nodelist *ndlp,
11401 			   struct sli4_wcqe_xri_aborted *axri)
11402 {
11403 	uint32_t ext_status = 0;
11404 
11405 	if (!ndlp) {
11406 		lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
11407 				"3115 Node Context not found, driver "
11408 				"ignoring abts err event\n");
11409 		return;
11410 	}
11411 
11412 	lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
11413 			"3116 Port generated FCP XRI ABORT event on "
11414 			"vpi %d rpi %d xri x%x status 0x%x parameter x%x\n",
11415 			ndlp->vport->vpi, phba->sli4_hba.rpi_ids[ndlp->nlp_rpi],
11416 			bf_get(lpfc_wcqe_xa_xri, axri),
11417 			bf_get(lpfc_wcqe_xa_status, axri),
11418 			axri->parameter);
11419 
11420 	/*
11421 	 * Catch the ABTS protocol failure case.  Older OCe FW releases returned
11422 	 * LOCAL_REJECT and 0 for a failed ABTS exchange and later OCe and
11423 	 * LPe FW releases returned LOCAL_REJECT and SEQUENCE_TIMEOUT.
11424 	 */
11425 	ext_status = axri->parameter & IOERR_PARAM_MASK;
11426 	if ((bf_get(lpfc_wcqe_xa_status, axri) == IOSTAT_LOCAL_REJECT) &&
11427 	    ((ext_status == IOERR_SEQUENCE_TIMEOUT) || (ext_status == 0)))
11428 		lpfc_sli_post_recovery_event(phba, ndlp);
11429 }
11430 
11431 /**
11432  * lpfc_sli_async_event_handler - ASYNC iocb handler function
11433  * @phba: Pointer to HBA context object.
11434  * @pring: Pointer to driver SLI ring object.
11435  * @iocbq: Pointer to iocb object.
11436  *
11437  * This function is called by the slow ring event handler
11438  * function when there is an ASYNC event iocb in the ring.
11439  * This function is called with no lock held.
11440  * Currently this function handles only temperature related
11441  * ASYNC events. The function decodes the temperature sensor
11442  * event message and posts events for the management applications.
11443  **/
11444 static void
11445 lpfc_sli_async_event_handler(struct lpfc_hba * phba,
11446 	struct lpfc_sli_ring * pring, struct lpfc_iocbq * iocbq)
11447 {
11448 	IOCB_t *icmd;
11449 	uint16_t evt_code;
11450 	struct temp_event temp_event_data;
11451 	struct Scsi_Host *shost;
11452 	uint32_t *iocb_w;
11453 
11454 	icmd = &iocbq->iocb;
11455 	evt_code = icmd->un.asyncstat.evt_code;
11456 
11457 	switch (evt_code) {
11458 	case ASYNC_TEMP_WARN:
11459 	case ASYNC_TEMP_SAFE:
11460 		temp_event_data.data = (uint32_t) icmd->ulpContext;
11461 		temp_event_data.event_type = FC_REG_TEMPERATURE_EVENT;
11462 		if (evt_code == ASYNC_TEMP_WARN) {
11463 			temp_event_data.event_code = LPFC_THRESHOLD_TEMP;
11464 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
11465 				"0347 Adapter is very hot, please take "
11466 				"corrective action. temperature : %d Celsius\n",
11467 				(uint32_t) icmd->ulpContext);
11468 		} else {
11469 			temp_event_data.event_code = LPFC_NORMAL_TEMP;
11470 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
11471 				"0340 Adapter temperature is OK now. "
11472 				"temperature : %d Celsius\n",
11473 				(uint32_t) icmd->ulpContext);
11474 		}
11475 
11476 		/* Send temperature change event to applications */
11477 		shost = lpfc_shost_from_vport(phba->pport);
11478 		fc_host_post_vendor_event(shost, fc_get_event_number(),
11479 			sizeof(temp_event_data), (char *) &temp_event_data,
11480 			LPFC_NL_VENDOR_ID);
11481 		break;
11482 	case ASYNC_STATUS_CN:
11483 		lpfc_sli_abts_err_handler(phba, iocbq);
11484 		break;
11485 	default:
11486 		iocb_w = (uint32_t *) icmd;
11487 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
11488 			"0346 Ring %d handler: unexpected ASYNC_STATUS"
11489 			" evt_code 0x%x\n"
11490 			"W0  0x%08x W1  0x%08x W2  0x%08x W3  0x%08x\n"
11491 			"W4  0x%08x W5  0x%08x W6  0x%08x W7  0x%08x\n"
11492 			"W8  0x%08x W9  0x%08x W10 0x%08x W11 0x%08x\n"
11493 			"W12 0x%08x W13 0x%08x W14 0x%08x W15 0x%08x\n",
11494 			pring->ringno, icmd->un.asyncstat.evt_code,
11495 			iocb_w[0], iocb_w[1], iocb_w[2], iocb_w[3],
11496 			iocb_w[4], iocb_w[5], iocb_w[6], iocb_w[7],
11497 			iocb_w[8], iocb_w[9], iocb_w[10], iocb_w[11],
11498 			iocb_w[12], iocb_w[13], iocb_w[14], iocb_w[15]);
11499 
11500 		break;
11501 	}
11502 }
11503 
11504 
11505 /**
11506  * lpfc_sli4_setup - SLI ring setup function
11507  * @phba: Pointer to HBA context object.
11508  *
11509  * lpfc_sli_setup sets up rings of the SLI interface with
11510  * number of iocbs per ring and iotags. This function is
11511  * called while driver attach to the HBA and before the
11512  * interrupts are enabled. So there is no need for locking.
11513  *
11514  * This function always returns 0.
11515  **/
11516 int
11517 lpfc_sli4_setup(struct lpfc_hba *phba)
11518 {
11519 	struct lpfc_sli_ring *pring;
11520 
11521 	pring = phba->sli4_hba.els_wq->pring;
11522 	pring->num_mask = LPFC_MAX_RING_MASK;
11523 	pring->prt[0].profile = 0;	/* Mask 0 */
11524 	pring->prt[0].rctl = FC_RCTL_ELS_REQ;
11525 	pring->prt[0].type = FC_TYPE_ELS;
11526 	pring->prt[0].lpfc_sli_rcv_unsol_event =
11527 	    lpfc_els_unsol_event;
11528 	pring->prt[1].profile = 0;	/* Mask 1 */
11529 	pring->prt[1].rctl = FC_RCTL_ELS_REP;
11530 	pring->prt[1].type = FC_TYPE_ELS;
11531 	pring->prt[1].lpfc_sli_rcv_unsol_event =
11532 	    lpfc_els_unsol_event;
11533 	pring->prt[2].profile = 0;	/* Mask 2 */
11534 	/* NameServer Inquiry */
11535 	pring->prt[2].rctl = FC_RCTL_DD_UNSOL_CTL;
11536 	/* NameServer */
11537 	pring->prt[2].type = FC_TYPE_CT;
11538 	pring->prt[2].lpfc_sli_rcv_unsol_event =
11539 	    lpfc_ct_unsol_event;
11540 	pring->prt[3].profile = 0;	/* Mask 3 */
11541 	/* NameServer response */
11542 	pring->prt[3].rctl = FC_RCTL_DD_SOL_CTL;
11543 	/* NameServer */
11544 	pring->prt[3].type = FC_TYPE_CT;
11545 	pring->prt[3].lpfc_sli_rcv_unsol_event =
11546 	    lpfc_ct_unsol_event;
11547 	return 0;
11548 }
11549 
11550 /**
11551  * lpfc_sli_setup - SLI ring setup function
11552  * @phba: Pointer to HBA context object.
11553  *
11554  * lpfc_sli_setup sets up rings of the SLI interface with
11555  * number of iocbs per ring and iotags. This function is
11556  * called while driver attach to the HBA and before the
11557  * interrupts are enabled. So there is no need for locking.
11558  *
11559  * This function always returns 0. SLI3 only.
11560  **/
11561 int
11562 lpfc_sli_setup(struct lpfc_hba *phba)
11563 {
11564 	int i, totiocbsize = 0;
11565 	struct lpfc_sli *psli = &phba->sli;
11566 	struct lpfc_sli_ring *pring;
11567 
11568 	psli->num_rings = MAX_SLI3_CONFIGURED_RINGS;
11569 	psli->sli_flag = 0;
11570 
11571 	psli->iocbq_lookup = NULL;
11572 	psli->iocbq_lookup_len = 0;
11573 	psli->last_iotag = 0;
11574 
11575 	for (i = 0; i < psli->num_rings; i++) {
11576 		pring = &psli->sli3_ring[i];
11577 		switch (i) {
11578 		case LPFC_FCP_RING:	/* ring 0 - FCP */
11579 			/* numCiocb and numRiocb are used in config_port */
11580 			pring->sli.sli3.numCiocb = SLI2_IOCB_CMD_R0_ENTRIES;
11581 			pring->sli.sli3.numRiocb = SLI2_IOCB_RSP_R0_ENTRIES;
11582 			pring->sli.sli3.numCiocb +=
11583 				SLI2_IOCB_CMD_R1XTRA_ENTRIES;
11584 			pring->sli.sli3.numRiocb +=
11585 				SLI2_IOCB_RSP_R1XTRA_ENTRIES;
11586 			pring->sli.sli3.numCiocb +=
11587 				SLI2_IOCB_CMD_R3XTRA_ENTRIES;
11588 			pring->sli.sli3.numRiocb +=
11589 				SLI2_IOCB_RSP_R3XTRA_ENTRIES;
11590 			pring->sli.sli3.sizeCiocb = (phba->sli_rev == 3) ?
11591 							SLI3_IOCB_CMD_SIZE :
11592 							SLI2_IOCB_CMD_SIZE;
11593 			pring->sli.sli3.sizeRiocb = (phba->sli_rev == 3) ?
11594 							SLI3_IOCB_RSP_SIZE :
11595 							SLI2_IOCB_RSP_SIZE;
11596 			pring->iotag_ctr = 0;
11597 			pring->iotag_max =
11598 			    (phba->cfg_hba_queue_depth * 2);
11599 			pring->fast_iotag = pring->iotag_max;
11600 			pring->num_mask = 0;
11601 			break;
11602 		case LPFC_EXTRA_RING:	/* ring 1 - EXTRA */
11603 			/* numCiocb and numRiocb are used in config_port */
11604 			pring->sli.sli3.numCiocb = SLI2_IOCB_CMD_R1_ENTRIES;
11605 			pring->sli.sli3.numRiocb = SLI2_IOCB_RSP_R1_ENTRIES;
11606 			pring->sli.sli3.sizeCiocb = (phba->sli_rev == 3) ?
11607 							SLI3_IOCB_CMD_SIZE :
11608 							SLI2_IOCB_CMD_SIZE;
11609 			pring->sli.sli3.sizeRiocb = (phba->sli_rev == 3) ?
11610 							SLI3_IOCB_RSP_SIZE :
11611 							SLI2_IOCB_RSP_SIZE;
11612 			pring->iotag_max = phba->cfg_hba_queue_depth;
11613 			pring->num_mask = 0;
11614 			break;
11615 		case LPFC_ELS_RING:	/* ring 2 - ELS / CT */
11616 			/* numCiocb and numRiocb are used in config_port */
11617 			pring->sli.sli3.numCiocb = SLI2_IOCB_CMD_R2_ENTRIES;
11618 			pring->sli.sli3.numRiocb = SLI2_IOCB_RSP_R2_ENTRIES;
11619 			pring->sli.sli3.sizeCiocb = (phba->sli_rev == 3) ?
11620 							SLI3_IOCB_CMD_SIZE :
11621 							SLI2_IOCB_CMD_SIZE;
11622 			pring->sli.sli3.sizeRiocb = (phba->sli_rev == 3) ?
11623 							SLI3_IOCB_RSP_SIZE :
11624 							SLI2_IOCB_RSP_SIZE;
11625 			pring->fast_iotag = 0;
11626 			pring->iotag_ctr = 0;
11627 			pring->iotag_max = 4096;
11628 			pring->lpfc_sli_rcv_async_status =
11629 				lpfc_sli_async_event_handler;
11630 			pring->num_mask = LPFC_MAX_RING_MASK;
11631 			pring->prt[0].profile = 0;	/* Mask 0 */
11632 			pring->prt[0].rctl = FC_RCTL_ELS_REQ;
11633 			pring->prt[0].type = FC_TYPE_ELS;
11634 			pring->prt[0].lpfc_sli_rcv_unsol_event =
11635 			    lpfc_els_unsol_event;
11636 			pring->prt[1].profile = 0;	/* Mask 1 */
11637 			pring->prt[1].rctl = FC_RCTL_ELS_REP;
11638 			pring->prt[1].type = FC_TYPE_ELS;
11639 			pring->prt[1].lpfc_sli_rcv_unsol_event =
11640 			    lpfc_els_unsol_event;
11641 			pring->prt[2].profile = 0;	/* Mask 2 */
11642 			/* NameServer Inquiry */
11643 			pring->prt[2].rctl = FC_RCTL_DD_UNSOL_CTL;
11644 			/* NameServer */
11645 			pring->prt[2].type = FC_TYPE_CT;
11646 			pring->prt[2].lpfc_sli_rcv_unsol_event =
11647 			    lpfc_ct_unsol_event;
11648 			pring->prt[3].profile = 0;	/* Mask 3 */
11649 			/* NameServer response */
11650 			pring->prt[3].rctl = FC_RCTL_DD_SOL_CTL;
11651 			/* NameServer */
11652 			pring->prt[3].type = FC_TYPE_CT;
11653 			pring->prt[3].lpfc_sli_rcv_unsol_event =
11654 			    lpfc_ct_unsol_event;
11655 			break;
11656 		}
11657 		totiocbsize += (pring->sli.sli3.numCiocb *
11658 			pring->sli.sli3.sizeCiocb) +
11659 			(pring->sli.sli3.numRiocb * pring->sli.sli3.sizeRiocb);
11660 	}
11661 	if (totiocbsize > MAX_SLIM_IOCB_SIZE) {
11662 		/* Too many cmd / rsp ring entries in SLI2 SLIM */
11663 		printk(KERN_ERR "%d:0462 Too many cmd / rsp ring entries in "
11664 		       "SLI2 SLIM Data: x%x x%lx\n",
11665 		       phba->brd_no, totiocbsize,
11666 		       (unsigned long) MAX_SLIM_IOCB_SIZE);
11667 	}
11668 	if (phba->cfg_multi_ring_support == 2)
11669 		lpfc_extra_ring_setup(phba);
11670 
11671 	return 0;
11672 }
11673 
11674 /**
11675  * lpfc_sli4_queue_init - Queue initialization function
11676  * @phba: Pointer to HBA context object.
11677  *
11678  * lpfc_sli4_queue_init sets up mailbox queues and iocb queues for each
11679  * ring. This function also initializes ring indices of each ring.
11680  * This function is called during the initialization of the SLI
11681  * interface of an HBA.
11682  * This function is called with no lock held and always returns
11683  * 1.
11684  **/
11685 void
11686 lpfc_sli4_queue_init(struct lpfc_hba *phba)
11687 {
11688 	struct lpfc_sli *psli;
11689 	struct lpfc_sli_ring *pring;
11690 	int i;
11691 
11692 	psli = &phba->sli;
11693 	spin_lock_irq(&phba->hbalock);
11694 	INIT_LIST_HEAD(&psli->mboxq);
11695 	INIT_LIST_HEAD(&psli->mboxq_cmpl);
11696 	/* Initialize list headers for txq and txcmplq as double linked lists */
11697 	for (i = 0; i < phba->cfg_hdw_queue; i++) {
11698 		pring = phba->sli4_hba.hdwq[i].io_wq->pring;
11699 		pring->flag = 0;
11700 		pring->ringno = LPFC_FCP_RING;
11701 		pring->txcmplq_cnt = 0;
11702 		INIT_LIST_HEAD(&pring->txq);
11703 		INIT_LIST_HEAD(&pring->txcmplq);
11704 		INIT_LIST_HEAD(&pring->iocb_continueq);
11705 		spin_lock_init(&pring->ring_lock);
11706 	}
11707 	pring = phba->sli4_hba.els_wq->pring;
11708 	pring->flag = 0;
11709 	pring->ringno = LPFC_ELS_RING;
11710 	pring->txcmplq_cnt = 0;
11711 	INIT_LIST_HEAD(&pring->txq);
11712 	INIT_LIST_HEAD(&pring->txcmplq);
11713 	INIT_LIST_HEAD(&pring->iocb_continueq);
11714 	spin_lock_init(&pring->ring_lock);
11715 
11716 	if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) {
11717 		pring = phba->sli4_hba.nvmels_wq->pring;
11718 		pring->flag = 0;
11719 		pring->ringno = LPFC_ELS_RING;
11720 		pring->txcmplq_cnt = 0;
11721 		INIT_LIST_HEAD(&pring->txq);
11722 		INIT_LIST_HEAD(&pring->txcmplq);
11723 		INIT_LIST_HEAD(&pring->iocb_continueq);
11724 		spin_lock_init(&pring->ring_lock);
11725 	}
11726 
11727 	spin_unlock_irq(&phba->hbalock);
11728 }
11729 
11730 /**
11731  * lpfc_sli_queue_init - Queue initialization function
11732  * @phba: Pointer to HBA context object.
11733  *
11734  * lpfc_sli_queue_init sets up mailbox queues and iocb queues for each
11735  * ring. This function also initializes ring indices of each ring.
11736  * This function is called during the initialization of the SLI
11737  * interface of an HBA.
11738  * This function is called with no lock held and always returns
11739  * 1.
11740  **/
11741 void
11742 lpfc_sli_queue_init(struct lpfc_hba *phba)
11743 {
11744 	struct lpfc_sli *psli;
11745 	struct lpfc_sli_ring *pring;
11746 	int i;
11747 
11748 	psli = &phba->sli;
11749 	spin_lock_irq(&phba->hbalock);
11750 	INIT_LIST_HEAD(&psli->mboxq);
11751 	INIT_LIST_HEAD(&psli->mboxq_cmpl);
11752 	/* Initialize list headers for txq and txcmplq as double linked lists */
11753 	for (i = 0; i < psli->num_rings; i++) {
11754 		pring = &psli->sli3_ring[i];
11755 		pring->ringno = i;
11756 		pring->sli.sli3.next_cmdidx  = 0;
11757 		pring->sli.sli3.local_getidx = 0;
11758 		pring->sli.sli3.cmdidx = 0;
11759 		INIT_LIST_HEAD(&pring->iocb_continueq);
11760 		INIT_LIST_HEAD(&pring->iocb_continue_saveq);
11761 		INIT_LIST_HEAD(&pring->postbufq);
11762 		pring->flag = 0;
11763 		INIT_LIST_HEAD(&pring->txq);
11764 		INIT_LIST_HEAD(&pring->txcmplq);
11765 		spin_lock_init(&pring->ring_lock);
11766 	}
11767 	spin_unlock_irq(&phba->hbalock);
11768 }
11769 
11770 /**
11771  * lpfc_sli_mbox_sys_flush - Flush mailbox command sub-system
11772  * @phba: Pointer to HBA context object.
11773  *
11774  * This routine flushes the mailbox command subsystem. It will unconditionally
11775  * flush all the mailbox commands in the three possible stages in the mailbox
11776  * command sub-system: pending mailbox command queue; the outstanding mailbox
11777  * command; and completed mailbox command queue. It is caller's responsibility
11778  * to make sure that the driver is in the proper state to flush the mailbox
11779  * command sub-system. Namely, the posting of mailbox commands into the
11780  * pending mailbox command queue from the various clients must be stopped;
11781  * either the HBA is in a state that it will never works on the outstanding
11782  * mailbox command (such as in EEH or ERATT conditions) or the outstanding
11783  * mailbox command has been completed.
11784  **/
11785 static void
11786 lpfc_sli_mbox_sys_flush(struct lpfc_hba *phba)
11787 {
11788 	LIST_HEAD(completions);
11789 	struct lpfc_sli *psli = &phba->sli;
11790 	LPFC_MBOXQ_t *pmb;
11791 	unsigned long iflag;
11792 
11793 	/* Disable softirqs, including timers from obtaining phba->hbalock */
11794 	local_bh_disable();
11795 
11796 	/* Flush all the mailbox commands in the mbox system */
11797 	spin_lock_irqsave(&phba->hbalock, iflag);
11798 
11799 	/* The pending mailbox command queue */
11800 	list_splice_init(&phba->sli.mboxq, &completions);
11801 	/* The outstanding active mailbox command */
11802 	if (psli->mbox_active) {
11803 		list_add_tail(&psli->mbox_active->list, &completions);
11804 		psli->mbox_active = NULL;
11805 		psli->sli_flag &= ~LPFC_SLI_MBOX_ACTIVE;
11806 	}
11807 	/* The completed mailbox command queue */
11808 	list_splice_init(&phba->sli.mboxq_cmpl, &completions);
11809 	spin_unlock_irqrestore(&phba->hbalock, iflag);
11810 
11811 	/* Enable softirqs again, done with phba->hbalock */
11812 	local_bh_enable();
11813 
11814 	/* Return all flushed mailbox commands with MBX_NOT_FINISHED status */
11815 	while (!list_empty(&completions)) {
11816 		list_remove_head(&completions, pmb, LPFC_MBOXQ_t, list);
11817 		pmb->u.mb.mbxStatus = MBX_NOT_FINISHED;
11818 		if (pmb->mbox_cmpl)
11819 			pmb->mbox_cmpl(phba, pmb);
11820 	}
11821 }
11822 
11823 /**
11824  * lpfc_sli_host_down - Vport cleanup function
11825  * @vport: Pointer to virtual port object.
11826  *
11827  * lpfc_sli_host_down is called to clean up the resources
11828  * associated with a vport before destroying virtual
11829  * port data structures.
11830  * This function does following operations:
11831  * - Free discovery resources associated with this virtual
11832  *   port.
11833  * - Free iocbs associated with this virtual port in
11834  *   the txq.
11835  * - Send abort for all iocb commands associated with this
11836  *   vport in txcmplq.
11837  *
11838  * This function is called with no lock held and always returns 1.
11839  **/
11840 int
11841 lpfc_sli_host_down(struct lpfc_vport *vport)
11842 {
11843 	LIST_HEAD(completions);
11844 	struct lpfc_hba *phba = vport->phba;
11845 	struct lpfc_sli *psli = &phba->sli;
11846 	struct lpfc_queue *qp = NULL;
11847 	struct lpfc_sli_ring *pring;
11848 	struct lpfc_iocbq *iocb, *next_iocb;
11849 	int i;
11850 	unsigned long flags = 0;
11851 	uint16_t prev_pring_flag;
11852 
11853 	lpfc_cleanup_discovery_resources(vport);
11854 
11855 	spin_lock_irqsave(&phba->hbalock, flags);
11856 
11857 	/*
11858 	 * Error everything on the txq since these iocbs
11859 	 * have not been given to the FW yet.
11860 	 * Also issue ABTS for everything on the txcmplq
11861 	 */
11862 	if (phba->sli_rev != LPFC_SLI_REV4) {
11863 		for (i = 0; i < psli->num_rings; i++) {
11864 			pring = &psli->sli3_ring[i];
11865 			prev_pring_flag = pring->flag;
11866 			/* Only slow rings */
11867 			if (pring->ringno == LPFC_ELS_RING) {
11868 				pring->flag |= LPFC_DEFERRED_RING_EVENT;
11869 				/* Set the lpfc data pending flag */
11870 				set_bit(LPFC_DATA_READY, &phba->data_flags);
11871 			}
11872 			list_for_each_entry_safe(iocb, next_iocb,
11873 						 &pring->txq, list) {
11874 				if (iocb->vport != vport)
11875 					continue;
11876 				list_move_tail(&iocb->list, &completions);
11877 			}
11878 			list_for_each_entry_safe(iocb, next_iocb,
11879 						 &pring->txcmplq, list) {
11880 				if (iocb->vport != vport)
11881 					continue;
11882 				lpfc_sli_issue_abort_iotag(phba, pring, iocb,
11883 							   NULL);
11884 			}
11885 			pring->flag = prev_pring_flag;
11886 		}
11887 	} else {
11888 		list_for_each_entry(qp, &phba->sli4_hba.lpfc_wq_list, wq_list) {
11889 			pring = qp->pring;
11890 			if (!pring)
11891 				continue;
11892 			if (pring == phba->sli4_hba.els_wq->pring) {
11893 				pring->flag |= LPFC_DEFERRED_RING_EVENT;
11894 				/* Set the lpfc data pending flag */
11895 				set_bit(LPFC_DATA_READY, &phba->data_flags);
11896 			}
11897 			prev_pring_flag = pring->flag;
11898 			spin_lock(&pring->ring_lock);
11899 			list_for_each_entry_safe(iocb, next_iocb,
11900 						 &pring->txq, list) {
11901 				if (iocb->vport != vport)
11902 					continue;
11903 				list_move_tail(&iocb->list, &completions);
11904 			}
11905 			spin_unlock(&pring->ring_lock);
11906 			list_for_each_entry_safe(iocb, next_iocb,
11907 						 &pring->txcmplq, list) {
11908 				if (iocb->vport != vport)
11909 					continue;
11910 				lpfc_sli_issue_abort_iotag(phba, pring, iocb,
11911 							   NULL);
11912 			}
11913 			pring->flag = prev_pring_flag;
11914 		}
11915 	}
11916 	spin_unlock_irqrestore(&phba->hbalock, flags);
11917 
11918 	/* Make sure HBA is alive */
11919 	lpfc_issue_hb_tmo(phba);
11920 
11921 	/* Cancel all the IOCBs from the completions list */
11922 	lpfc_sli_cancel_iocbs(phba, &completions, IOSTAT_LOCAL_REJECT,
11923 			      IOERR_SLI_DOWN);
11924 	return 1;
11925 }
11926 
11927 /**
11928  * lpfc_sli_hba_down - Resource cleanup function for the HBA
11929  * @phba: Pointer to HBA context object.
11930  *
11931  * This function cleans up all iocb, buffers, mailbox commands
11932  * while shutting down the HBA. This function is called with no
11933  * lock held and always returns 1.
11934  * This function does the following to cleanup driver resources:
11935  * - Free discovery resources for each virtual port
11936  * - Cleanup any pending fabric iocbs
11937  * - Iterate through the iocb txq and free each entry
11938  *   in the list.
11939  * - Free up any buffer posted to the HBA
11940  * - Free mailbox commands in the mailbox queue.
11941  **/
11942 int
11943 lpfc_sli_hba_down(struct lpfc_hba *phba)
11944 {
11945 	LIST_HEAD(completions);
11946 	struct lpfc_sli *psli = &phba->sli;
11947 	struct lpfc_queue *qp = NULL;
11948 	struct lpfc_sli_ring *pring;
11949 	struct lpfc_dmabuf *buf_ptr;
11950 	unsigned long flags = 0;
11951 	int i;
11952 
11953 	/* Shutdown the mailbox command sub-system */
11954 	lpfc_sli_mbox_sys_shutdown(phba, LPFC_MBX_WAIT);
11955 
11956 	lpfc_hba_down_prep(phba);
11957 
11958 	/* Disable softirqs, including timers from obtaining phba->hbalock */
11959 	local_bh_disable();
11960 
11961 	lpfc_fabric_abort_hba(phba);
11962 
11963 	spin_lock_irqsave(&phba->hbalock, flags);
11964 
11965 	/*
11966 	 * Error everything on the txq since these iocbs
11967 	 * have not been given to the FW yet.
11968 	 */
11969 	if (phba->sli_rev != LPFC_SLI_REV4) {
11970 		for (i = 0; i < psli->num_rings; i++) {
11971 			pring = &psli->sli3_ring[i];
11972 			/* Only slow rings */
11973 			if (pring->ringno == LPFC_ELS_RING) {
11974 				pring->flag |= LPFC_DEFERRED_RING_EVENT;
11975 				/* Set the lpfc data pending flag */
11976 				set_bit(LPFC_DATA_READY, &phba->data_flags);
11977 			}
11978 			list_splice_init(&pring->txq, &completions);
11979 		}
11980 	} else {
11981 		list_for_each_entry(qp, &phba->sli4_hba.lpfc_wq_list, wq_list) {
11982 			pring = qp->pring;
11983 			if (!pring)
11984 				continue;
11985 			spin_lock(&pring->ring_lock);
11986 			list_splice_init(&pring->txq, &completions);
11987 			spin_unlock(&pring->ring_lock);
11988 			if (pring == phba->sli4_hba.els_wq->pring) {
11989 				pring->flag |= LPFC_DEFERRED_RING_EVENT;
11990 				/* Set the lpfc data pending flag */
11991 				set_bit(LPFC_DATA_READY, &phba->data_flags);
11992 			}
11993 		}
11994 	}
11995 	spin_unlock_irqrestore(&phba->hbalock, flags);
11996 
11997 	/* Cancel all the IOCBs from the completions list */
11998 	lpfc_sli_cancel_iocbs(phba, &completions, IOSTAT_LOCAL_REJECT,
11999 			      IOERR_SLI_DOWN);
12000 
12001 	spin_lock_irqsave(&phba->hbalock, flags);
12002 	list_splice_init(&phba->elsbuf, &completions);
12003 	phba->elsbuf_cnt = 0;
12004 	phba->elsbuf_prev_cnt = 0;
12005 	spin_unlock_irqrestore(&phba->hbalock, flags);
12006 
12007 	while (!list_empty(&completions)) {
12008 		list_remove_head(&completions, buf_ptr,
12009 			struct lpfc_dmabuf, list);
12010 		lpfc_mbuf_free(phba, buf_ptr->virt, buf_ptr->phys);
12011 		kfree(buf_ptr);
12012 	}
12013 
12014 	/* Enable softirqs again, done with phba->hbalock */
12015 	local_bh_enable();
12016 
12017 	/* Return any active mbox cmds */
12018 	del_timer_sync(&psli->mbox_tmo);
12019 
12020 	spin_lock_irqsave(&phba->pport->work_port_lock, flags);
12021 	phba->pport->work_port_events &= ~WORKER_MBOX_TMO;
12022 	spin_unlock_irqrestore(&phba->pport->work_port_lock, flags);
12023 
12024 	return 1;
12025 }
12026 
12027 /**
12028  * lpfc_sli_pcimem_bcopy - SLI memory copy function
12029  * @srcp: Source memory pointer.
12030  * @destp: Destination memory pointer.
12031  * @cnt: Number of words required to be copied.
12032  *
12033  * This function is used for copying data between driver memory
12034  * and the SLI memory. This function also changes the endianness
12035  * of each word if native endianness is different from SLI
12036  * endianness. This function can be called with or without
12037  * lock.
12038  **/
12039 void
12040 lpfc_sli_pcimem_bcopy(void *srcp, void *destp, uint32_t cnt)
12041 {
12042 	uint32_t *src = srcp;
12043 	uint32_t *dest = destp;
12044 	uint32_t ldata;
12045 	int i;
12046 
12047 	for (i = 0; i < (int)cnt; i += sizeof (uint32_t)) {
12048 		ldata = *src;
12049 		ldata = le32_to_cpu(ldata);
12050 		*dest = ldata;
12051 		src++;
12052 		dest++;
12053 	}
12054 }
12055 
12056 
12057 /**
12058  * lpfc_sli_bemem_bcopy - SLI memory copy function
12059  * @srcp: Source memory pointer.
12060  * @destp: Destination memory pointer.
12061  * @cnt: Number of words required to be copied.
12062  *
12063  * This function is used for copying data between a data structure
12064  * with big endian representation to local endianness.
12065  * This function can be called with or without lock.
12066  **/
12067 void
12068 lpfc_sli_bemem_bcopy(void *srcp, void *destp, uint32_t cnt)
12069 {
12070 	uint32_t *src = srcp;
12071 	uint32_t *dest = destp;
12072 	uint32_t ldata;
12073 	int i;
12074 
12075 	for (i = 0; i < (int)cnt; i += sizeof(uint32_t)) {
12076 		ldata = *src;
12077 		ldata = be32_to_cpu(ldata);
12078 		*dest = ldata;
12079 		src++;
12080 		dest++;
12081 	}
12082 }
12083 
12084 /**
12085  * lpfc_sli_ringpostbuf_put - Function to add a buffer to postbufq
12086  * @phba: Pointer to HBA context object.
12087  * @pring: Pointer to driver SLI ring object.
12088  * @mp: Pointer to driver buffer object.
12089  *
12090  * This function is called with no lock held.
12091  * It always return zero after adding the buffer to the postbufq
12092  * buffer list.
12093  **/
12094 int
12095 lpfc_sli_ringpostbuf_put(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
12096 			 struct lpfc_dmabuf *mp)
12097 {
12098 	/* Stick struct lpfc_dmabuf at end of postbufq so driver can look it up
12099 	   later */
12100 	spin_lock_irq(&phba->hbalock);
12101 	list_add_tail(&mp->list, &pring->postbufq);
12102 	pring->postbufq_cnt++;
12103 	spin_unlock_irq(&phba->hbalock);
12104 	return 0;
12105 }
12106 
12107 /**
12108  * lpfc_sli_get_buffer_tag - allocates a tag for a CMD_QUE_XRI64_CX buffer
12109  * @phba: Pointer to HBA context object.
12110  *
12111  * When HBQ is enabled, buffers are searched based on tags. This function
12112  * allocates a tag for buffer posted using CMD_QUE_XRI64_CX iocb. The
12113  * tag is bit wise or-ed with QUE_BUFTAG_BIT to make sure that the tag
12114  * does not conflict with tags of buffer posted for unsolicited events.
12115  * The function returns the allocated tag. The function is called with
12116  * no locks held.
12117  **/
12118 uint32_t
12119 lpfc_sli_get_buffer_tag(struct lpfc_hba *phba)
12120 {
12121 	spin_lock_irq(&phba->hbalock);
12122 	phba->buffer_tag_count++;
12123 	/*
12124 	 * Always set the QUE_BUFTAG_BIT to distiguish between
12125 	 * a tag assigned by HBQ.
12126 	 */
12127 	phba->buffer_tag_count |= QUE_BUFTAG_BIT;
12128 	spin_unlock_irq(&phba->hbalock);
12129 	return phba->buffer_tag_count;
12130 }
12131 
12132 /**
12133  * lpfc_sli_ring_taggedbuf_get - find HBQ buffer associated with given tag
12134  * @phba: Pointer to HBA context object.
12135  * @pring: Pointer to driver SLI ring object.
12136  * @tag: Buffer tag.
12137  *
12138  * Buffers posted using CMD_QUE_XRI64_CX iocb are in pring->postbufq
12139  * list. After HBA DMA data to these buffers, CMD_IOCB_RET_XRI64_CX
12140  * iocb is posted to the response ring with the tag of the buffer.
12141  * This function searches the pring->postbufq list using the tag
12142  * to find buffer associated with CMD_IOCB_RET_XRI64_CX
12143  * iocb. If the buffer is found then lpfc_dmabuf object of the
12144  * buffer is returned to the caller else NULL is returned.
12145  * This function is called with no lock held.
12146  **/
12147 struct lpfc_dmabuf *
12148 lpfc_sli_ring_taggedbuf_get(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
12149 			uint32_t tag)
12150 {
12151 	struct lpfc_dmabuf *mp, *next_mp;
12152 	struct list_head *slp = &pring->postbufq;
12153 
12154 	/* Search postbufq, from the beginning, looking for a match on tag */
12155 	spin_lock_irq(&phba->hbalock);
12156 	list_for_each_entry_safe(mp, next_mp, &pring->postbufq, list) {
12157 		if (mp->buffer_tag == tag) {
12158 			list_del_init(&mp->list);
12159 			pring->postbufq_cnt--;
12160 			spin_unlock_irq(&phba->hbalock);
12161 			return mp;
12162 		}
12163 	}
12164 
12165 	spin_unlock_irq(&phba->hbalock);
12166 	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
12167 			"0402 Cannot find virtual addr for buffer tag on "
12168 			"ring %d Data x%lx x%px x%px x%x\n",
12169 			pring->ringno, (unsigned long) tag,
12170 			slp->next, slp->prev, pring->postbufq_cnt);
12171 
12172 	return NULL;
12173 }
12174 
12175 /**
12176  * lpfc_sli_ringpostbuf_get - search buffers for unsolicited CT and ELS events
12177  * @phba: Pointer to HBA context object.
12178  * @pring: Pointer to driver SLI ring object.
12179  * @phys: DMA address of the buffer.
12180  *
12181  * This function searches the buffer list using the dma_address
12182  * of unsolicited event to find the driver's lpfc_dmabuf object
12183  * corresponding to the dma_address. The function returns the
12184  * lpfc_dmabuf object if a buffer is found else it returns NULL.
12185  * This function is called by the ct and els unsolicited event
12186  * handlers to get the buffer associated with the unsolicited
12187  * event.
12188  *
12189  * This function is called with no lock held.
12190  **/
12191 struct lpfc_dmabuf *
12192 lpfc_sli_ringpostbuf_get(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
12193 			 dma_addr_t phys)
12194 {
12195 	struct lpfc_dmabuf *mp, *next_mp;
12196 	struct list_head *slp = &pring->postbufq;
12197 
12198 	/* Search postbufq, from the beginning, looking for a match on phys */
12199 	spin_lock_irq(&phba->hbalock);
12200 	list_for_each_entry_safe(mp, next_mp, &pring->postbufq, list) {
12201 		if (mp->phys == phys) {
12202 			list_del_init(&mp->list);
12203 			pring->postbufq_cnt--;
12204 			spin_unlock_irq(&phba->hbalock);
12205 			return mp;
12206 		}
12207 	}
12208 
12209 	spin_unlock_irq(&phba->hbalock);
12210 	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
12211 			"0410 Cannot find virtual addr for mapped buf on "
12212 			"ring %d Data x%llx x%px x%px x%x\n",
12213 			pring->ringno, (unsigned long long)phys,
12214 			slp->next, slp->prev, pring->postbufq_cnt);
12215 	return NULL;
12216 }
12217 
12218 /**
12219  * lpfc_sli_abort_els_cmpl - Completion handler for the els abort iocbs
12220  * @phba: Pointer to HBA context object.
12221  * @cmdiocb: Pointer to driver command iocb object.
12222  * @rspiocb: Pointer to driver response iocb object.
12223  *
12224  * This function is the completion handler for the abort iocbs for
12225  * ELS commands. This function is called from the ELS ring event
12226  * handler with no lock held. This function frees memory resources
12227  * associated with the abort iocb.
12228  **/
12229 static void
12230 lpfc_sli_abort_els_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
12231 			struct lpfc_iocbq *rspiocb)
12232 {
12233 	IOCB_t *irsp = &rspiocb->iocb;
12234 	uint16_t abort_iotag, abort_context;
12235 	struct lpfc_iocbq *abort_iocb = NULL;
12236 
12237 	if (irsp->ulpStatus) {
12238 
12239 		/*
12240 		 * Assume that the port already completed and returned, or
12241 		 * will return the iocb. Just Log the message.
12242 		 */
12243 		abort_context = cmdiocb->iocb.un.acxri.abortContextTag;
12244 		abort_iotag = cmdiocb->iocb.un.acxri.abortIoTag;
12245 
12246 		spin_lock_irq(&phba->hbalock);
12247 		if (phba->sli_rev < LPFC_SLI_REV4) {
12248 			if (irsp->ulpCommand == CMD_ABORT_XRI_CX &&
12249 			    irsp->ulpStatus == IOSTAT_LOCAL_REJECT &&
12250 			    irsp->un.ulpWord[4] == IOERR_ABORT_REQUESTED) {
12251 				spin_unlock_irq(&phba->hbalock);
12252 				goto release_iocb;
12253 			}
12254 			if (abort_iotag != 0 &&
12255 				abort_iotag <= phba->sli.last_iotag)
12256 				abort_iocb =
12257 					phba->sli.iocbq_lookup[abort_iotag];
12258 		} else
12259 			/* For sli4 the abort_tag is the XRI,
12260 			 * so the abort routine puts the iotag  of the iocb
12261 			 * being aborted in the context field of the abort
12262 			 * IOCB.
12263 			 */
12264 			abort_iocb = phba->sli.iocbq_lookup[abort_context];
12265 
12266 		lpfc_printf_log(phba, KERN_WARNING, LOG_ELS | LOG_SLI,
12267 				"0327 Cannot abort els iocb x%px "
12268 				"with tag %x context %x, abort status %x, "
12269 				"abort code %x\n",
12270 				abort_iocb, abort_iotag, abort_context,
12271 				irsp->ulpStatus, irsp->un.ulpWord[4]);
12272 
12273 		spin_unlock_irq(&phba->hbalock);
12274 	}
12275 release_iocb:
12276 	lpfc_sli_release_iocbq(phba, cmdiocb);
12277 	return;
12278 }
12279 
12280 /**
12281  * lpfc_ignore_els_cmpl - Completion handler for aborted ELS command
12282  * @phba: Pointer to HBA context object.
12283  * @cmdiocb: Pointer to driver command iocb object.
12284  * @rspiocb: Pointer to driver response iocb object.
12285  *
12286  * The function is called from SLI ring event handler with no
12287  * lock held. This function is the completion handler for ELS commands
12288  * which are aborted. The function frees memory resources used for
12289  * the aborted ELS commands.
12290  **/
12291 void
12292 lpfc_ignore_els_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
12293 		     struct lpfc_iocbq *rspiocb)
12294 {
12295 	struct lpfc_nodelist *ndlp = (struct lpfc_nodelist *) cmdiocb->context1;
12296 	IOCB_t *irsp = &rspiocb->iocb;
12297 
12298 	/* ELS cmd tag <ulpIoTag> completes */
12299 	lpfc_printf_log(phba, KERN_INFO, LOG_ELS,
12300 			"0139 Ignoring ELS cmd tag x%x completion Data: "
12301 			"x%x x%x x%x\n",
12302 			irsp->ulpIoTag, irsp->ulpStatus,
12303 			irsp->un.ulpWord[4], irsp->ulpTimeout);
12304 	/*
12305 	 * Deref the ndlp after free_iocb. sli_release_iocb will access the ndlp
12306 	 * if exchange is busy.
12307 	 */
12308 	if (cmdiocb->iocb.ulpCommand == CMD_GEN_REQUEST64_CR)
12309 		lpfc_ct_free_iocb(phba, cmdiocb);
12310 	else
12311 		lpfc_els_free_iocb(phba, cmdiocb);
12312 
12313 	lpfc_nlp_put(ndlp);
12314 }
12315 
12316 /**
12317  * lpfc_sli_issue_abort_iotag - Abort function for a command iocb
12318  * @phba: Pointer to HBA context object.
12319  * @pring: Pointer to driver SLI ring object.
12320  * @cmdiocb: Pointer to driver command iocb object.
12321  * @cmpl: completion function.
12322  *
12323  * This function issues an abort iocb for the provided command iocb. In case
12324  * of unloading, the abort iocb will not be issued to commands on the ELS
12325  * ring. Instead, the callback function shall be changed to those commands
12326  * so that nothing happens when them finishes. This function is called with
12327  * hbalock held andno ring_lock held (SLI4). The function returns IOCB_SUCCESS
12328  * when the command iocb is an abort request.
12329  *
12330  **/
12331 int
12332 lpfc_sli_issue_abort_iotag(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
12333 			   struct lpfc_iocbq *cmdiocb, void *cmpl)
12334 {
12335 	struct lpfc_vport *vport = cmdiocb->vport;
12336 	struct lpfc_iocbq *abtsiocbp;
12337 	IOCB_t *icmd = NULL;
12338 	IOCB_t *iabt = NULL;
12339 	int retval = IOCB_ERROR;
12340 	unsigned long iflags;
12341 	struct lpfc_nodelist *ndlp;
12342 
12343 	/*
12344 	 * There are certain command types we don't want to abort.  And we
12345 	 * don't want to abort commands that are already in the process of
12346 	 * being aborted.
12347 	 */
12348 	icmd = &cmdiocb->iocb;
12349 	if (icmd->ulpCommand == CMD_ABORT_XRI_CN ||
12350 	    icmd->ulpCommand == CMD_CLOSE_XRI_CN ||
12351 	    cmdiocb->iocb_flag & LPFC_DRIVER_ABORTED)
12352 		return IOCB_ABORTING;
12353 
12354 	if (!pring) {
12355 		if (cmdiocb->iocb_flag & LPFC_IO_FABRIC)
12356 			cmdiocb->fabric_iocb_cmpl = lpfc_ignore_els_cmpl;
12357 		else
12358 			cmdiocb->iocb_cmpl = lpfc_ignore_els_cmpl;
12359 		return retval;
12360 	}
12361 
12362 	/*
12363 	 * If we're unloading, don't abort iocb on the ELS ring, but change
12364 	 * the callback so that nothing happens when it finishes.
12365 	 */
12366 	if ((vport->load_flag & FC_UNLOADING) &&
12367 	    pring->ringno == LPFC_ELS_RING) {
12368 		if (cmdiocb->iocb_flag & LPFC_IO_FABRIC)
12369 			cmdiocb->fabric_iocb_cmpl = lpfc_ignore_els_cmpl;
12370 		else
12371 			cmdiocb->iocb_cmpl = lpfc_ignore_els_cmpl;
12372 		return retval;
12373 	}
12374 
12375 	/* issue ABTS for this IOCB based on iotag */
12376 	abtsiocbp = __lpfc_sli_get_iocbq(phba);
12377 	if (abtsiocbp == NULL)
12378 		return IOCB_NORESOURCE;
12379 
12380 	/* This signals the response to set the correct status
12381 	 * before calling the completion handler
12382 	 */
12383 	cmdiocb->iocb_flag |= LPFC_DRIVER_ABORTED;
12384 
12385 	iabt = &abtsiocbp->iocb;
12386 	iabt->un.acxri.abortType = ABORT_TYPE_ABTS;
12387 	iabt->un.acxri.abortContextTag = icmd->ulpContext;
12388 	if (phba->sli_rev == LPFC_SLI_REV4) {
12389 		iabt->un.acxri.abortIoTag = cmdiocb->sli4_xritag;
12390 		if (pring->ringno == LPFC_ELS_RING)
12391 			iabt->un.acxri.abortContextTag = cmdiocb->iotag;
12392 	} else {
12393 		iabt->un.acxri.abortIoTag = icmd->ulpIoTag;
12394 		if (pring->ringno == LPFC_ELS_RING) {
12395 			ndlp = (struct lpfc_nodelist *)(cmdiocb->context1);
12396 			iabt->un.acxri.abortContextTag = ndlp->nlp_rpi;
12397 		}
12398 	}
12399 	iabt->ulpLe = 1;
12400 	iabt->ulpClass = icmd->ulpClass;
12401 
12402 	/* ABTS WQE must go to the same WQ as the WQE to be aborted */
12403 	abtsiocbp->hba_wqidx = cmdiocb->hba_wqidx;
12404 	if (cmdiocb->iocb_flag & LPFC_IO_FCP) {
12405 		abtsiocbp->iocb_flag |= LPFC_IO_FCP;
12406 		abtsiocbp->iocb_flag |= LPFC_USE_FCPWQIDX;
12407 	}
12408 	if (cmdiocb->iocb_flag & LPFC_IO_FOF)
12409 		abtsiocbp->iocb_flag |= LPFC_IO_FOF;
12410 
12411 	if (phba->link_state >= LPFC_LINK_UP)
12412 		iabt->ulpCommand = CMD_ABORT_XRI_CN;
12413 	else
12414 		iabt->ulpCommand = CMD_CLOSE_XRI_CN;
12415 
12416 	if (cmpl)
12417 		abtsiocbp->iocb_cmpl = cmpl;
12418 	else
12419 		abtsiocbp->iocb_cmpl = lpfc_sli_abort_els_cmpl;
12420 	abtsiocbp->vport = vport;
12421 
12422 	if (phba->sli_rev == LPFC_SLI_REV4) {
12423 		pring = lpfc_sli4_calc_ring(phba, abtsiocbp);
12424 		if (unlikely(pring == NULL))
12425 			goto abort_iotag_exit;
12426 		/* Note: both hbalock and ring_lock need to be set here */
12427 		spin_lock_irqsave(&pring->ring_lock, iflags);
12428 		retval = __lpfc_sli_issue_iocb(phba, pring->ringno,
12429 			abtsiocbp, 0);
12430 		spin_unlock_irqrestore(&pring->ring_lock, iflags);
12431 	} else {
12432 		retval = __lpfc_sli_issue_iocb(phba, pring->ringno,
12433 			abtsiocbp, 0);
12434 	}
12435 
12436 abort_iotag_exit:
12437 
12438 	lpfc_printf_vlog(vport, KERN_INFO, LOG_SLI,
12439 			 "0339 Abort xri x%x, original iotag x%x, "
12440 			 "abort cmd iotag x%x retval x%x\n",
12441 			 iabt->un.acxri.abortIoTag,
12442 			 iabt->un.acxri.abortContextTag,
12443 			 abtsiocbp->iotag, retval);
12444 
12445 	if (retval) {
12446 		cmdiocb->iocb_flag &= ~LPFC_DRIVER_ABORTED;
12447 		__lpfc_sli_release_iocbq(phba, abtsiocbp);
12448 	}
12449 
12450 	/*
12451 	 * Caller to this routine should check for IOCB_ERROR
12452 	 * and handle it properly.  This routine no longer removes
12453 	 * iocb off txcmplq and call compl in case of IOCB_ERROR.
12454 	 */
12455 	return retval;
12456 }
12457 
12458 /**
12459  * lpfc_sli_hba_iocb_abort - Abort all iocbs to an hba.
12460  * @phba: pointer to lpfc HBA data structure.
12461  *
12462  * This routine will abort all pending and outstanding iocbs to an HBA.
12463  **/
12464 void
12465 lpfc_sli_hba_iocb_abort(struct lpfc_hba *phba)
12466 {
12467 	struct lpfc_sli *psli = &phba->sli;
12468 	struct lpfc_sli_ring *pring;
12469 	struct lpfc_queue *qp = NULL;
12470 	int i;
12471 
12472 	if (phba->sli_rev != LPFC_SLI_REV4) {
12473 		for (i = 0; i < psli->num_rings; i++) {
12474 			pring = &psli->sli3_ring[i];
12475 			lpfc_sli_abort_iocb_ring(phba, pring);
12476 		}
12477 		return;
12478 	}
12479 	list_for_each_entry(qp, &phba->sli4_hba.lpfc_wq_list, wq_list) {
12480 		pring = qp->pring;
12481 		if (!pring)
12482 			continue;
12483 		lpfc_sli_abort_iocb_ring(phba, pring);
12484 	}
12485 }
12486 
12487 /**
12488  * lpfc_sli_validate_fcp_iocb - find commands associated with a vport or LUN
12489  * @iocbq: Pointer to driver iocb object.
12490  * @vport: Pointer to driver virtual port object.
12491  * @tgt_id: SCSI ID of the target.
12492  * @lun_id: LUN ID of the scsi device.
12493  * @ctx_cmd: LPFC_CTX_LUN/LPFC_CTX_TGT/LPFC_CTX_HOST
12494  *
12495  * This function acts as an iocb filter for functions which abort or count
12496  * all FCP iocbs pending on a lun/SCSI target/SCSI host. It will return
12497  * 0 if the filtering criteria is met for the given iocb and will return
12498  * 1 if the filtering criteria is not met.
12499  * If ctx_cmd == LPFC_CTX_LUN, the function returns 0 only if the
12500  * given iocb is for the SCSI device specified by vport, tgt_id and
12501  * lun_id parameter.
12502  * If ctx_cmd == LPFC_CTX_TGT,  the function returns 0 only if the
12503  * given iocb is for the SCSI target specified by vport and tgt_id
12504  * parameters.
12505  * If ctx_cmd == LPFC_CTX_HOST, the function returns 0 only if the
12506  * given iocb is for the SCSI host associated with the given vport.
12507  * This function is called with no locks held.
12508  **/
12509 static int
12510 lpfc_sli_validate_fcp_iocb(struct lpfc_iocbq *iocbq, struct lpfc_vport *vport,
12511 			   uint16_t tgt_id, uint64_t lun_id,
12512 			   lpfc_ctx_cmd ctx_cmd)
12513 {
12514 	struct lpfc_io_buf *lpfc_cmd;
12515 	IOCB_t *icmd = NULL;
12516 	int rc = 1;
12517 
12518 	if (!iocbq || iocbq->vport != vport)
12519 		return rc;
12520 
12521 	if (!(iocbq->iocb_flag & LPFC_IO_FCP) ||
12522 	    !(iocbq->iocb_flag & LPFC_IO_ON_TXCMPLQ) ||
12523 	      iocbq->iocb_flag & LPFC_DRIVER_ABORTED)
12524 		return rc;
12525 
12526 	icmd = &iocbq->iocb;
12527 	if (icmd->ulpCommand == CMD_ABORT_XRI_CN ||
12528 	    icmd->ulpCommand == CMD_CLOSE_XRI_CN)
12529 		return rc;
12530 
12531 	lpfc_cmd = container_of(iocbq, struct lpfc_io_buf, cur_iocbq);
12532 
12533 	if (lpfc_cmd->pCmd == NULL)
12534 		return rc;
12535 
12536 	switch (ctx_cmd) {
12537 	case LPFC_CTX_LUN:
12538 		if ((lpfc_cmd->rdata) && (lpfc_cmd->rdata->pnode) &&
12539 		    (lpfc_cmd->rdata->pnode->nlp_sid == tgt_id) &&
12540 		    (scsilun_to_int(&lpfc_cmd->fcp_cmnd->fcp_lun) == lun_id))
12541 			rc = 0;
12542 		break;
12543 	case LPFC_CTX_TGT:
12544 		if ((lpfc_cmd->rdata) && (lpfc_cmd->rdata->pnode) &&
12545 		    (lpfc_cmd->rdata->pnode->nlp_sid == tgt_id))
12546 			rc = 0;
12547 		break;
12548 	case LPFC_CTX_HOST:
12549 		rc = 0;
12550 		break;
12551 	default:
12552 		printk(KERN_ERR "%s: Unknown context cmd type, value %d\n",
12553 			__func__, ctx_cmd);
12554 		break;
12555 	}
12556 
12557 	return rc;
12558 }
12559 
12560 /**
12561  * lpfc_sli_sum_iocb - Function to count the number of FCP iocbs pending
12562  * @vport: Pointer to virtual port.
12563  * @tgt_id: SCSI ID of the target.
12564  * @lun_id: LUN ID of the scsi device.
12565  * @ctx_cmd: LPFC_CTX_LUN/LPFC_CTX_TGT/LPFC_CTX_HOST.
12566  *
12567  * This function returns number of FCP commands pending for the vport.
12568  * When ctx_cmd == LPFC_CTX_LUN, the function returns number of FCP
12569  * commands pending on the vport associated with SCSI device specified
12570  * by tgt_id and lun_id parameters.
12571  * When ctx_cmd == LPFC_CTX_TGT, the function returns number of FCP
12572  * commands pending on the vport associated with SCSI target specified
12573  * by tgt_id parameter.
12574  * When ctx_cmd == LPFC_CTX_HOST, the function returns number of FCP
12575  * commands pending on the vport.
12576  * This function returns the number of iocbs which satisfy the filter.
12577  * This function is called without any lock held.
12578  **/
12579 int
12580 lpfc_sli_sum_iocb(struct lpfc_vport *vport, uint16_t tgt_id, uint64_t lun_id,
12581 		  lpfc_ctx_cmd ctx_cmd)
12582 {
12583 	struct lpfc_hba *phba = vport->phba;
12584 	struct lpfc_iocbq *iocbq;
12585 	int sum, i;
12586 
12587 	spin_lock_irq(&phba->hbalock);
12588 	for (i = 1, sum = 0; i <= phba->sli.last_iotag; i++) {
12589 		iocbq = phba->sli.iocbq_lookup[i];
12590 
12591 		if (lpfc_sli_validate_fcp_iocb (iocbq, vport, tgt_id, lun_id,
12592 						ctx_cmd) == 0)
12593 			sum++;
12594 	}
12595 	spin_unlock_irq(&phba->hbalock);
12596 
12597 	return sum;
12598 }
12599 
12600 /**
12601  * lpfc_sli4_abort_fcp_cmpl - Completion handler function for aborted FCP IOCBs
12602  * @phba: Pointer to HBA context object
12603  * @cmdiocb: Pointer to command iocb object.
12604  * @wcqe: pointer to the complete wcqe
12605  *
12606  * This function is called when an aborted FCP iocb completes. This
12607  * function is called by the ring event handler with no lock held.
12608  * This function frees the iocb. It is called for sli-4 adapters.
12609  **/
12610 void
12611 lpfc_sli4_abort_fcp_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
12612 			 struct lpfc_wcqe_complete *wcqe)
12613 {
12614 	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
12615 			"3017 ABORT_XRI_CN completing on rpi x%x "
12616 			"original iotag x%x, abort cmd iotag x%x "
12617 			"status 0x%x, reason 0x%x\n",
12618 			cmdiocb->iocb.un.acxri.abortContextTag,
12619 			cmdiocb->iocb.un.acxri.abortIoTag,
12620 			cmdiocb->iotag,
12621 			(bf_get(lpfc_wcqe_c_status, wcqe)
12622 			& LPFC_IOCB_STATUS_MASK),
12623 			wcqe->parameter);
12624 	lpfc_sli_release_iocbq(phba, cmdiocb);
12625 }
12626 
12627 /**
12628  * lpfc_sli_abort_fcp_cmpl - Completion handler function for aborted FCP IOCBs
12629  * @phba: Pointer to HBA context object
12630  * @cmdiocb: Pointer to command iocb object.
12631  * @rspiocb: Pointer to response iocb object.
12632  *
12633  * This function is called when an aborted FCP iocb completes. This
12634  * function is called by the ring event handler with no lock held.
12635  * This function frees the iocb.
12636  **/
12637 void
12638 lpfc_sli_abort_fcp_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
12639 			struct lpfc_iocbq *rspiocb)
12640 {
12641 	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
12642 			"3096 ABORT_XRI_CN completing on rpi x%x "
12643 			"original iotag x%x, abort cmd iotag x%x "
12644 			"status 0x%x, reason 0x%x\n",
12645 			cmdiocb->iocb.un.acxri.abortContextTag,
12646 			cmdiocb->iocb.un.acxri.abortIoTag,
12647 			cmdiocb->iotag, rspiocb->iocb.ulpStatus,
12648 			rspiocb->iocb.un.ulpWord[4]);
12649 	lpfc_sli_release_iocbq(phba, cmdiocb);
12650 	return;
12651 }
12652 
12653 /**
12654  * lpfc_sli_abort_iocb - issue abort for all commands on a host/target/LUN
12655  * @vport: Pointer to virtual port.
12656  * @tgt_id: SCSI ID of the target.
12657  * @lun_id: LUN ID of the scsi device.
12658  * @abort_cmd: LPFC_CTX_LUN/LPFC_CTX_TGT/LPFC_CTX_HOST.
12659  *
12660  * This function sends an abort command for every SCSI command
12661  * associated with the given virtual port pending on the ring
12662  * filtered by lpfc_sli_validate_fcp_iocb function.
12663  * When abort_cmd == LPFC_CTX_LUN, the function sends abort only to the
12664  * FCP iocbs associated with lun specified by tgt_id and lun_id
12665  * parameters
12666  * When abort_cmd == LPFC_CTX_TGT, the function sends abort only to the
12667  * FCP iocbs associated with SCSI target specified by tgt_id parameter.
12668  * When abort_cmd == LPFC_CTX_HOST, the function sends abort to all
12669  * FCP iocbs associated with virtual port.
12670  * The pring used for SLI3 is sli3_ring[LPFC_FCP_RING], for SLI4
12671  * lpfc_sli4_calc_ring is used.
12672  * This function returns number of iocbs it failed to abort.
12673  * This function is called with no locks held.
12674  **/
12675 int
12676 lpfc_sli_abort_iocb(struct lpfc_vport *vport, u16 tgt_id, u64 lun_id,
12677 		    lpfc_ctx_cmd abort_cmd)
12678 {
12679 	struct lpfc_hba *phba = vport->phba;
12680 	struct lpfc_sli_ring *pring = NULL;
12681 	struct lpfc_iocbq *iocbq;
12682 	int errcnt = 0, ret_val = 0;
12683 	unsigned long iflags;
12684 	int i;
12685 	void *fcp_cmpl = NULL;
12686 
12687 	/* all I/Os are in process of being flushed */
12688 	if (phba->hba_flag & HBA_IOQ_FLUSH)
12689 		return errcnt;
12690 
12691 	for (i = 1; i <= phba->sli.last_iotag; i++) {
12692 		iocbq = phba->sli.iocbq_lookup[i];
12693 
12694 		if (lpfc_sli_validate_fcp_iocb(iocbq, vport, tgt_id, lun_id,
12695 					       abort_cmd) != 0)
12696 			continue;
12697 
12698 		spin_lock_irqsave(&phba->hbalock, iflags);
12699 		if (phba->sli_rev == LPFC_SLI_REV3) {
12700 			pring = &phba->sli.sli3_ring[LPFC_FCP_RING];
12701 			fcp_cmpl = lpfc_sli_abort_fcp_cmpl;
12702 		} else if (phba->sli_rev == LPFC_SLI_REV4) {
12703 			pring = lpfc_sli4_calc_ring(phba, iocbq);
12704 			fcp_cmpl = lpfc_sli4_abort_fcp_cmpl;
12705 		}
12706 		ret_val = lpfc_sli_issue_abort_iotag(phba, pring, iocbq,
12707 						     fcp_cmpl);
12708 		spin_unlock_irqrestore(&phba->hbalock, iflags);
12709 		if (ret_val != IOCB_SUCCESS)
12710 			errcnt++;
12711 	}
12712 
12713 	return errcnt;
12714 }
12715 
12716 /**
12717  * lpfc_sli_abort_taskmgmt - issue abort for all commands on a host/target/LUN
12718  * @vport: Pointer to virtual port.
12719  * @pring: Pointer to driver SLI ring object.
12720  * @tgt_id: SCSI ID of the target.
12721  * @lun_id: LUN ID of the scsi device.
12722  * @cmd: LPFC_CTX_LUN/LPFC_CTX_TGT/LPFC_CTX_HOST.
12723  *
12724  * This function sends an abort command for every SCSI command
12725  * associated with the given virtual port pending on the ring
12726  * filtered by lpfc_sli_validate_fcp_iocb function.
12727  * When taskmgmt_cmd == LPFC_CTX_LUN, the function sends abort only to the
12728  * FCP iocbs associated with lun specified by tgt_id and lun_id
12729  * parameters
12730  * When taskmgmt_cmd == LPFC_CTX_TGT, the function sends abort only to the
12731  * FCP iocbs associated with SCSI target specified by tgt_id parameter.
12732  * When taskmgmt_cmd == LPFC_CTX_HOST, the function sends abort to all
12733  * FCP iocbs associated with virtual port.
12734  * This function returns number of iocbs it aborted .
12735  * This function is called with no locks held right after a taskmgmt
12736  * command is sent.
12737  **/
12738 int
12739 lpfc_sli_abort_taskmgmt(struct lpfc_vport *vport, struct lpfc_sli_ring *pring,
12740 			uint16_t tgt_id, uint64_t lun_id, lpfc_ctx_cmd cmd)
12741 {
12742 	struct lpfc_hba *phba = vport->phba;
12743 	struct lpfc_io_buf *lpfc_cmd;
12744 	struct lpfc_iocbq *abtsiocbq;
12745 	struct lpfc_nodelist *ndlp;
12746 	struct lpfc_iocbq *iocbq;
12747 	IOCB_t *icmd;
12748 	int sum, i, ret_val;
12749 	unsigned long iflags;
12750 	struct lpfc_sli_ring *pring_s4 = NULL;
12751 
12752 	spin_lock_irqsave(&phba->hbalock, iflags);
12753 
12754 	/* all I/Os are in process of being flushed */
12755 	if (phba->hba_flag & HBA_IOQ_FLUSH) {
12756 		spin_unlock_irqrestore(&phba->hbalock, iflags);
12757 		return 0;
12758 	}
12759 	sum = 0;
12760 
12761 	for (i = 1; i <= phba->sli.last_iotag; i++) {
12762 		iocbq = phba->sli.iocbq_lookup[i];
12763 
12764 		if (lpfc_sli_validate_fcp_iocb(iocbq, vport, tgt_id, lun_id,
12765 					       cmd) != 0)
12766 			continue;
12767 
12768 		/* Guard against IO completion being called at same time */
12769 		lpfc_cmd = container_of(iocbq, struct lpfc_io_buf, cur_iocbq);
12770 		spin_lock(&lpfc_cmd->buf_lock);
12771 
12772 		if (!lpfc_cmd->pCmd) {
12773 			spin_unlock(&lpfc_cmd->buf_lock);
12774 			continue;
12775 		}
12776 
12777 		if (phba->sli_rev == LPFC_SLI_REV4) {
12778 			pring_s4 =
12779 			    phba->sli4_hba.hdwq[iocbq->hba_wqidx].io_wq->pring;
12780 			if (!pring_s4) {
12781 				spin_unlock(&lpfc_cmd->buf_lock);
12782 				continue;
12783 			}
12784 			/* Note: both hbalock and ring_lock must be set here */
12785 			spin_lock(&pring_s4->ring_lock);
12786 		}
12787 
12788 		/*
12789 		 * If the iocbq is already being aborted, don't take a second
12790 		 * action, but do count it.
12791 		 */
12792 		if ((iocbq->iocb_flag & LPFC_DRIVER_ABORTED) ||
12793 		    !(iocbq->iocb_flag & LPFC_IO_ON_TXCMPLQ)) {
12794 			if (phba->sli_rev == LPFC_SLI_REV4)
12795 				spin_unlock(&pring_s4->ring_lock);
12796 			spin_unlock(&lpfc_cmd->buf_lock);
12797 			continue;
12798 		}
12799 
12800 		/* issue ABTS for this IOCB based on iotag */
12801 		abtsiocbq = __lpfc_sli_get_iocbq(phba);
12802 		if (!abtsiocbq) {
12803 			if (phba->sli_rev == LPFC_SLI_REV4)
12804 				spin_unlock(&pring_s4->ring_lock);
12805 			spin_unlock(&lpfc_cmd->buf_lock);
12806 			continue;
12807 		}
12808 
12809 		icmd = &iocbq->iocb;
12810 		abtsiocbq->iocb.un.acxri.abortType = ABORT_TYPE_ABTS;
12811 		abtsiocbq->iocb.un.acxri.abortContextTag = icmd->ulpContext;
12812 		if (phba->sli_rev == LPFC_SLI_REV4)
12813 			abtsiocbq->iocb.un.acxri.abortIoTag =
12814 							 iocbq->sli4_xritag;
12815 		else
12816 			abtsiocbq->iocb.un.acxri.abortIoTag = icmd->ulpIoTag;
12817 		abtsiocbq->iocb.ulpLe = 1;
12818 		abtsiocbq->iocb.ulpClass = icmd->ulpClass;
12819 		abtsiocbq->vport = vport;
12820 
12821 		/* ABTS WQE must go to the same WQ as the WQE to be aborted */
12822 		abtsiocbq->hba_wqidx = iocbq->hba_wqidx;
12823 		if (iocbq->iocb_flag & LPFC_IO_FCP)
12824 			abtsiocbq->iocb_flag |= LPFC_USE_FCPWQIDX;
12825 		if (iocbq->iocb_flag & LPFC_IO_FOF)
12826 			abtsiocbq->iocb_flag |= LPFC_IO_FOF;
12827 
12828 		ndlp = lpfc_cmd->rdata->pnode;
12829 
12830 		if (lpfc_is_link_up(phba) &&
12831 		    (ndlp && ndlp->nlp_state == NLP_STE_MAPPED_NODE))
12832 			abtsiocbq->iocb.ulpCommand = CMD_ABORT_XRI_CN;
12833 		else
12834 			abtsiocbq->iocb.ulpCommand = CMD_CLOSE_XRI_CN;
12835 
12836 		/* Setup callback routine and issue the command. */
12837 		abtsiocbq->iocb_cmpl = lpfc_sli_abort_fcp_cmpl;
12838 
12839 		/*
12840 		 * Indicate the IO is being aborted by the driver and set
12841 		 * the caller's flag into the aborted IO.
12842 		 */
12843 		iocbq->iocb_flag |= LPFC_DRIVER_ABORTED;
12844 
12845 		if (phba->sli_rev == LPFC_SLI_REV4) {
12846 			ret_val = __lpfc_sli_issue_iocb(phba, pring_s4->ringno,
12847 							abtsiocbq, 0);
12848 			spin_unlock(&pring_s4->ring_lock);
12849 		} else {
12850 			ret_val = __lpfc_sli_issue_iocb(phba, pring->ringno,
12851 							abtsiocbq, 0);
12852 		}
12853 
12854 		spin_unlock(&lpfc_cmd->buf_lock);
12855 
12856 		if (ret_val == IOCB_ERROR)
12857 			__lpfc_sli_release_iocbq(phba, abtsiocbq);
12858 		else
12859 			sum++;
12860 	}
12861 	spin_unlock_irqrestore(&phba->hbalock, iflags);
12862 	return sum;
12863 }
12864 
12865 /**
12866  * lpfc_sli_wake_iocb_wait - lpfc_sli_issue_iocb_wait's completion handler
12867  * @phba: Pointer to HBA context object.
12868  * @cmdiocbq: Pointer to command iocb.
12869  * @rspiocbq: Pointer to response iocb.
12870  *
12871  * This function is the completion handler for iocbs issued using
12872  * lpfc_sli_issue_iocb_wait function. This function is called by the
12873  * ring event handler function without any lock held. This function
12874  * can be called from both worker thread context and interrupt
12875  * context. This function also can be called from other thread which
12876  * cleans up the SLI layer objects.
12877  * This function copy the contents of the response iocb to the
12878  * response iocb memory object provided by the caller of
12879  * lpfc_sli_issue_iocb_wait and then wakes up the thread which
12880  * sleeps for the iocb completion.
12881  **/
12882 static void
12883 lpfc_sli_wake_iocb_wait(struct lpfc_hba *phba,
12884 			struct lpfc_iocbq *cmdiocbq,
12885 			struct lpfc_iocbq *rspiocbq)
12886 {
12887 	wait_queue_head_t *pdone_q;
12888 	unsigned long iflags;
12889 	struct lpfc_io_buf *lpfc_cmd;
12890 
12891 	spin_lock_irqsave(&phba->hbalock, iflags);
12892 	if (cmdiocbq->iocb_flag & LPFC_IO_WAKE_TMO) {
12893 
12894 		/*
12895 		 * A time out has occurred for the iocb.  If a time out
12896 		 * completion handler has been supplied, call it.  Otherwise,
12897 		 * just free the iocbq.
12898 		 */
12899 
12900 		spin_unlock_irqrestore(&phba->hbalock, iflags);
12901 		cmdiocbq->iocb_cmpl = cmdiocbq->wait_iocb_cmpl;
12902 		cmdiocbq->wait_iocb_cmpl = NULL;
12903 		if (cmdiocbq->iocb_cmpl)
12904 			(cmdiocbq->iocb_cmpl)(phba, cmdiocbq, NULL);
12905 		else
12906 			lpfc_sli_release_iocbq(phba, cmdiocbq);
12907 		return;
12908 	}
12909 
12910 	cmdiocbq->iocb_flag |= LPFC_IO_WAKE;
12911 	if (cmdiocbq->context2 && rspiocbq)
12912 		memcpy(&((struct lpfc_iocbq *)cmdiocbq->context2)->iocb,
12913 		       &rspiocbq->iocb, sizeof(IOCB_t));
12914 
12915 	/* Set the exchange busy flag for task management commands */
12916 	if ((cmdiocbq->iocb_flag & LPFC_IO_FCP) &&
12917 		!(cmdiocbq->iocb_flag & LPFC_IO_LIBDFC)) {
12918 		lpfc_cmd = container_of(cmdiocbq, struct lpfc_io_buf,
12919 			cur_iocbq);
12920 		if (rspiocbq && (rspiocbq->iocb_flag & LPFC_EXCHANGE_BUSY))
12921 			lpfc_cmd->flags |= LPFC_SBUF_XBUSY;
12922 		else
12923 			lpfc_cmd->flags &= ~LPFC_SBUF_XBUSY;
12924 	}
12925 
12926 	pdone_q = cmdiocbq->context_un.wait_queue;
12927 	if (pdone_q)
12928 		wake_up(pdone_q);
12929 	spin_unlock_irqrestore(&phba->hbalock, iflags);
12930 	return;
12931 }
12932 
12933 /**
12934  * lpfc_chk_iocb_flg - Test IOCB flag with lock held.
12935  * @phba: Pointer to HBA context object..
12936  * @piocbq: Pointer to command iocb.
12937  * @flag: Flag to test.
12938  *
12939  * This routine grabs the hbalock and then test the iocb_flag to
12940  * see if the passed in flag is set.
12941  * Returns:
12942  * 1 if flag is set.
12943  * 0 if flag is not set.
12944  **/
12945 static int
12946 lpfc_chk_iocb_flg(struct lpfc_hba *phba,
12947 		 struct lpfc_iocbq *piocbq, uint32_t flag)
12948 {
12949 	unsigned long iflags;
12950 	int ret;
12951 
12952 	spin_lock_irqsave(&phba->hbalock, iflags);
12953 	ret = piocbq->iocb_flag & flag;
12954 	spin_unlock_irqrestore(&phba->hbalock, iflags);
12955 	return ret;
12956 
12957 }
12958 
12959 /**
12960  * lpfc_sli_issue_iocb_wait - Synchronous function to issue iocb commands
12961  * @phba: Pointer to HBA context object..
12962  * @ring_number: Ring number
12963  * @piocb: Pointer to command iocb.
12964  * @prspiocbq: Pointer to response iocb.
12965  * @timeout: Timeout in number of seconds.
12966  *
12967  * This function issues the iocb to firmware and waits for the
12968  * iocb to complete. The iocb_cmpl field of the shall be used
12969  * to handle iocbs which time out. If the field is NULL, the
12970  * function shall free the iocbq structure.  If more clean up is
12971  * needed, the caller is expected to provide a completion function
12972  * that will provide the needed clean up.  If the iocb command is
12973  * not completed within timeout seconds, the function will either
12974  * free the iocbq structure (if iocb_cmpl == NULL) or execute the
12975  * completion function set in the iocb_cmpl field and then return
12976  * a status of IOCB_TIMEDOUT.  The caller should not free the iocb
12977  * resources if this function returns IOCB_TIMEDOUT.
12978  * The function waits for the iocb completion using an
12979  * non-interruptible wait.
12980  * This function will sleep while waiting for iocb completion.
12981  * So, this function should not be called from any context which
12982  * does not allow sleeping. Due to the same reason, this function
12983  * cannot be called with interrupt disabled.
12984  * This function assumes that the iocb completions occur while
12985  * this function sleep. So, this function cannot be called from
12986  * the thread which process iocb completion for this ring.
12987  * This function clears the iocb_flag of the iocb object before
12988  * issuing the iocb and the iocb completion handler sets this
12989  * flag and wakes this thread when the iocb completes.
12990  * The contents of the response iocb will be copied to prspiocbq
12991  * by the completion handler when the command completes.
12992  * This function returns IOCB_SUCCESS when success.
12993  * This function is called with no lock held.
12994  **/
12995 int
12996 lpfc_sli_issue_iocb_wait(struct lpfc_hba *phba,
12997 			 uint32_t ring_number,
12998 			 struct lpfc_iocbq *piocb,
12999 			 struct lpfc_iocbq *prspiocbq,
13000 			 uint32_t timeout)
13001 {
13002 	DECLARE_WAIT_QUEUE_HEAD_ONSTACK(done_q);
13003 	long timeleft, timeout_req = 0;
13004 	int retval = IOCB_SUCCESS;
13005 	uint32_t creg_val;
13006 	struct lpfc_iocbq *iocb;
13007 	int txq_cnt = 0;
13008 	int txcmplq_cnt = 0;
13009 	struct lpfc_sli_ring *pring;
13010 	unsigned long iflags;
13011 	bool iocb_completed = true;
13012 
13013 	if (phba->sli_rev >= LPFC_SLI_REV4)
13014 		pring = lpfc_sli4_calc_ring(phba, piocb);
13015 	else
13016 		pring = &phba->sli.sli3_ring[ring_number];
13017 	/*
13018 	 * If the caller has provided a response iocbq buffer, then context2
13019 	 * is NULL or its an error.
13020 	 */
13021 	if (prspiocbq) {
13022 		if (piocb->context2)
13023 			return IOCB_ERROR;
13024 		piocb->context2 = prspiocbq;
13025 	}
13026 
13027 	piocb->wait_iocb_cmpl = piocb->iocb_cmpl;
13028 	piocb->iocb_cmpl = lpfc_sli_wake_iocb_wait;
13029 	piocb->context_un.wait_queue = &done_q;
13030 	piocb->iocb_flag &= ~(LPFC_IO_WAKE | LPFC_IO_WAKE_TMO);
13031 
13032 	if (phba->cfg_poll & DISABLE_FCP_RING_INT) {
13033 		if (lpfc_readl(phba->HCregaddr, &creg_val))
13034 			return IOCB_ERROR;
13035 		creg_val |= (HC_R0INT_ENA << LPFC_FCP_RING);
13036 		writel(creg_val, phba->HCregaddr);
13037 		readl(phba->HCregaddr); /* flush */
13038 	}
13039 
13040 	retval = lpfc_sli_issue_iocb(phba, ring_number, piocb,
13041 				     SLI_IOCB_RET_IOCB);
13042 	if (retval == IOCB_SUCCESS) {
13043 		timeout_req = msecs_to_jiffies(timeout * 1000);
13044 		timeleft = wait_event_timeout(done_q,
13045 				lpfc_chk_iocb_flg(phba, piocb, LPFC_IO_WAKE),
13046 				timeout_req);
13047 		spin_lock_irqsave(&phba->hbalock, iflags);
13048 		if (!(piocb->iocb_flag & LPFC_IO_WAKE)) {
13049 
13050 			/*
13051 			 * IOCB timed out.  Inform the wake iocb wait
13052 			 * completion function and set local status
13053 			 */
13054 
13055 			iocb_completed = false;
13056 			piocb->iocb_flag |= LPFC_IO_WAKE_TMO;
13057 		}
13058 		spin_unlock_irqrestore(&phba->hbalock, iflags);
13059 		if (iocb_completed) {
13060 			lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
13061 					"0331 IOCB wake signaled\n");
13062 			/* Note: we are not indicating if the IOCB has a success
13063 			 * status or not - that's for the caller to check.
13064 			 * IOCB_SUCCESS means just that the command was sent and
13065 			 * completed. Not that it completed successfully.
13066 			 * */
13067 		} else if (timeleft == 0) {
13068 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
13069 					"0338 IOCB wait timeout error - no "
13070 					"wake response Data x%x\n", timeout);
13071 			retval = IOCB_TIMEDOUT;
13072 		} else {
13073 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
13074 					"0330 IOCB wake NOT set, "
13075 					"Data x%x x%lx\n",
13076 					timeout, (timeleft / jiffies));
13077 			retval = IOCB_TIMEDOUT;
13078 		}
13079 	} else if (retval == IOCB_BUSY) {
13080 		if (phba->cfg_log_verbose & LOG_SLI) {
13081 			list_for_each_entry(iocb, &pring->txq, list) {
13082 				txq_cnt++;
13083 			}
13084 			list_for_each_entry(iocb, &pring->txcmplq, list) {
13085 				txcmplq_cnt++;
13086 			}
13087 			lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
13088 				"2818 Max IOCBs %d txq cnt %d txcmplq cnt %d\n",
13089 				phba->iocb_cnt, txq_cnt, txcmplq_cnt);
13090 		}
13091 		return retval;
13092 	} else {
13093 		lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
13094 				"0332 IOCB wait issue failed, Data x%x\n",
13095 				retval);
13096 		retval = IOCB_ERROR;
13097 	}
13098 
13099 	if (phba->cfg_poll & DISABLE_FCP_RING_INT) {
13100 		if (lpfc_readl(phba->HCregaddr, &creg_val))
13101 			return IOCB_ERROR;
13102 		creg_val &= ~(HC_R0INT_ENA << LPFC_FCP_RING);
13103 		writel(creg_val, phba->HCregaddr);
13104 		readl(phba->HCregaddr); /* flush */
13105 	}
13106 
13107 	if (prspiocbq)
13108 		piocb->context2 = NULL;
13109 
13110 	piocb->context_un.wait_queue = NULL;
13111 	piocb->iocb_cmpl = NULL;
13112 	return retval;
13113 }
13114 
13115 /**
13116  * lpfc_sli_issue_mbox_wait - Synchronous function to issue mailbox
13117  * @phba: Pointer to HBA context object.
13118  * @pmboxq: Pointer to driver mailbox object.
13119  * @timeout: Timeout in number of seconds.
13120  *
13121  * This function issues the mailbox to firmware and waits for the
13122  * mailbox command to complete. If the mailbox command is not
13123  * completed within timeout seconds, it returns MBX_TIMEOUT.
13124  * The function waits for the mailbox completion using an
13125  * interruptible wait. If the thread is woken up due to a
13126  * signal, MBX_TIMEOUT error is returned to the caller. Caller
13127  * should not free the mailbox resources, if this function returns
13128  * MBX_TIMEOUT.
13129  * This function will sleep while waiting for mailbox completion.
13130  * So, this function should not be called from any context which
13131  * does not allow sleeping. Due to the same reason, this function
13132  * cannot be called with interrupt disabled.
13133  * This function assumes that the mailbox completion occurs while
13134  * this function sleep. So, this function cannot be called from
13135  * the worker thread which processes mailbox completion.
13136  * This function is called in the context of HBA management
13137  * applications.
13138  * This function returns MBX_SUCCESS when successful.
13139  * This function is called with no lock held.
13140  **/
13141 int
13142 lpfc_sli_issue_mbox_wait(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq,
13143 			 uint32_t timeout)
13144 {
13145 	struct completion mbox_done;
13146 	int retval;
13147 	unsigned long flag;
13148 
13149 	pmboxq->mbox_flag &= ~LPFC_MBX_WAKE;
13150 	/* setup wake call as IOCB callback */
13151 	pmboxq->mbox_cmpl = lpfc_sli_wake_mbox_wait;
13152 
13153 	/* setup context3 field to pass wait_queue pointer to wake function  */
13154 	init_completion(&mbox_done);
13155 	pmboxq->context3 = &mbox_done;
13156 	/* now issue the command */
13157 	retval = lpfc_sli_issue_mbox(phba, pmboxq, MBX_NOWAIT);
13158 	if (retval == MBX_BUSY || retval == MBX_SUCCESS) {
13159 		wait_for_completion_timeout(&mbox_done,
13160 					    msecs_to_jiffies(timeout * 1000));
13161 
13162 		spin_lock_irqsave(&phba->hbalock, flag);
13163 		pmboxq->context3 = NULL;
13164 		/*
13165 		 * if LPFC_MBX_WAKE flag is set the mailbox is completed
13166 		 * else do not free the resources.
13167 		 */
13168 		if (pmboxq->mbox_flag & LPFC_MBX_WAKE) {
13169 			retval = MBX_SUCCESS;
13170 		} else {
13171 			retval = MBX_TIMEOUT;
13172 			pmboxq->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
13173 		}
13174 		spin_unlock_irqrestore(&phba->hbalock, flag);
13175 	}
13176 	return retval;
13177 }
13178 
13179 /**
13180  * lpfc_sli_mbox_sys_shutdown - shutdown mailbox command sub-system
13181  * @phba: Pointer to HBA context.
13182  * @mbx_action: Mailbox shutdown options.
13183  *
13184  * This function is called to shutdown the driver's mailbox sub-system.
13185  * It first marks the mailbox sub-system is in a block state to prevent
13186  * the asynchronous mailbox command from issued off the pending mailbox
13187  * command queue. If the mailbox command sub-system shutdown is due to
13188  * HBA error conditions such as EEH or ERATT, this routine shall invoke
13189  * the mailbox sub-system flush routine to forcefully bring down the
13190  * mailbox sub-system. Otherwise, if it is due to normal condition (such
13191  * as with offline or HBA function reset), this routine will wait for the
13192  * outstanding mailbox command to complete before invoking the mailbox
13193  * sub-system flush routine to gracefully bring down mailbox sub-system.
13194  **/
13195 void
13196 lpfc_sli_mbox_sys_shutdown(struct lpfc_hba *phba, int mbx_action)
13197 {
13198 	struct lpfc_sli *psli = &phba->sli;
13199 	unsigned long timeout;
13200 
13201 	if (mbx_action == LPFC_MBX_NO_WAIT) {
13202 		/* delay 100ms for port state */
13203 		msleep(100);
13204 		lpfc_sli_mbox_sys_flush(phba);
13205 		return;
13206 	}
13207 	timeout = msecs_to_jiffies(LPFC_MBOX_TMO * 1000) + jiffies;
13208 
13209 	/* Disable softirqs, including timers from obtaining phba->hbalock */
13210 	local_bh_disable();
13211 
13212 	spin_lock_irq(&phba->hbalock);
13213 	psli->sli_flag |= LPFC_SLI_ASYNC_MBX_BLK;
13214 
13215 	if (psli->sli_flag & LPFC_SLI_ACTIVE) {
13216 		/* Determine how long we might wait for the active mailbox
13217 		 * command to be gracefully completed by firmware.
13218 		 */
13219 		if (phba->sli.mbox_active)
13220 			timeout = msecs_to_jiffies(lpfc_mbox_tmo_val(phba,
13221 						phba->sli.mbox_active) *
13222 						1000) + jiffies;
13223 		spin_unlock_irq(&phba->hbalock);
13224 
13225 		/* Enable softirqs again, done with phba->hbalock */
13226 		local_bh_enable();
13227 
13228 		while (phba->sli.mbox_active) {
13229 			/* Check active mailbox complete status every 2ms */
13230 			msleep(2);
13231 			if (time_after(jiffies, timeout))
13232 				/* Timeout, let the mailbox flush routine to
13233 				 * forcefully release active mailbox command
13234 				 */
13235 				break;
13236 		}
13237 	} else {
13238 		spin_unlock_irq(&phba->hbalock);
13239 
13240 		/* Enable softirqs again, done with phba->hbalock */
13241 		local_bh_enable();
13242 	}
13243 
13244 	lpfc_sli_mbox_sys_flush(phba);
13245 }
13246 
13247 /**
13248  * lpfc_sli_eratt_read - read sli-3 error attention events
13249  * @phba: Pointer to HBA context.
13250  *
13251  * This function is called to read the SLI3 device error attention registers
13252  * for possible error attention events. The caller must hold the hostlock
13253  * with spin_lock_irq().
13254  *
13255  * This function returns 1 when there is Error Attention in the Host Attention
13256  * Register and returns 0 otherwise.
13257  **/
13258 static int
13259 lpfc_sli_eratt_read(struct lpfc_hba *phba)
13260 {
13261 	uint32_t ha_copy;
13262 
13263 	/* Read chip Host Attention (HA) register */
13264 	if (lpfc_readl(phba->HAregaddr, &ha_copy))
13265 		goto unplug_err;
13266 
13267 	if (ha_copy & HA_ERATT) {
13268 		/* Read host status register to retrieve error event */
13269 		if (lpfc_sli_read_hs(phba))
13270 			goto unplug_err;
13271 
13272 		/* Check if there is a deferred error condition is active */
13273 		if ((HS_FFER1 & phba->work_hs) &&
13274 		    ((HS_FFER2 | HS_FFER3 | HS_FFER4 | HS_FFER5 |
13275 		      HS_FFER6 | HS_FFER7 | HS_FFER8) & phba->work_hs)) {
13276 			phba->hba_flag |= DEFER_ERATT;
13277 			/* Clear all interrupt enable conditions */
13278 			writel(0, phba->HCregaddr);
13279 			readl(phba->HCregaddr);
13280 		}
13281 
13282 		/* Set the driver HA work bitmap */
13283 		phba->work_ha |= HA_ERATT;
13284 		/* Indicate polling handles this ERATT */
13285 		phba->hba_flag |= HBA_ERATT_HANDLED;
13286 		return 1;
13287 	}
13288 	return 0;
13289 
13290 unplug_err:
13291 	/* Set the driver HS work bitmap */
13292 	phba->work_hs |= UNPLUG_ERR;
13293 	/* Set the driver HA work bitmap */
13294 	phba->work_ha |= HA_ERATT;
13295 	/* Indicate polling handles this ERATT */
13296 	phba->hba_flag |= HBA_ERATT_HANDLED;
13297 	return 1;
13298 }
13299 
13300 /**
13301  * lpfc_sli4_eratt_read - read sli-4 error attention events
13302  * @phba: Pointer to HBA context.
13303  *
13304  * This function is called to read the SLI4 device error attention registers
13305  * for possible error attention events. The caller must hold the hostlock
13306  * with spin_lock_irq().
13307  *
13308  * This function returns 1 when there is Error Attention in the Host Attention
13309  * Register and returns 0 otherwise.
13310  **/
13311 static int
13312 lpfc_sli4_eratt_read(struct lpfc_hba *phba)
13313 {
13314 	uint32_t uerr_sta_hi, uerr_sta_lo;
13315 	uint32_t if_type, portsmphr;
13316 	struct lpfc_register portstat_reg;
13317 
13318 	/*
13319 	 * For now, use the SLI4 device internal unrecoverable error
13320 	 * registers for error attention. This can be changed later.
13321 	 */
13322 	if_type = bf_get(lpfc_sli_intf_if_type, &phba->sli4_hba.sli_intf);
13323 	switch (if_type) {
13324 	case LPFC_SLI_INTF_IF_TYPE_0:
13325 		if (lpfc_readl(phba->sli4_hba.u.if_type0.UERRLOregaddr,
13326 			&uerr_sta_lo) ||
13327 			lpfc_readl(phba->sli4_hba.u.if_type0.UERRHIregaddr,
13328 			&uerr_sta_hi)) {
13329 			phba->work_hs |= UNPLUG_ERR;
13330 			phba->work_ha |= HA_ERATT;
13331 			phba->hba_flag |= HBA_ERATT_HANDLED;
13332 			return 1;
13333 		}
13334 		if ((~phba->sli4_hba.ue_mask_lo & uerr_sta_lo) ||
13335 		    (~phba->sli4_hba.ue_mask_hi & uerr_sta_hi)) {
13336 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
13337 					"1423 HBA Unrecoverable error: "
13338 					"uerr_lo_reg=0x%x, uerr_hi_reg=0x%x, "
13339 					"ue_mask_lo_reg=0x%x, "
13340 					"ue_mask_hi_reg=0x%x\n",
13341 					uerr_sta_lo, uerr_sta_hi,
13342 					phba->sli4_hba.ue_mask_lo,
13343 					phba->sli4_hba.ue_mask_hi);
13344 			phba->work_status[0] = uerr_sta_lo;
13345 			phba->work_status[1] = uerr_sta_hi;
13346 			phba->work_ha |= HA_ERATT;
13347 			phba->hba_flag |= HBA_ERATT_HANDLED;
13348 			return 1;
13349 		}
13350 		break;
13351 	case LPFC_SLI_INTF_IF_TYPE_2:
13352 	case LPFC_SLI_INTF_IF_TYPE_6:
13353 		if (lpfc_readl(phba->sli4_hba.u.if_type2.STATUSregaddr,
13354 			&portstat_reg.word0) ||
13355 			lpfc_readl(phba->sli4_hba.PSMPHRregaddr,
13356 			&portsmphr)){
13357 			phba->work_hs |= UNPLUG_ERR;
13358 			phba->work_ha |= HA_ERATT;
13359 			phba->hba_flag |= HBA_ERATT_HANDLED;
13360 			return 1;
13361 		}
13362 		if (bf_get(lpfc_sliport_status_err, &portstat_reg)) {
13363 			phba->work_status[0] =
13364 				readl(phba->sli4_hba.u.if_type2.ERR1regaddr);
13365 			phba->work_status[1] =
13366 				readl(phba->sli4_hba.u.if_type2.ERR2regaddr);
13367 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
13368 					"2885 Port Status Event: "
13369 					"port status reg 0x%x, "
13370 					"port smphr reg 0x%x, "
13371 					"error 1=0x%x, error 2=0x%x\n",
13372 					portstat_reg.word0,
13373 					portsmphr,
13374 					phba->work_status[0],
13375 					phba->work_status[1]);
13376 			phba->work_ha |= HA_ERATT;
13377 			phba->hba_flag |= HBA_ERATT_HANDLED;
13378 			return 1;
13379 		}
13380 		break;
13381 	case LPFC_SLI_INTF_IF_TYPE_1:
13382 	default:
13383 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
13384 				"2886 HBA Error Attention on unsupported "
13385 				"if type %d.", if_type);
13386 		return 1;
13387 	}
13388 
13389 	return 0;
13390 }
13391 
13392 /**
13393  * lpfc_sli_check_eratt - check error attention events
13394  * @phba: Pointer to HBA context.
13395  *
13396  * This function is called from timer soft interrupt context to check HBA's
13397  * error attention register bit for error attention events.
13398  *
13399  * This function returns 1 when there is Error Attention in the Host Attention
13400  * Register and returns 0 otherwise.
13401  **/
13402 int
13403 lpfc_sli_check_eratt(struct lpfc_hba *phba)
13404 {
13405 	uint32_t ha_copy;
13406 
13407 	/* If somebody is waiting to handle an eratt, don't process it
13408 	 * here. The brdkill function will do this.
13409 	 */
13410 	if (phba->link_flag & LS_IGNORE_ERATT)
13411 		return 0;
13412 
13413 	/* Check if interrupt handler handles this ERATT */
13414 	spin_lock_irq(&phba->hbalock);
13415 	if (phba->hba_flag & HBA_ERATT_HANDLED) {
13416 		/* Interrupt handler has handled ERATT */
13417 		spin_unlock_irq(&phba->hbalock);
13418 		return 0;
13419 	}
13420 
13421 	/*
13422 	 * If there is deferred error attention, do not check for error
13423 	 * attention
13424 	 */
13425 	if (unlikely(phba->hba_flag & DEFER_ERATT)) {
13426 		spin_unlock_irq(&phba->hbalock);
13427 		return 0;
13428 	}
13429 
13430 	/* If PCI channel is offline, don't process it */
13431 	if (unlikely(pci_channel_offline(phba->pcidev))) {
13432 		spin_unlock_irq(&phba->hbalock);
13433 		return 0;
13434 	}
13435 
13436 	switch (phba->sli_rev) {
13437 	case LPFC_SLI_REV2:
13438 	case LPFC_SLI_REV3:
13439 		/* Read chip Host Attention (HA) register */
13440 		ha_copy = lpfc_sli_eratt_read(phba);
13441 		break;
13442 	case LPFC_SLI_REV4:
13443 		/* Read device Uncoverable Error (UERR) registers */
13444 		ha_copy = lpfc_sli4_eratt_read(phba);
13445 		break;
13446 	default:
13447 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
13448 				"0299 Invalid SLI revision (%d)\n",
13449 				phba->sli_rev);
13450 		ha_copy = 0;
13451 		break;
13452 	}
13453 	spin_unlock_irq(&phba->hbalock);
13454 
13455 	return ha_copy;
13456 }
13457 
13458 /**
13459  * lpfc_intr_state_check - Check device state for interrupt handling
13460  * @phba: Pointer to HBA context.
13461  *
13462  * This inline routine checks whether a device or its PCI slot is in a state
13463  * that the interrupt should be handled.
13464  *
13465  * This function returns 0 if the device or the PCI slot is in a state that
13466  * interrupt should be handled, otherwise -EIO.
13467  */
13468 static inline int
13469 lpfc_intr_state_check(struct lpfc_hba *phba)
13470 {
13471 	/* If the pci channel is offline, ignore all the interrupts */
13472 	if (unlikely(pci_channel_offline(phba->pcidev)))
13473 		return -EIO;
13474 
13475 	/* Update device level interrupt statistics */
13476 	phba->sli.slistat.sli_intr++;
13477 
13478 	/* Ignore all interrupts during initialization. */
13479 	if (unlikely(phba->link_state < LPFC_LINK_DOWN))
13480 		return -EIO;
13481 
13482 	return 0;
13483 }
13484 
13485 /**
13486  * lpfc_sli_sp_intr_handler - Slow-path interrupt handler to SLI-3 device
13487  * @irq: Interrupt number.
13488  * @dev_id: The device context pointer.
13489  *
13490  * This function is directly called from the PCI layer as an interrupt
13491  * service routine when device with SLI-3 interface spec is enabled with
13492  * MSI-X multi-message interrupt mode and there are slow-path events in
13493  * the HBA. However, when the device is enabled with either MSI or Pin-IRQ
13494  * interrupt mode, this function is called as part of the device-level
13495  * interrupt handler. When the PCI slot is in error recovery or the HBA
13496  * is undergoing initialization, the interrupt handler will not process
13497  * the interrupt. The link attention and ELS ring attention events are
13498  * handled by the worker thread. The interrupt handler signals the worker
13499  * thread and returns for these events. This function is called without
13500  * any lock held. It gets the hbalock to access and update SLI data
13501  * structures.
13502  *
13503  * This function returns IRQ_HANDLED when interrupt is handled else it
13504  * returns IRQ_NONE.
13505  **/
13506 irqreturn_t
13507 lpfc_sli_sp_intr_handler(int irq, void *dev_id)
13508 {
13509 	struct lpfc_hba  *phba;
13510 	uint32_t ha_copy, hc_copy;
13511 	uint32_t work_ha_copy;
13512 	unsigned long status;
13513 	unsigned long iflag;
13514 	uint32_t control;
13515 
13516 	MAILBOX_t *mbox, *pmbox;
13517 	struct lpfc_vport *vport;
13518 	struct lpfc_nodelist *ndlp;
13519 	struct lpfc_dmabuf *mp;
13520 	LPFC_MBOXQ_t *pmb;
13521 	int rc;
13522 
13523 	/*
13524 	 * Get the driver's phba structure from the dev_id and
13525 	 * assume the HBA is not interrupting.
13526 	 */
13527 	phba = (struct lpfc_hba *)dev_id;
13528 
13529 	if (unlikely(!phba))
13530 		return IRQ_NONE;
13531 
13532 	/*
13533 	 * Stuff needs to be attented to when this function is invoked as an
13534 	 * individual interrupt handler in MSI-X multi-message interrupt mode
13535 	 */
13536 	if (phba->intr_type == MSIX) {
13537 		/* Check device state for handling interrupt */
13538 		if (lpfc_intr_state_check(phba))
13539 			return IRQ_NONE;
13540 		/* Need to read HA REG for slow-path events */
13541 		spin_lock_irqsave(&phba->hbalock, iflag);
13542 		if (lpfc_readl(phba->HAregaddr, &ha_copy))
13543 			goto unplug_error;
13544 		/* If somebody is waiting to handle an eratt don't process it
13545 		 * here. The brdkill function will do this.
13546 		 */
13547 		if (phba->link_flag & LS_IGNORE_ERATT)
13548 			ha_copy &= ~HA_ERATT;
13549 		/* Check the need for handling ERATT in interrupt handler */
13550 		if (ha_copy & HA_ERATT) {
13551 			if (phba->hba_flag & HBA_ERATT_HANDLED)
13552 				/* ERATT polling has handled ERATT */
13553 				ha_copy &= ~HA_ERATT;
13554 			else
13555 				/* Indicate interrupt handler handles ERATT */
13556 				phba->hba_flag |= HBA_ERATT_HANDLED;
13557 		}
13558 
13559 		/*
13560 		 * If there is deferred error attention, do not check for any
13561 		 * interrupt.
13562 		 */
13563 		if (unlikely(phba->hba_flag & DEFER_ERATT)) {
13564 			spin_unlock_irqrestore(&phba->hbalock, iflag);
13565 			return IRQ_NONE;
13566 		}
13567 
13568 		/* Clear up only attention source related to slow-path */
13569 		if (lpfc_readl(phba->HCregaddr, &hc_copy))
13570 			goto unplug_error;
13571 
13572 		writel(hc_copy & ~(HC_MBINT_ENA | HC_R2INT_ENA |
13573 			HC_LAINT_ENA | HC_ERINT_ENA),
13574 			phba->HCregaddr);
13575 		writel((ha_copy & (HA_MBATT | HA_R2_CLR_MSK)),
13576 			phba->HAregaddr);
13577 		writel(hc_copy, phba->HCregaddr);
13578 		readl(phba->HAregaddr); /* flush */
13579 		spin_unlock_irqrestore(&phba->hbalock, iflag);
13580 	} else
13581 		ha_copy = phba->ha_copy;
13582 
13583 	work_ha_copy = ha_copy & phba->work_ha_mask;
13584 
13585 	if (work_ha_copy) {
13586 		if (work_ha_copy & HA_LATT) {
13587 			if (phba->sli.sli_flag & LPFC_PROCESS_LA) {
13588 				/*
13589 				 * Turn off Link Attention interrupts
13590 				 * until CLEAR_LA done
13591 				 */
13592 				spin_lock_irqsave(&phba->hbalock, iflag);
13593 				phba->sli.sli_flag &= ~LPFC_PROCESS_LA;
13594 				if (lpfc_readl(phba->HCregaddr, &control))
13595 					goto unplug_error;
13596 				control &= ~HC_LAINT_ENA;
13597 				writel(control, phba->HCregaddr);
13598 				readl(phba->HCregaddr); /* flush */
13599 				spin_unlock_irqrestore(&phba->hbalock, iflag);
13600 			}
13601 			else
13602 				work_ha_copy &= ~HA_LATT;
13603 		}
13604 
13605 		if (work_ha_copy & ~(HA_ERATT | HA_MBATT | HA_LATT)) {
13606 			/*
13607 			 * Turn off Slow Rings interrupts, LPFC_ELS_RING is
13608 			 * the only slow ring.
13609 			 */
13610 			status = (work_ha_copy &
13611 				(HA_RXMASK  << (4*LPFC_ELS_RING)));
13612 			status >>= (4*LPFC_ELS_RING);
13613 			if (status & HA_RXMASK) {
13614 				spin_lock_irqsave(&phba->hbalock, iflag);
13615 				if (lpfc_readl(phba->HCregaddr, &control))
13616 					goto unplug_error;
13617 
13618 				lpfc_debugfs_slow_ring_trc(phba,
13619 				"ISR slow ring:   ctl:x%x stat:x%x isrcnt:x%x",
13620 				control, status,
13621 				(uint32_t)phba->sli.slistat.sli_intr);
13622 
13623 				if (control & (HC_R0INT_ENA << LPFC_ELS_RING)) {
13624 					lpfc_debugfs_slow_ring_trc(phba,
13625 						"ISR Disable ring:"
13626 						"pwork:x%x hawork:x%x wait:x%x",
13627 						phba->work_ha, work_ha_copy,
13628 						(uint32_t)((unsigned long)
13629 						&phba->work_waitq));
13630 
13631 					control &=
13632 					    ~(HC_R0INT_ENA << LPFC_ELS_RING);
13633 					writel(control, phba->HCregaddr);
13634 					readl(phba->HCregaddr); /* flush */
13635 				}
13636 				else {
13637 					lpfc_debugfs_slow_ring_trc(phba,
13638 						"ISR slow ring:   pwork:"
13639 						"x%x hawork:x%x wait:x%x",
13640 						phba->work_ha, work_ha_copy,
13641 						(uint32_t)((unsigned long)
13642 						&phba->work_waitq));
13643 				}
13644 				spin_unlock_irqrestore(&phba->hbalock, iflag);
13645 			}
13646 		}
13647 		spin_lock_irqsave(&phba->hbalock, iflag);
13648 		if (work_ha_copy & HA_ERATT) {
13649 			if (lpfc_sli_read_hs(phba))
13650 				goto unplug_error;
13651 			/*
13652 			 * Check if there is a deferred error condition
13653 			 * is active
13654 			 */
13655 			if ((HS_FFER1 & phba->work_hs) &&
13656 				((HS_FFER2 | HS_FFER3 | HS_FFER4 | HS_FFER5 |
13657 				  HS_FFER6 | HS_FFER7 | HS_FFER8) &
13658 				  phba->work_hs)) {
13659 				phba->hba_flag |= DEFER_ERATT;
13660 				/* Clear all interrupt enable conditions */
13661 				writel(0, phba->HCregaddr);
13662 				readl(phba->HCregaddr);
13663 			}
13664 		}
13665 
13666 		if ((work_ha_copy & HA_MBATT) && (phba->sli.mbox_active)) {
13667 			pmb = phba->sli.mbox_active;
13668 			pmbox = &pmb->u.mb;
13669 			mbox = phba->mbox;
13670 			vport = pmb->vport;
13671 
13672 			/* First check out the status word */
13673 			lpfc_sli_pcimem_bcopy(mbox, pmbox, sizeof(uint32_t));
13674 			if (pmbox->mbxOwner != OWN_HOST) {
13675 				spin_unlock_irqrestore(&phba->hbalock, iflag);
13676 				/*
13677 				 * Stray Mailbox Interrupt, mbxCommand <cmd>
13678 				 * mbxStatus <status>
13679 				 */
13680 				lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
13681 						"(%d):0304 Stray Mailbox "
13682 						"Interrupt mbxCommand x%x "
13683 						"mbxStatus x%x\n",
13684 						(vport ? vport->vpi : 0),
13685 						pmbox->mbxCommand,
13686 						pmbox->mbxStatus);
13687 				/* clear mailbox attention bit */
13688 				work_ha_copy &= ~HA_MBATT;
13689 			} else {
13690 				phba->sli.mbox_active = NULL;
13691 				spin_unlock_irqrestore(&phba->hbalock, iflag);
13692 				phba->last_completion_time = jiffies;
13693 				del_timer(&phba->sli.mbox_tmo);
13694 				if (pmb->mbox_cmpl) {
13695 					lpfc_sli_pcimem_bcopy(mbox, pmbox,
13696 							MAILBOX_CMD_SIZE);
13697 					if (pmb->out_ext_byte_len &&
13698 						pmb->ctx_buf)
13699 						lpfc_sli_pcimem_bcopy(
13700 						phba->mbox_ext,
13701 						pmb->ctx_buf,
13702 						pmb->out_ext_byte_len);
13703 				}
13704 				if (pmb->mbox_flag & LPFC_MBX_IMED_UNREG) {
13705 					pmb->mbox_flag &= ~LPFC_MBX_IMED_UNREG;
13706 
13707 					lpfc_debugfs_disc_trc(vport,
13708 						LPFC_DISC_TRC_MBOX_VPORT,
13709 						"MBOX dflt rpi: : "
13710 						"status:x%x rpi:x%x",
13711 						(uint32_t)pmbox->mbxStatus,
13712 						pmbox->un.varWords[0], 0);
13713 
13714 					if (!pmbox->mbxStatus) {
13715 						mp = (struct lpfc_dmabuf *)
13716 							(pmb->ctx_buf);
13717 						ndlp = (struct lpfc_nodelist *)
13718 							pmb->ctx_ndlp;
13719 
13720 						/* Reg_LOGIN of dflt RPI was
13721 						 * successful. new lets get
13722 						 * rid of the RPI using the
13723 						 * same mbox buffer.
13724 						 */
13725 						lpfc_unreg_login(phba,
13726 							vport->vpi,
13727 							pmbox->un.varWords[0],
13728 							pmb);
13729 						pmb->mbox_cmpl =
13730 							lpfc_mbx_cmpl_dflt_rpi;
13731 						pmb->ctx_buf = mp;
13732 						pmb->ctx_ndlp = ndlp;
13733 						pmb->vport = vport;
13734 						rc = lpfc_sli_issue_mbox(phba,
13735 								pmb,
13736 								MBX_NOWAIT);
13737 						if (rc != MBX_BUSY)
13738 							lpfc_printf_log(phba,
13739 							KERN_ERR,
13740 							LOG_TRACE_EVENT,
13741 							"0350 rc should have"
13742 							"been MBX_BUSY\n");
13743 						if (rc != MBX_NOT_FINISHED)
13744 							goto send_current_mbox;
13745 					}
13746 				}
13747 				spin_lock_irqsave(
13748 						&phba->pport->work_port_lock,
13749 						iflag);
13750 				phba->pport->work_port_events &=
13751 					~WORKER_MBOX_TMO;
13752 				spin_unlock_irqrestore(
13753 						&phba->pport->work_port_lock,
13754 						iflag);
13755 
13756 				/* Do NOT queue MBX_HEARTBEAT to the worker
13757 				 * thread for processing.
13758 				 */
13759 				if (pmbox->mbxCommand == MBX_HEARTBEAT) {
13760 					/* Process mbox now */
13761 					phba->sli.mbox_active = NULL;
13762 					phba->sli.sli_flag &=
13763 						~LPFC_SLI_MBOX_ACTIVE;
13764 					if (pmb->mbox_cmpl)
13765 						pmb->mbox_cmpl(phba, pmb);
13766 				} else {
13767 					/* Queue to worker thread to process */
13768 					lpfc_mbox_cmpl_put(phba, pmb);
13769 				}
13770 			}
13771 		} else
13772 			spin_unlock_irqrestore(&phba->hbalock, iflag);
13773 
13774 		if ((work_ha_copy & HA_MBATT) &&
13775 		    (phba->sli.mbox_active == NULL)) {
13776 send_current_mbox:
13777 			/* Process next mailbox command if there is one */
13778 			do {
13779 				rc = lpfc_sli_issue_mbox(phba, NULL,
13780 							 MBX_NOWAIT);
13781 			} while (rc == MBX_NOT_FINISHED);
13782 			if (rc != MBX_SUCCESS)
13783 				lpfc_printf_log(phba, KERN_ERR,
13784 						LOG_TRACE_EVENT,
13785 						"0349 rc should be "
13786 						"MBX_SUCCESS\n");
13787 		}
13788 
13789 		spin_lock_irqsave(&phba->hbalock, iflag);
13790 		phba->work_ha |= work_ha_copy;
13791 		spin_unlock_irqrestore(&phba->hbalock, iflag);
13792 		lpfc_worker_wake_up(phba);
13793 	}
13794 	return IRQ_HANDLED;
13795 unplug_error:
13796 	spin_unlock_irqrestore(&phba->hbalock, iflag);
13797 	return IRQ_HANDLED;
13798 
13799 } /* lpfc_sli_sp_intr_handler */
13800 
13801 /**
13802  * lpfc_sli_fp_intr_handler - Fast-path interrupt handler to SLI-3 device.
13803  * @irq: Interrupt number.
13804  * @dev_id: The device context pointer.
13805  *
13806  * This function is directly called from the PCI layer as an interrupt
13807  * service routine when device with SLI-3 interface spec is enabled with
13808  * MSI-X multi-message interrupt mode and there is a fast-path FCP IOCB
13809  * ring event in the HBA. However, when the device is enabled with either
13810  * MSI or Pin-IRQ interrupt mode, this function is called as part of the
13811  * device-level interrupt handler. When the PCI slot is in error recovery
13812  * or the HBA is undergoing initialization, the interrupt handler will not
13813  * process the interrupt. The SCSI FCP fast-path ring event are handled in
13814  * the intrrupt context. This function is called without any lock held.
13815  * It gets the hbalock to access and update SLI data structures.
13816  *
13817  * This function returns IRQ_HANDLED when interrupt is handled else it
13818  * returns IRQ_NONE.
13819  **/
13820 irqreturn_t
13821 lpfc_sli_fp_intr_handler(int irq, void *dev_id)
13822 {
13823 	struct lpfc_hba  *phba;
13824 	uint32_t ha_copy;
13825 	unsigned long status;
13826 	unsigned long iflag;
13827 	struct lpfc_sli_ring *pring;
13828 
13829 	/* Get the driver's phba structure from the dev_id and
13830 	 * assume the HBA is not interrupting.
13831 	 */
13832 	phba = (struct lpfc_hba *) dev_id;
13833 
13834 	if (unlikely(!phba))
13835 		return IRQ_NONE;
13836 
13837 	/*
13838 	 * Stuff needs to be attented to when this function is invoked as an
13839 	 * individual interrupt handler in MSI-X multi-message interrupt mode
13840 	 */
13841 	if (phba->intr_type == MSIX) {
13842 		/* Check device state for handling interrupt */
13843 		if (lpfc_intr_state_check(phba))
13844 			return IRQ_NONE;
13845 		/* Need to read HA REG for FCP ring and other ring events */
13846 		if (lpfc_readl(phba->HAregaddr, &ha_copy))
13847 			return IRQ_HANDLED;
13848 		/* Clear up only attention source related to fast-path */
13849 		spin_lock_irqsave(&phba->hbalock, iflag);
13850 		/*
13851 		 * If there is deferred error attention, do not check for
13852 		 * any interrupt.
13853 		 */
13854 		if (unlikely(phba->hba_flag & DEFER_ERATT)) {
13855 			spin_unlock_irqrestore(&phba->hbalock, iflag);
13856 			return IRQ_NONE;
13857 		}
13858 		writel((ha_copy & (HA_R0_CLR_MSK | HA_R1_CLR_MSK)),
13859 			phba->HAregaddr);
13860 		readl(phba->HAregaddr); /* flush */
13861 		spin_unlock_irqrestore(&phba->hbalock, iflag);
13862 	} else
13863 		ha_copy = phba->ha_copy;
13864 
13865 	/*
13866 	 * Process all events on FCP ring. Take the optimized path for FCP IO.
13867 	 */
13868 	ha_copy &= ~(phba->work_ha_mask);
13869 
13870 	status = (ha_copy & (HA_RXMASK << (4*LPFC_FCP_RING)));
13871 	status >>= (4*LPFC_FCP_RING);
13872 	pring = &phba->sli.sli3_ring[LPFC_FCP_RING];
13873 	if (status & HA_RXMASK)
13874 		lpfc_sli_handle_fast_ring_event(phba, pring, status);
13875 
13876 	if (phba->cfg_multi_ring_support == 2) {
13877 		/*
13878 		 * Process all events on extra ring. Take the optimized path
13879 		 * for extra ring IO.
13880 		 */
13881 		status = (ha_copy & (HA_RXMASK << (4*LPFC_EXTRA_RING)));
13882 		status >>= (4*LPFC_EXTRA_RING);
13883 		if (status & HA_RXMASK) {
13884 			lpfc_sli_handle_fast_ring_event(phba,
13885 					&phba->sli.sli3_ring[LPFC_EXTRA_RING],
13886 					status);
13887 		}
13888 	}
13889 	return IRQ_HANDLED;
13890 }  /* lpfc_sli_fp_intr_handler */
13891 
13892 /**
13893  * lpfc_sli_intr_handler - Device-level interrupt handler to SLI-3 device
13894  * @irq: Interrupt number.
13895  * @dev_id: The device context pointer.
13896  *
13897  * This function is the HBA device-level interrupt handler to device with
13898  * SLI-3 interface spec, called from the PCI layer when either MSI or
13899  * Pin-IRQ interrupt mode is enabled and there is an event in the HBA which
13900  * requires driver attention. This function invokes the slow-path interrupt
13901  * attention handling function and fast-path interrupt attention handling
13902  * function in turn to process the relevant HBA attention events. This
13903  * function is called without any lock held. It gets the hbalock to access
13904  * and update SLI data structures.
13905  *
13906  * This function returns IRQ_HANDLED when interrupt is handled, else it
13907  * returns IRQ_NONE.
13908  **/
13909 irqreturn_t
13910 lpfc_sli_intr_handler(int irq, void *dev_id)
13911 {
13912 	struct lpfc_hba  *phba;
13913 	irqreturn_t sp_irq_rc, fp_irq_rc;
13914 	unsigned long status1, status2;
13915 	uint32_t hc_copy;
13916 
13917 	/*
13918 	 * Get the driver's phba structure from the dev_id and
13919 	 * assume the HBA is not interrupting.
13920 	 */
13921 	phba = (struct lpfc_hba *) dev_id;
13922 
13923 	if (unlikely(!phba))
13924 		return IRQ_NONE;
13925 
13926 	/* Check device state for handling interrupt */
13927 	if (lpfc_intr_state_check(phba))
13928 		return IRQ_NONE;
13929 
13930 	spin_lock(&phba->hbalock);
13931 	if (lpfc_readl(phba->HAregaddr, &phba->ha_copy)) {
13932 		spin_unlock(&phba->hbalock);
13933 		return IRQ_HANDLED;
13934 	}
13935 
13936 	if (unlikely(!phba->ha_copy)) {
13937 		spin_unlock(&phba->hbalock);
13938 		return IRQ_NONE;
13939 	} else if (phba->ha_copy & HA_ERATT) {
13940 		if (phba->hba_flag & HBA_ERATT_HANDLED)
13941 			/* ERATT polling has handled ERATT */
13942 			phba->ha_copy &= ~HA_ERATT;
13943 		else
13944 			/* Indicate interrupt handler handles ERATT */
13945 			phba->hba_flag |= HBA_ERATT_HANDLED;
13946 	}
13947 
13948 	/*
13949 	 * If there is deferred error attention, do not check for any interrupt.
13950 	 */
13951 	if (unlikely(phba->hba_flag & DEFER_ERATT)) {
13952 		spin_unlock(&phba->hbalock);
13953 		return IRQ_NONE;
13954 	}
13955 
13956 	/* Clear attention sources except link and error attentions */
13957 	if (lpfc_readl(phba->HCregaddr, &hc_copy)) {
13958 		spin_unlock(&phba->hbalock);
13959 		return IRQ_HANDLED;
13960 	}
13961 	writel(hc_copy & ~(HC_MBINT_ENA | HC_R0INT_ENA | HC_R1INT_ENA
13962 		| HC_R2INT_ENA | HC_LAINT_ENA | HC_ERINT_ENA),
13963 		phba->HCregaddr);
13964 	writel((phba->ha_copy & ~(HA_LATT | HA_ERATT)), phba->HAregaddr);
13965 	writel(hc_copy, phba->HCregaddr);
13966 	readl(phba->HAregaddr); /* flush */
13967 	spin_unlock(&phba->hbalock);
13968 
13969 	/*
13970 	 * Invokes slow-path host attention interrupt handling as appropriate.
13971 	 */
13972 
13973 	/* status of events with mailbox and link attention */
13974 	status1 = phba->ha_copy & (HA_MBATT | HA_LATT | HA_ERATT);
13975 
13976 	/* status of events with ELS ring */
13977 	status2 = (phba->ha_copy & (HA_RXMASK  << (4*LPFC_ELS_RING)));
13978 	status2 >>= (4*LPFC_ELS_RING);
13979 
13980 	if (status1 || (status2 & HA_RXMASK))
13981 		sp_irq_rc = lpfc_sli_sp_intr_handler(irq, dev_id);
13982 	else
13983 		sp_irq_rc = IRQ_NONE;
13984 
13985 	/*
13986 	 * Invoke fast-path host attention interrupt handling as appropriate.
13987 	 */
13988 
13989 	/* status of events with FCP ring */
13990 	status1 = (phba->ha_copy & (HA_RXMASK << (4*LPFC_FCP_RING)));
13991 	status1 >>= (4*LPFC_FCP_RING);
13992 
13993 	/* status of events with extra ring */
13994 	if (phba->cfg_multi_ring_support == 2) {
13995 		status2 = (phba->ha_copy & (HA_RXMASK << (4*LPFC_EXTRA_RING)));
13996 		status2 >>= (4*LPFC_EXTRA_RING);
13997 	} else
13998 		status2 = 0;
13999 
14000 	if ((status1 & HA_RXMASK) || (status2 & HA_RXMASK))
14001 		fp_irq_rc = lpfc_sli_fp_intr_handler(irq, dev_id);
14002 	else
14003 		fp_irq_rc = IRQ_NONE;
14004 
14005 	/* Return device-level interrupt handling status */
14006 	return (sp_irq_rc == IRQ_HANDLED) ? sp_irq_rc : fp_irq_rc;
14007 }  /* lpfc_sli_intr_handler */
14008 
14009 /**
14010  * lpfc_sli4_els_xri_abort_event_proc - Process els xri abort event
14011  * @phba: pointer to lpfc hba data structure.
14012  *
14013  * This routine is invoked by the worker thread to process all the pending
14014  * SLI4 els abort xri events.
14015  **/
14016 void lpfc_sli4_els_xri_abort_event_proc(struct lpfc_hba *phba)
14017 {
14018 	struct lpfc_cq_event *cq_event;
14019 	unsigned long iflags;
14020 
14021 	/* First, declare the els xri abort event has been handled */
14022 	spin_lock_irqsave(&phba->hbalock, iflags);
14023 	phba->hba_flag &= ~ELS_XRI_ABORT_EVENT;
14024 	spin_unlock_irqrestore(&phba->hbalock, iflags);
14025 
14026 	/* Now, handle all the els xri abort events */
14027 	spin_lock_irqsave(&phba->sli4_hba.els_xri_abrt_list_lock, iflags);
14028 	while (!list_empty(&phba->sli4_hba.sp_els_xri_aborted_work_queue)) {
14029 		/* Get the first event from the head of the event queue */
14030 		list_remove_head(&phba->sli4_hba.sp_els_xri_aborted_work_queue,
14031 				 cq_event, struct lpfc_cq_event, list);
14032 		spin_unlock_irqrestore(&phba->sli4_hba.els_xri_abrt_list_lock,
14033 				       iflags);
14034 		/* Notify aborted XRI for ELS work queue */
14035 		lpfc_sli4_els_xri_aborted(phba, &cq_event->cqe.wcqe_axri);
14036 
14037 		/* Free the event processed back to the free pool */
14038 		lpfc_sli4_cq_event_release(phba, cq_event);
14039 		spin_lock_irqsave(&phba->sli4_hba.els_xri_abrt_list_lock,
14040 				  iflags);
14041 	}
14042 	spin_unlock_irqrestore(&phba->sli4_hba.els_xri_abrt_list_lock, iflags);
14043 }
14044 
14045 /**
14046  * lpfc_sli4_iocb_param_transfer - Transfer pIocbOut and cmpl status to pIocbIn
14047  * @phba: pointer to lpfc hba data structure
14048  * @pIocbIn: pointer to the rspiocbq
14049  * @pIocbOut: pointer to the cmdiocbq
14050  * @wcqe: pointer to the complete wcqe
14051  *
14052  * This routine transfers the fields of a command iocbq to a response iocbq
14053  * by copying all the IOCB fields from command iocbq and transferring the
14054  * completion status information from the complete wcqe.
14055  **/
14056 static void
14057 lpfc_sli4_iocb_param_transfer(struct lpfc_hba *phba,
14058 			      struct lpfc_iocbq *pIocbIn,
14059 			      struct lpfc_iocbq *pIocbOut,
14060 			      struct lpfc_wcqe_complete *wcqe)
14061 {
14062 	int numBdes, i;
14063 	unsigned long iflags;
14064 	uint32_t status, max_response;
14065 	struct lpfc_dmabuf *dmabuf;
14066 	struct ulp_bde64 *bpl, bde;
14067 	size_t offset = offsetof(struct lpfc_iocbq, iocb);
14068 
14069 	memcpy((char *)pIocbIn + offset, (char *)pIocbOut + offset,
14070 	       sizeof(struct lpfc_iocbq) - offset);
14071 	/* Map WCQE parameters into irspiocb parameters */
14072 	status = bf_get(lpfc_wcqe_c_status, wcqe);
14073 	pIocbIn->iocb.ulpStatus = (status & LPFC_IOCB_STATUS_MASK);
14074 	if (pIocbOut->iocb_flag & LPFC_IO_FCP)
14075 		if (pIocbIn->iocb.ulpStatus == IOSTAT_FCP_RSP_ERROR)
14076 			pIocbIn->iocb.un.fcpi.fcpi_parm =
14077 					pIocbOut->iocb.un.fcpi.fcpi_parm -
14078 					wcqe->total_data_placed;
14079 		else
14080 			pIocbIn->iocb.un.ulpWord[4] = wcqe->parameter;
14081 	else {
14082 		pIocbIn->iocb.un.ulpWord[4] = wcqe->parameter;
14083 		switch (pIocbOut->iocb.ulpCommand) {
14084 		case CMD_ELS_REQUEST64_CR:
14085 			dmabuf = (struct lpfc_dmabuf *)pIocbOut->context3;
14086 			bpl  = (struct ulp_bde64 *)dmabuf->virt;
14087 			bde.tus.w = le32_to_cpu(bpl[1].tus.w);
14088 			max_response = bde.tus.f.bdeSize;
14089 			break;
14090 		case CMD_GEN_REQUEST64_CR:
14091 			max_response = 0;
14092 			if (!pIocbOut->context3)
14093 				break;
14094 			numBdes = pIocbOut->iocb.un.genreq64.bdl.bdeSize/
14095 					sizeof(struct ulp_bde64);
14096 			dmabuf = (struct lpfc_dmabuf *)pIocbOut->context3;
14097 			bpl = (struct ulp_bde64 *)dmabuf->virt;
14098 			for (i = 0; i < numBdes; i++) {
14099 				bde.tus.w = le32_to_cpu(bpl[i].tus.w);
14100 				if (bde.tus.f.bdeFlags != BUFF_TYPE_BDE_64)
14101 					max_response += bde.tus.f.bdeSize;
14102 			}
14103 			break;
14104 		default:
14105 			max_response = wcqe->total_data_placed;
14106 			break;
14107 		}
14108 		if (max_response < wcqe->total_data_placed)
14109 			pIocbIn->iocb.un.genreq64.bdl.bdeSize = max_response;
14110 		else
14111 			pIocbIn->iocb.un.genreq64.bdl.bdeSize =
14112 				wcqe->total_data_placed;
14113 	}
14114 
14115 	/* Convert BG errors for completion status */
14116 	if (status == CQE_STATUS_DI_ERROR) {
14117 		pIocbIn->iocb.ulpStatus = IOSTAT_LOCAL_REJECT;
14118 
14119 		if (bf_get(lpfc_wcqe_c_bg_edir, wcqe))
14120 			pIocbIn->iocb.un.ulpWord[4] = IOERR_RX_DMA_FAILED;
14121 		else
14122 			pIocbIn->iocb.un.ulpWord[4] = IOERR_TX_DMA_FAILED;
14123 
14124 		pIocbIn->iocb.unsli3.sli3_bg.bgstat = 0;
14125 		if (bf_get(lpfc_wcqe_c_bg_ge, wcqe)) /* Guard Check failed */
14126 			pIocbIn->iocb.unsli3.sli3_bg.bgstat |=
14127 				BGS_GUARD_ERR_MASK;
14128 		if (bf_get(lpfc_wcqe_c_bg_ae, wcqe)) /* App Tag Check failed */
14129 			pIocbIn->iocb.unsli3.sli3_bg.bgstat |=
14130 				BGS_APPTAG_ERR_MASK;
14131 		if (bf_get(lpfc_wcqe_c_bg_re, wcqe)) /* Ref Tag Check failed */
14132 			pIocbIn->iocb.unsli3.sli3_bg.bgstat |=
14133 				BGS_REFTAG_ERR_MASK;
14134 
14135 		/* Check to see if there was any good data before the error */
14136 		if (bf_get(lpfc_wcqe_c_bg_tdpv, wcqe)) {
14137 			pIocbIn->iocb.unsli3.sli3_bg.bgstat |=
14138 				BGS_HI_WATER_MARK_PRESENT_MASK;
14139 			pIocbIn->iocb.unsli3.sli3_bg.bghm =
14140 				wcqe->total_data_placed;
14141 		}
14142 
14143 		/*
14144 		* Set ALL the error bits to indicate we don't know what
14145 		* type of error it is.
14146 		*/
14147 		if (!pIocbIn->iocb.unsli3.sli3_bg.bgstat)
14148 			pIocbIn->iocb.unsli3.sli3_bg.bgstat |=
14149 				(BGS_REFTAG_ERR_MASK | BGS_APPTAG_ERR_MASK |
14150 				BGS_GUARD_ERR_MASK);
14151 	}
14152 
14153 	/* Pick up HBA exchange busy condition */
14154 	if (bf_get(lpfc_wcqe_c_xb, wcqe)) {
14155 		spin_lock_irqsave(&phba->hbalock, iflags);
14156 		pIocbIn->iocb_flag |= LPFC_EXCHANGE_BUSY;
14157 		spin_unlock_irqrestore(&phba->hbalock, iflags);
14158 	}
14159 }
14160 
14161 /**
14162  * lpfc_sli4_els_wcqe_to_rspiocbq - Get response iocbq from els wcqe
14163  * @phba: Pointer to HBA context object.
14164  * @irspiocbq: Pointer to work-queue completion queue entry.
14165  *
14166  * This routine handles an ELS work-queue completion event and construct
14167  * a pseudo response ELS IODBQ from the SLI4 ELS WCQE for the common
14168  * discovery engine to handle.
14169  *
14170  * Return: Pointer to the receive IOCBQ, NULL otherwise.
14171  **/
14172 static struct lpfc_iocbq *
14173 lpfc_sli4_els_wcqe_to_rspiocbq(struct lpfc_hba *phba,
14174 			       struct lpfc_iocbq *irspiocbq)
14175 {
14176 	struct lpfc_sli_ring *pring;
14177 	struct lpfc_iocbq *cmdiocbq;
14178 	struct lpfc_wcqe_complete *wcqe;
14179 	unsigned long iflags;
14180 
14181 	pring = lpfc_phba_elsring(phba);
14182 	if (unlikely(!pring))
14183 		return NULL;
14184 
14185 	wcqe = &irspiocbq->cq_event.cqe.wcqe_cmpl;
14186 	pring->stats.iocb_event++;
14187 	/* Look up the ELS command IOCB and create pseudo response IOCB */
14188 	cmdiocbq = lpfc_sli_iocbq_lookup_by_tag(phba, pring,
14189 				bf_get(lpfc_wcqe_c_request_tag, wcqe));
14190 	if (unlikely(!cmdiocbq)) {
14191 		lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
14192 				"0386 ELS complete with no corresponding "
14193 				"cmdiocb: 0x%x 0x%x 0x%x 0x%x\n",
14194 				wcqe->word0, wcqe->total_data_placed,
14195 				wcqe->parameter, wcqe->word3);
14196 		lpfc_sli_release_iocbq(phba, irspiocbq);
14197 		return NULL;
14198 	}
14199 
14200 	spin_lock_irqsave(&pring->ring_lock, iflags);
14201 	/* Put the iocb back on the txcmplq */
14202 	lpfc_sli_ringtxcmpl_put(phba, pring, cmdiocbq);
14203 	spin_unlock_irqrestore(&pring->ring_lock, iflags);
14204 
14205 	/* Fake the irspiocbq and copy necessary response information */
14206 	lpfc_sli4_iocb_param_transfer(phba, irspiocbq, cmdiocbq, wcqe);
14207 
14208 	return irspiocbq;
14209 }
14210 
14211 inline struct lpfc_cq_event *
14212 lpfc_cq_event_setup(struct lpfc_hba *phba, void *entry, int size)
14213 {
14214 	struct lpfc_cq_event *cq_event;
14215 
14216 	/* Allocate a new internal CQ_EVENT entry */
14217 	cq_event = lpfc_sli4_cq_event_alloc(phba);
14218 	if (!cq_event) {
14219 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
14220 				"0602 Failed to alloc CQ_EVENT entry\n");
14221 		return NULL;
14222 	}
14223 
14224 	/* Move the CQE into the event */
14225 	memcpy(&cq_event->cqe, entry, size);
14226 	return cq_event;
14227 }
14228 
14229 /**
14230  * lpfc_sli4_sp_handle_async_event - Handle an asynchronous event
14231  * @phba: Pointer to HBA context object.
14232  * @mcqe: Pointer to mailbox completion queue entry.
14233  *
14234  * This routine process a mailbox completion queue entry with asynchronous
14235  * event.
14236  *
14237  * Return: true if work posted to worker thread, otherwise false.
14238  **/
14239 static bool
14240 lpfc_sli4_sp_handle_async_event(struct lpfc_hba *phba, struct lpfc_mcqe *mcqe)
14241 {
14242 	struct lpfc_cq_event *cq_event;
14243 	unsigned long iflags;
14244 
14245 	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
14246 			"0392 Async Event: word0:x%x, word1:x%x, "
14247 			"word2:x%x, word3:x%x\n", mcqe->word0,
14248 			mcqe->mcqe_tag0, mcqe->mcqe_tag1, mcqe->trailer);
14249 
14250 	cq_event = lpfc_cq_event_setup(phba, mcqe, sizeof(struct lpfc_mcqe));
14251 	if (!cq_event)
14252 		return false;
14253 
14254 	spin_lock_irqsave(&phba->sli4_hba.asynce_list_lock, iflags);
14255 	list_add_tail(&cq_event->list, &phba->sli4_hba.sp_asynce_work_queue);
14256 	spin_unlock_irqrestore(&phba->sli4_hba.asynce_list_lock, iflags);
14257 
14258 	/* Set the async event flag */
14259 	spin_lock_irqsave(&phba->hbalock, iflags);
14260 	phba->hba_flag |= ASYNC_EVENT;
14261 	spin_unlock_irqrestore(&phba->hbalock, iflags);
14262 
14263 	return true;
14264 }
14265 
14266 /**
14267  * lpfc_sli4_sp_handle_mbox_event - Handle a mailbox completion event
14268  * @phba: Pointer to HBA context object.
14269  * @mcqe: Pointer to mailbox completion queue entry.
14270  *
14271  * This routine process a mailbox completion queue entry with mailbox
14272  * completion event.
14273  *
14274  * Return: true if work posted to worker thread, otherwise false.
14275  **/
14276 static bool
14277 lpfc_sli4_sp_handle_mbox_event(struct lpfc_hba *phba, struct lpfc_mcqe *mcqe)
14278 {
14279 	uint32_t mcqe_status;
14280 	MAILBOX_t *mbox, *pmbox;
14281 	struct lpfc_mqe *mqe;
14282 	struct lpfc_vport *vport;
14283 	struct lpfc_nodelist *ndlp;
14284 	struct lpfc_dmabuf *mp;
14285 	unsigned long iflags;
14286 	LPFC_MBOXQ_t *pmb;
14287 	bool workposted = false;
14288 	int rc;
14289 
14290 	/* If not a mailbox complete MCQE, out by checking mailbox consume */
14291 	if (!bf_get(lpfc_trailer_completed, mcqe))
14292 		goto out_no_mqe_complete;
14293 
14294 	/* Get the reference to the active mbox command */
14295 	spin_lock_irqsave(&phba->hbalock, iflags);
14296 	pmb = phba->sli.mbox_active;
14297 	if (unlikely(!pmb)) {
14298 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
14299 				"1832 No pending MBOX command to handle\n");
14300 		spin_unlock_irqrestore(&phba->hbalock, iflags);
14301 		goto out_no_mqe_complete;
14302 	}
14303 	spin_unlock_irqrestore(&phba->hbalock, iflags);
14304 	mqe = &pmb->u.mqe;
14305 	pmbox = (MAILBOX_t *)&pmb->u.mqe;
14306 	mbox = phba->mbox;
14307 	vport = pmb->vport;
14308 
14309 	/* Reset heartbeat timer */
14310 	phba->last_completion_time = jiffies;
14311 	del_timer(&phba->sli.mbox_tmo);
14312 
14313 	/* Move mbox data to caller's mailbox region, do endian swapping */
14314 	if (pmb->mbox_cmpl && mbox)
14315 		lpfc_sli4_pcimem_bcopy(mbox, mqe, sizeof(struct lpfc_mqe));
14316 
14317 	/*
14318 	 * For mcqe errors, conditionally move a modified error code to
14319 	 * the mbox so that the error will not be missed.
14320 	 */
14321 	mcqe_status = bf_get(lpfc_mcqe_status, mcqe);
14322 	if (mcqe_status != MB_CQE_STATUS_SUCCESS) {
14323 		if (bf_get(lpfc_mqe_status, mqe) == MBX_SUCCESS)
14324 			bf_set(lpfc_mqe_status, mqe,
14325 			       (LPFC_MBX_ERROR_RANGE | mcqe_status));
14326 	}
14327 	if (pmb->mbox_flag & LPFC_MBX_IMED_UNREG) {
14328 		pmb->mbox_flag &= ~LPFC_MBX_IMED_UNREG;
14329 		lpfc_debugfs_disc_trc(vport, LPFC_DISC_TRC_MBOX_VPORT,
14330 				      "MBOX dflt rpi: status:x%x rpi:x%x",
14331 				      mcqe_status,
14332 				      pmbox->un.varWords[0], 0);
14333 		if (mcqe_status == MB_CQE_STATUS_SUCCESS) {
14334 			mp = (struct lpfc_dmabuf *)(pmb->ctx_buf);
14335 			ndlp = (struct lpfc_nodelist *)pmb->ctx_ndlp;
14336 
14337 			/* Reg_LOGIN of dflt RPI was successful. Mark the
14338 			 * node as having an UNREG_LOGIN in progress to stop
14339 			 * an unsolicited PLOGI from the same NPortId from
14340 			 * starting another mailbox transaction.
14341 			 */
14342 			spin_lock_irqsave(&ndlp->lock, iflags);
14343 			ndlp->nlp_flag |= NLP_UNREG_INP;
14344 			spin_unlock_irqrestore(&ndlp->lock, iflags);
14345 			lpfc_unreg_login(phba, vport->vpi,
14346 					 pmbox->un.varWords[0], pmb);
14347 			pmb->mbox_cmpl = lpfc_mbx_cmpl_dflt_rpi;
14348 			pmb->ctx_buf = mp;
14349 
14350 			/* No reference taken here.  This is a default
14351 			 * RPI reg/immediate unreg cycle. The reference was
14352 			 * taken in the reg rpi path and is released when
14353 			 * this mailbox completes.
14354 			 */
14355 			pmb->ctx_ndlp = ndlp;
14356 			pmb->vport = vport;
14357 			rc = lpfc_sli_issue_mbox(phba, pmb, MBX_NOWAIT);
14358 			if (rc != MBX_BUSY)
14359 				lpfc_printf_log(phba, KERN_ERR,
14360 						LOG_TRACE_EVENT,
14361 						"0385 rc should "
14362 						"have been MBX_BUSY\n");
14363 			if (rc != MBX_NOT_FINISHED)
14364 				goto send_current_mbox;
14365 		}
14366 	}
14367 	spin_lock_irqsave(&phba->pport->work_port_lock, iflags);
14368 	phba->pport->work_port_events &= ~WORKER_MBOX_TMO;
14369 	spin_unlock_irqrestore(&phba->pport->work_port_lock, iflags);
14370 
14371 	/* Do NOT queue MBX_HEARTBEAT to the worker thread for processing. */
14372 	if (pmbox->mbxCommand == MBX_HEARTBEAT) {
14373 		spin_lock_irqsave(&phba->hbalock, iflags);
14374 		/* Release the mailbox command posting token */
14375 		phba->sli.sli_flag &= ~LPFC_SLI_MBOX_ACTIVE;
14376 		phba->sli.mbox_active = NULL;
14377 		if (bf_get(lpfc_trailer_consumed, mcqe))
14378 			lpfc_sli4_mq_release(phba->sli4_hba.mbx_wq);
14379 		spin_unlock_irqrestore(&phba->hbalock, iflags);
14380 
14381 		/* Post the next mbox command, if there is one */
14382 		lpfc_sli4_post_async_mbox(phba);
14383 
14384 		/* Process cmpl now */
14385 		if (pmb->mbox_cmpl)
14386 			pmb->mbox_cmpl(phba, pmb);
14387 		return false;
14388 	}
14389 
14390 	/* There is mailbox completion work to queue to the worker thread */
14391 	spin_lock_irqsave(&phba->hbalock, iflags);
14392 	__lpfc_mbox_cmpl_put(phba, pmb);
14393 	phba->work_ha |= HA_MBATT;
14394 	spin_unlock_irqrestore(&phba->hbalock, iflags);
14395 	workposted = true;
14396 
14397 send_current_mbox:
14398 	spin_lock_irqsave(&phba->hbalock, iflags);
14399 	/* Release the mailbox command posting token */
14400 	phba->sli.sli_flag &= ~LPFC_SLI_MBOX_ACTIVE;
14401 	/* Setting active mailbox pointer need to be in sync to flag clear */
14402 	phba->sli.mbox_active = NULL;
14403 	if (bf_get(lpfc_trailer_consumed, mcqe))
14404 		lpfc_sli4_mq_release(phba->sli4_hba.mbx_wq);
14405 	spin_unlock_irqrestore(&phba->hbalock, iflags);
14406 	/* Wake up worker thread to post the next pending mailbox command */
14407 	lpfc_worker_wake_up(phba);
14408 	return workposted;
14409 
14410 out_no_mqe_complete:
14411 	spin_lock_irqsave(&phba->hbalock, iflags);
14412 	if (bf_get(lpfc_trailer_consumed, mcqe))
14413 		lpfc_sli4_mq_release(phba->sli4_hba.mbx_wq);
14414 	spin_unlock_irqrestore(&phba->hbalock, iflags);
14415 	return false;
14416 }
14417 
14418 /**
14419  * lpfc_sli4_sp_handle_mcqe - Process a mailbox completion queue entry
14420  * @phba: Pointer to HBA context object.
14421  * @cq: Pointer to associated CQ
14422  * @cqe: Pointer to mailbox completion queue entry.
14423  *
14424  * This routine process a mailbox completion queue entry, it invokes the
14425  * proper mailbox complete handling or asynchronous event handling routine
14426  * according to the MCQE's async bit.
14427  *
14428  * Return: true if work posted to worker thread, otherwise false.
14429  **/
14430 static bool
14431 lpfc_sli4_sp_handle_mcqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
14432 			 struct lpfc_cqe *cqe)
14433 {
14434 	struct lpfc_mcqe mcqe;
14435 	bool workposted;
14436 
14437 	cq->CQ_mbox++;
14438 
14439 	/* Copy the mailbox MCQE and convert endian order as needed */
14440 	lpfc_sli4_pcimem_bcopy(cqe, &mcqe, sizeof(struct lpfc_mcqe));
14441 
14442 	/* Invoke the proper event handling routine */
14443 	if (!bf_get(lpfc_trailer_async, &mcqe))
14444 		workposted = lpfc_sli4_sp_handle_mbox_event(phba, &mcqe);
14445 	else
14446 		workposted = lpfc_sli4_sp_handle_async_event(phba, &mcqe);
14447 	return workposted;
14448 }
14449 
14450 /**
14451  * lpfc_sli4_sp_handle_els_wcqe - Handle els work-queue completion event
14452  * @phba: Pointer to HBA context object.
14453  * @cq: Pointer to associated CQ
14454  * @wcqe: Pointer to work-queue completion queue entry.
14455  *
14456  * This routine handles an ELS work-queue completion event.
14457  *
14458  * Return: true if work posted to worker thread, otherwise false.
14459  **/
14460 static bool
14461 lpfc_sli4_sp_handle_els_wcqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
14462 			     struct lpfc_wcqe_complete *wcqe)
14463 {
14464 	struct lpfc_iocbq *irspiocbq;
14465 	unsigned long iflags;
14466 	struct lpfc_sli_ring *pring = cq->pring;
14467 	int txq_cnt = 0;
14468 	int txcmplq_cnt = 0;
14469 
14470 	/* Check for response status */
14471 	if (unlikely(bf_get(lpfc_wcqe_c_status, wcqe))) {
14472 		/* Log the error status */
14473 		lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
14474 				"0357 ELS CQE error: status=x%x: "
14475 				"CQE: %08x %08x %08x %08x\n",
14476 				bf_get(lpfc_wcqe_c_status, wcqe),
14477 				wcqe->word0, wcqe->total_data_placed,
14478 				wcqe->parameter, wcqe->word3);
14479 	}
14480 
14481 	/* Get an irspiocbq for later ELS response processing use */
14482 	irspiocbq = lpfc_sli_get_iocbq(phba);
14483 	if (!irspiocbq) {
14484 		if (!list_empty(&pring->txq))
14485 			txq_cnt++;
14486 		if (!list_empty(&pring->txcmplq))
14487 			txcmplq_cnt++;
14488 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
14489 			"0387 NO IOCBQ data: txq_cnt=%d iocb_cnt=%d "
14490 			"els_txcmplq_cnt=%d\n",
14491 			txq_cnt, phba->iocb_cnt,
14492 			txcmplq_cnt);
14493 		return false;
14494 	}
14495 
14496 	/* Save off the slow-path queue event for work thread to process */
14497 	memcpy(&irspiocbq->cq_event.cqe.wcqe_cmpl, wcqe, sizeof(*wcqe));
14498 	spin_lock_irqsave(&phba->hbalock, iflags);
14499 	list_add_tail(&irspiocbq->cq_event.list,
14500 		      &phba->sli4_hba.sp_queue_event);
14501 	phba->hba_flag |= HBA_SP_QUEUE_EVT;
14502 	spin_unlock_irqrestore(&phba->hbalock, iflags);
14503 
14504 	return true;
14505 }
14506 
14507 /**
14508  * lpfc_sli4_sp_handle_rel_wcqe - Handle slow-path WQ entry consumed event
14509  * @phba: Pointer to HBA context object.
14510  * @wcqe: Pointer to work-queue completion queue entry.
14511  *
14512  * This routine handles slow-path WQ entry consumed event by invoking the
14513  * proper WQ release routine to the slow-path WQ.
14514  **/
14515 static void
14516 lpfc_sli4_sp_handle_rel_wcqe(struct lpfc_hba *phba,
14517 			     struct lpfc_wcqe_release *wcqe)
14518 {
14519 	/* sanity check on queue memory */
14520 	if (unlikely(!phba->sli4_hba.els_wq))
14521 		return;
14522 	/* Check for the slow-path ELS work queue */
14523 	if (bf_get(lpfc_wcqe_r_wq_id, wcqe) == phba->sli4_hba.els_wq->queue_id)
14524 		lpfc_sli4_wq_release(phba->sli4_hba.els_wq,
14525 				     bf_get(lpfc_wcqe_r_wqe_index, wcqe));
14526 	else
14527 		lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
14528 				"2579 Slow-path wqe consume event carries "
14529 				"miss-matched qid: wcqe-qid=x%x, sp-qid=x%x\n",
14530 				bf_get(lpfc_wcqe_r_wqe_index, wcqe),
14531 				phba->sli4_hba.els_wq->queue_id);
14532 }
14533 
14534 /**
14535  * lpfc_sli4_sp_handle_abort_xri_wcqe - Handle a xri abort event
14536  * @phba: Pointer to HBA context object.
14537  * @cq: Pointer to a WQ completion queue.
14538  * @wcqe: Pointer to work-queue completion queue entry.
14539  *
14540  * This routine handles an XRI abort event.
14541  *
14542  * Return: true if work posted to worker thread, otherwise false.
14543  **/
14544 static bool
14545 lpfc_sli4_sp_handle_abort_xri_wcqe(struct lpfc_hba *phba,
14546 				   struct lpfc_queue *cq,
14547 				   struct sli4_wcqe_xri_aborted *wcqe)
14548 {
14549 	bool workposted = false;
14550 	struct lpfc_cq_event *cq_event;
14551 	unsigned long iflags;
14552 
14553 	switch (cq->subtype) {
14554 	case LPFC_IO:
14555 		lpfc_sli4_io_xri_aborted(phba, wcqe, cq->hdwq);
14556 		if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) {
14557 			/* Notify aborted XRI for NVME work queue */
14558 			if (phba->nvmet_support)
14559 				lpfc_sli4_nvmet_xri_aborted(phba, wcqe);
14560 		}
14561 		workposted = false;
14562 		break;
14563 	case LPFC_NVME_LS: /* NVME LS uses ELS resources */
14564 	case LPFC_ELS:
14565 		cq_event = lpfc_cq_event_setup(phba, wcqe, sizeof(*wcqe));
14566 		if (!cq_event) {
14567 			workposted = false;
14568 			break;
14569 		}
14570 		cq_event->hdwq = cq->hdwq;
14571 		spin_lock_irqsave(&phba->sli4_hba.els_xri_abrt_list_lock,
14572 				  iflags);
14573 		list_add_tail(&cq_event->list,
14574 			      &phba->sli4_hba.sp_els_xri_aborted_work_queue);
14575 		/* Set the els xri abort event flag */
14576 		phba->hba_flag |= ELS_XRI_ABORT_EVENT;
14577 		spin_unlock_irqrestore(&phba->sli4_hba.els_xri_abrt_list_lock,
14578 				       iflags);
14579 		workposted = true;
14580 		break;
14581 	default:
14582 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
14583 				"0603 Invalid CQ subtype %d: "
14584 				"%08x %08x %08x %08x\n",
14585 				cq->subtype, wcqe->word0, wcqe->parameter,
14586 				wcqe->word2, wcqe->word3);
14587 		workposted = false;
14588 		break;
14589 	}
14590 	return workposted;
14591 }
14592 
14593 #define FC_RCTL_MDS_DIAGS	0xF4
14594 
14595 /**
14596  * lpfc_sli4_sp_handle_rcqe - Process a receive-queue completion queue entry
14597  * @phba: Pointer to HBA context object.
14598  * @rcqe: Pointer to receive-queue completion queue entry.
14599  *
14600  * This routine process a receive-queue completion queue entry.
14601  *
14602  * Return: true if work posted to worker thread, otherwise false.
14603  **/
14604 static bool
14605 lpfc_sli4_sp_handle_rcqe(struct lpfc_hba *phba, struct lpfc_rcqe *rcqe)
14606 {
14607 	bool workposted = false;
14608 	struct fc_frame_header *fc_hdr;
14609 	struct lpfc_queue *hrq = phba->sli4_hba.hdr_rq;
14610 	struct lpfc_queue *drq = phba->sli4_hba.dat_rq;
14611 	struct lpfc_nvmet_tgtport *tgtp;
14612 	struct hbq_dmabuf *dma_buf;
14613 	uint32_t status, rq_id;
14614 	unsigned long iflags;
14615 
14616 	/* sanity check on queue memory */
14617 	if (unlikely(!hrq) || unlikely(!drq))
14618 		return workposted;
14619 
14620 	if (bf_get(lpfc_cqe_code, rcqe) == CQE_CODE_RECEIVE_V1)
14621 		rq_id = bf_get(lpfc_rcqe_rq_id_v1, rcqe);
14622 	else
14623 		rq_id = bf_get(lpfc_rcqe_rq_id, rcqe);
14624 	if (rq_id != hrq->queue_id)
14625 		goto out;
14626 
14627 	status = bf_get(lpfc_rcqe_status, rcqe);
14628 	switch (status) {
14629 	case FC_STATUS_RQ_BUF_LEN_EXCEEDED:
14630 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
14631 				"2537 Receive Frame Truncated!!\n");
14632 		fallthrough;
14633 	case FC_STATUS_RQ_SUCCESS:
14634 		spin_lock_irqsave(&phba->hbalock, iflags);
14635 		lpfc_sli4_rq_release(hrq, drq);
14636 		dma_buf = lpfc_sli_hbqbuf_get(&phba->hbqs[0].hbq_buffer_list);
14637 		if (!dma_buf) {
14638 			hrq->RQ_no_buf_found++;
14639 			spin_unlock_irqrestore(&phba->hbalock, iflags);
14640 			goto out;
14641 		}
14642 		hrq->RQ_rcv_buf++;
14643 		hrq->RQ_buf_posted--;
14644 		memcpy(&dma_buf->cq_event.cqe.rcqe_cmpl, rcqe, sizeof(*rcqe));
14645 
14646 		fc_hdr = (struct fc_frame_header *)dma_buf->hbuf.virt;
14647 
14648 		if (fc_hdr->fh_r_ctl == FC_RCTL_MDS_DIAGS ||
14649 		    fc_hdr->fh_r_ctl == FC_RCTL_DD_UNSOL_DATA) {
14650 			spin_unlock_irqrestore(&phba->hbalock, iflags);
14651 			/* Handle MDS Loopback frames */
14652 			if  (!(phba->pport->load_flag & FC_UNLOADING))
14653 				lpfc_sli4_handle_mds_loopback(phba->pport,
14654 							      dma_buf);
14655 			else
14656 				lpfc_in_buf_free(phba, &dma_buf->dbuf);
14657 			break;
14658 		}
14659 
14660 		/* save off the frame for the work thread to process */
14661 		list_add_tail(&dma_buf->cq_event.list,
14662 			      &phba->sli4_hba.sp_queue_event);
14663 		/* Frame received */
14664 		phba->hba_flag |= HBA_SP_QUEUE_EVT;
14665 		spin_unlock_irqrestore(&phba->hbalock, iflags);
14666 		workposted = true;
14667 		break;
14668 	case FC_STATUS_INSUFF_BUF_FRM_DISC:
14669 		if (phba->nvmet_support) {
14670 			tgtp = phba->targetport->private;
14671 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
14672 					"6402 RQE Error x%x, posted %d err_cnt "
14673 					"%d: %x %x %x\n",
14674 					status, hrq->RQ_buf_posted,
14675 					hrq->RQ_no_posted_buf,
14676 					atomic_read(&tgtp->rcv_fcp_cmd_in),
14677 					atomic_read(&tgtp->rcv_fcp_cmd_out),
14678 					atomic_read(&tgtp->xmt_fcp_release));
14679 		}
14680 		fallthrough;
14681 
14682 	case FC_STATUS_INSUFF_BUF_NEED_BUF:
14683 		hrq->RQ_no_posted_buf++;
14684 		/* Post more buffers if possible */
14685 		spin_lock_irqsave(&phba->hbalock, iflags);
14686 		phba->hba_flag |= HBA_POST_RECEIVE_BUFFER;
14687 		spin_unlock_irqrestore(&phba->hbalock, iflags);
14688 		workposted = true;
14689 		break;
14690 	}
14691 out:
14692 	return workposted;
14693 }
14694 
14695 /**
14696  * lpfc_sli4_sp_handle_cqe - Process a slow path completion queue entry
14697  * @phba: Pointer to HBA context object.
14698  * @cq: Pointer to the completion queue.
14699  * @cqe: Pointer to a completion queue entry.
14700  *
14701  * This routine process a slow-path work-queue or receive queue completion queue
14702  * entry.
14703  *
14704  * Return: true if work posted to worker thread, otherwise false.
14705  **/
14706 static bool
14707 lpfc_sli4_sp_handle_cqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
14708 			 struct lpfc_cqe *cqe)
14709 {
14710 	struct lpfc_cqe cqevt;
14711 	bool workposted = false;
14712 
14713 	/* Copy the work queue CQE and convert endian order if needed */
14714 	lpfc_sli4_pcimem_bcopy(cqe, &cqevt, sizeof(struct lpfc_cqe));
14715 
14716 	/* Check and process for different type of WCQE and dispatch */
14717 	switch (bf_get(lpfc_cqe_code, &cqevt)) {
14718 	case CQE_CODE_COMPL_WQE:
14719 		/* Process the WQ/RQ complete event */
14720 		phba->last_completion_time = jiffies;
14721 		workposted = lpfc_sli4_sp_handle_els_wcqe(phba, cq,
14722 				(struct lpfc_wcqe_complete *)&cqevt);
14723 		break;
14724 	case CQE_CODE_RELEASE_WQE:
14725 		/* Process the WQ release event */
14726 		lpfc_sli4_sp_handle_rel_wcqe(phba,
14727 				(struct lpfc_wcqe_release *)&cqevt);
14728 		break;
14729 	case CQE_CODE_XRI_ABORTED:
14730 		/* Process the WQ XRI abort event */
14731 		phba->last_completion_time = jiffies;
14732 		workposted = lpfc_sli4_sp_handle_abort_xri_wcqe(phba, cq,
14733 				(struct sli4_wcqe_xri_aborted *)&cqevt);
14734 		break;
14735 	case CQE_CODE_RECEIVE:
14736 	case CQE_CODE_RECEIVE_V1:
14737 		/* Process the RQ event */
14738 		phba->last_completion_time = jiffies;
14739 		workposted = lpfc_sli4_sp_handle_rcqe(phba,
14740 				(struct lpfc_rcqe *)&cqevt);
14741 		break;
14742 	default:
14743 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
14744 				"0388 Not a valid WCQE code: x%x\n",
14745 				bf_get(lpfc_cqe_code, &cqevt));
14746 		break;
14747 	}
14748 	return workposted;
14749 }
14750 
14751 /**
14752  * lpfc_sli4_sp_handle_eqe - Process a slow-path event queue entry
14753  * @phba: Pointer to HBA context object.
14754  * @eqe: Pointer to fast-path event queue entry.
14755  * @speq: Pointer to slow-path event queue.
14756  *
14757  * This routine process a event queue entry from the slow-path event queue.
14758  * It will check the MajorCode and MinorCode to determine this is for a
14759  * completion event on a completion queue, if not, an error shall be logged
14760  * and just return. Otherwise, it will get to the corresponding completion
14761  * queue and process all the entries on that completion queue, rearm the
14762  * completion queue, and then return.
14763  *
14764  **/
14765 static void
14766 lpfc_sli4_sp_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe,
14767 	struct lpfc_queue *speq)
14768 {
14769 	struct lpfc_queue *cq = NULL, *childq;
14770 	uint16_t cqid;
14771 	int ret = 0;
14772 
14773 	/* Get the reference to the corresponding CQ */
14774 	cqid = bf_get_le32(lpfc_eqe_resource_id, eqe);
14775 
14776 	list_for_each_entry(childq, &speq->child_list, list) {
14777 		if (childq->queue_id == cqid) {
14778 			cq = childq;
14779 			break;
14780 		}
14781 	}
14782 	if (unlikely(!cq)) {
14783 		if (phba->sli.sli_flag & LPFC_SLI_ACTIVE)
14784 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
14785 					"0365 Slow-path CQ identifier "
14786 					"(%d) does not exist\n", cqid);
14787 		return;
14788 	}
14789 
14790 	/* Save EQ associated with this CQ */
14791 	cq->assoc_qp = speq;
14792 
14793 	if (is_kdump_kernel())
14794 		ret = queue_work(phba->wq, &cq->spwork);
14795 	else
14796 		ret = queue_work_on(cq->chann, phba->wq, &cq->spwork);
14797 
14798 	if (!ret)
14799 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
14800 				"0390 Cannot schedule queue work "
14801 				"for CQ eqcqid=%d, cqid=%d on CPU %d\n",
14802 				cqid, cq->queue_id, raw_smp_processor_id());
14803 }
14804 
14805 /**
14806  * __lpfc_sli4_process_cq - Process elements of a CQ
14807  * @phba: Pointer to HBA context object.
14808  * @cq: Pointer to CQ to be processed
14809  * @handler: Routine to process each cqe
14810  * @delay: Pointer to usdelay to set in case of rescheduling of the handler
14811  * @poll_mode: Polling mode we were called from
14812  *
14813  * This routine processes completion queue entries in a CQ. While a valid
14814  * queue element is found, the handler is called. During processing checks
14815  * are made for periodic doorbell writes to let the hardware know of
14816  * element consumption.
14817  *
14818  * If the max limit on cqes to process is hit, or there are no more valid
14819  * entries, the loop stops. If we processed a sufficient number of elements,
14820  * meaning there is sufficient load, rather than rearming and generating
14821  * another interrupt, a cq rescheduling delay will be set. A delay of 0
14822  * indicates no rescheduling.
14823  *
14824  * Returns True if work scheduled, False otherwise.
14825  **/
14826 static bool
14827 __lpfc_sli4_process_cq(struct lpfc_hba *phba, struct lpfc_queue *cq,
14828 	bool (*handler)(struct lpfc_hba *, struct lpfc_queue *,
14829 			struct lpfc_cqe *), unsigned long *delay,
14830 			enum lpfc_poll_mode poll_mode)
14831 {
14832 	struct lpfc_cqe *cqe;
14833 	bool workposted = false;
14834 	int count = 0, consumed = 0;
14835 	bool arm = true;
14836 
14837 	/* default - no reschedule */
14838 	*delay = 0;
14839 
14840 	if (cmpxchg(&cq->queue_claimed, 0, 1) != 0)
14841 		goto rearm_and_exit;
14842 
14843 	/* Process all the entries to the CQ */
14844 	cq->q_flag = 0;
14845 	cqe = lpfc_sli4_cq_get(cq);
14846 	while (cqe) {
14847 		workposted |= handler(phba, cq, cqe);
14848 		__lpfc_sli4_consume_cqe(phba, cq, cqe);
14849 
14850 		consumed++;
14851 		if (!(++count % cq->max_proc_limit))
14852 			break;
14853 
14854 		if (!(count % cq->notify_interval)) {
14855 			phba->sli4_hba.sli4_write_cq_db(phba, cq, consumed,
14856 						LPFC_QUEUE_NOARM);
14857 			consumed = 0;
14858 			cq->assoc_qp->q_flag |= HBA_EQ_DELAY_CHK;
14859 		}
14860 
14861 		if (count == LPFC_NVMET_CQ_NOTIFY)
14862 			cq->q_flag |= HBA_NVMET_CQ_NOTIFY;
14863 
14864 		cqe = lpfc_sli4_cq_get(cq);
14865 	}
14866 	if (count >= phba->cfg_cq_poll_threshold) {
14867 		*delay = 1;
14868 		arm = false;
14869 	}
14870 
14871 	/* Note: complete the irq_poll softirq before rearming CQ */
14872 	if (poll_mode == LPFC_IRQ_POLL)
14873 		irq_poll_complete(&cq->iop);
14874 
14875 	/* Track the max number of CQEs processed in 1 EQ */
14876 	if (count > cq->CQ_max_cqe)
14877 		cq->CQ_max_cqe = count;
14878 
14879 	cq->assoc_qp->EQ_cqe_cnt += count;
14880 
14881 	/* Catch the no cq entry condition */
14882 	if (unlikely(count == 0))
14883 		lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
14884 				"0369 No entry from completion queue "
14885 				"qid=%d\n", cq->queue_id);
14886 
14887 	xchg(&cq->queue_claimed, 0);
14888 
14889 rearm_and_exit:
14890 	phba->sli4_hba.sli4_write_cq_db(phba, cq, consumed,
14891 			arm ?  LPFC_QUEUE_REARM : LPFC_QUEUE_NOARM);
14892 
14893 	return workposted;
14894 }
14895 
14896 /**
14897  * __lpfc_sli4_sp_process_cq - Process a slow-path event queue entry
14898  * @cq: pointer to CQ to process
14899  *
14900  * This routine calls the cq processing routine with a handler specific
14901  * to the type of queue bound to it.
14902  *
14903  * The CQ routine returns two values: the first is the calling status,
14904  * which indicates whether work was queued to the  background discovery
14905  * thread. If true, the routine should wakeup the discovery thread;
14906  * the second is the delay parameter. If non-zero, rather than rearming
14907  * the CQ and yet another interrupt, the CQ handler should be queued so
14908  * that it is processed in a subsequent polling action. The value of
14909  * the delay indicates when to reschedule it.
14910  **/
14911 static void
14912 __lpfc_sli4_sp_process_cq(struct lpfc_queue *cq)
14913 {
14914 	struct lpfc_hba *phba = cq->phba;
14915 	unsigned long delay;
14916 	bool workposted = false;
14917 	int ret = 0;
14918 
14919 	/* Process and rearm the CQ */
14920 	switch (cq->type) {
14921 	case LPFC_MCQ:
14922 		workposted |= __lpfc_sli4_process_cq(phba, cq,
14923 						lpfc_sli4_sp_handle_mcqe,
14924 						&delay, LPFC_QUEUE_WORK);
14925 		break;
14926 	case LPFC_WCQ:
14927 		if (cq->subtype == LPFC_IO)
14928 			workposted |= __lpfc_sli4_process_cq(phba, cq,
14929 						lpfc_sli4_fp_handle_cqe,
14930 						&delay, LPFC_QUEUE_WORK);
14931 		else
14932 			workposted |= __lpfc_sli4_process_cq(phba, cq,
14933 						lpfc_sli4_sp_handle_cqe,
14934 						&delay, LPFC_QUEUE_WORK);
14935 		break;
14936 	default:
14937 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
14938 				"0370 Invalid completion queue type (%d)\n",
14939 				cq->type);
14940 		return;
14941 	}
14942 
14943 	if (delay) {
14944 		if (is_kdump_kernel())
14945 			ret = queue_delayed_work(phba->wq, &cq->sched_spwork,
14946 						delay);
14947 		else
14948 			ret = queue_delayed_work_on(cq->chann, phba->wq,
14949 						&cq->sched_spwork, delay);
14950 		if (!ret)
14951 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
14952 				"0394 Cannot schedule queue work "
14953 				"for cqid=%d on CPU %d\n",
14954 				cq->queue_id, cq->chann);
14955 	}
14956 
14957 	/* wake up worker thread if there are works to be done */
14958 	if (workposted)
14959 		lpfc_worker_wake_up(phba);
14960 }
14961 
14962 /**
14963  * lpfc_sli4_sp_process_cq - slow-path work handler when started by
14964  *   interrupt
14965  * @work: pointer to work element
14966  *
14967  * translates from the work handler and calls the slow-path handler.
14968  **/
14969 static void
14970 lpfc_sli4_sp_process_cq(struct work_struct *work)
14971 {
14972 	struct lpfc_queue *cq = container_of(work, struct lpfc_queue, spwork);
14973 
14974 	__lpfc_sli4_sp_process_cq(cq);
14975 }
14976 
14977 /**
14978  * lpfc_sli4_dly_sp_process_cq - slow-path work handler when started by timer
14979  * @work: pointer to work element
14980  *
14981  * translates from the work handler and calls the slow-path handler.
14982  **/
14983 static void
14984 lpfc_sli4_dly_sp_process_cq(struct work_struct *work)
14985 {
14986 	struct lpfc_queue *cq = container_of(to_delayed_work(work),
14987 					struct lpfc_queue, sched_spwork);
14988 
14989 	__lpfc_sli4_sp_process_cq(cq);
14990 }
14991 
14992 /**
14993  * lpfc_sli4_fp_handle_fcp_wcqe - Process fast-path work queue completion entry
14994  * @phba: Pointer to HBA context object.
14995  * @cq: Pointer to associated CQ
14996  * @wcqe: Pointer to work-queue completion queue entry.
14997  *
14998  * This routine process a fast-path work queue completion entry from fast-path
14999  * event queue for FCP command response completion.
15000  **/
15001 static void
15002 lpfc_sli4_fp_handle_fcp_wcqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
15003 			     struct lpfc_wcqe_complete *wcqe)
15004 {
15005 	struct lpfc_sli_ring *pring = cq->pring;
15006 	struct lpfc_iocbq *cmdiocbq;
15007 	struct lpfc_iocbq irspiocbq;
15008 	unsigned long iflags;
15009 
15010 	/* Check for response status */
15011 	if (unlikely(bf_get(lpfc_wcqe_c_status, wcqe))) {
15012 		/* If resource errors reported from HBA, reduce queue
15013 		 * depth of the SCSI device.
15014 		 */
15015 		if (((bf_get(lpfc_wcqe_c_status, wcqe) ==
15016 		     IOSTAT_LOCAL_REJECT)) &&
15017 		    ((wcqe->parameter & IOERR_PARAM_MASK) ==
15018 		     IOERR_NO_RESOURCES))
15019 			phba->lpfc_rampdown_queue_depth(phba);
15020 
15021 		/* Log the cmpl status */
15022 		lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
15023 				"0373 FCP CQE cmpl: status=x%x: "
15024 				"CQE: %08x %08x %08x %08x\n",
15025 				bf_get(lpfc_wcqe_c_status, wcqe),
15026 				wcqe->word0, wcqe->total_data_placed,
15027 				wcqe->parameter, wcqe->word3);
15028 	}
15029 
15030 	/* Look up the FCP command IOCB and create pseudo response IOCB */
15031 	spin_lock_irqsave(&pring->ring_lock, iflags);
15032 	pring->stats.iocb_event++;
15033 	spin_unlock_irqrestore(&pring->ring_lock, iflags);
15034 	cmdiocbq = lpfc_sli_iocbq_lookup_by_tag(phba, pring,
15035 				bf_get(lpfc_wcqe_c_request_tag, wcqe));
15036 	if (unlikely(!cmdiocbq)) {
15037 		lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
15038 				"0374 FCP complete with no corresponding "
15039 				"cmdiocb: iotag (%d)\n",
15040 				bf_get(lpfc_wcqe_c_request_tag, wcqe));
15041 		return;
15042 	}
15043 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
15044 	cmdiocbq->isr_timestamp = cq->isr_timestamp;
15045 #endif
15046 	if (cmdiocbq->iocb_cmpl == NULL) {
15047 		if (cmdiocbq->wqe_cmpl) {
15048 			/* For FCP the flag is cleared in wqe_cmpl */
15049 			if (!(cmdiocbq->iocb_flag & LPFC_IO_FCP) &&
15050 			    cmdiocbq->iocb_flag & LPFC_DRIVER_ABORTED) {
15051 				spin_lock_irqsave(&phba->hbalock, iflags);
15052 				cmdiocbq->iocb_flag &= ~LPFC_DRIVER_ABORTED;
15053 				spin_unlock_irqrestore(&phba->hbalock, iflags);
15054 			}
15055 
15056 			/* Pass the cmd_iocb and the wcqe to the upper layer */
15057 			(cmdiocbq->wqe_cmpl)(phba, cmdiocbq, wcqe);
15058 			return;
15059 		}
15060 		lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
15061 				"0375 FCP cmdiocb not callback function "
15062 				"iotag: (%d)\n",
15063 				bf_get(lpfc_wcqe_c_request_tag, wcqe));
15064 		return;
15065 	}
15066 
15067 	/* Only SLI4 non-IO commands stil use IOCB */
15068 	/* Fake the irspiocb and copy necessary response information */
15069 	lpfc_sli4_iocb_param_transfer(phba, &irspiocbq, cmdiocbq, wcqe);
15070 
15071 	if (cmdiocbq->iocb_flag & LPFC_DRIVER_ABORTED) {
15072 		spin_lock_irqsave(&phba->hbalock, iflags);
15073 		cmdiocbq->iocb_flag &= ~LPFC_DRIVER_ABORTED;
15074 		spin_unlock_irqrestore(&phba->hbalock, iflags);
15075 	}
15076 
15077 	/* Pass the cmd_iocb and the rsp state to the upper layer */
15078 	(cmdiocbq->iocb_cmpl)(phba, cmdiocbq, &irspiocbq);
15079 }
15080 
15081 /**
15082  * lpfc_sli4_fp_handle_rel_wcqe - Handle fast-path WQ entry consumed event
15083  * @phba: Pointer to HBA context object.
15084  * @cq: Pointer to completion queue.
15085  * @wcqe: Pointer to work-queue completion queue entry.
15086  *
15087  * This routine handles an fast-path WQ entry consumed event by invoking the
15088  * proper WQ release routine to the slow-path WQ.
15089  **/
15090 static void
15091 lpfc_sli4_fp_handle_rel_wcqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
15092 			     struct lpfc_wcqe_release *wcqe)
15093 {
15094 	struct lpfc_queue *childwq;
15095 	bool wqid_matched = false;
15096 	uint16_t hba_wqid;
15097 
15098 	/* Check for fast-path FCP work queue release */
15099 	hba_wqid = bf_get(lpfc_wcqe_r_wq_id, wcqe);
15100 	list_for_each_entry(childwq, &cq->child_list, list) {
15101 		if (childwq->queue_id == hba_wqid) {
15102 			lpfc_sli4_wq_release(childwq,
15103 					bf_get(lpfc_wcqe_r_wqe_index, wcqe));
15104 			if (childwq->q_flag & HBA_NVMET_WQFULL)
15105 				lpfc_nvmet_wqfull_process(phba, childwq);
15106 			wqid_matched = true;
15107 			break;
15108 		}
15109 	}
15110 	/* Report warning log message if no match found */
15111 	if (wqid_matched != true)
15112 		lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
15113 				"2580 Fast-path wqe consume event carries "
15114 				"miss-matched qid: wcqe-qid=x%x\n", hba_wqid);
15115 }
15116 
15117 /**
15118  * lpfc_sli4_nvmet_handle_rcqe - Process a receive-queue completion queue entry
15119  * @phba: Pointer to HBA context object.
15120  * @cq: Pointer to completion queue.
15121  * @rcqe: Pointer to receive-queue completion queue entry.
15122  *
15123  * This routine process a receive-queue completion queue entry.
15124  *
15125  * Return: true if work posted to worker thread, otherwise false.
15126  **/
15127 static bool
15128 lpfc_sli4_nvmet_handle_rcqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
15129 			    struct lpfc_rcqe *rcqe)
15130 {
15131 	bool workposted = false;
15132 	struct lpfc_queue *hrq;
15133 	struct lpfc_queue *drq;
15134 	struct rqb_dmabuf *dma_buf;
15135 	struct fc_frame_header *fc_hdr;
15136 	struct lpfc_nvmet_tgtport *tgtp;
15137 	uint32_t status, rq_id;
15138 	unsigned long iflags;
15139 	uint32_t fctl, idx;
15140 
15141 	if ((phba->nvmet_support == 0) ||
15142 	    (phba->sli4_hba.nvmet_cqset == NULL))
15143 		return workposted;
15144 
15145 	idx = cq->queue_id - phba->sli4_hba.nvmet_cqset[0]->queue_id;
15146 	hrq = phba->sli4_hba.nvmet_mrq_hdr[idx];
15147 	drq = phba->sli4_hba.nvmet_mrq_data[idx];
15148 
15149 	/* sanity check on queue memory */
15150 	if (unlikely(!hrq) || unlikely(!drq))
15151 		return workposted;
15152 
15153 	if (bf_get(lpfc_cqe_code, rcqe) == CQE_CODE_RECEIVE_V1)
15154 		rq_id = bf_get(lpfc_rcqe_rq_id_v1, rcqe);
15155 	else
15156 		rq_id = bf_get(lpfc_rcqe_rq_id, rcqe);
15157 
15158 	if ((phba->nvmet_support == 0) ||
15159 	    (rq_id != hrq->queue_id))
15160 		return workposted;
15161 
15162 	status = bf_get(lpfc_rcqe_status, rcqe);
15163 	switch (status) {
15164 	case FC_STATUS_RQ_BUF_LEN_EXCEEDED:
15165 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
15166 				"6126 Receive Frame Truncated!!\n");
15167 		fallthrough;
15168 	case FC_STATUS_RQ_SUCCESS:
15169 		spin_lock_irqsave(&phba->hbalock, iflags);
15170 		lpfc_sli4_rq_release(hrq, drq);
15171 		dma_buf = lpfc_sli_rqbuf_get(phba, hrq);
15172 		if (!dma_buf) {
15173 			hrq->RQ_no_buf_found++;
15174 			spin_unlock_irqrestore(&phba->hbalock, iflags);
15175 			goto out;
15176 		}
15177 		spin_unlock_irqrestore(&phba->hbalock, iflags);
15178 		hrq->RQ_rcv_buf++;
15179 		hrq->RQ_buf_posted--;
15180 		fc_hdr = (struct fc_frame_header *)dma_buf->hbuf.virt;
15181 
15182 		/* Just some basic sanity checks on FCP Command frame */
15183 		fctl = (fc_hdr->fh_f_ctl[0] << 16 |
15184 			fc_hdr->fh_f_ctl[1] << 8 |
15185 			fc_hdr->fh_f_ctl[2]);
15186 		if (((fctl &
15187 		    (FC_FC_FIRST_SEQ | FC_FC_END_SEQ | FC_FC_SEQ_INIT)) !=
15188 		    (FC_FC_FIRST_SEQ | FC_FC_END_SEQ | FC_FC_SEQ_INIT)) ||
15189 		    (fc_hdr->fh_seq_cnt != 0)) /* 0 byte swapped is still 0 */
15190 			goto drop;
15191 
15192 		if (fc_hdr->fh_type == FC_TYPE_FCP) {
15193 			dma_buf->bytes_recv = bf_get(lpfc_rcqe_length, rcqe);
15194 			lpfc_nvmet_unsol_fcp_event(
15195 				phba, idx, dma_buf, cq->isr_timestamp,
15196 				cq->q_flag & HBA_NVMET_CQ_NOTIFY);
15197 			return false;
15198 		}
15199 drop:
15200 		lpfc_rq_buf_free(phba, &dma_buf->hbuf);
15201 		break;
15202 	case FC_STATUS_INSUFF_BUF_FRM_DISC:
15203 		if (phba->nvmet_support) {
15204 			tgtp = phba->targetport->private;
15205 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
15206 					"6401 RQE Error x%x, posted %d err_cnt "
15207 					"%d: %x %x %x\n",
15208 					status, hrq->RQ_buf_posted,
15209 					hrq->RQ_no_posted_buf,
15210 					atomic_read(&tgtp->rcv_fcp_cmd_in),
15211 					atomic_read(&tgtp->rcv_fcp_cmd_out),
15212 					atomic_read(&tgtp->xmt_fcp_release));
15213 		}
15214 		fallthrough;
15215 
15216 	case FC_STATUS_INSUFF_BUF_NEED_BUF:
15217 		hrq->RQ_no_posted_buf++;
15218 		/* Post more buffers if possible */
15219 		break;
15220 	}
15221 out:
15222 	return workposted;
15223 }
15224 
15225 /**
15226  * lpfc_sli4_fp_handle_cqe - Process fast-path work queue completion entry
15227  * @phba: adapter with cq
15228  * @cq: Pointer to the completion queue.
15229  * @cqe: Pointer to fast-path completion queue entry.
15230  *
15231  * This routine process a fast-path work queue completion entry from fast-path
15232  * event queue for FCP command response completion.
15233  *
15234  * Return: true if work posted to worker thread, otherwise false.
15235  **/
15236 static bool
15237 lpfc_sli4_fp_handle_cqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
15238 			 struct lpfc_cqe *cqe)
15239 {
15240 	struct lpfc_wcqe_release wcqe;
15241 	bool workposted = false;
15242 
15243 	/* Copy the work queue CQE and convert endian order if needed */
15244 	lpfc_sli4_pcimem_bcopy(cqe, &wcqe, sizeof(struct lpfc_cqe));
15245 
15246 	/* Check and process for different type of WCQE and dispatch */
15247 	switch (bf_get(lpfc_wcqe_c_code, &wcqe)) {
15248 	case CQE_CODE_COMPL_WQE:
15249 	case CQE_CODE_NVME_ERSP:
15250 		cq->CQ_wq++;
15251 		/* Process the WQ complete event */
15252 		phba->last_completion_time = jiffies;
15253 		if (cq->subtype == LPFC_IO || cq->subtype == LPFC_NVME_LS)
15254 			lpfc_sli4_fp_handle_fcp_wcqe(phba, cq,
15255 				(struct lpfc_wcqe_complete *)&wcqe);
15256 		break;
15257 	case CQE_CODE_RELEASE_WQE:
15258 		cq->CQ_release_wqe++;
15259 		/* Process the WQ release event */
15260 		lpfc_sli4_fp_handle_rel_wcqe(phba, cq,
15261 				(struct lpfc_wcqe_release *)&wcqe);
15262 		break;
15263 	case CQE_CODE_XRI_ABORTED:
15264 		cq->CQ_xri_aborted++;
15265 		/* Process the WQ XRI abort event */
15266 		phba->last_completion_time = jiffies;
15267 		workposted = lpfc_sli4_sp_handle_abort_xri_wcqe(phba, cq,
15268 				(struct sli4_wcqe_xri_aborted *)&wcqe);
15269 		break;
15270 	case CQE_CODE_RECEIVE_V1:
15271 	case CQE_CODE_RECEIVE:
15272 		phba->last_completion_time = jiffies;
15273 		if (cq->subtype == LPFC_NVMET) {
15274 			workposted = lpfc_sli4_nvmet_handle_rcqe(
15275 				phba, cq, (struct lpfc_rcqe *)&wcqe);
15276 		}
15277 		break;
15278 	default:
15279 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
15280 				"0144 Not a valid CQE code: x%x\n",
15281 				bf_get(lpfc_wcqe_c_code, &wcqe));
15282 		break;
15283 	}
15284 	return workposted;
15285 }
15286 
15287 /**
15288  * lpfc_sli4_sched_cq_work - Schedules cq work
15289  * @phba: Pointer to HBA context object.
15290  * @cq: Pointer to CQ
15291  * @cqid: CQ ID
15292  *
15293  * This routine checks the poll mode of the CQ corresponding to
15294  * cq->chann, then either schedules a softirq or queue_work to complete
15295  * cq work.
15296  *
15297  * queue_work path is taken if in NVMET mode, or if poll_mode is in
15298  * LPFC_QUEUE_WORK mode.  Otherwise, softirq path is taken.
15299  *
15300  **/
15301 static void lpfc_sli4_sched_cq_work(struct lpfc_hba *phba,
15302 				    struct lpfc_queue *cq, uint16_t cqid)
15303 {
15304 	int ret = 0;
15305 
15306 	switch (cq->poll_mode) {
15307 	case LPFC_IRQ_POLL:
15308 		/* CGN mgmt is mutually exclusive from softirq processing */
15309 		if (phba->cmf_active_mode == LPFC_CFG_OFF) {
15310 			irq_poll_sched(&cq->iop);
15311 			break;
15312 		}
15313 		fallthrough;
15314 	case LPFC_QUEUE_WORK:
15315 	default:
15316 		if (is_kdump_kernel())
15317 			ret = queue_work(phba->wq, &cq->irqwork);
15318 		else
15319 			ret = queue_work_on(cq->chann, phba->wq, &cq->irqwork);
15320 		if (!ret)
15321 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
15322 					"0383 Cannot schedule queue work "
15323 					"for CQ eqcqid=%d, cqid=%d on CPU %d\n",
15324 					cqid, cq->queue_id,
15325 					raw_smp_processor_id());
15326 	}
15327 }
15328 
15329 /**
15330  * lpfc_sli4_hba_handle_eqe - Process a fast-path event queue entry
15331  * @phba: Pointer to HBA context object.
15332  * @eq: Pointer to the queue structure.
15333  * @eqe: Pointer to fast-path event queue entry.
15334  *
15335  * This routine process a event queue entry from the fast-path event queue.
15336  * It will check the MajorCode and MinorCode to determine this is for a
15337  * completion event on a completion queue, if not, an error shall be logged
15338  * and just return. Otherwise, it will get to the corresponding completion
15339  * queue and process all the entries on the completion queue, rearm the
15340  * completion queue, and then return.
15341  **/
15342 static void
15343 lpfc_sli4_hba_handle_eqe(struct lpfc_hba *phba, struct lpfc_queue *eq,
15344 			 struct lpfc_eqe *eqe)
15345 {
15346 	struct lpfc_queue *cq = NULL;
15347 	uint32_t qidx = eq->hdwq;
15348 	uint16_t cqid, id;
15349 
15350 	if (unlikely(bf_get_le32(lpfc_eqe_major_code, eqe) != 0)) {
15351 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
15352 				"0366 Not a valid completion "
15353 				"event: majorcode=x%x, minorcode=x%x\n",
15354 				bf_get_le32(lpfc_eqe_major_code, eqe),
15355 				bf_get_le32(lpfc_eqe_minor_code, eqe));
15356 		return;
15357 	}
15358 
15359 	/* Get the reference to the corresponding CQ */
15360 	cqid = bf_get_le32(lpfc_eqe_resource_id, eqe);
15361 
15362 	/* Use the fast lookup method first */
15363 	if (cqid <= phba->sli4_hba.cq_max) {
15364 		cq = phba->sli4_hba.cq_lookup[cqid];
15365 		if (cq)
15366 			goto  work_cq;
15367 	}
15368 
15369 	/* Next check for NVMET completion */
15370 	if (phba->cfg_nvmet_mrq && phba->sli4_hba.nvmet_cqset) {
15371 		id = phba->sli4_hba.nvmet_cqset[0]->queue_id;
15372 		if ((cqid >= id) && (cqid < (id + phba->cfg_nvmet_mrq))) {
15373 			/* Process NVMET unsol rcv */
15374 			cq = phba->sli4_hba.nvmet_cqset[cqid - id];
15375 			goto  process_cq;
15376 		}
15377 	}
15378 
15379 	if (phba->sli4_hba.nvmels_cq &&
15380 	    (cqid == phba->sli4_hba.nvmels_cq->queue_id)) {
15381 		/* Process NVME unsol rcv */
15382 		cq = phba->sli4_hba.nvmels_cq;
15383 	}
15384 
15385 	/* Otherwise this is a Slow path event */
15386 	if (cq == NULL) {
15387 		lpfc_sli4_sp_handle_eqe(phba, eqe,
15388 					phba->sli4_hba.hdwq[qidx].hba_eq);
15389 		return;
15390 	}
15391 
15392 process_cq:
15393 	if (unlikely(cqid != cq->queue_id)) {
15394 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
15395 				"0368 Miss-matched fast-path completion "
15396 				"queue identifier: eqcqid=%d, fcpcqid=%d\n",
15397 				cqid, cq->queue_id);
15398 		return;
15399 	}
15400 
15401 work_cq:
15402 #if defined(CONFIG_SCSI_LPFC_DEBUG_FS)
15403 	if (phba->ktime_on)
15404 		cq->isr_timestamp = ktime_get_ns();
15405 	else
15406 		cq->isr_timestamp = 0;
15407 #endif
15408 	lpfc_sli4_sched_cq_work(phba, cq, cqid);
15409 }
15410 
15411 /**
15412  * __lpfc_sli4_hba_process_cq - Process a fast-path event queue entry
15413  * @cq: Pointer to CQ to be processed
15414  * @poll_mode: Enum lpfc_poll_state to determine poll mode
15415  *
15416  * This routine calls the cq processing routine with the handler for
15417  * fast path CQEs.
15418  *
15419  * The CQ routine returns two values: the first is the calling status,
15420  * which indicates whether work was queued to the  background discovery
15421  * thread. If true, the routine should wakeup the discovery thread;
15422  * the second is the delay parameter. If non-zero, rather than rearming
15423  * the CQ and yet another interrupt, the CQ handler should be queued so
15424  * that it is processed in a subsequent polling action. The value of
15425  * the delay indicates when to reschedule it.
15426  **/
15427 static void
15428 __lpfc_sli4_hba_process_cq(struct lpfc_queue *cq,
15429 			   enum lpfc_poll_mode poll_mode)
15430 {
15431 	struct lpfc_hba *phba = cq->phba;
15432 	unsigned long delay;
15433 	bool workposted = false;
15434 	int ret = 0;
15435 
15436 	/* process and rearm the CQ */
15437 	workposted |= __lpfc_sli4_process_cq(phba, cq, lpfc_sli4_fp_handle_cqe,
15438 					     &delay, poll_mode);
15439 
15440 	if (delay) {
15441 		if (is_kdump_kernel())
15442 			ret = queue_delayed_work(phba->wq, &cq->sched_irqwork,
15443 						delay);
15444 		else
15445 			ret = queue_delayed_work_on(cq->chann, phba->wq,
15446 						&cq->sched_irqwork, delay);
15447 		if (!ret)
15448 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
15449 					"0367 Cannot schedule queue work "
15450 					"for cqid=%d on CPU %d\n",
15451 					cq->queue_id, cq->chann);
15452 	}
15453 
15454 	/* wake up worker thread if there are works to be done */
15455 	if (workposted)
15456 		lpfc_worker_wake_up(phba);
15457 }
15458 
15459 /**
15460  * lpfc_sli4_hba_process_cq - fast-path work handler when started by
15461  *   interrupt
15462  * @work: pointer to work element
15463  *
15464  * translates from the work handler and calls the fast-path handler.
15465  **/
15466 static void
15467 lpfc_sli4_hba_process_cq(struct work_struct *work)
15468 {
15469 	struct lpfc_queue *cq = container_of(work, struct lpfc_queue, irqwork);
15470 
15471 	__lpfc_sli4_hba_process_cq(cq, LPFC_QUEUE_WORK);
15472 }
15473 
15474 /**
15475  * lpfc_sli4_dly_hba_process_cq - fast-path work handler when started by timer
15476  * @work: pointer to work element
15477  *
15478  * translates from the work handler and calls the fast-path handler.
15479  **/
15480 static void
15481 lpfc_sli4_dly_hba_process_cq(struct work_struct *work)
15482 {
15483 	struct lpfc_queue *cq = container_of(to_delayed_work(work),
15484 					struct lpfc_queue, sched_irqwork);
15485 
15486 	__lpfc_sli4_hba_process_cq(cq, LPFC_QUEUE_WORK);
15487 }
15488 
15489 /**
15490  * lpfc_sli4_hba_intr_handler - HBA interrupt handler to SLI-4 device
15491  * @irq: Interrupt number.
15492  * @dev_id: The device context pointer.
15493  *
15494  * This function is directly called from the PCI layer as an interrupt
15495  * service routine when device with SLI-4 interface spec is enabled with
15496  * MSI-X multi-message interrupt mode and there is a fast-path FCP IOCB
15497  * ring event in the HBA. However, when the device is enabled with either
15498  * MSI or Pin-IRQ interrupt mode, this function is called as part of the
15499  * device-level interrupt handler. When the PCI slot is in error recovery
15500  * or the HBA is undergoing initialization, the interrupt handler will not
15501  * process the interrupt. The SCSI FCP fast-path ring event are handled in
15502  * the intrrupt context. This function is called without any lock held.
15503  * It gets the hbalock to access and update SLI data structures. Note that,
15504  * the FCP EQ to FCP CQ are one-to-one map such that the FCP EQ index is
15505  * equal to that of FCP CQ index.
15506  *
15507  * The link attention and ELS ring attention events are handled
15508  * by the worker thread. The interrupt handler signals the worker thread
15509  * and returns for these events. This function is called without any lock
15510  * held. It gets the hbalock to access and update SLI data structures.
15511  *
15512  * This function returns IRQ_HANDLED when interrupt is handled else it
15513  * returns IRQ_NONE.
15514  **/
15515 irqreturn_t
15516 lpfc_sli4_hba_intr_handler(int irq, void *dev_id)
15517 {
15518 	struct lpfc_hba *phba;
15519 	struct lpfc_hba_eq_hdl *hba_eq_hdl;
15520 	struct lpfc_queue *fpeq;
15521 	unsigned long iflag;
15522 	int ecount = 0;
15523 	int hba_eqidx;
15524 	struct lpfc_eq_intr_info *eqi;
15525 
15526 	/* Get the driver's phba structure from the dev_id */
15527 	hba_eq_hdl = (struct lpfc_hba_eq_hdl *)dev_id;
15528 	phba = hba_eq_hdl->phba;
15529 	hba_eqidx = hba_eq_hdl->idx;
15530 
15531 	if (unlikely(!phba))
15532 		return IRQ_NONE;
15533 	if (unlikely(!phba->sli4_hba.hdwq))
15534 		return IRQ_NONE;
15535 
15536 	/* Get to the EQ struct associated with this vector */
15537 	fpeq = phba->sli4_hba.hba_eq_hdl[hba_eqidx].eq;
15538 	if (unlikely(!fpeq))
15539 		return IRQ_NONE;
15540 
15541 	/* Check device state for handling interrupt */
15542 	if (unlikely(lpfc_intr_state_check(phba))) {
15543 		/* Check again for link_state with lock held */
15544 		spin_lock_irqsave(&phba->hbalock, iflag);
15545 		if (phba->link_state < LPFC_LINK_DOWN)
15546 			/* Flush, clear interrupt, and rearm the EQ */
15547 			lpfc_sli4_eqcq_flush(phba, fpeq);
15548 		spin_unlock_irqrestore(&phba->hbalock, iflag);
15549 		return IRQ_NONE;
15550 	}
15551 
15552 	eqi = this_cpu_ptr(phba->sli4_hba.eq_info);
15553 	eqi->icnt++;
15554 
15555 	fpeq->last_cpu = raw_smp_processor_id();
15556 
15557 	if (eqi->icnt > LPFC_EQD_ISR_TRIGGER &&
15558 	    fpeq->q_flag & HBA_EQ_DELAY_CHK &&
15559 	    phba->cfg_auto_imax &&
15560 	    fpeq->q_mode != LPFC_MAX_AUTO_EQ_DELAY &&
15561 	    phba->sli.sli_flag & LPFC_SLI_USE_EQDR)
15562 		lpfc_sli4_mod_hba_eq_delay(phba, fpeq, LPFC_MAX_AUTO_EQ_DELAY);
15563 
15564 	/* process and rearm the EQ */
15565 	ecount = lpfc_sli4_process_eq(phba, fpeq, LPFC_QUEUE_REARM);
15566 
15567 	if (unlikely(ecount == 0)) {
15568 		fpeq->EQ_no_entry++;
15569 		if (phba->intr_type == MSIX)
15570 			/* MSI-X treated interrupt served as no EQ share INT */
15571 			lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
15572 					"0358 MSI-X interrupt with no EQE\n");
15573 		else
15574 			/* Non MSI-X treated on interrupt as EQ share INT */
15575 			return IRQ_NONE;
15576 	}
15577 
15578 	return IRQ_HANDLED;
15579 } /* lpfc_sli4_hba_intr_handler */
15580 
15581 /**
15582  * lpfc_sli4_intr_handler - Device-level interrupt handler for SLI-4 device
15583  * @irq: Interrupt number.
15584  * @dev_id: The device context pointer.
15585  *
15586  * This function is the device-level interrupt handler to device with SLI-4
15587  * interface spec, called from the PCI layer when either MSI or Pin-IRQ
15588  * interrupt mode is enabled and there is an event in the HBA which requires
15589  * driver attention. This function invokes the slow-path interrupt attention
15590  * handling function and fast-path interrupt attention handling function in
15591  * turn to process the relevant HBA attention events. This function is called
15592  * without any lock held. It gets the hbalock to access and update SLI data
15593  * structures.
15594  *
15595  * This function returns IRQ_HANDLED when interrupt is handled, else it
15596  * returns IRQ_NONE.
15597  **/
15598 irqreturn_t
15599 lpfc_sli4_intr_handler(int irq, void *dev_id)
15600 {
15601 	struct lpfc_hba  *phba;
15602 	irqreturn_t hba_irq_rc;
15603 	bool hba_handled = false;
15604 	int qidx;
15605 
15606 	/* Get the driver's phba structure from the dev_id */
15607 	phba = (struct lpfc_hba *)dev_id;
15608 
15609 	if (unlikely(!phba))
15610 		return IRQ_NONE;
15611 
15612 	/*
15613 	 * Invoke fast-path host attention interrupt handling as appropriate.
15614 	 */
15615 	for (qidx = 0; qidx < phba->cfg_irq_chann; qidx++) {
15616 		hba_irq_rc = lpfc_sli4_hba_intr_handler(irq,
15617 					&phba->sli4_hba.hba_eq_hdl[qidx]);
15618 		if (hba_irq_rc == IRQ_HANDLED)
15619 			hba_handled |= true;
15620 	}
15621 
15622 	return (hba_handled == true) ? IRQ_HANDLED : IRQ_NONE;
15623 } /* lpfc_sli4_intr_handler */
15624 
15625 void lpfc_sli4_poll_hbtimer(struct timer_list *t)
15626 {
15627 	struct lpfc_hba *phba = from_timer(phba, t, cpuhp_poll_timer);
15628 	struct lpfc_queue *eq;
15629 	int i = 0;
15630 
15631 	rcu_read_lock();
15632 
15633 	list_for_each_entry_rcu(eq, &phba->poll_list, _poll_list)
15634 		i += lpfc_sli4_poll_eq(eq, LPFC_POLL_SLOWPATH);
15635 	if (!list_empty(&phba->poll_list))
15636 		mod_timer(&phba->cpuhp_poll_timer,
15637 			  jiffies + msecs_to_jiffies(LPFC_POLL_HB));
15638 
15639 	rcu_read_unlock();
15640 }
15641 
15642 inline int lpfc_sli4_poll_eq(struct lpfc_queue *eq, uint8_t path)
15643 {
15644 	struct lpfc_hba *phba = eq->phba;
15645 	int i = 0;
15646 
15647 	/*
15648 	 * Unlocking an irq is one of the entry point to check
15649 	 * for re-schedule, but we are good for io submission
15650 	 * path as midlayer does a get_cpu to glue us in. Flush
15651 	 * out the invalidate queue so we can see the updated
15652 	 * value for flag.
15653 	 */
15654 	smp_rmb();
15655 
15656 	if (READ_ONCE(eq->mode) == LPFC_EQ_POLL)
15657 		/* We will not likely get the completion for the caller
15658 		 * during this iteration but i guess that's fine.
15659 		 * Future io's coming on this eq should be able to
15660 		 * pick it up.  As for the case of single io's, they
15661 		 * will be handled through a sched from polling timer
15662 		 * function which is currently triggered every 1msec.
15663 		 */
15664 		i = lpfc_sli4_process_eq(phba, eq, LPFC_QUEUE_NOARM);
15665 
15666 	return i;
15667 }
15668 
15669 static inline void lpfc_sli4_add_to_poll_list(struct lpfc_queue *eq)
15670 {
15671 	struct lpfc_hba *phba = eq->phba;
15672 
15673 	/* kickstart slowpath processing if needed */
15674 	if (list_empty(&phba->poll_list))
15675 		mod_timer(&phba->cpuhp_poll_timer,
15676 			  jiffies + msecs_to_jiffies(LPFC_POLL_HB));
15677 
15678 	list_add_rcu(&eq->_poll_list, &phba->poll_list);
15679 	synchronize_rcu();
15680 }
15681 
15682 static inline void lpfc_sli4_remove_from_poll_list(struct lpfc_queue *eq)
15683 {
15684 	struct lpfc_hba *phba = eq->phba;
15685 
15686 	/* Disable slowpath processing for this eq.  Kick start the eq
15687 	 * by RE-ARMING the eq's ASAP
15688 	 */
15689 	list_del_rcu(&eq->_poll_list);
15690 	synchronize_rcu();
15691 
15692 	if (list_empty(&phba->poll_list))
15693 		del_timer_sync(&phba->cpuhp_poll_timer);
15694 }
15695 
15696 void lpfc_sli4_cleanup_poll_list(struct lpfc_hba *phba)
15697 {
15698 	struct lpfc_queue *eq, *next;
15699 
15700 	list_for_each_entry_safe(eq, next, &phba->poll_list, _poll_list)
15701 		list_del(&eq->_poll_list);
15702 
15703 	INIT_LIST_HEAD(&phba->poll_list);
15704 	synchronize_rcu();
15705 }
15706 
15707 static inline void
15708 __lpfc_sli4_switch_eqmode(struct lpfc_queue *eq, uint8_t mode)
15709 {
15710 	if (mode == eq->mode)
15711 		return;
15712 	/*
15713 	 * currently this function is only called during a hotplug
15714 	 * event and the cpu on which this function is executing
15715 	 * is going offline.  By now the hotplug has instructed
15716 	 * the scheduler to remove this cpu from cpu active mask.
15717 	 * So we don't need to work about being put aside by the
15718 	 * scheduler for a high priority process.  Yes, the inte-
15719 	 * rrupts could come but they are known to retire ASAP.
15720 	 */
15721 
15722 	/* Disable polling in the fastpath */
15723 	WRITE_ONCE(eq->mode, mode);
15724 	/* flush out the store buffer */
15725 	smp_wmb();
15726 
15727 	/*
15728 	 * Add this eq to the polling list and start polling. For
15729 	 * a grace period both interrupt handler and poller will
15730 	 * try to process the eq _but_ that's fine.  We have a
15731 	 * synchronization mechanism in place (queue_claimed) to
15732 	 * deal with it.  This is just a draining phase for int-
15733 	 * errupt handler (not eq's) as we have guranteed through
15734 	 * barrier that all the CPUs have seen the new CQ_POLLED
15735 	 * state. which will effectively disable the REARMING of
15736 	 * the EQ.  The whole idea is eq's die off eventually as
15737 	 * we are not rearming EQ's anymore.
15738 	 */
15739 	mode ? lpfc_sli4_add_to_poll_list(eq) :
15740 	       lpfc_sli4_remove_from_poll_list(eq);
15741 }
15742 
15743 void lpfc_sli4_start_polling(struct lpfc_queue *eq)
15744 {
15745 	__lpfc_sli4_switch_eqmode(eq, LPFC_EQ_POLL);
15746 }
15747 
15748 void lpfc_sli4_stop_polling(struct lpfc_queue *eq)
15749 {
15750 	struct lpfc_hba *phba = eq->phba;
15751 
15752 	__lpfc_sli4_switch_eqmode(eq, LPFC_EQ_INTERRUPT);
15753 
15754 	/* Kick start for the pending io's in h/w.
15755 	 * Once we switch back to interrupt processing on a eq
15756 	 * the io path completion will only arm eq's when it
15757 	 * receives a completion.  But since eq's are in disa-
15758 	 * rmed state it doesn't receive a completion.  This
15759 	 * creates a deadlock scenaro.
15760 	 */
15761 	phba->sli4_hba.sli4_write_eq_db(phba, eq, 0, LPFC_QUEUE_REARM);
15762 }
15763 
15764 /**
15765  * lpfc_sli4_queue_free - free a queue structure and associated memory
15766  * @queue: The queue structure to free.
15767  *
15768  * This function frees a queue structure and the DMAable memory used for
15769  * the host resident queue. This function must be called after destroying the
15770  * queue on the HBA.
15771  **/
15772 void
15773 lpfc_sli4_queue_free(struct lpfc_queue *queue)
15774 {
15775 	struct lpfc_dmabuf *dmabuf;
15776 
15777 	if (!queue)
15778 		return;
15779 
15780 	if (!list_empty(&queue->wq_list))
15781 		list_del(&queue->wq_list);
15782 
15783 	while (!list_empty(&queue->page_list)) {
15784 		list_remove_head(&queue->page_list, dmabuf, struct lpfc_dmabuf,
15785 				 list);
15786 		dma_free_coherent(&queue->phba->pcidev->dev, queue->page_size,
15787 				  dmabuf->virt, dmabuf->phys);
15788 		kfree(dmabuf);
15789 	}
15790 	if (queue->rqbp) {
15791 		lpfc_free_rq_buffer(queue->phba, queue);
15792 		kfree(queue->rqbp);
15793 	}
15794 
15795 	if (!list_empty(&queue->cpu_list))
15796 		list_del(&queue->cpu_list);
15797 
15798 	kfree(queue);
15799 	return;
15800 }
15801 
15802 /**
15803  * lpfc_sli4_queue_alloc - Allocate and initialize a queue structure
15804  * @phba: The HBA that this queue is being created on.
15805  * @page_size: The size of a queue page
15806  * @entry_size: The size of each queue entry for this queue.
15807  * @entry_count: The number of entries that this queue will handle.
15808  * @cpu: The cpu that will primarily utilize this queue.
15809  *
15810  * This function allocates a queue structure and the DMAable memory used for
15811  * the host resident queue. This function must be called before creating the
15812  * queue on the HBA.
15813  **/
15814 struct lpfc_queue *
15815 lpfc_sli4_queue_alloc(struct lpfc_hba *phba, uint32_t page_size,
15816 		      uint32_t entry_size, uint32_t entry_count, int cpu)
15817 {
15818 	struct lpfc_queue *queue;
15819 	struct lpfc_dmabuf *dmabuf;
15820 	uint32_t hw_page_size = phba->sli4_hba.pc_sli4_params.if_page_sz;
15821 	uint16_t x, pgcnt;
15822 
15823 	if (!phba->sli4_hba.pc_sli4_params.supported)
15824 		hw_page_size = page_size;
15825 
15826 	pgcnt = ALIGN(entry_size * entry_count, hw_page_size) / hw_page_size;
15827 
15828 	/* If needed, Adjust page count to match the max the adapter supports */
15829 	if (pgcnt > phba->sli4_hba.pc_sli4_params.wqpcnt)
15830 		pgcnt = phba->sli4_hba.pc_sli4_params.wqpcnt;
15831 
15832 	queue = kzalloc_node(sizeof(*queue) + (sizeof(void *) * pgcnt),
15833 			     GFP_KERNEL, cpu_to_node(cpu));
15834 	if (!queue)
15835 		return NULL;
15836 
15837 	INIT_LIST_HEAD(&queue->list);
15838 	INIT_LIST_HEAD(&queue->_poll_list);
15839 	INIT_LIST_HEAD(&queue->wq_list);
15840 	INIT_LIST_HEAD(&queue->wqfull_list);
15841 	INIT_LIST_HEAD(&queue->page_list);
15842 	INIT_LIST_HEAD(&queue->child_list);
15843 	INIT_LIST_HEAD(&queue->cpu_list);
15844 
15845 	/* Set queue parameters now.  If the system cannot provide memory
15846 	 * resources, the free routine needs to know what was allocated.
15847 	 */
15848 	queue->page_count = pgcnt;
15849 	queue->q_pgs = (void **)&queue[1];
15850 	queue->entry_cnt_per_pg = hw_page_size / entry_size;
15851 	queue->entry_size = entry_size;
15852 	queue->entry_count = entry_count;
15853 	queue->page_size = hw_page_size;
15854 	queue->phba = phba;
15855 
15856 	for (x = 0; x < queue->page_count; x++) {
15857 		dmabuf = kzalloc_node(sizeof(*dmabuf), GFP_KERNEL,
15858 				      dev_to_node(&phba->pcidev->dev));
15859 		if (!dmabuf)
15860 			goto out_fail;
15861 		dmabuf->virt = dma_alloc_coherent(&phba->pcidev->dev,
15862 						  hw_page_size, &dmabuf->phys,
15863 						  GFP_KERNEL);
15864 		if (!dmabuf->virt) {
15865 			kfree(dmabuf);
15866 			goto out_fail;
15867 		}
15868 		dmabuf->buffer_tag = x;
15869 		list_add_tail(&dmabuf->list, &queue->page_list);
15870 		/* use lpfc_sli4_qe to index a paritcular entry in this page */
15871 		queue->q_pgs[x] = dmabuf->virt;
15872 	}
15873 	INIT_WORK(&queue->irqwork, lpfc_sli4_hba_process_cq);
15874 	INIT_WORK(&queue->spwork, lpfc_sli4_sp_process_cq);
15875 	INIT_DELAYED_WORK(&queue->sched_irqwork, lpfc_sli4_dly_hba_process_cq);
15876 	INIT_DELAYED_WORK(&queue->sched_spwork, lpfc_sli4_dly_sp_process_cq);
15877 
15878 	/* notify_interval will be set during q creation */
15879 
15880 	return queue;
15881 out_fail:
15882 	lpfc_sli4_queue_free(queue);
15883 	return NULL;
15884 }
15885 
15886 /**
15887  * lpfc_dual_chute_pci_bar_map - Map pci base address register to host memory
15888  * @phba: HBA structure that indicates port to create a queue on.
15889  * @pci_barset: PCI BAR set flag.
15890  *
15891  * This function shall perform iomap of the specified PCI BAR address to host
15892  * memory address if not already done so and return it. The returned host
15893  * memory address can be NULL.
15894  */
15895 static void __iomem *
15896 lpfc_dual_chute_pci_bar_map(struct lpfc_hba *phba, uint16_t pci_barset)
15897 {
15898 	if (!phba->pcidev)
15899 		return NULL;
15900 
15901 	switch (pci_barset) {
15902 	case WQ_PCI_BAR_0_AND_1:
15903 		return phba->pci_bar0_memmap_p;
15904 	case WQ_PCI_BAR_2_AND_3:
15905 		return phba->pci_bar2_memmap_p;
15906 	case WQ_PCI_BAR_4_AND_5:
15907 		return phba->pci_bar4_memmap_p;
15908 	default:
15909 		break;
15910 	}
15911 	return NULL;
15912 }
15913 
15914 /**
15915  * lpfc_modify_hba_eq_delay - Modify Delay Multiplier on EQs
15916  * @phba: HBA structure that EQs are on.
15917  * @startq: The starting EQ index to modify
15918  * @numq: The number of EQs (consecutive indexes) to modify
15919  * @usdelay: amount of delay
15920  *
15921  * This function revises the EQ delay on 1 or more EQs. The EQ delay
15922  * is set either by writing to a register (if supported by the SLI Port)
15923  * or by mailbox command. The mailbox command allows several EQs to be
15924  * updated at once.
15925  *
15926  * The @phba struct is used to send a mailbox command to HBA. The @startq
15927  * is used to get the starting EQ index to change. The @numq value is
15928  * used to specify how many consecutive EQ indexes, starting at EQ index,
15929  * are to be changed. This function is asynchronous and will wait for any
15930  * mailbox commands to finish before returning.
15931  *
15932  * On success this function will return a zero. If unable to allocate
15933  * enough memory this function will return -ENOMEM. If a mailbox command
15934  * fails this function will return -ENXIO. Note: on ENXIO, some EQs may
15935  * have had their delay multipler changed.
15936  **/
15937 void
15938 lpfc_modify_hba_eq_delay(struct lpfc_hba *phba, uint32_t startq,
15939 			 uint32_t numq, uint32_t usdelay)
15940 {
15941 	struct lpfc_mbx_modify_eq_delay *eq_delay;
15942 	LPFC_MBOXQ_t *mbox;
15943 	struct lpfc_queue *eq;
15944 	int cnt = 0, rc, length;
15945 	uint32_t shdr_status, shdr_add_status;
15946 	uint32_t dmult;
15947 	int qidx;
15948 	union lpfc_sli4_cfg_shdr *shdr;
15949 
15950 	if (startq >= phba->cfg_irq_chann)
15951 		return;
15952 
15953 	if (usdelay > 0xFFFF) {
15954 		lpfc_printf_log(phba, KERN_INFO, LOG_INIT | LOG_FCP | LOG_NVME,
15955 				"6429 usdelay %d too large. Scaled down to "
15956 				"0xFFFF.\n", usdelay);
15957 		usdelay = 0xFFFF;
15958 	}
15959 
15960 	/* set values by EQ_DELAY register if supported */
15961 	if (phba->sli.sli_flag & LPFC_SLI_USE_EQDR) {
15962 		for (qidx = startq; qidx < phba->cfg_irq_chann; qidx++) {
15963 			eq = phba->sli4_hba.hba_eq_hdl[qidx].eq;
15964 			if (!eq)
15965 				continue;
15966 
15967 			lpfc_sli4_mod_hba_eq_delay(phba, eq, usdelay);
15968 
15969 			if (++cnt >= numq)
15970 				break;
15971 		}
15972 		return;
15973 	}
15974 
15975 	/* Otherwise, set values by mailbox cmd */
15976 
15977 	mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
15978 	if (!mbox) {
15979 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
15980 				"6428 Failed allocating mailbox cmd buffer."
15981 				" EQ delay was not set.\n");
15982 		return;
15983 	}
15984 	length = (sizeof(struct lpfc_mbx_modify_eq_delay) -
15985 		  sizeof(struct lpfc_sli4_cfg_mhdr));
15986 	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
15987 			 LPFC_MBOX_OPCODE_MODIFY_EQ_DELAY,
15988 			 length, LPFC_SLI4_MBX_EMBED);
15989 	eq_delay = &mbox->u.mqe.un.eq_delay;
15990 
15991 	/* Calculate delay multiper from maximum interrupt per second */
15992 	dmult = (usdelay * LPFC_DMULT_CONST) / LPFC_SEC_TO_USEC;
15993 	if (dmult)
15994 		dmult--;
15995 	if (dmult > LPFC_DMULT_MAX)
15996 		dmult = LPFC_DMULT_MAX;
15997 
15998 	for (qidx = startq; qidx < phba->cfg_irq_chann; qidx++) {
15999 		eq = phba->sli4_hba.hba_eq_hdl[qidx].eq;
16000 		if (!eq)
16001 			continue;
16002 		eq->q_mode = usdelay;
16003 		eq_delay->u.request.eq[cnt].eq_id = eq->queue_id;
16004 		eq_delay->u.request.eq[cnt].phase = 0;
16005 		eq_delay->u.request.eq[cnt].delay_multi = dmult;
16006 
16007 		if (++cnt >= numq)
16008 			break;
16009 	}
16010 	eq_delay->u.request.num_eq = cnt;
16011 
16012 	mbox->vport = phba->pport;
16013 	mbox->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
16014 	mbox->ctx_buf = NULL;
16015 	mbox->ctx_ndlp = NULL;
16016 	rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL);
16017 	shdr = (union lpfc_sli4_cfg_shdr *) &eq_delay->header.cfg_shdr;
16018 	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
16019 	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
16020 	if (shdr_status || shdr_add_status || rc) {
16021 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
16022 				"2512 MODIFY_EQ_DELAY mailbox failed with "
16023 				"status x%x add_status x%x, mbx status x%x\n",
16024 				shdr_status, shdr_add_status, rc);
16025 	}
16026 	mempool_free(mbox, phba->mbox_mem_pool);
16027 	return;
16028 }
16029 
16030 /**
16031  * lpfc_eq_create - Create an Event Queue on the HBA
16032  * @phba: HBA structure that indicates port to create a queue on.
16033  * @eq: The queue structure to use to create the event queue.
16034  * @imax: The maximum interrupt per second limit.
16035  *
16036  * This function creates an event queue, as detailed in @eq, on a port,
16037  * described by @phba by sending an EQ_CREATE mailbox command to the HBA.
16038  *
16039  * The @phba struct is used to send mailbox command to HBA. The @eq struct
16040  * is used to get the entry count and entry size that are necessary to
16041  * determine the number of pages to allocate and use for this queue. This
16042  * function will send the EQ_CREATE mailbox command to the HBA to setup the
16043  * event queue. This function is asynchronous and will wait for the mailbox
16044  * command to finish before continuing.
16045  *
16046  * On success this function will return a zero. If unable to allocate enough
16047  * memory this function will return -ENOMEM. If the queue create mailbox command
16048  * fails this function will return -ENXIO.
16049  **/
16050 int
16051 lpfc_eq_create(struct lpfc_hba *phba, struct lpfc_queue *eq, uint32_t imax)
16052 {
16053 	struct lpfc_mbx_eq_create *eq_create;
16054 	LPFC_MBOXQ_t *mbox;
16055 	int rc, length, status = 0;
16056 	struct lpfc_dmabuf *dmabuf;
16057 	uint32_t shdr_status, shdr_add_status;
16058 	union lpfc_sli4_cfg_shdr *shdr;
16059 	uint16_t dmult;
16060 	uint32_t hw_page_size = phba->sli4_hba.pc_sli4_params.if_page_sz;
16061 
16062 	/* sanity check on queue memory */
16063 	if (!eq)
16064 		return -ENODEV;
16065 	if (!phba->sli4_hba.pc_sli4_params.supported)
16066 		hw_page_size = SLI4_PAGE_SIZE;
16067 
16068 	mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
16069 	if (!mbox)
16070 		return -ENOMEM;
16071 	length = (sizeof(struct lpfc_mbx_eq_create) -
16072 		  sizeof(struct lpfc_sli4_cfg_mhdr));
16073 	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
16074 			 LPFC_MBOX_OPCODE_EQ_CREATE,
16075 			 length, LPFC_SLI4_MBX_EMBED);
16076 	eq_create = &mbox->u.mqe.un.eq_create;
16077 	shdr = (union lpfc_sli4_cfg_shdr *) &eq_create->header.cfg_shdr;
16078 	bf_set(lpfc_mbx_eq_create_num_pages, &eq_create->u.request,
16079 	       eq->page_count);
16080 	bf_set(lpfc_eq_context_size, &eq_create->u.request.context,
16081 	       LPFC_EQE_SIZE);
16082 	bf_set(lpfc_eq_context_valid, &eq_create->u.request.context, 1);
16083 
16084 	/* Use version 2 of CREATE_EQ if eqav is set */
16085 	if (phba->sli4_hba.pc_sli4_params.eqav) {
16086 		bf_set(lpfc_mbox_hdr_version, &shdr->request,
16087 		       LPFC_Q_CREATE_VERSION_2);
16088 		bf_set(lpfc_eq_context_autovalid, &eq_create->u.request.context,
16089 		       phba->sli4_hba.pc_sli4_params.eqav);
16090 	}
16091 
16092 	/* don't setup delay multiplier using EQ_CREATE */
16093 	dmult = 0;
16094 	bf_set(lpfc_eq_context_delay_multi, &eq_create->u.request.context,
16095 	       dmult);
16096 	switch (eq->entry_count) {
16097 	default:
16098 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
16099 				"0360 Unsupported EQ count. (%d)\n",
16100 				eq->entry_count);
16101 		if (eq->entry_count < 256) {
16102 			status = -EINVAL;
16103 			goto out;
16104 		}
16105 		fallthrough;	/* otherwise default to smallest count */
16106 	case 256:
16107 		bf_set(lpfc_eq_context_count, &eq_create->u.request.context,
16108 		       LPFC_EQ_CNT_256);
16109 		break;
16110 	case 512:
16111 		bf_set(lpfc_eq_context_count, &eq_create->u.request.context,
16112 		       LPFC_EQ_CNT_512);
16113 		break;
16114 	case 1024:
16115 		bf_set(lpfc_eq_context_count, &eq_create->u.request.context,
16116 		       LPFC_EQ_CNT_1024);
16117 		break;
16118 	case 2048:
16119 		bf_set(lpfc_eq_context_count, &eq_create->u.request.context,
16120 		       LPFC_EQ_CNT_2048);
16121 		break;
16122 	case 4096:
16123 		bf_set(lpfc_eq_context_count, &eq_create->u.request.context,
16124 		       LPFC_EQ_CNT_4096);
16125 		break;
16126 	}
16127 	list_for_each_entry(dmabuf, &eq->page_list, list) {
16128 		memset(dmabuf->virt, 0, hw_page_size);
16129 		eq_create->u.request.page[dmabuf->buffer_tag].addr_lo =
16130 					putPaddrLow(dmabuf->phys);
16131 		eq_create->u.request.page[dmabuf->buffer_tag].addr_hi =
16132 					putPaddrHigh(dmabuf->phys);
16133 	}
16134 	mbox->vport = phba->pport;
16135 	mbox->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
16136 	mbox->ctx_buf = NULL;
16137 	mbox->ctx_ndlp = NULL;
16138 	rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL);
16139 	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
16140 	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
16141 	if (shdr_status || shdr_add_status || rc) {
16142 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
16143 				"2500 EQ_CREATE mailbox failed with "
16144 				"status x%x add_status x%x, mbx status x%x\n",
16145 				shdr_status, shdr_add_status, rc);
16146 		status = -ENXIO;
16147 	}
16148 	eq->type = LPFC_EQ;
16149 	eq->subtype = LPFC_NONE;
16150 	eq->queue_id = bf_get(lpfc_mbx_eq_create_q_id, &eq_create->u.response);
16151 	if (eq->queue_id == 0xFFFF)
16152 		status = -ENXIO;
16153 	eq->host_index = 0;
16154 	eq->notify_interval = LPFC_EQ_NOTIFY_INTRVL;
16155 	eq->max_proc_limit = LPFC_EQ_MAX_PROC_LIMIT;
16156 out:
16157 	mempool_free(mbox, phba->mbox_mem_pool);
16158 	return status;
16159 }
16160 
16161 static int lpfc_cq_poll_hdler(struct irq_poll *iop, int budget)
16162 {
16163 	struct lpfc_queue *cq = container_of(iop, struct lpfc_queue, iop);
16164 
16165 	__lpfc_sli4_hba_process_cq(cq, LPFC_IRQ_POLL);
16166 
16167 	return 1;
16168 }
16169 
16170 /**
16171  * lpfc_cq_create - Create a Completion Queue on the HBA
16172  * @phba: HBA structure that indicates port to create a queue on.
16173  * @cq: The queue structure to use to create the completion queue.
16174  * @eq: The event queue to bind this completion queue to.
16175  * @type: Type of queue (EQ, GCQ, MCQ, WCQ, etc).
16176  * @subtype: Functional purpose of the queue (MBOX, IO, ELS, NVMET, etc).
16177  *
16178  * This function creates a completion queue, as detailed in @wq, on a port,
16179  * described by @phba by sending a CQ_CREATE mailbox command to the HBA.
16180  *
16181  * The @phba struct is used to send mailbox command to HBA. The @cq struct
16182  * is used to get the entry count and entry size that are necessary to
16183  * determine the number of pages to allocate and use for this queue. The @eq
16184  * is used to indicate which event queue to bind this completion queue to. This
16185  * function will send the CQ_CREATE mailbox command to the HBA to setup the
16186  * completion queue. This function is asynchronous and will wait for the mailbox
16187  * command to finish before continuing.
16188  *
16189  * On success this function will return a zero. If unable to allocate enough
16190  * memory this function will return -ENOMEM. If the queue create mailbox command
16191  * fails this function will return -ENXIO.
16192  **/
16193 int
16194 lpfc_cq_create(struct lpfc_hba *phba, struct lpfc_queue *cq,
16195 	       struct lpfc_queue *eq, uint32_t type, uint32_t subtype)
16196 {
16197 	struct lpfc_mbx_cq_create *cq_create;
16198 	struct lpfc_dmabuf *dmabuf;
16199 	LPFC_MBOXQ_t *mbox;
16200 	int rc, length, status = 0;
16201 	uint32_t shdr_status, shdr_add_status;
16202 	union lpfc_sli4_cfg_shdr *shdr;
16203 
16204 	/* sanity check on queue memory */
16205 	if (!cq || !eq)
16206 		return -ENODEV;
16207 
16208 	mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
16209 	if (!mbox)
16210 		return -ENOMEM;
16211 	length = (sizeof(struct lpfc_mbx_cq_create) -
16212 		  sizeof(struct lpfc_sli4_cfg_mhdr));
16213 	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
16214 			 LPFC_MBOX_OPCODE_CQ_CREATE,
16215 			 length, LPFC_SLI4_MBX_EMBED);
16216 	cq_create = &mbox->u.mqe.un.cq_create;
16217 	shdr = (union lpfc_sli4_cfg_shdr *) &cq_create->header.cfg_shdr;
16218 	bf_set(lpfc_mbx_cq_create_num_pages, &cq_create->u.request,
16219 		    cq->page_count);
16220 	bf_set(lpfc_cq_context_event, &cq_create->u.request.context, 1);
16221 	bf_set(lpfc_cq_context_valid, &cq_create->u.request.context, 1);
16222 	bf_set(lpfc_mbox_hdr_version, &shdr->request,
16223 	       phba->sli4_hba.pc_sli4_params.cqv);
16224 	if (phba->sli4_hba.pc_sli4_params.cqv == LPFC_Q_CREATE_VERSION_2) {
16225 		bf_set(lpfc_mbx_cq_create_page_size, &cq_create->u.request,
16226 		       (cq->page_size / SLI4_PAGE_SIZE));
16227 		bf_set(lpfc_cq_eq_id_2, &cq_create->u.request.context,
16228 		       eq->queue_id);
16229 		bf_set(lpfc_cq_context_autovalid, &cq_create->u.request.context,
16230 		       phba->sli4_hba.pc_sli4_params.cqav);
16231 	} else {
16232 		bf_set(lpfc_cq_eq_id, &cq_create->u.request.context,
16233 		       eq->queue_id);
16234 	}
16235 	switch (cq->entry_count) {
16236 	case 2048:
16237 	case 4096:
16238 		if (phba->sli4_hba.pc_sli4_params.cqv ==
16239 		    LPFC_Q_CREATE_VERSION_2) {
16240 			cq_create->u.request.context.lpfc_cq_context_count =
16241 				cq->entry_count;
16242 			bf_set(lpfc_cq_context_count,
16243 			       &cq_create->u.request.context,
16244 			       LPFC_CQ_CNT_WORD7);
16245 			break;
16246 		}
16247 		fallthrough;
16248 	default:
16249 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
16250 				"0361 Unsupported CQ count: "
16251 				"entry cnt %d sz %d pg cnt %d\n",
16252 				cq->entry_count, cq->entry_size,
16253 				cq->page_count);
16254 		if (cq->entry_count < 256) {
16255 			status = -EINVAL;
16256 			goto out;
16257 		}
16258 		fallthrough;	/* otherwise default to smallest count */
16259 	case 256:
16260 		bf_set(lpfc_cq_context_count, &cq_create->u.request.context,
16261 		       LPFC_CQ_CNT_256);
16262 		break;
16263 	case 512:
16264 		bf_set(lpfc_cq_context_count, &cq_create->u.request.context,
16265 		       LPFC_CQ_CNT_512);
16266 		break;
16267 	case 1024:
16268 		bf_set(lpfc_cq_context_count, &cq_create->u.request.context,
16269 		       LPFC_CQ_CNT_1024);
16270 		break;
16271 	}
16272 	list_for_each_entry(dmabuf, &cq->page_list, list) {
16273 		memset(dmabuf->virt, 0, cq->page_size);
16274 		cq_create->u.request.page[dmabuf->buffer_tag].addr_lo =
16275 					putPaddrLow(dmabuf->phys);
16276 		cq_create->u.request.page[dmabuf->buffer_tag].addr_hi =
16277 					putPaddrHigh(dmabuf->phys);
16278 	}
16279 	rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL);
16280 
16281 	/* The IOCTL status is embedded in the mailbox subheader. */
16282 	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
16283 	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
16284 	if (shdr_status || shdr_add_status || rc) {
16285 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
16286 				"2501 CQ_CREATE mailbox failed with "
16287 				"status x%x add_status x%x, mbx status x%x\n",
16288 				shdr_status, shdr_add_status, rc);
16289 		status = -ENXIO;
16290 		goto out;
16291 	}
16292 	cq->queue_id = bf_get(lpfc_mbx_cq_create_q_id, &cq_create->u.response);
16293 	if (cq->queue_id == 0xFFFF) {
16294 		status = -ENXIO;
16295 		goto out;
16296 	}
16297 	/* link the cq onto the parent eq child list */
16298 	list_add_tail(&cq->list, &eq->child_list);
16299 	/* Set up completion queue's type and subtype */
16300 	cq->type = type;
16301 	cq->subtype = subtype;
16302 	cq->queue_id = bf_get(lpfc_mbx_cq_create_q_id, &cq_create->u.response);
16303 	cq->assoc_qid = eq->queue_id;
16304 	cq->assoc_qp = eq;
16305 	cq->host_index = 0;
16306 	cq->notify_interval = LPFC_CQ_NOTIFY_INTRVL;
16307 	cq->max_proc_limit = min(phba->cfg_cq_max_proc_limit, cq->entry_count);
16308 
16309 	if (cq->queue_id > phba->sli4_hba.cq_max)
16310 		phba->sli4_hba.cq_max = cq->queue_id;
16311 
16312 	irq_poll_init(&cq->iop, LPFC_IRQ_POLL_WEIGHT, lpfc_cq_poll_hdler);
16313 out:
16314 	mempool_free(mbox, phba->mbox_mem_pool);
16315 	return status;
16316 }
16317 
16318 /**
16319  * lpfc_cq_create_set - Create a set of Completion Queues on the HBA for MRQ
16320  * @phba: HBA structure that indicates port to create a queue on.
16321  * @cqp: The queue structure array to use to create the completion queues.
16322  * @hdwq: The hardware queue array  with the EQ to bind completion queues to.
16323  * @type: Type of queue (EQ, GCQ, MCQ, WCQ, etc).
16324  * @subtype: Functional purpose of the queue (MBOX, IO, ELS, NVMET, etc).
16325  *
16326  * This function creates a set of  completion queue, s to support MRQ
16327  * as detailed in @cqp, on a port,
16328  * described by @phba by sending a CREATE_CQ_SET mailbox command to the HBA.
16329  *
16330  * The @phba struct is used to send mailbox command to HBA. The @cq struct
16331  * is used to get the entry count and entry size that are necessary to
16332  * determine the number of pages to allocate and use for this queue. The @eq
16333  * is used to indicate which event queue to bind this completion queue to. This
16334  * function will send the CREATE_CQ_SET mailbox command to the HBA to setup the
16335  * completion queue. This function is asynchronous and will wait for the mailbox
16336  * command to finish before continuing.
16337  *
16338  * On success this function will return a zero. If unable to allocate enough
16339  * memory this function will return -ENOMEM. If the queue create mailbox command
16340  * fails this function will return -ENXIO.
16341  **/
16342 int
16343 lpfc_cq_create_set(struct lpfc_hba *phba, struct lpfc_queue **cqp,
16344 		   struct lpfc_sli4_hdw_queue *hdwq, uint32_t type,
16345 		   uint32_t subtype)
16346 {
16347 	struct lpfc_queue *cq;
16348 	struct lpfc_queue *eq;
16349 	struct lpfc_mbx_cq_create_set *cq_set;
16350 	struct lpfc_dmabuf *dmabuf;
16351 	LPFC_MBOXQ_t *mbox;
16352 	int rc, length, alloclen, status = 0;
16353 	int cnt, idx, numcq, page_idx = 0;
16354 	uint32_t shdr_status, shdr_add_status;
16355 	union lpfc_sli4_cfg_shdr *shdr;
16356 	uint32_t hw_page_size = phba->sli4_hba.pc_sli4_params.if_page_sz;
16357 
16358 	/* sanity check on queue memory */
16359 	numcq = phba->cfg_nvmet_mrq;
16360 	if (!cqp || !hdwq || !numcq)
16361 		return -ENODEV;
16362 
16363 	mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
16364 	if (!mbox)
16365 		return -ENOMEM;
16366 
16367 	length = sizeof(struct lpfc_mbx_cq_create_set);
16368 	length += ((numcq * cqp[0]->page_count) *
16369 		   sizeof(struct dma_address));
16370 	alloclen = lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_FCOE,
16371 			LPFC_MBOX_OPCODE_FCOE_CQ_CREATE_SET, length,
16372 			LPFC_SLI4_MBX_NEMBED);
16373 	if (alloclen < length) {
16374 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
16375 				"3098 Allocated DMA memory size (%d) is "
16376 				"less than the requested DMA memory size "
16377 				"(%d)\n", alloclen, length);
16378 		status = -ENOMEM;
16379 		goto out;
16380 	}
16381 	cq_set = mbox->sge_array->addr[0];
16382 	shdr = (union lpfc_sli4_cfg_shdr *)&cq_set->cfg_shdr;
16383 	bf_set(lpfc_mbox_hdr_version, &shdr->request, 0);
16384 
16385 	for (idx = 0; idx < numcq; idx++) {
16386 		cq = cqp[idx];
16387 		eq = hdwq[idx].hba_eq;
16388 		if (!cq || !eq) {
16389 			status = -ENOMEM;
16390 			goto out;
16391 		}
16392 		if (!phba->sli4_hba.pc_sli4_params.supported)
16393 			hw_page_size = cq->page_size;
16394 
16395 		switch (idx) {
16396 		case 0:
16397 			bf_set(lpfc_mbx_cq_create_set_page_size,
16398 			       &cq_set->u.request,
16399 			       (hw_page_size / SLI4_PAGE_SIZE));
16400 			bf_set(lpfc_mbx_cq_create_set_num_pages,
16401 			       &cq_set->u.request, cq->page_count);
16402 			bf_set(lpfc_mbx_cq_create_set_evt,
16403 			       &cq_set->u.request, 1);
16404 			bf_set(lpfc_mbx_cq_create_set_valid,
16405 			       &cq_set->u.request, 1);
16406 			bf_set(lpfc_mbx_cq_create_set_cqe_size,
16407 			       &cq_set->u.request, 0);
16408 			bf_set(lpfc_mbx_cq_create_set_num_cq,
16409 			       &cq_set->u.request, numcq);
16410 			bf_set(lpfc_mbx_cq_create_set_autovalid,
16411 			       &cq_set->u.request,
16412 			       phba->sli4_hba.pc_sli4_params.cqav);
16413 			switch (cq->entry_count) {
16414 			case 2048:
16415 			case 4096:
16416 				if (phba->sli4_hba.pc_sli4_params.cqv ==
16417 				    LPFC_Q_CREATE_VERSION_2) {
16418 					bf_set(lpfc_mbx_cq_create_set_cqe_cnt,
16419 					       &cq_set->u.request,
16420 						cq->entry_count);
16421 					bf_set(lpfc_mbx_cq_create_set_cqe_cnt,
16422 					       &cq_set->u.request,
16423 					       LPFC_CQ_CNT_WORD7);
16424 					break;
16425 				}
16426 				fallthrough;
16427 			default:
16428 				lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
16429 						"3118 Bad CQ count. (%d)\n",
16430 						cq->entry_count);
16431 				if (cq->entry_count < 256) {
16432 					status = -EINVAL;
16433 					goto out;
16434 				}
16435 				fallthrough;	/* otherwise default to smallest */
16436 			case 256:
16437 				bf_set(lpfc_mbx_cq_create_set_cqe_cnt,
16438 				       &cq_set->u.request, LPFC_CQ_CNT_256);
16439 				break;
16440 			case 512:
16441 				bf_set(lpfc_mbx_cq_create_set_cqe_cnt,
16442 				       &cq_set->u.request, LPFC_CQ_CNT_512);
16443 				break;
16444 			case 1024:
16445 				bf_set(lpfc_mbx_cq_create_set_cqe_cnt,
16446 				       &cq_set->u.request, LPFC_CQ_CNT_1024);
16447 				break;
16448 			}
16449 			bf_set(lpfc_mbx_cq_create_set_eq_id0,
16450 			       &cq_set->u.request, eq->queue_id);
16451 			break;
16452 		case 1:
16453 			bf_set(lpfc_mbx_cq_create_set_eq_id1,
16454 			       &cq_set->u.request, eq->queue_id);
16455 			break;
16456 		case 2:
16457 			bf_set(lpfc_mbx_cq_create_set_eq_id2,
16458 			       &cq_set->u.request, eq->queue_id);
16459 			break;
16460 		case 3:
16461 			bf_set(lpfc_mbx_cq_create_set_eq_id3,
16462 			       &cq_set->u.request, eq->queue_id);
16463 			break;
16464 		case 4:
16465 			bf_set(lpfc_mbx_cq_create_set_eq_id4,
16466 			       &cq_set->u.request, eq->queue_id);
16467 			break;
16468 		case 5:
16469 			bf_set(lpfc_mbx_cq_create_set_eq_id5,
16470 			       &cq_set->u.request, eq->queue_id);
16471 			break;
16472 		case 6:
16473 			bf_set(lpfc_mbx_cq_create_set_eq_id6,
16474 			       &cq_set->u.request, eq->queue_id);
16475 			break;
16476 		case 7:
16477 			bf_set(lpfc_mbx_cq_create_set_eq_id7,
16478 			       &cq_set->u.request, eq->queue_id);
16479 			break;
16480 		case 8:
16481 			bf_set(lpfc_mbx_cq_create_set_eq_id8,
16482 			       &cq_set->u.request, eq->queue_id);
16483 			break;
16484 		case 9:
16485 			bf_set(lpfc_mbx_cq_create_set_eq_id9,
16486 			       &cq_set->u.request, eq->queue_id);
16487 			break;
16488 		case 10:
16489 			bf_set(lpfc_mbx_cq_create_set_eq_id10,
16490 			       &cq_set->u.request, eq->queue_id);
16491 			break;
16492 		case 11:
16493 			bf_set(lpfc_mbx_cq_create_set_eq_id11,
16494 			       &cq_set->u.request, eq->queue_id);
16495 			break;
16496 		case 12:
16497 			bf_set(lpfc_mbx_cq_create_set_eq_id12,
16498 			       &cq_set->u.request, eq->queue_id);
16499 			break;
16500 		case 13:
16501 			bf_set(lpfc_mbx_cq_create_set_eq_id13,
16502 			       &cq_set->u.request, eq->queue_id);
16503 			break;
16504 		case 14:
16505 			bf_set(lpfc_mbx_cq_create_set_eq_id14,
16506 			       &cq_set->u.request, eq->queue_id);
16507 			break;
16508 		case 15:
16509 			bf_set(lpfc_mbx_cq_create_set_eq_id15,
16510 			       &cq_set->u.request, eq->queue_id);
16511 			break;
16512 		}
16513 
16514 		/* link the cq onto the parent eq child list */
16515 		list_add_tail(&cq->list, &eq->child_list);
16516 		/* Set up completion queue's type and subtype */
16517 		cq->type = type;
16518 		cq->subtype = subtype;
16519 		cq->assoc_qid = eq->queue_id;
16520 		cq->assoc_qp = eq;
16521 		cq->host_index = 0;
16522 		cq->notify_interval = LPFC_CQ_NOTIFY_INTRVL;
16523 		cq->max_proc_limit = min(phba->cfg_cq_max_proc_limit,
16524 					 cq->entry_count);
16525 		cq->chann = idx;
16526 
16527 		rc = 0;
16528 		list_for_each_entry(dmabuf, &cq->page_list, list) {
16529 			memset(dmabuf->virt, 0, hw_page_size);
16530 			cnt = page_idx + dmabuf->buffer_tag;
16531 			cq_set->u.request.page[cnt].addr_lo =
16532 					putPaddrLow(dmabuf->phys);
16533 			cq_set->u.request.page[cnt].addr_hi =
16534 					putPaddrHigh(dmabuf->phys);
16535 			rc++;
16536 		}
16537 		page_idx += rc;
16538 	}
16539 
16540 	rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL);
16541 
16542 	/* The IOCTL status is embedded in the mailbox subheader. */
16543 	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
16544 	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
16545 	if (shdr_status || shdr_add_status || rc) {
16546 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
16547 				"3119 CQ_CREATE_SET mailbox failed with "
16548 				"status x%x add_status x%x, mbx status x%x\n",
16549 				shdr_status, shdr_add_status, rc);
16550 		status = -ENXIO;
16551 		goto out;
16552 	}
16553 	rc = bf_get(lpfc_mbx_cq_create_set_base_id, &cq_set->u.response);
16554 	if (rc == 0xFFFF) {
16555 		status = -ENXIO;
16556 		goto out;
16557 	}
16558 
16559 	for (idx = 0; idx < numcq; idx++) {
16560 		cq = cqp[idx];
16561 		cq->queue_id = rc + idx;
16562 		if (cq->queue_id > phba->sli4_hba.cq_max)
16563 			phba->sli4_hba.cq_max = cq->queue_id;
16564 	}
16565 
16566 out:
16567 	lpfc_sli4_mbox_cmd_free(phba, mbox);
16568 	return status;
16569 }
16570 
16571 /**
16572  * lpfc_mq_create_fb_init - Send MCC_CREATE without async events registration
16573  * @phba: HBA structure that indicates port to create a queue on.
16574  * @mq: The queue structure to use to create the mailbox queue.
16575  * @mbox: An allocated pointer to type LPFC_MBOXQ_t
16576  * @cq: The completion queue to associate with this cq.
16577  *
16578  * This function provides failback (fb) functionality when the
16579  * mq_create_ext fails on older FW generations.  It's purpose is identical
16580  * to mq_create_ext otherwise.
16581  *
16582  * This routine cannot fail as all attributes were previously accessed and
16583  * initialized in mq_create_ext.
16584  **/
16585 static void
16586 lpfc_mq_create_fb_init(struct lpfc_hba *phba, struct lpfc_queue *mq,
16587 		       LPFC_MBOXQ_t *mbox, struct lpfc_queue *cq)
16588 {
16589 	struct lpfc_mbx_mq_create *mq_create;
16590 	struct lpfc_dmabuf *dmabuf;
16591 	int length;
16592 
16593 	length = (sizeof(struct lpfc_mbx_mq_create) -
16594 		  sizeof(struct lpfc_sli4_cfg_mhdr));
16595 	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
16596 			 LPFC_MBOX_OPCODE_MQ_CREATE,
16597 			 length, LPFC_SLI4_MBX_EMBED);
16598 	mq_create = &mbox->u.mqe.un.mq_create;
16599 	bf_set(lpfc_mbx_mq_create_num_pages, &mq_create->u.request,
16600 	       mq->page_count);
16601 	bf_set(lpfc_mq_context_cq_id, &mq_create->u.request.context,
16602 	       cq->queue_id);
16603 	bf_set(lpfc_mq_context_valid, &mq_create->u.request.context, 1);
16604 	switch (mq->entry_count) {
16605 	case 16:
16606 		bf_set(lpfc_mq_context_ring_size, &mq_create->u.request.context,
16607 		       LPFC_MQ_RING_SIZE_16);
16608 		break;
16609 	case 32:
16610 		bf_set(lpfc_mq_context_ring_size, &mq_create->u.request.context,
16611 		       LPFC_MQ_RING_SIZE_32);
16612 		break;
16613 	case 64:
16614 		bf_set(lpfc_mq_context_ring_size, &mq_create->u.request.context,
16615 		       LPFC_MQ_RING_SIZE_64);
16616 		break;
16617 	case 128:
16618 		bf_set(lpfc_mq_context_ring_size, &mq_create->u.request.context,
16619 		       LPFC_MQ_RING_SIZE_128);
16620 		break;
16621 	}
16622 	list_for_each_entry(dmabuf, &mq->page_list, list) {
16623 		mq_create->u.request.page[dmabuf->buffer_tag].addr_lo =
16624 			putPaddrLow(dmabuf->phys);
16625 		mq_create->u.request.page[dmabuf->buffer_tag].addr_hi =
16626 			putPaddrHigh(dmabuf->phys);
16627 	}
16628 }
16629 
16630 /**
16631  * lpfc_mq_create - Create a mailbox Queue on the HBA
16632  * @phba: HBA structure that indicates port to create a queue on.
16633  * @mq: The queue structure to use to create the mailbox queue.
16634  * @cq: The completion queue to associate with this cq.
16635  * @subtype: The queue's subtype.
16636  *
16637  * This function creates a mailbox queue, as detailed in @mq, on a port,
16638  * described by @phba by sending a MQ_CREATE mailbox command to the HBA.
16639  *
16640  * The @phba struct is used to send mailbox command to HBA. The @cq struct
16641  * is used to get the entry count and entry size that are necessary to
16642  * determine the number of pages to allocate and use for this queue. This
16643  * function will send the MQ_CREATE mailbox command to the HBA to setup the
16644  * mailbox queue. This function is asynchronous and will wait for the mailbox
16645  * command to finish before continuing.
16646  *
16647  * On success this function will return a zero. If unable to allocate enough
16648  * memory this function will return -ENOMEM. If the queue create mailbox command
16649  * fails this function will return -ENXIO.
16650  **/
16651 int32_t
16652 lpfc_mq_create(struct lpfc_hba *phba, struct lpfc_queue *mq,
16653 	       struct lpfc_queue *cq, uint32_t subtype)
16654 {
16655 	struct lpfc_mbx_mq_create *mq_create;
16656 	struct lpfc_mbx_mq_create_ext *mq_create_ext;
16657 	struct lpfc_dmabuf *dmabuf;
16658 	LPFC_MBOXQ_t *mbox;
16659 	int rc, length, status = 0;
16660 	uint32_t shdr_status, shdr_add_status;
16661 	union lpfc_sli4_cfg_shdr *shdr;
16662 	uint32_t hw_page_size = phba->sli4_hba.pc_sli4_params.if_page_sz;
16663 
16664 	/* sanity check on queue memory */
16665 	if (!mq || !cq)
16666 		return -ENODEV;
16667 	if (!phba->sli4_hba.pc_sli4_params.supported)
16668 		hw_page_size = SLI4_PAGE_SIZE;
16669 
16670 	mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
16671 	if (!mbox)
16672 		return -ENOMEM;
16673 	length = (sizeof(struct lpfc_mbx_mq_create_ext) -
16674 		  sizeof(struct lpfc_sli4_cfg_mhdr));
16675 	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
16676 			 LPFC_MBOX_OPCODE_MQ_CREATE_EXT,
16677 			 length, LPFC_SLI4_MBX_EMBED);
16678 
16679 	mq_create_ext = &mbox->u.mqe.un.mq_create_ext;
16680 	shdr = (union lpfc_sli4_cfg_shdr *) &mq_create_ext->header.cfg_shdr;
16681 	bf_set(lpfc_mbx_mq_create_ext_num_pages,
16682 	       &mq_create_ext->u.request, mq->page_count);
16683 	bf_set(lpfc_mbx_mq_create_ext_async_evt_link,
16684 	       &mq_create_ext->u.request, 1);
16685 	bf_set(lpfc_mbx_mq_create_ext_async_evt_fip,
16686 	       &mq_create_ext->u.request, 1);
16687 	bf_set(lpfc_mbx_mq_create_ext_async_evt_group5,
16688 	       &mq_create_ext->u.request, 1);
16689 	bf_set(lpfc_mbx_mq_create_ext_async_evt_fc,
16690 	       &mq_create_ext->u.request, 1);
16691 	bf_set(lpfc_mbx_mq_create_ext_async_evt_sli,
16692 	       &mq_create_ext->u.request, 1);
16693 	bf_set(lpfc_mq_context_valid, &mq_create_ext->u.request.context, 1);
16694 	bf_set(lpfc_mbox_hdr_version, &shdr->request,
16695 	       phba->sli4_hba.pc_sli4_params.mqv);
16696 	if (phba->sli4_hba.pc_sli4_params.mqv == LPFC_Q_CREATE_VERSION_1)
16697 		bf_set(lpfc_mbx_mq_create_ext_cq_id, &mq_create_ext->u.request,
16698 		       cq->queue_id);
16699 	else
16700 		bf_set(lpfc_mq_context_cq_id, &mq_create_ext->u.request.context,
16701 		       cq->queue_id);
16702 	switch (mq->entry_count) {
16703 	default:
16704 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
16705 				"0362 Unsupported MQ count. (%d)\n",
16706 				mq->entry_count);
16707 		if (mq->entry_count < 16) {
16708 			status = -EINVAL;
16709 			goto out;
16710 		}
16711 		fallthrough;	/* otherwise default to smallest count */
16712 	case 16:
16713 		bf_set(lpfc_mq_context_ring_size,
16714 		       &mq_create_ext->u.request.context,
16715 		       LPFC_MQ_RING_SIZE_16);
16716 		break;
16717 	case 32:
16718 		bf_set(lpfc_mq_context_ring_size,
16719 		       &mq_create_ext->u.request.context,
16720 		       LPFC_MQ_RING_SIZE_32);
16721 		break;
16722 	case 64:
16723 		bf_set(lpfc_mq_context_ring_size,
16724 		       &mq_create_ext->u.request.context,
16725 		       LPFC_MQ_RING_SIZE_64);
16726 		break;
16727 	case 128:
16728 		bf_set(lpfc_mq_context_ring_size,
16729 		       &mq_create_ext->u.request.context,
16730 		       LPFC_MQ_RING_SIZE_128);
16731 		break;
16732 	}
16733 	list_for_each_entry(dmabuf, &mq->page_list, list) {
16734 		memset(dmabuf->virt, 0, hw_page_size);
16735 		mq_create_ext->u.request.page[dmabuf->buffer_tag].addr_lo =
16736 					putPaddrLow(dmabuf->phys);
16737 		mq_create_ext->u.request.page[dmabuf->buffer_tag].addr_hi =
16738 					putPaddrHigh(dmabuf->phys);
16739 	}
16740 	rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL);
16741 	mq->queue_id = bf_get(lpfc_mbx_mq_create_q_id,
16742 			      &mq_create_ext->u.response);
16743 	if (rc != MBX_SUCCESS) {
16744 		lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
16745 				"2795 MQ_CREATE_EXT failed with "
16746 				"status x%x. Failback to MQ_CREATE.\n",
16747 				rc);
16748 		lpfc_mq_create_fb_init(phba, mq, mbox, cq);
16749 		mq_create = &mbox->u.mqe.un.mq_create;
16750 		rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL);
16751 		shdr = (union lpfc_sli4_cfg_shdr *) &mq_create->header.cfg_shdr;
16752 		mq->queue_id = bf_get(lpfc_mbx_mq_create_q_id,
16753 				      &mq_create->u.response);
16754 	}
16755 
16756 	/* The IOCTL status is embedded in the mailbox subheader. */
16757 	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
16758 	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
16759 	if (shdr_status || shdr_add_status || rc) {
16760 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
16761 				"2502 MQ_CREATE mailbox failed with "
16762 				"status x%x add_status x%x, mbx status x%x\n",
16763 				shdr_status, shdr_add_status, rc);
16764 		status = -ENXIO;
16765 		goto out;
16766 	}
16767 	if (mq->queue_id == 0xFFFF) {
16768 		status = -ENXIO;
16769 		goto out;
16770 	}
16771 	mq->type = LPFC_MQ;
16772 	mq->assoc_qid = cq->queue_id;
16773 	mq->subtype = subtype;
16774 	mq->host_index = 0;
16775 	mq->hba_index = 0;
16776 
16777 	/* link the mq onto the parent cq child list */
16778 	list_add_tail(&mq->list, &cq->child_list);
16779 out:
16780 	mempool_free(mbox, phba->mbox_mem_pool);
16781 	return status;
16782 }
16783 
16784 /**
16785  * lpfc_wq_create - Create a Work Queue on the HBA
16786  * @phba: HBA structure that indicates port to create a queue on.
16787  * @wq: The queue structure to use to create the work queue.
16788  * @cq: The completion queue to bind this work queue to.
16789  * @subtype: The subtype of the work queue indicating its functionality.
16790  *
16791  * This function creates a work queue, as detailed in @wq, on a port, described
16792  * by @phba by sending a WQ_CREATE mailbox command to the HBA.
16793  *
16794  * The @phba struct is used to send mailbox command to HBA. The @wq struct
16795  * is used to get the entry count and entry size that are necessary to
16796  * determine the number of pages to allocate and use for this queue. The @cq
16797  * is used to indicate which completion queue to bind this work queue to. This
16798  * function will send the WQ_CREATE mailbox command to the HBA to setup the
16799  * work queue. This function is asynchronous and will wait for the mailbox
16800  * command to finish before continuing.
16801  *
16802  * On success this function will return a zero. If unable to allocate enough
16803  * memory this function will return -ENOMEM. If the queue create mailbox command
16804  * fails this function will return -ENXIO.
16805  **/
16806 int
16807 lpfc_wq_create(struct lpfc_hba *phba, struct lpfc_queue *wq,
16808 	       struct lpfc_queue *cq, uint32_t subtype)
16809 {
16810 	struct lpfc_mbx_wq_create *wq_create;
16811 	struct lpfc_dmabuf *dmabuf;
16812 	LPFC_MBOXQ_t *mbox;
16813 	int rc, length, status = 0;
16814 	uint32_t shdr_status, shdr_add_status;
16815 	union lpfc_sli4_cfg_shdr *shdr;
16816 	uint32_t hw_page_size = phba->sli4_hba.pc_sli4_params.if_page_sz;
16817 	struct dma_address *page;
16818 	void __iomem *bar_memmap_p;
16819 	uint32_t db_offset;
16820 	uint16_t pci_barset;
16821 	uint8_t dpp_barset;
16822 	uint32_t dpp_offset;
16823 	uint8_t wq_create_version;
16824 #ifdef CONFIG_X86
16825 	unsigned long pg_addr;
16826 #endif
16827 
16828 	/* sanity check on queue memory */
16829 	if (!wq || !cq)
16830 		return -ENODEV;
16831 	if (!phba->sli4_hba.pc_sli4_params.supported)
16832 		hw_page_size = wq->page_size;
16833 
16834 	mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
16835 	if (!mbox)
16836 		return -ENOMEM;
16837 	length = (sizeof(struct lpfc_mbx_wq_create) -
16838 		  sizeof(struct lpfc_sli4_cfg_mhdr));
16839 	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_FCOE,
16840 			 LPFC_MBOX_OPCODE_FCOE_WQ_CREATE,
16841 			 length, LPFC_SLI4_MBX_EMBED);
16842 	wq_create = &mbox->u.mqe.un.wq_create;
16843 	shdr = (union lpfc_sli4_cfg_shdr *) &wq_create->header.cfg_shdr;
16844 	bf_set(lpfc_mbx_wq_create_num_pages, &wq_create->u.request,
16845 		    wq->page_count);
16846 	bf_set(lpfc_mbx_wq_create_cq_id, &wq_create->u.request,
16847 		    cq->queue_id);
16848 
16849 	/* wqv is the earliest version supported, NOT the latest */
16850 	bf_set(lpfc_mbox_hdr_version, &shdr->request,
16851 	       phba->sli4_hba.pc_sli4_params.wqv);
16852 
16853 	if ((phba->sli4_hba.pc_sli4_params.wqsize & LPFC_WQ_SZ128_SUPPORT) ||
16854 	    (wq->page_size > SLI4_PAGE_SIZE))
16855 		wq_create_version = LPFC_Q_CREATE_VERSION_1;
16856 	else
16857 		wq_create_version = LPFC_Q_CREATE_VERSION_0;
16858 
16859 	switch (wq_create_version) {
16860 	case LPFC_Q_CREATE_VERSION_1:
16861 		bf_set(lpfc_mbx_wq_create_wqe_count, &wq_create->u.request_1,
16862 		       wq->entry_count);
16863 		bf_set(lpfc_mbox_hdr_version, &shdr->request,
16864 		       LPFC_Q_CREATE_VERSION_1);
16865 
16866 		switch (wq->entry_size) {
16867 		default:
16868 		case 64:
16869 			bf_set(lpfc_mbx_wq_create_wqe_size,
16870 			       &wq_create->u.request_1,
16871 			       LPFC_WQ_WQE_SIZE_64);
16872 			break;
16873 		case 128:
16874 			bf_set(lpfc_mbx_wq_create_wqe_size,
16875 			       &wq_create->u.request_1,
16876 			       LPFC_WQ_WQE_SIZE_128);
16877 			break;
16878 		}
16879 		/* Request DPP by default */
16880 		bf_set(lpfc_mbx_wq_create_dpp_req, &wq_create->u.request_1, 1);
16881 		bf_set(lpfc_mbx_wq_create_page_size,
16882 		       &wq_create->u.request_1,
16883 		       (wq->page_size / SLI4_PAGE_SIZE));
16884 		page = wq_create->u.request_1.page;
16885 		break;
16886 	default:
16887 		page = wq_create->u.request.page;
16888 		break;
16889 	}
16890 
16891 	list_for_each_entry(dmabuf, &wq->page_list, list) {
16892 		memset(dmabuf->virt, 0, hw_page_size);
16893 		page[dmabuf->buffer_tag].addr_lo = putPaddrLow(dmabuf->phys);
16894 		page[dmabuf->buffer_tag].addr_hi = putPaddrHigh(dmabuf->phys);
16895 	}
16896 
16897 	if (phba->sli4_hba.fw_func_mode & LPFC_DUA_MODE)
16898 		bf_set(lpfc_mbx_wq_create_dua, &wq_create->u.request, 1);
16899 
16900 	rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL);
16901 	/* The IOCTL status is embedded in the mailbox subheader. */
16902 	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
16903 	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
16904 	if (shdr_status || shdr_add_status || rc) {
16905 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
16906 				"2503 WQ_CREATE mailbox failed with "
16907 				"status x%x add_status x%x, mbx status x%x\n",
16908 				shdr_status, shdr_add_status, rc);
16909 		status = -ENXIO;
16910 		goto out;
16911 	}
16912 
16913 	if (wq_create_version == LPFC_Q_CREATE_VERSION_0)
16914 		wq->queue_id = bf_get(lpfc_mbx_wq_create_q_id,
16915 					&wq_create->u.response);
16916 	else
16917 		wq->queue_id = bf_get(lpfc_mbx_wq_create_v1_q_id,
16918 					&wq_create->u.response_1);
16919 
16920 	if (wq->queue_id == 0xFFFF) {
16921 		status = -ENXIO;
16922 		goto out;
16923 	}
16924 
16925 	wq->db_format = LPFC_DB_LIST_FORMAT;
16926 	if (wq_create_version == LPFC_Q_CREATE_VERSION_0) {
16927 		if (phba->sli4_hba.fw_func_mode & LPFC_DUA_MODE) {
16928 			wq->db_format = bf_get(lpfc_mbx_wq_create_db_format,
16929 					       &wq_create->u.response);
16930 			if ((wq->db_format != LPFC_DB_LIST_FORMAT) &&
16931 			    (wq->db_format != LPFC_DB_RING_FORMAT)) {
16932 				lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
16933 						"3265 WQ[%d] doorbell format "
16934 						"not supported: x%x\n",
16935 						wq->queue_id, wq->db_format);
16936 				status = -EINVAL;
16937 				goto out;
16938 			}
16939 			pci_barset = bf_get(lpfc_mbx_wq_create_bar_set,
16940 					    &wq_create->u.response);
16941 			bar_memmap_p = lpfc_dual_chute_pci_bar_map(phba,
16942 								   pci_barset);
16943 			if (!bar_memmap_p) {
16944 				lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
16945 						"3263 WQ[%d] failed to memmap "
16946 						"pci barset:x%x\n",
16947 						wq->queue_id, pci_barset);
16948 				status = -ENOMEM;
16949 				goto out;
16950 			}
16951 			db_offset = wq_create->u.response.doorbell_offset;
16952 			if ((db_offset != LPFC_ULP0_WQ_DOORBELL) &&
16953 			    (db_offset != LPFC_ULP1_WQ_DOORBELL)) {
16954 				lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
16955 						"3252 WQ[%d] doorbell offset "
16956 						"not supported: x%x\n",
16957 						wq->queue_id, db_offset);
16958 				status = -EINVAL;
16959 				goto out;
16960 			}
16961 			wq->db_regaddr = bar_memmap_p + db_offset;
16962 			lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
16963 					"3264 WQ[%d]: barset:x%x, offset:x%x, "
16964 					"format:x%x\n", wq->queue_id,
16965 					pci_barset, db_offset, wq->db_format);
16966 		} else
16967 			wq->db_regaddr = phba->sli4_hba.WQDBregaddr;
16968 	} else {
16969 		/* Check if DPP was honored by the firmware */
16970 		wq->dpp_enable = bf_get(lpfc_mbx_wq_create_dpp_rsp,
16971 				    &wq_create->u.response_1);
16972 		if (wq->dpp_enable) {
16973 			pci_barset = bf_get(lpfc_mbx_wq_create_v1_bar_set,
16974 					    &wq_create->u.response_1);
16975 			bar_memmap_p = lpfc_dual_chute_pci_bar_map(phba,
16976 								   pci_barset);
16977 			if (!bar_memmap_p) {
16978 				lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
16979 						"3267 WQ[%d] failed to memmap "
16980 						"pci barset:x%x\n",
16981 						wq->queue_id, pci_barset);
16982 				status = -ENOMEM;
16983 				goto out;
16984 			}
16985 			db_offset = wq_create->u.response_1.doorbell_offset;
16986 			wq->db_regaddr = bar_memmap_p + db_offset;
16987 			wq->dpp_id = bf_get(lpfc_mbx_wq_create_dpp_id,
16988 					    &wq_create->u.response_1);
16989 			dpp_barset = bf_get(lpfc_mbx_wq_create_dpp_bar,
16990 					    &wq_create->u.response_1);
16991 			bar_memmap_p = lpfc_dual_chute_pci_bar_map(phba,
16992 								   dpp_barset);
16993 			if (!bar_memmap_p) {
16994 				lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
16995 						"3268 WQ[%d] failed to memmap "
16996 						"pci barset:x%x\n",
16997 						wq->queue_id, dpp_barset);
16998 				status = -ENOMEM;
16999 				goto out;
17000 			}
17001 			dpp_offset = wq_create->u.response_1.dpp_offset;
17002 			wq->dpp_regaddr = bar_memmap_p + dpp_offset;
17003 			lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
17004 					"3271 WQ[%d]: barset:x%x, offset:x%x, "
17005 					"dpp_id:x%x dpp_barset:x%x "
17006 					"dpp_offset:x%x\n",
17007 					wq->queue_id, pci_barset, db_offset,
17008 					wq->dpp_id, dpp_barset, dpp_offset);
17009 
17010 #ifdef CONFIG_X86
17011 			/* Enable combined writes for DPP aperture */
17012 			pg_addr = (unsigned long)(wq->dpp_regaddr) & PAGE_MASK;
17013 			rc = set_memory_wc(pg_addr, 1);
17014 			if (rc) {
17015 				lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
17016 					"3272 Cannot setup Combined "
17017 					"Write on WQ[%d] - disable DPP\n",
17018 					wq->queue_id);
17019 				phba->cfg_enable_dpp = 0;
17020 			}
17021 #else
17022 			phba->cfg_enable_dpp = 0;
17023 #endif
17024 		} else
17025 			wq->db_regaddr = phba->sli4_hba.WQDBregaddr;
17026 	}
17027 	wq->pring = kzalloc(sizeof(struct lpfc_sli_ring), GFP_KERNEL);
17028 	if (wq->pring == NULL) {
17029 		status = -ENOMEM;
17030 		goto out;
17031 	}
17032 	wq->type = LPFC_WQ;
17033 	wq->assoc_qid = cq->queue_id;
17034 	wq->subtype = subtype;
17035 	wq->host_index = 0;
17036 	wq->hba_index = 0;
17037 	wq->notify_interval = LPFC_WQ_NOTIFY_INTRVL;
17038 
17039 	/* link the wq onto the parent cq child list */
17040 	list_add_tail(&wq->list, &cq->child_list);
17041 out:
17042 	mempool_free(mbox, phba->mbox_mem_pool);
17043 	return status;
17044 }
17045 
17046 /**
17047  * lpfc_rq_create - Create a Receive Queue on the HBA
17048  * @phba: HBA structure that indicates port to create a queue on.
17049  * @hrq: The queue structure to use to create the header receive queue.
17050  * @drq: The queue structure to use to create the data receive queue.
17051  * @cq: The completion queue to bind this work queue to.
17052  * @subtype: The subtype of the work queue indicating its functionality.
17053  *
17054  * This function creates a receive buffer queue pair , as detailed in @hrq and
17055  * @drq, on a port, described by @phba by sending a RQ_CREATE mailbox command
17056  * to the HBA.
17057  *
17058  * The @phba struct is used to send mailbox command to HBA. The @drq and @hrq
17059  * struct is used to get the entry count that is necessary to determine the
17060  * number of pages to use for this queue. The @cq is used to indicate which
17061  * completion queue to bind received buffers that are posted to these queues to.
17062  * This function will send the RQ_CREATE mailbox command to the HBA to setup the
17063  * receive queue pair. This function is asynchronous and will wait for the
17064  * mailbox command to finish before continuing.
17065  *
17066  * On success this function will return a zero. If unable to allocate enough
17067  * memory this function will return -ENOMEM. If the queue create mailbox command
17068  * fails this function will return -ENXIO.
17069  **/
17070 int
17071 lpfc_rq_create(struct lpfc_hba *phba, struct lpfc_queue *hrq,
17072 	       struct lpfc_queue *drq, struct lpfc_queue *cq, uint32_t subtype)
17073 {
17074 	struct lpfc_mbx_rq_create *rq_create;
17075 	struct lpfc_dmabuf *dmabuf;
17076 	LPFC_MBOXQ_t *mbox;
17077 	int rc, length, status = 0;
17078 	uint32_t shdr_status, shdr_add_status;
17079 	union lpfc_sli4_cfg_shdr *shdr;
17080 	uint32_t hw_page_size = phba->sli4_hba.pc_sli4_params.if_page_sz;
17081 	void __iomem *bar_memmap_p;
17082 	uint32_t db_offset;
17083 	uint16_t pci_barset;
17084 
17085 	/* sanity check on queue memory */
17086 	if (!hrq || !drq || !cq)
17087 		return -ENODEV;
17088 	if (!phba->sli4_hba.pc_sli4_params.supported)
17089 		hw_page_size = SLI4_PAGE_SIZE;
17090 
17091 	if (hrq->entry_count != drq->entry_count)
17092 		return -EINVAL;
17093 	mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
17094 	if (!mbox)
17095 		return -ENOMEM;
17096 	length = (sizeof(struct lpfc_mbx_rq_create) -
17097 		  sizeof(struct lpfc_sli4_cfg_mhdr));
17098 	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_FCOE,
17099 			 LPFC_MBOX_OPCODE_FCOE_RQ_CREATE,
17100 			 length, LPFC_SLI4_MBX_EMBED);
17101 	rq_create = &mbox->u.mqe.un.rq_create;
17102 	shdr = (union lpfc_sli4_cfg_shdr *) &rq_create->header.cfg_shdr;
17103 	bf_set(lpfc_mbox_hdr_version, &shdr->request,
17104 	       phba->sli4_hba.pc_sli4_params.rqv);
17105 	if (phba->sli4_hba.pc_sli4_params.rqv == LPFC_Q_CREATE_VERSION_1) {
17106 		bf_set(lpfc_rq_context_rqe_count_1,
17107 		       &rq_create->u.request.context,
17108 		       hrq->entry_count);
17109 		rq_create->u.request.context.buffer_size = LPFC_HDR_BUF_SIZE;
17110 		bf_set(lpfc_rq_context_rqe_size,
17111 		       &rq_create->u.request.context,
17112 		       LPFC_RQE_SIZE_8);
17113 		bf_set(lpfc_rq_context_page_size,
17114 		       &rq_create->u.request.context,
17115 		       LPFC_RQ_PAGE_SIZE_4096);
17116 	} else {
17117 		switch (hrq->entry_count) {
17118 		default:
17119 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
17120 					"2535 Unsupported RQ count. (%d)\n",
17121 					hrq->entry_count);
17122 			if (hrq->entry_count < 512) {
17123 				status = -EINVAL;
17124 				goto out;
17125 			}
17126 			fallthrough;	/* otherwise default to smallest count */
17127 		case 512:
17128 			bf_set(lpfc_rq_context_rqe_count,
17129 			       &rq_create->u.request.context,
17130 			       LPFC_RQ_RING_SIZE_512);
17131 			break;
17132 		case 1024:
17133 			bf_set(lpfc_rq_context_rqe_count,
17134 			       &rq_create->u.request.context,
17135 			       LPFC_RQ_RING_SIZE_1024);
17136 			break;
17137 		case 2048:
17138 			bf_set(lpfc_rq_context_rqe_count,
17139 			       &rq_create->u.request.context,
17140 			       LPFC_RQ_RING_SIZE_2048);
17141 			break;
17142 		case 4096:
17143 			bf_set(lpfc_rq_context_rqe_count,
17144 			       &rq_create->u.request.context,
17145 			       LPFC_RQ_RING_SIZE_4096);
17146 			break;
17147 		}
17148 		bf_set(lpfc_rq_context_buf_size, &rq_create->u.request.context,
17149 		       LPFC_HDR_BUF_SIZE);
17150 	}
17151 	bf_set(lpfc_rq_context_cq_id, &rq_create->u.request.context,
17152 	       cq->queue_id);
17153 	bf_set(lpfc_mbx_rq_create_num_pages, &rq_create->u.request,
17154 	       hrq->page_count);
17155 	list_for_each_entry(dmabuf, &hrq->page_list, list) {
17156 		memset(dmabuf->virt, 0, hw_page_size);
17157 		rq_create->u.request.page[dmabuf->buffer_tag].addr_lo =
17158 					putPaddrLow(dmabuf->phys);
17159 		rq_create->u.request.page[dmabuf->buffer_tag].addr_hi =
17160 					putPaddrHigh(dmabuf->phys);
17161 	}
17162 	if (phba->sli4_hba.fw_func_mode & LPFC_DUA_MODE)
17163 		bf_set(lpfc_mbx_rq_create_dua, &rq_create->u.request, 1);
17164 
17165 	rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL);
17166 	/* The IOCTL status is embedded in the mailbox subheader. */
17167 	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
17168 	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
17169 	if (shdr_status || shdr_add_status || rc) {
17170 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
17171 				"2504 RQ_CREATE mailbox failed with "
17172 				"status x%x add_status x%x, mbx status x%x\n",
17173 				shdr_status, shdr_add_status, rc);
17174 		status = -ENXIO;
17175 		goto out;
17176 	}
17177 	hrq->queue_id = bf_get(lpfc_mbx_rq_create_q_id, &rq_create->u.response);
17178 	if (hrq->queue_id == 0xFFFF) {
17179 		status = -ENXIO;
17180 		goto out;
17181 	}
17182 
17183 	if (phba->sli4_hba.fw_func_mode & LPFC_DUA_MODE) {
17184 		hrq->db_format = bf_get(lpfc_mbx_rq_create_db_format,
17185 					&rq_create->u.response);
17186 		if ((hrq->db_format != LPFC_DB_LIST_FORMAT) &&
17187 		    (hrq->db_format != LPFC_DB_RING_FORMAT)) {
17188 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
17189 					"3262 RQ [%d] doorbell format not "
17190 					"supported: x%x\n", hrq->queue_id,
17191 					hrq->db_format);
17192 			status = -EINVAL;
17193 			goto out;
17194 		}
17195 
17196 		pci_barset = bf_get(lpfc_mbx_rq_create_bar_set,
17197 				    &rq_create->u.response);
17198 		bar_memmap_p = lpfc_dual_chute_pci_bar_map(phba, pci_barset);
17199 		if (!bar_memmap_p) {
17200 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
17201 					"3269 RQ[%d] failed to memmap pci "
17202 					"barset:x%x\n", hrq->queue_id,
17203 					pci_barset);
17204 			status = -ENOMEM;
17205 			goto out;
17206 		}
17207 
17208 		db_offset = rq_create->u.response.doorbell_offset;
17209 		if ((db_offset != LPFC_ULP0_RQ_DOORBELL) &&
17210 		    (db_offset != LPFC_ULP1_RQ_DOORBELL)) {
17211 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
17212 					"3270 RQ[%d] doorbell offset not "
17213 					"supported: x%x\n", hrq->queue_id,
17214 					db_offset);
17215 			status = -EINVAL;
17216 			goto out;
17217 		}
17218 		hrq->db_regaddr = bar_memmap_p + db_offset;
17219 		lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
17220 				"3266 RQ[qid:%d]: barset:x%x, offset:x%x, "
17221 				"format:x%x\n", hrq->queue_id, pci_barset,
17222 				db_offset, hrq->db_format);
17223 	} else {
17224 		hrq->db_format = LPFC_DB_RING_FORMAT;
17225 		hrq->db_regaddr = phba->sli4_hba.RQDBregaddr;
17226 	}
17227 	hrq->type = LPFC_HRQ;
17228 	hrq->assoc_qid = cq->queue_id;
17229 	hrq->subtype = subtype;
17230 	hrq->host_index = 0;
17231 	hrq->hba_index = 0;
17232 	hrq->notify_interval = LPFC_RQ_NOTIFY_INTRVL;
17233 
17234 	/* now create the data queue */
17235 	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_FCOE,
17236 			 LPFC_MBOX_OPCODE_FCOE_RQ_CREATE,
17237 			 length, LPFC_SLI4_MBX_EMBED);
17238 	bf_set(lpfc_mbox_hdr_version, &shdr->request,
17239 	       phba->sli4_hba.pc_sli4_params.rqv);
17240 	if (phba->sli4_hba.pc_sli4_params.rqv == LPFC_Q_CREATE_VERSION_1) {
17241 		bf_set(lpfc_rq_context_rqe_count_1,
17242 		       &rq_create->u.request.context, hrq->entry_count);
17243 		if (subtype == LPFC_NVMET)
17244 			rq_create->u.request.context.buffer_size =
17245 				LPFC_NVMET_DATA_BUF_SIZE;
17246 		else
17247 			rq_create->u.request.context.buffer_size =
17248 				LPFC_DATA_BUF_SIZE;
17249 		bf_set(lpfc_rq_context_rqe_size, &rq_create->u.request.context,
17250 		       LPFC_RQE_SIZE_8);
17251 		bf_set(lpfc_rq_context_page_size, &rq_create->u.request.context,
17252 		       (PAGE_SIZE/SLI4_PAGE_SIZE));
17253 	} else {
17254 		switch (drq->entry_count) {
17255 		default:
17256 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
17257 					"2536 Unsupported RQ count. (%d)\n",
17258 					drq->entry_count);
17259 			if (drq->entry_count < 512) {
17260 				status = -EINVAL;
17261 				goto out;
17262 			}
17263 			fallthrough;	/* otherwise default to smallest count */
17264 		case 512:
17265 			bf_set(lpfc_rq_context_rqe_count,
17266 			       &rq_create->u.request.context,
17267 			       LPFC_RQ_RING_SIZE_512);
17268 			break;
17269 		case 1024:
17270 			bf_set(lpfc_rq_context_rqe_count,
17271 			       &rq_create->u.request.context,
17272 			       LPFC_RQ_RING_SIZE_1024);
17273 			break;
17274 		case 2048:
17275 			bf_set(lpfc_rq_context_rqe_count,
17276 			       &rq_create->u.request.context,
17277 			       LPFC_RQ_RING_SIZE_2048);
17278 			break;
17279 		case 4096:
17280 			bf_set(lpfc_rq_context_rqe_count,
17281 			       &rq_create->u.request.context,
17282 			       LPFC_RQ_RING_SIZE_4096);
17283 			break;
17284 		}
17285 		if (subtype == LPFC_NVMET)
17286 			bf_set(lpfc_rq_context_buf_size,
17287 			       &rq_create->u.request.context,
17288 			       LPFC_NVMET_DATA_BUF_SIZE);
17289 		else
17290 			bf_set(lpfc_rq_context_buf_size,
17291 			       &rq_create->u.request.context,
17292 			       LPFC_DATA_BUF_SIZE);
17293 	}
17294 	bf_set(lpfc_rq_context_cq_id, &rq_create->u.request.context,
17295 	       cq->queue_id);
17296 	bf_set(lpfc_mbx_rq_create_num_pages, &rq_create->u.request,
17297 	       drq->page_count);
17298 	list_for_each_entry(dmabuf, &drq->page_list, list) {
17299 		rq_create->u.request.page[dmabuf->buffer_tag].addr_lo =
17300 					putPaddrLow(dmabuf->phys);
17301 		rq_create->u.request.page[dmabuf->buffer_tag].addr_hi =
17302 					putPaddrHigh(dmabuf->phys);
17303 	}
17304 	if (phba->sli4_hba.fw_func_mode & LPFC_DUA_MODE)
17305 		bf_set(lpfc_mbx_rq_create_dua, &rq_create->u.request, 1);
17306 	rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL);
17307 	/* The IOCTL status is embedded in the mailbox subheader. */
17308 	shdr = (union lpfc_sli4_cfg_shdr *) &rq_create->header.cfg_shdr;
17309 	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
17310 	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
17311 	if (shdr_status || shdr_add_status || rc) {
17312 		status = -ENXIO;
17313 		goto out;
17314 	}
17315 	drq->queue_id = bf_get(lpfc_mbx_rq_create_q_id, &rq_create->u.response);
17316 	if (drq->queue_id == 0xFFFF) {
17317 		status = -ENXIO;
17318 		goto out;
17319 	}
17320 	drq->type = LPFC_DRQ;
17321 	drq->assoc_qid = cq->queue_id;
17322 	drq->subtype = subtype;
17323 	drq->host_index = 0;
17324 	drq->hba_index = 0;
17325 	drq->notify_interval = LPFC_RQ_NOTIFY_INTRVL;
17326 
17327 	/* link the header and data RQs onto the parent cq child list */
17328 	list_add_tail(&hrq->list, &cq->child_list);
17329 	list_add_tail(&drq->list, &cq->child_list);
17330 
17331 out:
17332 	mempool_free(mbox, phba->mbox_mem_pool);
17333 	return status;
17334 }
17335 
17336 /**
17337  * lpfc_mrq_create - Create MRQ Receive Queues on the HBA
17338  * @phba: HBA structure that indicates port to create a queue on.
17339  * @hrqp: The queue structure array to use to create the header receive queues.
17340  * @drqp: The queue structure array to use to create the data receive queues.
17341  * @cqp: The completion queue array to bind these receive queues to.
17342  * @subtype: Functional purpose of the queue (MBOX, IO, ELS, NVMET, etc).
17343  *
17344  * This function creates a receive buffer queue pair , as detailed in @hrq and
17345  * @drq, on a port, described by @phba by sending a RQ_CREATE mailbox command
17346  * to the HBA.
17347  *
17348  * The @phba struct is used to send mailbox command to HBA. The @drq and @hrq
17349  * struct is used to get the entry count that is necessary to determine the
17350  * number of pages to use for this queue. The @cq is used to indicate which
17351  * completion queue to bind received buffers that are posted to these queues to.
17352  * This function will send the RQ_CREATE mailbox command to the HBA to setup the
17353  * receive queue pair. This function is asynchronous and will wait for the
17354  * mailbox command to finish before continuing.
17355  *
17356  * On success this function will return a zero. If unable to allocate enough
17357  * memory this function will return -ENOMEM. If the queue create mailbox command
17358  * fails this function will return -ENXIO.
17359  **/
17360 int
17361 lpfc_mrq_create(struct lpfc_hba *phba, struct lpfc_queue **hrqp,
17362 		struct lpfc_queue **drqp, struct lpfc_queue **cqp,
17363 		uint32_t subtype)
17364 {
17365 	struct lpfc_queue *hrq, *drq, *cq;
17366 	struct lpfc_mbx_rq_create_v2 *rq_create;
17367 	struct lpfc_dmabuf *dmabuf;
17368 	LPFC_MBOXQ_t *mbox;
17369 	int rc, length, alloclen, status = 0;
17370 	int cnt, idx, numrq, page_idx = 0;
17371 	uint32_t shdr_status, shdr_add_status;
17372 	union lpfc_sli4_cfg_shdr *shdr;
17373 	uint32_t hw_page_size = phba->sli4_hba.pc_sli4_params.if_page_sz;
17374 
17375 	numrq = phba->cfg_nvmet_mrq;
17376 	/* sanity check on array memory */
17377 	if (!hrqp || !drqp || !cqp || !numrq)
17378 		return -ENODEV;
17379 	if (!phba->sli4_hba.pc_sli4_params.supported)
17380 		hw_page_size = SLI4_PAGE_SIZE;
17381 
17382 	mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
17383 	if (!mbox)
17384 		return -ENOMEM;
17385 
17386 	length = sizeof(struct lpfc_mbx_rq_create_v2);
17387 	length += ((2 * numrq * hrqp[0]->page_count) *
17388 		   sizeof(struct dma_address));
17389 
17390 	alloclen = lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_FCOE,
17391 				    LPFC_MBOX_OPCODE_FCOE_RQ_CREATE, length,
17392 				    LPFC_SLI4_MBX_NEMBED);
17393 	if (alloclen < length) {
17394 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
17395 				"3099 Allocated DMA memory size (%d) is "
17396 				"less than the requested DMA memory size "
17397 				"(%d)\n", alloclen, length);
17398 		status = -ENOMEM;
17399 		goto out;
17400 	}
17401 
17402 
17403 
17404 	rq_create = mbox->sge_array->addr[0];
17405 	shdr = (union lpfc_sli4_cfg_shdr *)&rq_create->cfg_shdr;
17406 
17407 	bf_set(lpfc_mbox_hdr_version, &shdr->request, LPFC_Q_CREATE_VERSION_2);
17408 	cnt = 0;
17409 
17410 	for (idx = 0; idx < numrq; idx++) {
17411 		hrq = hrqp[idx];
17412 		drq = drqp[idx];
17413 		cq  = cqp[idx];
17414 
17415 		/* sanity check on queue memory */
17416 		if (!hrq || !drq || !cq) {
17417 			status = -ENODEV;
17418 			goto out;
17419 		}
17420 
17421 		if (hrq->entry_count != drq->entry_count) {
17422 			status = -EINVAL;
17423 			goto out;
17424 		}
17425 
17426 		if (idx == 0) {
17427 			bf_set(lpfc_mbx_rq_create_num_pages,
17428 			       &rq_create->u.request,
17429 			       hrq->page_count);
17430 			bf_set(lpfc_mbx_rq_create_rq_cnt,
17431 			       &rq_create->u.request, (numrq * 2));
17432 			bf_set(lpfc_mbx_rq_create_dnb, &rq_create->u.request,
17433 			       1);
17434 			bf_set(lpfc_rq_context_base_cq,
17435 			       &rq_create->u.request.context,
17436 			       cq->queue_id);
17437 			bf_set(lpfc_rq_context_data_size,
17438 			       &rq_create->u.request.context,
17439 			       LPFC_NVMET_DATA_BUF_SIZE);
17440 			bf_set(lpfc_rq_context_hdr_size,
17441 			       &rq_create->u.request.context,
17442 			       LPFC_HDR_BUF_SIZE);
17443 			bf_set(lpfc_rq_context_rqe_count_1,
17444 			       &rq_create->u.request.context,
17445 			       hrq->entry_count);
17446 			bf_set(lpfc_rq_context_rqe_size,
17447 			       &rq_create->u.request.context,
17448 			       LPFC_RQE_SIZE_8);
17449 			bf_set(lpfc_rq_context_page_size,
17450 			       &rq_create->u.request.context,
17451 			       (PAGE_SIZE/SLI4_PAGE_SIZE));
17452 		}
17453 		rc = 0;
17454 		list_for_each_entry(dmabuf, &hrq->page_list, list) {
17455 			memset(dmabuf->virt, 0, hw_page_size);
17456 			cnt = page_idx + dmabuf->buffer_tag;
17457 			rq_create->u.request.page[cnt].addr_lo =
17458 					putPaddrLow(dmabuf->phys);
17459 			rq_create->u.request.page[cnt].addr_hi =
17460 					putPaddrHigh(dmabuf->phys);
17461 			rc++;
17462 		}
17463 		page_idx += rc;
17464 
17465 		rc = 0;
17466 		list_for_each_entry(dmabuf, &drq->page_list, list) {
17467 			memset(dmabuf->virt, 0, hw_page_size);
17468 			cnt = page_idx + dmabuf->buffer_tag;
17469 			rq_create->u.request.page[cnt].addr_lo =
17470 					putPaddrLow(dmabuf->phys);
17471 			rq_create->u.request.page[cnt].addr_hi =
17472 					putPaddrHigh(dmabuf->phys);
17473 			rc++;
17474 		}
17475 		page_idx += rc;
17476 
17477 		hrq->db_format = LPFC_DB_RING_FORMAT;
17478 		hrq->db_regaddr = phba->sli4_hba.RQDBregaddr;
17479 		hrq->type = LPFC_HRQ;
17480 		hrq->assoc_qid = cq->queue_id;
17481 		hrq->subtype = subtype;
17482 		hrq->host_index = 0;
17483 		hrq->hba_index = 0;
17484 		hrq->notify_interval = LPFC_RQ_NOTIFY_INTRVL;
17485 
17486 		drq->db_format = LPFC_DB_RING_FORMAT;
17487 		drq->db_regaddr = phba->sli4_hba.RQDBregaddr;
17488 		drq->type = LPFC_DRQ;
17489 		drq->assoc_qid = cq->queue_id;
17490 		drq->subtype = subtype;
17491 		drq->host_index = 0;
17492 		drq->hba_index = 0;
17493 		drq->notify_interval = LPFC_RQ_NOTIFY_INTRVL;
17494 
17495 		list_add_tail(&hrq->list, &cq->child_list);
17496 		list_add_tail(&drq->list, &cq->child_list);
17497 	}
17498 
17499 	rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL);
17500 	/* The IOCTL status is embedded in the mailbox subheader. */
17501 	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
17502 	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
17503 	if (shdr_status || shdr_add_status || rc) {
17504 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
17505 				"3120 RQ_CREATE mailbox failed with "
17506 				"status x%x add_status x%x, mbx status x%x\n",
17507 				shdr_status, shdr_add_status, rc);
17508 		status = -ENXIO;
17509 		goto out;
17510 	}
17511 	rc = bf_get(lpfc_mbx_rq_create_q_id, &rq_create->u.response);
17512 	if (rc == 0xFFFF) {
17513 		status = -ENXIO;
17514 		goto out;
17515 	}
17516 
17517 	/* Initialize all RQs with associated queue id */
17518 	for (idx = 0; idx < numrq; idx++) {
17519 		hrq = hrqp[idx];
17520 		hrq->queue_id = rc + (2 * idx);
17521 		drq = drqp[idx];
17522 		drq->queue_id = rc + (2 * idx) + 1;
17523 	}
17524 
17525 out:
17526 	lpfc_sli4_mbox_cmd_free(phba, mbox);
17527 	return status;
17528 }
17529 
17530 /**
17531  * lpfc_eq_destroy - Destroy an event Queue on the HBA
17532  * @phba: HBA structure that indicates port to destroy a queue on.
17533  * @eq: The queue structure associated with the queue to destroy.
17534  *
17535  * This function destroys a queue, as detailed in @eq by sending an mailbox
17536  * command, specific to the type of queue, to the HBA.
17537  *
17538  * The @eq struct is used to get the queue ID of the queue to destroy.
17539  *
17540  * On success this function will return a zero. If the queue destroy mailbox
17541  * command fails this function will return -ENXIO.
17542  **/
17543 int
17544 lpfc_eq_destroy(struct lpfc_hba *phba, struct lpfc_queue *eq)
17545 {
17546 	LPFC_MBOXQ_t *mbox;
17547 	int rc, length, status = 0;
17548 	uint32_t shdr_status, shdr_add_status;
17549 	union lpfc_sli4_cfg_shdr *shdr;
17550 
17551 	/* sanity check on queue memory */
17552 	if (!eq)
17553 		return -ENODEV;
17554 
17555 	mbox = mempool_alloc(eq->phba->mbox_mem_pool, GFP_KERNEL);
17556 	if (!mbox)
17557 		return -ENOMEM;
17558 	length = (sizeof(struct lpfc_mbx_eq_destroy) -
17559 		  sizeof(struct lpfc_sli4_cfg_mhdr));
17560 	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
17561 			 LPFC_MBOX_OPCODE_EQ_DESTROY,
17562 			 length, LPFC_SLI4_MBX_EMBED);
17563 	bf_set(lpfc_mbx_eq_destroy_q_id, &mbox->u.mqe.un.eq_destroy.u.request,
17564 	       eq->queue_id);
17565 	mbox->vport = eq->phba->pport;
17566 	mbox->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
17567 
17568 	rc = lpfc_sli_issue_mbox(eq->phba, mbox, MBX_POLL);
17569 	/* The IOCTL status is embedded in the mailbox subheader. */
17570 	shdr = (union lpfc_sli4_cfg_shdr *)
17571 		&mbox->u.mqe.un.eq_destroy.header.cfg_shdr;
17572 	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
17573 	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
17574 	if (shdr_status || shdr_add_status || rc) {
17575 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
17576 				"2505 EQ_DESTROY mailbox failed with "
17577 				"status x%x add_status x%x, mbx status x%x\n",
17578 				shdr_status, shdr_add_status, rc);
17579 		status = -ENXIO;
17580 	}
17581 
17582 	/* Remove eq from any list */
17583 	list_del_init(&eq->list);
17584 	mempool_free(mbox, eq->phba->mbox_mem_pool);
17585 	return status;
17586 }
17587 
17588 /**
17589  * lpfc_cq_destroy - Destroy a Completion Queue on the HBA
17590  * @phba: HBA structure that indicates port to destroy a queue on.
17591  * @cq: The queue structure associated with the queue to destroy.
17592  *
17593  * This function destroys a queue, as detailed in @cq by sending an mailbox
17594  * command, specific to the type of queue, to the HBA.
17595  *
17596  * The @cq struct is used to get the queue ID of the queue to destroy.
17597  *
17598  * On success this function will return a zero. If the queue destroy mailbox
17599  * command fails this function will return -ENXIO.
17600  **/
17601 int
17602 lpfc_cq_destroy(struct lpfc_hba *phba, struct lpfc_queue *cq)
17603 {
17604 	LPFC_MBOXQ_t *mbox;
17605 	int rc, length, status = 0;
17606 	uint32_t shdr_status, shdr_add_status;
17607 	union lpfc_sli4_cfg_shdr *shdr;
17608 
17609 	/* sanity check on queue memory */
17610 	if (!cq)
17611 		return -ENODEV;
17612 	mbox = mempool_alloc(cq->phba->mbox_mem_pool, GFP_KERNEL);
17613 	if (!mbox)
17614 		return -ENOMEM;
17615 	length = (sizeof(struct lpfc_mbx_cq_destroy) -
17616 		  sizeof(struct lpfc_sli4_cfg_mhdr));
17617 	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
17618 			 LPFC_MBOX_OPCODE_CQ_DESTROY,
17619 			 length, LPFC_SLI4_MBX_EMBED);
17620 	bf_set(lpfc_mbx_cq_destroy_q_id, &mbox->u.mqe.un.cq_destroy.u.request,
17621 	       cq->queue_id);
17622 	mbox->vport = cq->phba->pport;
17623 	mbox->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
17624 	rc = lpfc_sli_issue_mbox(cq->phba, mbox, MBX_POLL);
17625 	/* The IOCTL status is embedded in the mailbox subheader. */
17626 	shdr = (union lpfc_sli4_cfg_shdr *)
17627 		&mbox->u.mqe.un.wq_create.header.cfg_shdr;
17628 	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
17629 	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
17630 	if (shdr_status || shdr_add_status || rc) {
17631 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
17632 				"2506 CQ_DESTROY mailbox failed with "
17633 				"status x%x add_status x%x, mbx status x%x\n",
17634 				shdr_status, shdr_add_status, rc);
17635 		status = -ENXIO;
17636 	}
17637 	/* Remove cq from any list */
17638 	list_del_init(&cq->list);
17639 	mempool_free(mbox, cq->phba->mbox_mem_pool);
17640 	return status;
17641 }
17642 
17643 /**
17644  * lpfc_mq_destroy - Destroy a Mailbox Queue on the HBA
17645  * @phba: HBA structure that indicates port to destroy a queue on.
17646  * @mq: The queue structure associated with the queue to destroy.
17647  *
17648  * This function destroys a queue, as detailed in @mq by sending an mailbox
17649  * command, specific to the type of queue, to the HBA.
17650  *
17651  * The @mq struct is used to get the queue ID of the queue to destroy.
17652  *
17653  * On success this function will return a zero. If the queue destroy mailbox
17654  * command fails this function will return -ENXIO.
17655  **/
17656 int
17657 lpfc_mq_destroy(struct lpfc_hba *phba, struct lpfc_queue *mq)
17658 {
17659 	LPFC_MBOXQ_t *mbox;
17660 	int rc, length, status = 0;
17661 	uint32_t shdr_status, shdr_add_status;
17662 	union lpfc_sli4_cfg_shdr *shdr;
17663 
17664 	/* sanity check on queue memory */
17665 	if (!mq)
17666 		return -ENODEV;
17667 	mbox = mempool_alloc(mq->phba->mbox_mem_pool, GFP_KERNEL);
17668 	if (!mbox)
17669 		return -ENOMEM;
17670 	length = (sizeof(struct lpfc_mbx_mq_destroy) -
17671 		  sizeof(struct lpfc_sli4_cfg_mhdr));
17672 	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
17673 			 LPFC_MBOX_OPCODE_MQ_DESTROY,
17674 			 length, LPFC_SLI4_MBX_EMBED);
17675 	bf_set(lpfc_mbx_mq_destroy_q_id, &mbox->u.mqe.un.mq_destroy.u.request,
17676 	       mq->queue_id);
17677 	mbox->vport = mq->phba->pport;
17678 	mbox->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
17679 	rc = lpfc_sli_issue_mbox(mq->phba, mbox, MBX_POLL);
17680 	/* The IOCTL status is embedded in the mailbox subheader. */
17681 	shdr = (union lpfc_sli4_cfg_shdr *)
17682 		&mbox->u.mqe.un.mq_destroy.header.cfg_shdr;
17683 	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
17684 	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
17685 	if (shdr_status || shdr_add_status || rc) {
17686 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
17687 				"2507 MQ_DESTROY mailbox failed with "
17688 				"status x%x add_status x%x, mbx status x%x\n",
17689 				shdr_status, shdr_add_status, rc);
17690 		status = -ENXIO;
17691 	}
17692 	/* Remove mq from any list */
17693 	list_del_init(&mq->list);
17694 	mempool_free(mbox, mq->phba->mbox_mem_pool);
17695 	return status;
17696 }
17697 
17698 /**
17699  * lpfc_wq_destroy - Destroy a Work Queue on the HBA
17700  * @phba: HBA structure that indicates port to destroy a queue on.
17701  * @wq: The queue structure associated with the queue to destroy.
17702  *
17703  * This function destroys a queue, as detailed in @wq by sending an mailbox
17704  * command, specific to the type of queue, to the HBA.
17705  *
17706  * The @wq struct is used to get the queue ID of the queue to destroy.
17707  *
17708  * On success this function will return a zero. If the queue destroy mailbox
17709  * command fails this function will return -ENXIO.
17710  **/
17711 int
17712 lpfc_wq_destroy(struct lpfc_hba *phba, struct lpfc_queue *wq)
17713 {
17714 	LPFC_MBOXQ_t *mbox;
17715 	int rc, length, status = 0;
17716 	uint32_t shdr_status, shdr_add_status;
17717 	union lpfc_sli4_cfg_shdr *shdr;
17718 
17719 	/* sanity check on queue memory */
17720 	if (!wq)
17721 		return -ENODEV;
17722 	mbox = mempool_alloc(wq->phba->mbox_mem_pool, GFP_KERNEL);
17723 	if (!mbox)
17724 		return -ENOMEM;
17725 	length = (sizeof(struct lpfc_mbx_wq_destroy) -
17726 		  sizeof(struct lpfc_sli4_cfg_mhdr));
17727 	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_FCOE,
17728 			 LPFC_MBOX_OPCODE_FCOE_WQ_DESTROY,
17729 			 length, LPFC_SLI4_MBX_EMBED);
17730 	bf_set(lpfc_mbx_wq_destroy_q_id, &mbox->u.mqe.un.wq_destroy.u.request,
17731 	       wq->queue_id);
17732 	mbox->vport = wq->phba->pport;
17733 	mbox->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
17734 	rc = lpfc_sli_issue_mbox(wq->phba, mbox, MBX_POLL);
17735 	shdr = (union lpfc_sli4_cfg_shdr *)
17736 		&mbox->u.mqe.un.wq_destroy.header.cfg_shdr;
17737 	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
17738 	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
17739 	if (shdr_status || shdr_add_status || rc) {
17740 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
17741 				"2508 WQ_DESTROY mailbox failed with "
17742 				"status x%x add_status x%x, mbx status x%x\n",
17743 				shdr_status, shdr_add_status, rc);
17744 		status = -ENXIO;
17745 	}
17746 	/* Remove wq from any list */
17747 	list_del_init(&wq->list);
17748 	kfree(wq->pring);
17749 	wq->pring = NULL;
17750 	mempool_free(mbox, wq->phba->mbox_mem_pool);
17751 	return status;
17752 }
17753 
17754 /**
17755  * lpfc_rq_destroy - Destroy a Receive Queue on the HBA
17756  * @phba: HBA structure that indicates port to destroy a queue on.
17757  * @hrq: The queue structure associated with the queue to destroy.
17758  * @drq: The queue structure associated with the queue to destroy.
17759  *
17760  * This function destroys a queue, as detailed in @rq by sending an mailbox
17761  * command, specific to the type of queue, to the HBA.
17762  *
17763  * The @rq struct is used to get the queue ID of the queue to destroy.
17764  *
17765  * On success this function will return a zero. If the queue destroy mailbox
17766  * command fails this function will return -ENXIO.
17767  **/
17768 int
17769 lpfc_rq_destroy(struct lpfc_hba *phba, struct lpfc_queue *hrq,
17770 		struct lpfc_queue *drq)
17771 {
17772 	LPFC_MBOXQ_t *mbox;
17773 	int rc, length, status = 0;
17774 	uint32_t shdr_status, shdr_add_status;
17775 	union lpfc_sli4_cfg_shdr *shdr;
17776 
17777 	/* sanity check on queue memory */
17778 	if (!hrq || !drq)
17779 		return -ENODEV;
17780 	mbox = mempool_alloc(hrq->phba->mbox_mem_pool, GFP_KERNEL);
17781 	if (!mbox)
17782 		return -ENOMEM;
17783 	length = (sizeof(struct lpfc_mbx_rq_destroy) -
17784 		  sizeof(struct lpfc_sli4_cfg_mhdr));
17785 	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_FCOE,
17786 			 LPFC_MBOX_OPCODE_FCOE_RQ_DESTROY,
17787 			 length, LPFC_SLI4_MBX_EMBED);
17788 	bf_set(lpfc_mbx_rq_destroy_q_id, &mbox->u.mqe.un.rq_destroy.u.request,
17789 	       hrq->queue_id);
17790 	mbox->vport = hrq->phba->pport;
17791 	mbox->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
17792 	rc = lpfc_sli_issue_mbox(hrq->phba, mbox, MBX_POLL);
17793 	/* The IOCTL status is embedded in the mailbox subheader. */
17794 	shdr = (union lpfc_sli4_cfg_shdr *)
17795 		&mbox->u.mqe.un.rq_destroy.header.cfg_shdr;
17796 	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
17797 	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
17798 	if (shdr_status || shdr_add_status || rc) {
17799 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
17800 				"2509 RQ_DESTROY mailbox failed with "
17801 				"status x%x add_status x%x, mbx status x%x\n",
17802 				shdr_status, shdr_add_status, rc);
17803 		mempool_free(mbox, hrq->phba->mbox_mem_pool);
17804 		return -ENXIO;
17805 	}
17806 	bf_set(lpfc_mbx_rq_destroy_q_id, &mbox->u.mqe.un.rq_destroy.u.request,
17807 	       drq->queue_id);
17808 	rc = lpfc_sli_issue_mbox(drq->phba, mbox, MBX_POLL);
17809 	shdr = (union lpfc_sli4_cfg_shdr *)
17810 		&mbox->u.mqe.un.rq_destroy.header.cfg_shdr;
17811 	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
17812 	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
17813 	if (shdr_status || shdr_add_status || rc) {
17814 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
17815 				"2510 RQ_DESTROY mailbox failed with "
17816 				"status x%x add_status x%x, mbx status x%x\n",
17817 				shdr_status, shdr_add_status, rc);
17818 		status = -ENXIO;
17819 	}
17820 	list_del_init(&hrq->list);
17821 	list_del_init(&drq->list);
17822 	mempool_free(mbox, hrq->phba->mbox_mem_pool);
17823 	return status;
17824 }
17825 
17826 /**
17827  * lpfc_sli4_post_sgl - Post scatter gather list for an XRI to HBA
17828  * @phba: The virtual port for which this call being executed.
17829  * @pdma_phys_addr0: Physical address of the 1st SGL page.
17830  * @pdma_phys_addr1: Physical address of the 2nd SGL page.
17831  * @xritag: the xritag that ties this io to the SGL pages.
17832  *
17833  * This routine will post the sgl pages for the IO that has the xritag
17834  * that is in the iocbq structure. The xritag is assigned during iocbq
17835  * creation and persists for as long as the driver is loaded.
17836  * if the caller has fewer than 256 scatter gather segments to map then
17837  * pdma_phys_addr1 should be 0.
17838  * If the caller needs to map more than 256 scatter gather segment then
17839  * pdma_phys_addr1 should be a valid physical address.
17840  * physical address for SGLs must be 64 byte aligned.
17841  * If you are going to map 2 SGL's then the first one must have 256 entries
17842  * the second sgl can have between 1 and 256 entries.
17843  *
17844  * Return codes:
17845  * 	0 - Success
17846  * 	-ENXIO, -ENOMEM - Failure
17847  **/
17848 int
17849 lpfc_sli4_post_sgl(struct lpfc_hba *phba,
17850 		dma_addr_t pdma_phys_addr0,
17851 		dma_addr_t pdma_phys_addr1,
17852 		uint16_t xritag)
17853 {
17854 	struct lpfc_mbx_post_sgl_pages *post_sgl_pages;
17855 	LPFC_MBOXQ_t *mbox;
17856 	int rc;
17857 	uint32_t shdr_status, shdr_add_status;
17858 	uint32_t mbox_tmo;
17859 	union lpfc_sli4_cfg_shdr *shdr;
17860 
17861 	if (xritag == NO_XRI) {
17862 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
17863 				"0364 Invalid param:\n");
17864 		return -EINVAL;
17865 	}
17866 
17867 	mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
17868 	if (!mbox)
17869 		return -ENOMEM;
17870 
17871 	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_FCOE,
17872 			LPFC_MBOX_OPCODE_FCOE_POST_SGL_PAGES,
17873 			sizeof(struct lpfc_mbx_post_sgl_pages) -
17874 			sizeof(struct lpfc_sli4_cfg_mhdr), LPFC_SLI4_MBX_EMBED);
17875 
17876 	post_sgl_pages = (struct lpfc_mbx_post_sgl_pages *)
17877 				&mbox->u.mqe.un.post_sgl_pages;
17878 	bf_set(lpfc_post_sgl_pages_xri, post_sgl_pages, xritag);
17879 	bf_set(lpfc_post_sgl_pages_xricnt, post_sgl_pages, 1);
17880 
17881 	post_sgl_pages->sgl_pg_pairs[0].sgl_pg0_addr_lo	=
17882 				cpu_to_le32(putPaddrLow(pdma_phys_addr0));
17883 	post_sgl_pages->sgl_pg_pairs[0].sgl_pg0_addr_hi =
17884 				cpu_to_le32(putPaddrHigh(pdma_phys_addr0));
17885 
17886 	post_sgl_pages->sgl_pg_pairs[0].sgl_pg1_addr_lo	=
17887 				cpu_to_le32(putPaddrLow(pdma_phys_addr1));
17888 	post_sgl_pages->sgl_pg_pairs[0].sgl_pg1_addr_hi =
17889 				cpu_to_le32(putPaddrHigh(pdma_phys_addr1));
17890 	if (!phba->sli4_hba.intr_enable)
17891 		rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL);
17892 	else {
17893 		mbox_tmo = lpfc_mbox_tmo_val(phba, mbox);
17894 		rc = lpfc_sli_issue_mbox_wait(phba, mbox, mbox_tmo);
17895 	}
17896 	/* The IOCTL status is embedded in the mailbox subheader. */
17897 	shdr = (union lpfc_sli4_cfg_shdr *) &post_sgl_pages->header.cfg_shdr;
17898 	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
17899 	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
17900 	if (!phba->sli4_hba.intr_enable)
17901 		mempool_free(mbox, phba->mbox_mem_pool);
17902 	else if (rc != MBX_TIMEOUT)
17903 		mempool_free(mbox, phba->mbox_mem_pool);
17904 	if (shdr_status || shdr_add_status || rc) {
17905 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
17906 				"2511 POST_SGL mailbox failed with "
17907 				"status x%x add_status x%x, mbx status x%x\n",
17908 				shdr_status, shdr_add_status, rc);
17909 	}
17910 	return 0;
17911 }
17912 
17913 /**
17914  * lpfc_sli4_alloc_xri - Get an available rpi in the device's range
17915  * @phba: pointer to lpfc hba data structure.
17916  *
17917  * This routine is invoked to post rpi header templates to the
17918  * HBA consistent with the SLI-4 interface spec.  This routine
17919  * posts a SLI4_PAGE_SIZE memory region to the port to hold up to
17920  * SLI4_PAGE_SIZE modulo 64 rpi context headers.
17921  *
17922  * Returns
17923  *	A nonzero rpi defined as rpi_base <= rpi < max_rpi if successful
17924  *	LPFC_RPI_ALLOC_ERROR if no rpis are available.
17925  **/
17926 static uint16_t
17927 lpfc_sli4_alloc_xri(struct lpfc_hba *phba)
17928 {
17929 	unsigned long xri;
17930 
17931 	/*
17932 	 * Fetch the next logical xri.  Because this index is logical,
17933 	 * the driver starts at 0 each time.
17934 	 */
17935 	spin_lock_irq(&phba->hbalock);
17936 	xri = find_next_zero_bit(phba->sli4_hba.xri_bmask,
17937 				 phba->sli4_hba.max_cfg_param.max_xri, 0);
17938 	if (xri >= phba->sli4_hba.max_cfg_param.max_xri) {
17939 		spin_unlock_irq(&phba->hbalock);
17940 		return NO_XRI;
17941 	} else {
17942 		set_bit(xri, phba->sli4_hba.xri_bmask);
17943 		phba->sli4_hba.max_cfg_param.xri_used++;
17944 	}
17945 	spin_unlock_irq(&phba->hbalock);
17946 	return xri;
17947 }
17948 
17949 /**
17950  * __lpfc_sli4_free_xri - Release an xri for reuse.
17951  * @phba: pointer to lpfc hba data structure.
17952  * @xri: xri to release.
17953  *
17954  * This routine is invoked to release an xri to the pool of
17955  * available rpis maintained by the driver.
17956  **/
17957 static void
17958 __lpfc_sli4_free_xri(struct lpfc_hba *phba, int xri)
17959 {
17960 	if (test_and_clear_bit(xri, phba->sli4_hba.xri_bmask)) {
17961 		phba->sli4_hba.max_cfg_param.xri_used--;
17962 	}
17963 }
17964 
17965 /**
17966  * lpfc_sli4_free_xri - Release an xri for reuse.
17967  * @phba: pointer to lpfc hba data structure.
17968  * @xri: xri to release.
17969  *
17970  * This routine is invoked to release an xri to the pool of
17971  * available rpis maintained by the driver.
17972  **/
17973 void
17974 lpfc_sli4_free_xri(struct lpfc_hba *phba, int xri)
17975 {
17976 	spin_lock_irq(&phba->hbalock);
17977 	__lpfc_sli4_free_xri(phba, xri);
17978 	spin_unlock_irq(&phba->hbalock);
17979 }
17980 
17981 /**
17982  * lpfc_sli4_next_xritag - Get an xritag for the io
17983  * @phba: Pointer to HBA context object.
17984  *
17985  * This function gets an xritag for the iocb. If there is no unused xritag
17986  * it will return 0xffff.
17987  * The function returns the allocated xritag if successful, else returns zero.
17988  * Zero is not a valid xritag.
17989  * The caller is not required to hold any lock.
17990  **/
17991 uint16_t
17992 lpfc_sli4_next_xritag(struct lpfc_hba *phba)
17993 {
17994 	uint16_t xri_index;
17995 
17996 	xri_index = lpfc_sli4_alloc_xri(phba);
17997 	if (xri_index == NO_XRI)
17998 		lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
17999 				"2004 Failed to allocate XRI.last XRITAG is %d"
18000 				" Max XRI is %d, Used XRI is %d\n",
18001 				xri_index,
18002 				phba->sli4_hba.max_cfg_param.max_xri,
18003 				phba->sli4_hba.max_cfg_param.xri_used);
18004 	return xri_index;
18005 }
18006 
18007 /**
18008  * lpfc_sli4_post_sgl_list - post a block of ELS sgls to the port.
18009  * @phba: pointer to lpfc hba data structure.
18010  * @post_sgl_list: pointer to els sgl entry list.
18011  * @post_cnt: number of els sgl entries on the list.
18012  *
18013  * This routine is invoked to post a block of driver's sgl pages to the
18014  * HBA using non-embedded mailbox command. No Lock is held. This routine
18015  * is only called when the driver is loading and after all IO has been
18016  * stopped.
18017  **/
18018 static int
18019 lpfc_sli4_post_sgl_list(struct lpfc_hba *phba,
18020 			    struct list_head *post_sgl_list,
18021 			    int post_cnt)
18022 {
18023 	struct lpfc_sglq *sglq_entry = NULL, *sglq_next = NULL;
18024 	struct lpfc_mbx_post_uembed_sgl_page1 *sgl;
18025 	struct sgl_page_pairs *sgl_pg_pairs;
18026 	void *viraddr;
18027 	LPFC_MBOXQ_t *mbox;
18028 	uint32_t reqlen, alloclen, pg_pairs;
18029 	uint32_t mbox_tmo;
18030 	uint16_t xritag_start = 0;
18031 	int rc = 0;
18032 	uint32_t shdr_status, shdr_add_status;
18033 	union lpfc_sli4_cfg_shdr *shdr;
18034 
18035 	reqlen = post_cnt * sizeof(struct sgl_page_pairs) +
18036 		 sizeof(union lpfc_sli4_cfg_shdr) + sizeof(uint32_t);
18037 	if (reqlen > SLI4_PAGE_SIZE) {
18038 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
18039 				"2559 Block sgl registration required DMA "
18040 				"size (%d) great than a page\n", reqlen);
18041 		return -ENOMEM;
18042 	}
18043 
18044 	mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
18045 	if (!mbox)
18046 		return -ENOMEM;
18047 
18048 	/* Allocate DMA memory and set up the non-embedded mailbox command */
18049 	alloclen = lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_FCOE,
18050 			 LPFC_MBOX_OPCODE_FCOE_POST_SGL_PAGES, reqlen,
18051 			 LPFC_SLI4_MBX_NEMBED);
18052 
18053 	if (alloclen < reqlen) {
18054 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
18055 				"0285 Allocated DMA memory size (%d) is "
18056 				"less than the requested DMA memory "
18057 				"size (%d)\n", alloclen, reqlen);
18058 		lpfc_sli4_mbox_cmd_free(phba, mbox);
18059 		return -ENOMEM;
18060 	}
18061 	/* Set up the SGL pages in the non-embedded DMA pages */
18062 	viraddr = mbox->sge_array->addr[0];
18063 	sgl = (struct lpfc_mbx_post_uembed_sgl_page1 *)viraddr;
18064 	sgl_pg_pairs = &sgl->sgl_pg_pairs;
18065 
18066 	pg_pairs = 0;
18067 	list_for_each_entry_safe(sglq_entry, sglq_next, post_sgl_list, list) {
18068 		/* Set up the sge entry */
18069 		sgl_pg_pairs->sgl_pg0_addr_lo =
18070 				cpu_to_le32(putPaddrLow(sglq_entry->phys));
18071 		sgl_pg_pairs->sgl_pg0_addr_hi =
18072 				cpu_to_le32(putPaddrHigh(sglq_entry->phys));
18073 		sgl_pg_pairs->sgl_pg1_addr_lo =
18074 				cpu_to_le32(putPaddrLow(0));
18075 		sgl_pg_pairs->sgl_pg1_addr_hi =
18076 				cpu_to_le32(putPaddrHigh(0));
18077 
18078 		/* Keep the first xritag on the list */
18079 		if (pg_pairs == 0)
18080 			xritag_start = sglq_entry->sli4_xritag;
18081 		sgl_pg_pairs++;
18082 		pg_pairs++;
18083 	}
18084 
18085 	/* Complete initialization and perform endian conversion. */
18086 	bf_set(lpfc_post_sgl_pages_xri, sgl, xritag_start);
18087 	bf_set(lpfc_post_sgl_pages_xricnt, sgl, post_cnt);
18088 	sgl->word0 = cpu_to_le32(sgl->word0);
18089 
18090 	if (!phba->sli4_hba.intr_enable)
18091 		rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL);
18092 	else {
18093 		mbox_tmo = lpfc_mbox_tmo_val(phba, mbox);
18094 		rc = lpfc_sli_issue_mbox_wait(phba, mbox, mbox_tmo);
18095 	}
18096 	shdr = (union lpfc_sli4_cfg_shdr *) &sgl->cfg_shdr;
18097 	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
18098 	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
18099 	if (!phba->sli4_hba.intr_enable)
18100 		lpfc_sli4_mbox_cmd_free(phba, mbox);
18101 	else if (rc != MBX_TIMEOUT)
18102 		lpfc_sli4_mbox_cmd_free(phba, mbox);
18103 	if (shdr_status || shdr_add_status || rc) {
18104 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
18105 				"2513 POST_SGL_BLOCK mailbox command failed "
18106 				"status x%x add_status x%x mbx status x%x\n",
18107 				shdr_status, shdr_add_status, rc);
18108 		rc = -ENXIO;
18109 	}
18110 	return rc;
18111 }
18112 
18113 /**
18114  * lpfc_sli4_post_io_sgl_block - post a block of nvme sgl list to firmware
18115  * @phba: pointer to lpfc hba data structure.
18116  * @nblist: pointer to nvme buffer list.
18117  * @count: number of scsi buffers on the list.
18118  *
18119  * This routine is invoked to post a block of @count scsi sgl pages from a
18120  * SCSI buffer list @nblist to the HBA using non-embedded mailbox command.
18121  * No Lock is held.
18122  *
18123  **/
18124 static int
18125 lpfc_sli4_post_io_sgl_block(struct lpfc_hba *phba, struct list_head *nblist,
18126 			    int count)
18127 {
18128 	struct lpfc_io_buf *lpfc_ncmd;
18129 	struct lpfc_mbx_post_uembed_sgl_page1 *sgl;
18130 	struct sgl_page_pairs *sgl_pg_pairs;
18131 	void *viraddr;
18132 	LPFC_MBOXQ_t *mbox;
18133 	uint32_t reqlen, alloclen, pg_pairs;
18134 	uint32_t mbox_tmo;
18135 	uint16_t xritag_start = 0;
18136 	int rc = 0;
18137 	uint32_t shdr_status, shdr_add_status;
18138 	dma_addr_t pdma_phys_bpl1;
18139 	union lpfc_sli4_cfg_shdr *shdr;
18140 
18141 	/* Calculate the requested length of the dma memory */
18142 	reqlen = count * sizeof(struct sgl_page_pairs) +
18143 		 sizeof(union lpfc_sli4_cfg_shdr) + sizeof(uint32_t);
18144 	if (reqlen > SLI4_PAGE_SIZE) {
18145 		lpfc_printf_log(phba, KERN_WARNING, LOG_INIT,
18146 				"6118 Block sgl registration required DMA "
18147 				"size (%d) great than a page\n", reqlen);
18148 		return -ENOMEM;
18149 	}
18150 	mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
18151 	if (!mbox) {
18152 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
18153 				"6119 Failed to allocate mbox cmd memory\n");
18154 		return -ENOMEM;
18155 	}
18156 
18157 	/* Allocate DMA memory and set up the non-embedded mailbox command */
18158 	alloclen = lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_FCOE,
18159 				    LPFC_MBOX_OPCODE_FCOE_POST_SGL_PAGES,
18160 				    reqlen, LPFC_SLI4_MBX_NEMBED);
18161 
18162 	if (alloclen < reqlen) {
18163 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
18164 				"6120 Allocated DMA memory size (%d) is "
18165 				"less than the requested DMA memory "
18166 				"size (%d)\n", alloclen, reqlen);
18167 		lpfc_sli4_mbox_cmd_free(phba, mbox);
18168 		return -ENOMEM;
18169 	}
18170 
18171 	/* Get the first SGE entry from the non-embedded DMA memory */
18172 	viraddr = mbox->sge_array->addr[0];
18173 
18174 	/* Set up the SGL pages in the non-embedded DMA pages */
18175 	sgl = (struct lpfc_mbx_post_uembed_sgl_page1 *)viraddr;
18176 	sgl_pg_pairs = &sgl->sgl_pg_pairs;
18177 
18178 	pg_pairs = 0;
18179 	list_for_each_entry(lpfc_ncmd, nblist, list) {
18180 		/* Set up the sge entry */
18181 		sgl_pg_pairs->sgl_pg0_addr_lo =
18182 			cpu_to_le32(putPaddrLow(lpfc_ncmd->dma_phys_sgl));
18183 		sgl_pg_pairs->sgl_pg0_addr_hi =
18184 			cpu_to_le32(putPaddrHigh(lpfc_ncmd->dma_phys_sgl));
18185 		if (phba->cfg_sg_dma_buf_size > SGL_PAGE_SIZE)
18186 			pdma_phys_bpl1 = lpfc_ncmd->dma_phys_sgl +
18187 						SGL_PAGE_SIZE;
18188 		else
18189 			pdma_phys_bpl1 = 0;
18190 		sgl_pg_pairs->sgl_pg1_addr_lo =
18191 			cpu_to_le32(putPaddrLow(pdma_phys_bpl1));
18192 		sgl_pg_pairs->sgl_pg1_addr_hi =
18193 			cpu_to_le32(putPaddrHigh(pdma_phys_bpl1));
18194 		/* Keep the first xritag on the list */
18195 		if (pg_pairs == 0)
18196 			xritag_start = lpfc_ncmd->cur_iocbq.sli4_xritag;
18197 		sgl_pg_pairs++;
18198 		pg_pairs++;
18199 	}
18200 	bf_set(lpfc_post_sgl_pages_xri, sgl, xritag_start);
18201 	bf_set(lpfc_post_sgl_pages_xricnt, sgl, pg_pairs);
18202 	/* Perform endian conversion if necessary */
18203 	sgl->word0 = cpu_to_le32(sgl->word0);
18204 
18205 	if (!phba->sli4_hba.intr_enable) {
18206 		rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL);
18207 	} else {
18208 		mbox_tmo = lpfc_mbox_tmo_val(phba, mbox);
18209 		rc = lpfc_sli_issue_mbox_wait(phba, mbox, mbox_tmo);
18210 	}
18211 	shdr = (union lpfc_sli4_cfg_shdr *)&sgl->cfg_shdr;
18212 	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
18213 	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
18214 	if (!phba->sli4_hba.intr_enable)
18215 		lpfc_sli4_mbox_cmd_free(phba, mbox);
18216 	else if (rc != MBX_TIMEOUT)
18217 		lpfc_sli4_mbox_cmd_free(phba, mbox);
18218 	if (shdr_status || shdr_add_status || rc) {
18219 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
18220 				"6125 POST_SGL_BLOCK mailbox command failed "
18221 				"status x%x add_status x%x mbx status x%x\n",
18222 				shdr_status, shdr_add_status, rc);
18223 		rc = -ENXIO;
18224 	}
18225 	return rc;
18226 }
18227 
18228 /**
18229  * lpfc_sli4_post_io_sgl_list - Post blocks of nvme buffer sgls from a list
18230  * @phba: pointer to lpfc hba data structure.
18231  * @post_nblist: pointer to the nvme buffer list.
18232  * @sb_count: number of nvme buffers.
18233  *
18234  * This routine walks a list of nvme buffers that was passed in. It attempts
18235  * to construct blocks of nvme buffer sgls which contains contiguous xris and
18236  * uses the non-embedded SGL block post mailbox commands to post to the port.
18237  * For single NVME buffer sgl with non-contiguous xri, if any, it shall use
18238  * embedded SGL post mailbox command for posting. The @post_nblist passed in
18239  * must be local list, thus no lock is needed when manipulate the list.
18240  *
18241  * Returns: 0 = failure, non-zero number of successfully posted buffers.
18242  **/
18243 int
18244 lpfc_sli4_post_io_sgl_list(struct lpfc_hba *phba,
18245 			   struct list_head *post_nblist, int sb_count)
18246 {
18247 	struct lpfc_io_buf *lpfc_ncmd, *lpfc_ncmd_next;
18248 	int status, sgl_size;
18249 	int post_cnt = 0, block_cnt = 0, num_posting = 0, num_posted = 0;
18250 	dma_addr_t pdma_phys_sgl1;
18251 	int last_xritag = NO_XRI;
18252 	int cur_xritag;
18253 	LIST_HEAD(prep_nblist);
18254 	LIST_HEAD(blck_nblist);
18255 	LIST_HEAD(nvme_nblist);
18256 
18257 	/* sanity check */
18258 	if (sb_count <= 0)
18259 		return -EINVAL;
18260 
18261 	sgl_size = phba->cfg_sg_dma_buf_size;
18262 	list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, post_nblist, list) {
18263 		list_del_init(&lpfc_ncmd->list);
18264 		block_cnt++;
18265 		if ((last_xritag != NO_XRI) &&
18266 		    (lpfc_ncmd->cur_iocbq.sli4_xritag != last_xritag + 1)) {
18267 			/* a hole in xri block, form a sgl posting block */
18268 			list_splice_init(&prep_nblist, &blck_nblist);
18269 			post_cnt = block_cnt - 1;
18270 			/* prepare list for next posting block */
18271 			list_add_tail(&lpfc_ncmd->list, &prep_nblist);
18272 			block_cnt = 1;
18273 		} else {
18274 			/* prepare list for next posting block */
18275 			list_add_tail(&lpfc_ncmd->list, &prep_nblist);
18276 			/* enough sgls for non-embed sgl mbox command */
18277 			if (block_cnt == LPFC_NEMBED_MBOX_SGL_CNT) {
18278 				list_splice_init(&prep_nblist, &blck_nblist);
18279 				post_cnt = block_cnt;
18280 				block_cnt = 0;
18281 			}
18282 		}
18283 		num_posting++;
18284 		last_xritag = lpfc_ncmd->cur_iocbq.sli4_xritag;
18285 
18286 		/* end of repost sgl list condition for NVME buffers */
18287 		if (num_posting == sb_count) {
18288 			if (post_cnt == 0) {
18289 				/* last sgl posting block */
18290 				list_splice_init(&prep_nblist, &blck_nblist);
18291 				post_cnt = block_cnt;
18292 			} else if (block_cnt == 1) {
18293 				/* last single sgl with non-contiguous xri */
18294 				if (sgl_size > SGL_PAGE_SIZE)
18295 					pdma_phys_sgl1 =
18296 						lpfc_ncmd->dma_phys_sgl +
18297 						SGL_PAGE_SIZE;
18298 				else
18299 					pdma_phys_sgl1 = 0;
18300 				cur_xritag = lpfc_ncmd->cur_iocbq.sli4_xritag;
18301 				status = lpfc_sli4_post_sgl(
18302 						phba, lpfc_ncmd->dma_phys_sgl,
18303 						pdma_phys_sgl1, cur_xritag);
18304 				if (status) {
18305 					/* Post error.  Buffer unavailable. */
18306 					lpfc_ncmd->flags |=
18307 						LPFC_SBUF_NOT_POSTED;
18308 				} else {
18309 					/* Post success. Bffer available. */
18310 					lpfc_ncmd->flags &=
18311 						~LPFC_SBUF_NOT_POSTED;
18312 					lpfc_ncmd->status = IOSTAT_SUCCESS;
18313 					num_posted++;
18314 				}
18315 				/* success, put on NVME buffer sgl list */
18316 				list_add_tail(&lpfc_ncmd->list, &nvme_nblist);
18317 			}
18318 		}
18319 
18320 		/* continue until a nembed page worth of sgls */
18321 		if (post_cnt == 0)
18322 			continue;
18323 
18324 		/* post block of NVME buffer list sgls */
18325 		status = lpfc_sli4_post_io_sgl_block(phba, &blck_nblist,
18326 						     post_cnt);
18327 
18328 		/* don't reset xirtag due to hole in xri block */
18329 		if (block_cnt == 0)
18330 			last_xritag = NO_XRI;
18331 
18332 		/* reset NVME buffer post count for next round of posting */
18333 		post_cnt = 0;
18334 
18335 		/* put posted NVME buffer-sgl posted on NVME buffer sgl list */
18336 		while (!list_empty(&blck_nblist)) {
18337 			list_remove_head(&blck_nblist, lpfc_ncmd,
18338 					 struct lpfc_io_buf, list);
18339 			if (status) {
18340 				/* Post error.  Mark buffer unavailable. */
18341 				lpfc_ncmd->flags |= LPFC_SBUF_NOT_POSTED;
18342 			} else {
18343 				/* Post success, Mark buffer available. */
18344 				lpfc_ncmd->flags &= ~LPFC_SBUF_NOT_POSTED;
18345 				lpfc_ncmd->status = IOSTAT_SUCCESS;
18346 				num_posted++;
18347 			}
18348 			list_add_tail(&lpfc_ncmd->list, &nvme_nblist);
18349 		}
18350 	}
18351 	/* Push NVME buffers with sgl posted to the available list */
18352 	lpfc_io_buf_replenish(phba, &nvme_nblist);
18353 
18354 	return num_posted;
18355 }
18356 
18357 /**
18358  * lpfc_fc_frame_check - Check that this frame is a valid frame to handle
18359  * @phba: pointer to lpfc_hba struct that the frame was received on
18360  * @fc_hdr: A pointer to the FC Header data (In Big Endian Format)
18361  *
18362  * This function checks the fields in the @fc_hdr to see if the FC frame is a
18363  * valid type of frame that the LPFC driver will handle. This function will
18364  * return a zero if the frame is a valid frame or a non zero value when the
18365  * frame does not pass the check.
18366  **/
18367 static int
18368 lpfc_fc_frame_check(struct lpfc_hba *phba, struct fc_frame_header *fc_hdr)
18369 {
18370 	/*  make rctl_names static to save stack space */
18371 	struct fc_vft_header *fc_vft_hdr;
18372 	uint32_t *header = (uint32_t *) fc_hdr;
18373 
18374 #define FC_RCTL_MDS_DIAGS	0xF4
18375 
18376 	switch (fc_hdr->fh_r_ctl) {
18377 	case FC_RCTL_DD_UNCAT:		/* uncategorized information */
18378 	case FC_RCTL_DD_SOL_DATA:	/* solicited data */
18379 	case FC_RCTL_DD_UNSOL_CTL:	/* unsolicited control */
18380 	case FC_RCTL_DD_SOL_CTL:	/* solicited control or reply */
18381 	case FC_RCTL_DD_UNSOL_DATA:	/* unsolicited data */
18382 	case FC_RCTL_DD_DATA_DESC:	/* data descriptor */
18383 	case FC_RCTL_DD_UNSOL_CMD:	/* unsolicited command */
18384 	case FC_RCTL_DD_CMD_STATUS:	/* command status */
18385 	case FC_RCTL_ELS_REQ:	/* extended link services request */
18386 	case FC_RCTL_ELS_REP:	/* extended link services reply */
18387 	case FC_RCTL_ELS4_REQ:	/* FC-4 ELS request */
18388 	case FC_RCTL_ELS4_REP:	/* FC-4 ELS reply */
18389 	case FC_RCTL_BA_NOP:  	/* basic link service NOP */
18390 	case FC_RCTL_BA_ABTS: 	/* basic link service abort */
18391 	case FC_RCTL_BA_RMC: 	/* remove connection */
18392 	case FC_RCTL_BA_ACC:	/* basic accept */
18393 	case FC_RCTL_BA_RJT:	/* basic reject */
18394 	case FC_RCTL_BA_PRMT:
18395 	case FC_RCTL_ACK_1:	/* acknowledge_1 */
18396 	case FC_RCTL_ACK_0:	/* acknowledge_0 */
18397 	case FC_RCTL_P_RJT:	/* port reject */
18398 	case FC_RCTL_F_RJT:	/* fabric reject */
18399 	case FC_RCTL_P_BSY:	/* port busy */
18400 	case FC_RCTL_F_BSY:	/* fabric busy to data frame */
18401 	case FC_RCTL_F_BSYL:	/* fabric busy to link control frame */
18402 	case FC_RCTL_LCR:	/* link credit reset */
18403 	case FC_RCTL_MDS_DIAGS: /* MDS Diagnostics */
18404 	case FC_RCTL_END:	/* end */
18405 		break;
18406 	case FC_RCTL_VFTH:	/* Virtual Fabric tagging Header */
18407 		fc_vft_hdr = (struct fc_vft_header *)fc_hdr;
18408 		fc_hdr = &((struct fc_frame_header *)fc_vft_hdr)[1];
18409 		return lpfc_fc_frame_check(phba, fc_hdr);
18410 	default:
18411 		goto drop;
18412 	}
18413 
18414 	switch (fc_hdr->fh_type) {
18415 	case FC_TYPE_BLS:
18416 	case FC_TYPE_ELS:
18417 	case FC_TYPE_FCP:
18418 	case FC_TYPE_CT:
18419 	case FC_TYPE_NVME:
18420 		break;
18421 	case FC_TYPE_IP:
18422 	case FC_TYPE_ILS:
18423 	default:
18424 		goto drop;
18425 	}
18426 
18427 	lpfc_printf_log(phba, KERN_INFO, LOG_ELS,
18428 			"2538 Received frame rctl:x%x, type:x%x, "
18429 			"frame Data:%08x %08x %08x %08x %08x %08x %08x\n",
18430 			fc_hdr->fh_r_ctl, fc_hdr->fh_type,
18431 			be32_to_cpu(header[0]), be32_to_cpu(header[1]),
18432 			be32_to_cpu(header[2]), be32_to_cpu(header[3]),
18433 			be32_to_cpu(header[4]), be32_to_cpu(header[5]),
18434 			be32_to_cpu(header[6]));
18435 	return 0;
18436 drop:
18437 	lpfc_printf_log(phba, KERN_WARNING, LOG_ELS,
18438 			"2539 Dropped frame rctl:x%x type:x%x\n",
18439 			fc_hdr->fh_r_ctl, fc_hdr->fh_type);
18440 	return 1;
18441 }
18442 
18443 /**
18444  * lpfc_fc_hdr_get_vfi - Get the VFI from an FC frame
18445  * @fc_hdr: A pointer to the FC Header data (In Big Endian Format)
18446  *
18447  * This function processes the FC header to retrieve the VFI from the VF
18448  * header, if one exists. This function will return the VFI if one exists
18449  * or 0 if no VSAN Header exists.
18450  **/
18451 static uint32_t
18452 lpfc_fc_hdr_get_vfi(struct fc_frame_header *fc_hdr)
18453 {
18454 	struct fc_vft_header *fc_vft_hdr = (struct fc_vft_header *)fc_hdr;
18455 
18456 	if (fc_hdr->fh_r_ctl != FC_RCTL_VFTH)
18457 		return 0;
18458 	return bf_get(fc_vft_hdr_vf_id, fc_vft_hdr);
18459 }
18460 
18461 /**
18462  * lpfc_fc_frame_to_vport - Finds the vport that a frame is destined to
18463  * @phba: Pointer to the HBA structure to search for the vport on
18464  * @fc_hdr: A pointer to the FC Header data (In Big Endian Format)
18465  * @fcfi: The FC Fabric ID that the frame came from
18466  * @did: Destination ID to match against
18467  *
18468  * This function searches the @phba for a vport that matches the content of the
18469  * @fc_hdr passed in and the @fcfi. This function uses the @fc_hdr to fetch the
18470  * VFI, if the Virtual Fabric Tagging Header exists, and the DID. This function
18471  * returns the matching vport pointer or NULL if unable to match frame to a
18472  * vport.
18473  **/
18474 static struct lpfc_vport *
18475 lpfc_fc_frame_to_vport(struct lpfc_hba *phba, struct fc_frame_header *fc_hdr,
18476 		       uint16_t fcfi, uint32_t did)
18477 {
18478 	struct lpfc_vport **vports;
18479 	struct lpfc_vport *vport = NULL;
18480 	int i;
18481 
18482 	if (did == Fabric_DID)
18483 		return phba->pport;
18484 	if ((phba->pport->fc_flag & FC_PT2PT) &&
18485 		!(phba->link_state == LPFC_HBA_READY))
18486 		return phba->pport;
18487 
18488 	vports = lpfc_create_vport_work_array(phba);
18489 	if (vports != NULL) {
18490 		for (i = 0; i <= phba->max_vpi && vports[i] != NULL; i++) {
18491 			if (phba->fcf.fcfi == fcfi &&
18492 			    vports[i]->vfi == lpfc_fc_hdr_get_vfi(fc_hdr) &&
18493 			    vports[i]->fc_myDID == did) {
18494 				vport = vports[i];
18495 				break;
18496 			}
18497 		}
18498 	}
18499 	lpfc_destroy_vport_work_array(phba, vports);
18500 	return vport;
18501 }
18502 
18503 /**
18504  * lpfc_update_rcv_time_stamp - Update vport's rcv seq time stamp
18505  * @vport: The vport to work on.
18506  *
18507  * This function updates the receive sequence time stamp for this vport. The
18508  * receive sequence time stamp indicates the time that the last frame of the
18509  * the sequence that has been idle for the longest amount of time was received.
18510  * the driver uses this time stamp to indicate if any received sequences have
18511  * timed out.
18512  **/
18513 static void
18514 lpfc_update_rcv_time_stamp(struct lpfc_vport *vport)
18515 {
18516 	struct lpfc_dmabuf *h_buf;
18517 	struct hbq_dmabuf *dmabuf = NULL;
18518 
18519 	/* get the oldest sequence on the rcv list */
18520 	h_buf = list_get_first(&vport->rcv_buffer_list,
18521 			       struct lpfc_dmabuf, list);
18522 	if (!h_buf)
18523 		return;
18524 	dmabuf = container_of(h_buf, struct hbq_dmabuf, hbuf);
18525 	vport->rcv_buffer_time_stamp = dmabuf->time_stamp;
18526 }
18527 
18528 /**
18529  * lpfc_cleanup_rcv_buffers - Cleans up all outstanding receive sequences.
18530  * @vport: The vport that the received sequences were sent to.
18531  *
18532  * This function cleans up all outstanding received sequences. This is called
18533  * by the driver when a link event or user action invalidates all the received
18534  * sequences.
18535  **/
18536 void
18537 lpfc_cleanup_rcv_buffers(struct lpfc_vport *vport)
18538 {
18539 	struct lpfc_dmabuf *h_buf, *hnext;
18540 	struct lpfc_dmabuf *d_buf, *dnext;
18541 	struct hbq_dmabuf *dmabuf = NULL;
18542 
18543 	/* start with the oldest sequence on the rcv list */
18544 	list_for_each_entry_safe(h_buf, hnext, &vport->rcv_buffer_list, list) {
18545 		dmabuf = container_of(h_buf, struct hbq_dmabuf, hbuf);
18546 		list_del_init(&dmabuf->hbuf.list);
18547 		list_for_each_entry_safe(d_buf, dnext,
18548 					 &dmabuf->dbuf.list, list) {
18549 			list_del_init(&d_buf->list);
18550 			lpfc_in_buf_free(vport->phba, d_buf);
18551 		}
18552 		lpfc_in_buf_free(vport->phba, &dmabuf->dbuf);
18553 	}
18554 }
18555 
18556 /**
18557  * lpfc_rcv_seq_check_edtov - Cleans up timed out receive sequences.
18558  * @vport: The vport that the received sequences were sent to.
18559  *
18560  * This function determines whether any received sequences have timed out by
18561  * first checking the vport's rcv_buffer_time_stamp. If this time_stamp
18562  * indicates that there is at least one timed out sequence this routine will
18563  * go through the received sequences one at a time from most inactive to most
18564  * active to determine which ones need to be cleaned up. Once it has determined
18565  * that a sequence needs to be cleaned up it will simply free up the resources
18566  * without sending an abort.
18567  **/
18568 void
18569 lpfc_rcv_seq_check_edtov(struct lpfc_vport *vport)
18570 {
18571 	struct lpfc_dmabuf *h_buf, *hnext;
18572 	struct lpfc_dmabuf *d_buf, *dnext;
18573 	struct hbq_dmabuf *dmabuf = NULL;
18574 	unsigned long timeout;
18575 	int abort_count = 0;
18576 
18577 	timeout = (msecs_to_jiffies(vport->phba->fc_edtov) +
18578 		   vport->rcv_buffer_time_stamp);
18579 	if (list_empty(&vport->rcv_buffer_list) ||
18580 	    time_before(jiffies, timeout))
18581 		return;
18582 	/* start with the oldest sequence on the rcv list */
18583 	list_for_each_entry_safe(h_buf, hnext, &vport->rcv_buffer_list, list) {
18584 		dmabuf = container_of(h_buf, struct hbq_dmabuf, hbuf);
18585 		timeout = (msecs_to_jiffies(vport->phba->fc_edtov) +
18586 			   dmabuf->time_stamp);
18587 		if (time_before(jiffies, timeout))
18588 			break;
18589 		abort_count++;
18590 		list_del_init(&dmabuf->hbuf.list);
18591 		list_for_each_entry_safe(d_buf, dnext,
18592 					 &dmabuf->dbuf.list, list) {
18593 			list_del_init(&d_buf->list);
18594 			lpfc_in_buf_free(vport->phba, d_buf);
18595 		}
18596 		lpfc_in_buf_free(vport->phba, &dmabuf->dbuf);
18597 	}
18598 	if (abort_count)
18599 		lpfc_update_rcv_time_stamp(vport);
18600 }
18601 
18602 /**
18603  * lpfc_fc_frame_add - Adds a frame to the vport's list of received sequences
18604  * @vport: pointer to a vitural port
18605  * @dmabuf: pointer to a dmabuf that describes the hdr and data of the FC frame
18606  *
18607  * This function searches through the existing incomplete sequences that have
18608  * been sent to this @vport. If the frame matches one of the incomplete
18609  * sequences then the dbuf in the @dmabuf is added to the list of frames that
18610  * make up that sequence. If no sequence is found that matches this frame then
18611  * the function will add the hbuf in the @dmabuf to the @vport's rcv_buffer_list
18612  * This function returns a pointer to the first dmabuf in the sequence list that
18613  * the frame was linked to.
18614  **/
18615 static struct hbq_dmabuf *
18616 lpfc_fc_frame_add(struct lpfc_vport *vport, struct hbq_dmabuf *dmabuf)
18617 {
18618 	struct fc_frame_header *new_hdr;
18619 	struct fc_frame_header *temp_hdr;
18620 	struct lpfc_dmabuf *d_buf;
18621 	struct lpfc_dmabuf *h_buf;
18622 	struct hbq_dmabuf *seq_dmabuf = NULL;
18623 	struct hbq_dmabuf *temp_dmabuf = NULL;
18624 	uint8_t	found = 0;
18625 
18626 	INIT_LIST_HEAD(&dmabuf->dbuf.list);
18627 	dmabuf->time_stamp = jiffies;
18628 	new_hdr = (struct fc_frame_header *)dmabuf->hbuf.virt;
18629 
18630 	/* Use the hdr_buf to find the sequence that this frame belongs to */
18631 	list_for_each_entry(h_buf, &vport->rcv_buffer_list, list) {
18632 		temp_hdr = (struct fc_frame_header *)h_buf->virt;
18633 		if ((temp_hdr->fh_seq_id != new_hdr->fh_seq_id) ||
18634 		    (temp_hdr->fh_ox_id != new_hdr->fh_ox_id) ||
18635 		    (memcmp(&temp_hdr->fh_s_id, &new_hdr->fh_s_id, 3)))
18636 			continue;
18637 		/* found a pending sequence that matches this frame */
18638 		seq_dmabuf = container_of(h_buf, struct hbq_dmabuf, hbuf);
18639 		break;
18640 	}
18641 	if (!seq_dmabuf) {
18642 		/*
18643 		 * This indicates first frame received for this sequence.
18644 		 * Queue the buffer on the vport's rcv_buffer_list.
18645 		 */
18646 		list_add_tail(&dmabuf->hbuf.list, &vport->rcv_buffer_list);
18647 		lpfc_update_rcv_time_stamp(vport);
18648 		return dmabuf;
18649 	}
18650 	temp_hdr = seq_dmabuf->hbuf.virt;
18651 	if (be16_to_cpu(new_hdr->fh_seq_cnt) <
18652 		be16_to_cpu(temp_hdr->fh_seq_cnt)) {
18653 		list_del_init(&seq_dmabuf->hbuf.list);
18654 		list_add_tail(&dmabuf->hbuf.list, &vport->rcv_buffer_list);
18655 		list_add_tail(&dmabuf->dbuf.list, &seq_dmabuf->dbuf.list);
18656 		lpfc_update_rcv_time_stamp(vport);
18657 		return dmabuf;
18658 	}
18659 	/* move this sequence to the tail to indicate a young sequence */
18660 	list_move_tail(&seq_dmabuf->hbuf.list, &vport->rcv_buffer_list);
18661 	seq_dmabuf->time_stamp = jiffies;
18662 	lpfc_update_rcv_time_stamp(vport);
18663 	if (list_empty(&seq_dmabuf->dbuf.list)) {
18664 		list_add_tail(&dmabuf->dbuf.list, &seq_dmabuf->dbuf.list);
18665 		return seq_dmabuf;
18666 	}
18667 	/* find the correct place in the sequence to insert this frame */
18668 	d_buf = list_entry(seq_dmabuf->dbuf.list.prev, typeof(*d_buf), list);
18669 	while (!found) {
18670 		temp_dmabuf = container_of(d_buf, struct hbq_dmabuf, dbuf);
18671 		temp_hdr = (struct fc_frame_header *)temp_dmabuf->hbuf.virt;
18672 		/*
18673 		 * If the frame's sequence count is greater than the frame on
18674 		 * the list then insert the frame right after this frame
18675 		 */
18676 		if (be16_to_cpu(new_hdr->fh_seq_cnt) >
18677 			be16_to_cpu(temp_hdr->fh_seq_cnt)) {
18678 			list_add(&dmabuf->dbuf.list, &temp_dmabuf->dbuf.list);
18679 			found = 1;
18680 			break;
18681 		}
18682 
18683 		if (&d_buf->list == &seq_dmabuf->dbuf.list)
18684 			break;
18685 		d_buf = list_entry(d_buf->list.prev, typeof(*d_buf), list);
18686 	}
18687 
18688 	if (found)
18689 		return seq_dmabuf;
18690 	return NULL;
18691 }
18692 
18693 /**
18694  * lpfc_sli4_abort_partial_seq - Abort partially assembled unsol sequence
18695  * @vport: pointer to a vitural port
18696  * @dmabuf: pointer to a dmabuf that describes the FC sequence
18697  *
18698  * This function tries to abort from the partially assembed sequence, described
18699  * by the information from basic abbort @dmabuf. It checks to see whether such
18700  * partially assembled sequence held by the driver. If so, it shall free up all
18701  * the frames from the partially assembled sequence.
18702  *
18703  * Return
18704  * true  -- if there is matching partially assembled sequence present and all
18705  *          the frames freed with the sequence;
18706  * false -- if there is no matching partially assembled sequence present so
18707  *          nothing got aborted in the lower layer driver
18708  **/
18709 static bool
18710 lpfc_sli4_abort_partial_seq(struct lpfc_vport *vport,
18711 			    struct hbq_dmabuf *dmabuf)
18712 {
18713 	struct fc_frame_header *new_hdr;
18714 	struct fc_frame_header *temp_hdr;
18715 	struct lpfc_dmabuf *d_buf, *n_buf, *h_buf;
18716 	struct hbq_dmabuf *seq_dmabuf = NULL;
18717 
18718 	/* Use the hdr_buf to find the sequence that matches this frame */
18719 	INIT_LIST_HEAD(&dmabuf->dbuf.list);
18720 	INIT_LIST_HEAD(&dmabuf->hbuf.list);
18721 	new_hdr = (struct fc_frame_header *)dmabuf->hbuf.virt;
18722 	list_for_each_entry(h_buf, &vport->rcv_buffer_list, list) {
18723 		temp_hdr = (struct fc_frame_header *)h_buf->virt;
18724 		if ((temp_hdr->fh_seq_id != new_hdr->fh_seq_id) ||
18725 		    (temp_hdr->fh_ox_id != new_hdr->fh_ox_id) ||
18726 		    (memcmp(&temp_hdr->fh_s_id, &new_hdr->fh_s_id, 3)))
18727 			continue;
18728 		/* found a pending sequence that matches this frame */
18729 		seq_dmabuf = container_of(h_buf, struct hbq_dmabuf, hbuf);
18730 		break;
18731 	}
18732 
18733 	/* Free up all the frames from the partially assembled sequence */
18734 	if (seq_dmabuf) {
18735 		list_for_each_entry_safe(d_buf, n_buf,
18736 					 &seq_dmabuf->dbuf.list, list) {
18737 			list_del_init(&d_buf->list);
18738 			lpfc_in_buf_free(vport->phba, d_buf);
18739 		}
18740 		return true;
18741 	}
18742 	return false;
18743 }
18744 
18745 /**
18746  * lpfc_sli4_abort_ulp_seq - Abort assembled unsol sequence from ulp
18747  * @vport: pointer to a vitural port
18748  * @dmabuf: pointer to a dmabuf that describes the FC sequence
18749  *
18750  * This function tries to abort from the assembed sequence from upper level
18751  * protocol, described by the information from basic abbort @dmabuf. It
18752  * checks to see whether such pending context exists at upper level protocol.
18753  * If so, it shall clean up the pending context.
18754  *
18755  * Return
18756  * true  -- if there is matching pending context of the sequence cleaned
18757  *          at ulp;
18758  * false -- if there is no matching pending context of the sequence present
18759  *          at ulp.
18760  **/
18761 static bool
18762 lpfc_sli4_abort_ulp_seq(struct lpfc_vport *vport, struct hbq_dmabuf *dmabuf)
18763 {
18764 	struct lpfc_hba *phba = vport->phba;
18765 	int handled;
18766 
18767 	/* Accepting abort at ulp with SLI4 only */
18768 	if (phba->sli_rev < LPFC_SLI_REV4)
18769 		return false;
18770 
18771 	/* Register all caring upper level protocols to attend abort */
18772 	handled = lpfc_ct_handle_unsol_abort(phba, dmabuf);
18773 	if (handled)
18774 		return true;
18775 
18776 	return false;
18777 }
18778 
18779 /**
18780  * lpfc_sli4_seq_abort_rsp_cmpl - BLS ABORT RSP seq abort iocb complete handler
18781  * @phba: Pointer to HBA context object.
18782  * @cmd_iocbq: pointer to the command iocbq structure.
18783  * @rsp_iocbq: pointer to the response iocbq structure.
18784  *
18785  * This function handles the sequence abort response iocb command complete
18786  * event. It properly releases the memory allocated to the sequence abort
18787  * accept iocb.
18788  **/
18789 static void
18790 lpfc_sli4_seq_abort_rsp_cmpl(struct lpfc_hba *phba,
18791 			     struct lpfc_iocbq *cmd_iocbq,
18792 			     struct lpfc_iocbq *rsp_iocbq)
18793 {
18794 	struct lpfc_nodelist *ndlp;
18795 
18796 	if (cmd_iocbq) {
18797 		ndlp = (struct lpfc_nodelist *)cmd_iocbq->context1;
18798 		lpfc_nlp_put(ndlp);
18799 		lpfc_sli_release_iocbq(phba, cmd_iocbq);
18800 	}
18801 
18802 	/* Failure means BLS ABORT RSP did not get delivered to remote node*/
18803 	if (rsp_iocbq && rsp_iocbq->iocb.ulpStatus)
18804 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
18805 			"3154 BLS ABORT RSP failed, data:  x%x/x%x\n",
18806 			rsp_iocbq->iocb.ulpStatus,
18807 			rsp_iocbq->iocb.un.ulpWord[4]);
18808 }
18809 
18810 /**
18811  * lpfc_sli4_xri_inrange - check xri is in range of xris owned by driver.
18812  * @phba: Pointer to HBA context object.
18813  * @xri: xri id in transaction.
18814  *
18815  * This function validates the xri maps to the known range of XRIs allocated an
18816  * used by the driver.
18817  **/
18818 uint16_t
18819 lpfc_sli4_xri_inrange(struct lpfc_hba *phba,
18820 		      uint16_t xri)
18821 {
18822 	uint16_t i;
18823 
18824 	for (i = 0; i < phba->sli4_hba.max_cfg_param.max_xri; i++) {
18825 		if (xri == phba->sli4_hba.xri_ids[i])
18826 			return i;
18827 	}
18828 	return NO_XRI;
18829 }
18830 
18831 /**
18832  * lpfc_sli4_seq_abort_rsp - bls rsp to sequence abort
18833  * @vport: pointer to a virtual port.
18834  * @fc_hdr: pointer to a FC frame header.
18835  * @aborted: was the partially assembled receive sequence successfully aborted
18836  *
18837  * This function sends a basic response to a previous unsol sequence abort
18838  * event after aborting the sequence handling.
18839  **/
18840 void
18841 lpfc_sli4_seq_abort_rsp(struct lpfc_vport *vport,
18842 			struct fc_frame_header *fc_hdr, bool aborted)
18843 {
18844 	struct lpfc_hba *phba = vport->phba;
18845 	struct lpfc_iocbq *ctiocb = NULL;
18846 	struct lpfc_nodelist *ndlp;
18847 	uint16_t oxid, rxid, xri, lxri;
18848 	uint32_t sid, fctl;
18849 	IOCB_t *icmd;
18850 	int rc;
18851 
18852 	if (!lpfc_is_link_up(phba))
18853 		return;
18854 
18855 	sid = sli4_sid_from_fc_hdr(fc_hdr);
18856 	oxid = be16_to_cpu(fc_hdr->fh_ox_id);
18857 	rxid = be16_to_cpu(fc_hdr->fh_rx_id);
18858 
18859 	ndlp = lpfc_findnode_did(vport, sid);
18860 	if (!ndlp) {
18861 		ndlp = lpfc_nlp_init(vport, sid);
18862 		if (!ndlp) {
18863 			lpfc_printf_vlog(vport, KERN_WARNING, LOG_ELS,
18864 					 "1268 Failed to allocate ndlp for "
18865 					 "oxid:x%x SID:x%x\n", oxid, sid);
18866 			return;
18867 		}
18868 		/* Put ndlp onto pport node list */
18869 		lpfc_enqueue_node(vport, ndlp);
18870 	}
18871 
18872 	/* Allocate buffer for rsp iocb */
18873 	ctiocb = lpfc_sli_get_iocbq(phba);
18874 	if (!ctiocb)
18875 		return;
18876 
18877 	/* Extract the F_CTL field from FC_HDR */
18878 	fctl = sli4_fctl_from_fc_hdr(fc_hdr);
18879 
18880 	icmd = &ctiocb->iocb;
18881 	icmd->un.xseq64.bdl.bdeSize = 0;
18882 	icmd->un.xseq64.bdl.ulpIoTag32 = 0;
18883 	icmd->un.xseq64.w5.hcsw.Dfctl = 0;
18884 	icmd->un.xseq64.w5.hcsw.Rctl = FC_RCTL_BA_ACC;
18885 	icmd->un.xseq64.w5.hcsw.Type = FC_TYPE_BLS;
18886 
18887 	/* Fill in the rest of iocb fields */
18888 	icmd->ulpCommand = CMD_XMIT_BLS_RSP64_CX;
18889 	icmd->ulpBdeCount = 0;
18890 	icmd->ulpLe = 1;
18891 	icmd->ulpClass = CLASS3;
18892 	icmd->ulpContext = phba->sli4_hba.rpi_ids[ndlp->nlp_rpi];
18893 	ctiocb->context1 = lpfc_nlp_get(ndlp);
18894 	if (!ctiocb->context1) {
18895 		lpfc_sli_release_iocbq(phba, ctiocb);
18896 		return;
18897 	}
18898 
18899 	ctiocb->vport = phba->pport;
18900 	ctiocb->iocb_cmpl = lpfc_sli4_seq_abort_rsp_cmpl;
18901 	ctiocb->sli4_lxritag = NO_XRI;
18902 	ctiocb->sli4_xritag = NO_XRI;
18903 
18904 	if (fctl & FC_FC_EX_CTX)
18905 		/* Exchange responder sent the abort so we
18906 		 * own the oxid.
18907 		 */
18908 		xri = oxid;
18909 	else
18910 		xri = rxid;
18911 	lxri = lpfc_sli4_xri_inrange(phba, xri);
18912 	if (lxri != NO_XRI)
18913 		lpfc_set_rrq_active(phba, ndlp, lxri,
18914 			(xri == oxid) ? rxid : oxid, 0);
18915 	/* For BA_ABTS from exchange responder, if the logical xri with
18916 	 * the oxid maps to the FCP XRI range, the port no longer has
18917 	 * that exchange context, send a BLS_RJT. Override the IOCB for
18918 	 * a BA_RJT.
18919 	 */
18920 	if ((fctl & FC_FC_EX_CTX) &&
18921 	    (lxri > lpfc_sli4_get_iocb_cnt(phba))) {
18922 		icmd->un.xseq64.w5.hcsw.Rctl = FC_RCTL_BA_RJT;
18923 		bf_set(lpfc_vndr_code, &icmd->un.bls_rsp, 0);
18924 		bf_set(lpfc_rsn_expln, &icmd->un.bls_rsp, FC_BA_RJT_INV_XID);
18925 		bf_set(lpfc_rsn_code, &icmd->un.bls_rsp, FC_BA_RJT_UNABLE);
18926 	}
18927 
18928 	/* If BA_ABTS failed to abort a partially assembled receive sequence,
18929 	 * the driver no longer has that exchange, send a BLS_RJT. Override
18930 	 * the IOCB for a BA_RJT.
18931 	 */
18932 	if (aborted == false) {
18933 		icmd->un.xseq64.w5.hcsw.Rctl = FC_RCTL_BA_RJT;
18934 		bf_set(lpfc_vndr_code, &icmd->un.bls_rsp, 0);
18935 		bf_set(lpfc_rsn_expln, &icmd->un.bls_rsp, FC_BA_RJT_INV_XID);
18936 		bf_set(lpfc_rsn_code, &icmd->un.bls_rsp, FC_BA_RJT_UNABLE);
18937 	}
18938 
18939 	if (fctl & FC_FC_EX_CTX) {
18940 		/* ABTS sent by responder to CT exchange, construction
18941 		 * of BA_ACC will use OX_ID from ABTS for the XRI_TAG
18942 		 * field and RX_ID from ABTS for RX_ID field.
18943 		 */
18944 		bf_set(lpfc_abts_orig, &icmd->un.bls_rsp, LPFC_ABTS_UNSOL_RSP);
18945 	} else {
18946 		/* ABTS sent by initiator to CT exchange, construction
18947 		 * of BA_ACC will need to allocate a new XRI as for the
18948 		 * XRI_TAG field.
18949 		 */
18950 		bf_set(lpfc_abts_orig, &icmd->un.bls_rsp, LPFC_ABTS_UNSOL_INT);
18951 	}
18952 	bf_set(lpfc_abts_rxid, &icmd->un.bls_rsp, rxid);
18953 	bf_set(lpfc_abts_oxid, &icmd->un.bls_rsp, oxid);
18954 
18955 	/* Xmit CT abts response on exchange <xid> */
18956 	lpfc_printf_vlog(vport, KERN_INFO, LOG_ELS,
18957 			 "1200 Send BLS cmd x%x on oxid x%x Data: x%x\n",
18958 			 icmd->un.xseq64.w5.hcsw.Rctl, oxid, phba->link_state);
18959 
18960 	rc = lpfc_sli_issue_iocb(phba, LPFC_ELS_RING, ctiocb, 0);
18961 	if (rc == IOCB_ERROR) {
18962 		lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
18963 				 "2925 Failed to issue CT ABTS RSP x%x on "
18964 				 "xri x%x, Data x%x\n",
18965 				 icmd->un.xseq64.w5.hcsw.Rctl, oxid,
18966 				 phba->link_state);
18967 		lpfc_nlp_put(ndlp);
18968 		ctiocb->context1 = NULL;
18969 		lpfc_sli_release_iocbq(phba, ctiocb);
18970 	}
18971 }
18972 
18973 /**
18974  * lpfc_sli4_handle_unsol_abort - Handle sli-4 unsolicited abort event
18975  * @vport: Pointer to the vport on which this sequence was received
18976  * @dmabuf: pointer to a dmabuf that describes the FC sequence
18977  *
18978  * This function handles an SLI-4 unsolicited abort event. If the unsolicited
18979  * receive sequence is only partially assembed by the driver, it shall abort
18980  * the partially assembled frames for the sequence. Otherwise, if the
18981  * unsolicited receive sequence has been completely assembled and passed to
18982  * the Upper Layer Protocol (ULP), it then mark the per oxid status for the
18983  * unsolicited sequence has been aborted. After that, it will issue a basic
18984  * accept to accept the abort.
18985  **/
18986 static void
18987 lpfc_sli4_handle_unsol_abort(struct lpfc_vport *vport,
18988 			     struct hbq_dmabuf *dmabuf)
18989 {
18990 	struct lpfc_hba *phba = vport->phba;
18991 	struct fc_frame_header fc_hdr;
18992 	uint32_t fctl;
18993 	bool aborted;
18994 
18995 	/* Make a copy of fc_hdr before the dmabuf being released */
18996 	memcpy(&fc_hdr, dmabuf->hbuf.virt, sizeof(struct fc_frame_header));
18997 	fctl = sli4_fctl_from_fc_hdr(&fc_hdr);
18998 
18999 	if (fctl & FC_FC_EX_CTX) {
19000 		/* ABTS by responder to exchange, no cleanup needed */
19001 		aborted = true;
19002 	} else {
19003 		/* ABTS by initiator to exchange, need to do cleanup */
19004 		aborted = lpfc_sli4_abort_partial_seq(vport, dmabuf);
19005 		if (aborted == false)
19006 			aborted = lpfc_sli4_abort_ulp_seq(vport, dmabuf);
19007 	}
19008 	lpfc_in_buf_free(phba, &dmabuf->dbuf);
19009 
19010 	if (phba->nvmet_support) {
19011 		lpfc_nvmet_rcv_unsol_abort(vport, &fc_hdr);
19012 		return;
19013 	}
19014 
19015 	/* Respond with BA_ACC or BA_RJT accordingly */
19016 	lpfc_sli4_seq_abort_rsp(vport, &fc_hdr, aborted);
19017 }
19018 
19019 /**
19020  * lpfc_seq_complete - Indicates if a sequence is complete
19021  * @dmabuf: pointer to a dmabuf that describes the FC sequence
19022  *
19023  * This function checks the sequence, starting with the frame described by
19024  * @dmabuf, to see if all the frames associated with this sequence are present.
19025  * the frames associated with this sequence are linked to the @dmabuf using the
19026  * dbuf list. This function looks for two major things. 1) That the first frame
19027  * has a sequence count of zero. 2) There is a frame with last frame of sequence
19028  * set. 3) That there are no holes in the sequence count. The function will
19029  * return 1 when the sequence is complete, otherwise it will return 0.
19030  **/
19031 static int
19032 lpfc_seq_complete(struct hbq_dmabuf *dmabuf)
19033 {
19034 	struct fc_frame_header *hdr;
19035 	struct lpfc_dmabuf *d_buf;
19036 	struct hbq_dmabuf *seq_dmabuf;
19037 	uint32_t fctl;
19038 	int seq_count = 0;
19039 
19040 	hdr = (struct fc_frame_header *)dmabuf->hbuf.virt;
19041 	/* make sure first fame of sequence has a sequence count of zero */
19042 	if (hdr->fh_seq_cnt != seq_count)
19043 		return 0;
19044 	fctl = (hdr->fh_f_ctl[0] << 16 |
19045 		hdr->fh_f_ctl[1] << 8 |
19046 		hdr->fh_f_ctl[2]);
19047 	/* If last frame of sequence we can return success. */
19048 	if (fctl & FC_FC_END_SEQ)
19049 		return 1;
19050 	list_for_each_entry(d_buf, &dmabuf->dbuf.list, list) {
19051 		seq_dmabuf = container_of(d_buf, struct hbq_dmabuf, dbuf);
19052 		hdr = (struct fc_frame_header *)seq_dmabuf->hbuf.virt;
19053 		/* If there is a hole in the sequence count then fail. */
19054 		if (++seq_count != be16_to_cpu(hdr->fh_seq_cnt))
19055 			return 0;
19056 		fctl = (hdr->fh_f_ctl[0] << 16 |
19057 			hdr->fh_f_ctl[1] << 8 |
19058 			hdr->fh_f_ctl[2]);
19059 		/* If last frame of sequence we can return success. */
19060 		if (fctl & FC_FC_END_SEQ)
19061 			return 1;
19062 	}
19063 	return 0;
19064 }
19065 
19066 /**
19067  * lpfc_prep_seq - Prep sequence for ULP processing
19068  * @vport: Pointer to the vport on which this sequence was received
19069  * @seq_dmabuf: pointer to a dmabuf that describes the FC sequence
19070  *
19071  * This function takes a sequence, described by a list of frames, and creates
19072  * a list of iocbq structures to describe the sequence. This iocbq list will be
19073  * used to issue to the generic unsolicited sequence handler. This routine
19074  * returns a pointer to the first iocbq in the list. If the function is unable
19075  * to allocate an iocbq then it throw out the received frames that were not
19076  * able to be described and return a pointer to the first iocbq. If unable to
19077  * allocate any iocbqs (including the first) this function will return NULL.
19078  **/
19079 static struct lpfc_iocbq *
19080 lpfc_prep_seq(struct lpfc_vport *vport, struct hbq_dmabuf *seq_dmabuf)
19081 {
19082 	struct hbq_dmabuf *hbq_buf;
19083 	struct lpfc_dmabuf *d_buf, *n_buf;
19084 	struct lpfc_iocbq *first_iocbq, *iocbq;
19085 	struct fc_frame_header *fc_hdr;
19086 	uint32_t sid;
19087 	uint32_t len, tot_len;
19088 	struct ulp_bde64 *pbde;
19089 
19090 	fc_hdr = (struct fc_frame_header *)seq_dmabuf->hbuf.virt;
19091 	/* remove from receive buffer list */
19092 	list_del_init(&seq_dmabuf->hbuf.list);
19093 	lpfc_update_rcv_time_stamp(vport);
19094 	/* get the Remote Port's SID */
19095 	sid = sli4_sid_from_fc_hdr(fc_hdr);
19096 	tot_len = 0;
19097 	/* Get an iocbq struct to fill in. */
19098 	first_iocbq = lpfc_sli_get_iocbq(vport->phba);
19099 	if (first_iocbq) {
19100 		/* Initialize the first IOCB. */
19101 		first_iocbq->iocb.unsli3.rcvsli3.acc_len = 0;
19102 		first_iocbq->iocb.ulpStatus = IOSTAT_SUCCESS;
19103 		first_iocbq->vport = vport;
19104 
19105 		/* Check FC Header to see what TYPE of frame we are rcv'ing */
19106 		if (sli4_type_from_fc_hdr(fc_hdr) == FC_TYPE_ELS) {
19107 			first_iocbq->iocb.ulpCommand = CMD_IOCB_RCV_ELS64_CX;
19108 			first_iocbq->iocb.un.rcvels.parmRo =
19109 				sli4_did_from_fc_hdr(fc_hdr);
19110 			first_iocbq->iocb.ulpPU = PARM_NPIV_DID;
19111 		} else
19112 			first_iocbq->iocb.ulpCommand = CMD_IOCB_RCV_SEQ64_CX;
19113 		first_iocbq->iocb.ulpContext = NO_XRI;
19114 		first_iocbq->iocb.unsli3.rcvsli3.ox_id =
19115 			be16_to_cpu(fc_hdr->fh_ox_id);
19116 		/* iocbq is prepped for internal consumption.  Physical vpi. */
19117 		first_iocbq->iocb.unsli3.rcvsli3.vpi =
19118 			vport->phba->vpi_ids[vport->vpi];
19119 		/* put the first buffer into the first IOCBq */
19120 		tot_len = bf_get(lpfc_rcqe_length,
19121 				       &seq_dmabuf->cq_event.cqe.rcqe_cmpl);
19122 
19123 		first_iocbq->context2 = &seq_dmabuf->dbuf;
19124 		first_iocbq->context3 = NULL;
19125 		first_iocbq->iocb.ulpBdeCount = 1;
19126 		if (tot_len > LPFC_DATA_BUF_SIZE)
19127 			first_iocbq->iocb.un.cont64[0].tus.f.bdeSize =
19128 							LPFC_DATA_BUF_SIZE;
19129 		else
19130 			first_iocbq->iocb.un.cont64[0].tus.f.bdeSize = tot_len;
19131 
19132 		first_iocbq->iocb.un.rcvels.remoteID = sid;
19133 
19134 		first_iocbq->iocb.unsli3.rcvsli3.acc_len = tot_len;
19135 	}
19136 	iocbq = first_iocbq;
19137 	/*
19138 	 * Each IOCBq can have two Buffers assigned, so go through the list
19139 	 * of buffers for this sequence and save two buffers in each IOCBq
19140 	 */
19141 	list_for_each_entry_safe(d_buf, n_buf, &seq_dmabuf->dbuf.list, list) {
19142 		if (!iocbq) {
19143 			lpfc_in_buf_free(vport->phba, d_buf);
19144 			continue;
19145 		}
19146 		if (!iocbq->context3) {
19147 			iocbq->context3 = d_buf;
19148 			iocbq->iocb.ulpBdeCount++;
19149 			/* We need to get the size out of the right CQE */
19150 			hbq_buf = container_of(d_buf, struct hbq_dmabuf, dbuf);
19151 			len = bf_get(lpfc_rcqe_length,
19152 				       &hbq_buf->cq_event.cqe.rcqe_cmpl);
19153 			pbde = (struct ulp_bde64 *)
19154 					&iocbq->iocb.unsli3.sli3Words[4];
19155 			if (len > LPFC_DATA_BUF_SIZE)
19156 				pbde->tus.f.bdeSize = LPFC_DATA_BUF_SIZE;
19157 			else
19158 				pbde->tus.f.bdeSize = len;
19159 
19160 			iocbq->iocb.unsli3.rcvsli3.acc_len += len;
19161 			tot_len += len;
19162 		} else {
19163 			iocbq = lpfc_sli_get_iocbq(vport->phba);
19164 			if (!iocbq) {
19165 				if (first_iocbq) {
19166 					first_iocbq->iocb.ulpStatus =
19167 							IOSTAT_FCP_RSP_ERROR;
19168 					first_iocbq->iocb.un.ulpWord[4] =
19169 							IOERR_NO_RESOURCES;
19170 				}
19171 				lpfc_in_buf_free(vport->phba, d_buf);
19172 				continue;
19173 			}
19174 			/* We need to get the size out of the right CQE */
19175 			hbq_buf = container_of(d_buf, struct hbq_dmabuf, dbuf);
19176 			len = bf_get(lpfc_rcqe_length,
19177 				       &hbq_buf->cq_event.cqe.rcqe_cmpl);
19178 			iocbq->context2 = d_buf;
19179 			iocbq->context3 = NULL;
19180 			iocbq->iocb.ulpBdeCount = 1;
19181 			if (len > LPFC_DATA_BUF_SIZE)
19182 				iocbq->iocb.un.cont64[0].tus.f.bdeSize =
19183 							LPFC_DATA_BUF_SIZE;
19184 			else
19185 				iocbq->iocb.un.cont64[0].tus.f.bdeSize = len;
19186 
19187 			tot_len += len;
19188 			iocbq->iocb.unsli3.rcvsli3.acc_len = tot_len;
19189 
19190 			iocbq->iocb.un.rcvels.remoteID = sid;
19191 			list_add_tail(&iocbq->list, &first_iocbq->list);
19192 		}
19193 	}
19194 	/* Free the sequence's header buffer */
19195 	if (!first_iocbq)
19196 		lpfc_in_buf_free(vport->phba, &seq_dmabuf->dbuf);
19197 
19198 	return first_iocbq;
19199 }
19200 
19201 static void
19202 lpfc_sli4_send_seq_to_ulp(struct lpfc_vport *vport,
19203 			  struct hbq_dmabuf *seq_dmabuf)
19204 {
19205 	struct fc_frame_header *fc_hdr;
19206 	struct lpfc_iocbq *iocbq, *curr_iocb, *next_iocb;
19207 	struct lpfc_hba *phba = vport->phba;
19208 
19209 	fc_hdr = (struct fc_frame_header *)seq_dmabuf->hbuf.virt;
19210 	iocbq = lpfc_prep_seq(vport, seq_dmabuf);
19211 	if (!iocbq) {
19212 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
19213 				"2707 Ring %d handler: Failed to allocate "
19214 				"iocb Rctl x%x Type x%x received\n",
19215 				LPFC_ELS_RING,
19216 				fc_hdr->fh_r_ctl, fc_hdr->fh_type);
19217 		return;
19218 	}
19219 	if (!lpfc_complete_unsol_iocb(phba,
19220 				      phba->sli4_hba.els_wq->pring,
19221 				      iocbq, fc_hdr->fh_r_ctl,
19222 				      fc_hdr->fh_type))
19223 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
19224 				"2540 Ring %d handler: unexpected Rctl "
19225 				"x%x Type x%x received\n",
19226 				LPFC_ELS_RING,
19227 				fc_hdr->fh_r_ctl, fc_hdr->fh_type);
19228 
19229 	/* Free iocb created in lpfc_prep_seq */
19230 	list_for_each_entry_safe(curr_iocb, next_iocb,
19231 		&iocbq->list, list) {
19232 		list_del_init(&curr_iocb->list);
19233 		lpfc_sli_release_iocbq(phba, curr_iocb);
19234 	}
19235 	lpfc_sli_release_iocbq(phba, iocbq);
19236 }
19237 
19238 static void
19239 lpfc_sli4_mds_loopback_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
19240 			    struct lpfc_iocbq *rspiocb)
19241 {
19242 	struct lpfc_dmabuf *pcmd = cmdiocb->context2;
19243 
19244 	if (pcmd && pcmd->virt)
19245 		dma_pool_free(phba->lpfc_drb_pool, pcmd->virt, pcmd->phys);
19246 	kfree(pcmd);
19247 	lpfc_sli_release_iocbq(phba, cmdiocb);
19248 	lpfc_drain_txq(phba);
19249 }
19250 
19251 static void
19252 lpfc_sli4_handle_mds_loopback(struct lpfc_vport *vport,
19253 			      struct hbq_dmabuf *dmabuf)
19254 {
19255 	struct fc_frame_header *fc_hdr;
19256 	struct lpfc_hba *phba = vport->phba;
19257 	struct lpfc_iocbq *iocbq = NULL;
19258 	union  lpfc_wqe *wqe;
19259 	struct lpfc_dmabuf *pcmd = NULL;
19260 	uint32_t frame_len;
19261 	int rc;
19262 	unsigned long iflags;
19263 
19264 	fc_hdr = (struct fc_frame_header *)dmabuf->hbuf.virt;
19265 	frame_len = bf_get(lpfc_rcqe_length, &dmabuf->cq_event.cqe.rcqe_cmpl);
19266 
19267 	/* Send the received frame back */
19268 	iocbq = lpfc_sli_get_iocbq(phba);
19269 	if (!iocbq) {
19270 		/* Queue cq event and wakeup worker thread to process it */
19271 		spin_lock_irqsave(&phba->hbalock, iflags);
19272 		list_add_tail(&dmabuf->cq_event.list,
19273 			      &phba->sli4_hba.sp_queue_event);
19274 		phba->hba_flag |= HBA_SP_QUEUE_EVT;
19275 		spin_unlock_irqrestore(&phba->hbalock, iflags);
19276 		lpfc_worker_wake_up(phba);
19277 		return;
19278 	}
19279 
19280 	/* Allocate buffer for command payload */
19281 	pcmd = kmalloc(sizeof(struct lpfc_dmabuf), GFP_KERNEL);
19282 	if (pcmd)
19283 		pcmd->virt = dma_pool_alloc(phba->lpfc_drb_pool, GFP_KERNEL,
19284 					    &pcmd->phys);
19285 	if (!pcmd || !pcmd->virt)
19286 		goto exit;
19287 
19288 	INIT_LIST_HEAD(&pcmd->list);
19289 
19290 	/* copyin the payload */
19291 	memcpy(pcmd->virt, dmabuf->dbuf.virt, frame_len);
19292 
19293 	/* fill in BDE's for command */
19294 	iocbq->iocb.un.xseq64.bdl.addrHigh = putPaddrHigh(pcmd->phys);
19295 	iocbq->iocb.un.xseq64.bdl.addrLow = putPaddrLow(pcmd->phys);
19296 	iocbq->iocb.un.xseq64.bdl.bdeFlags = BUFF_TYPE_BDE_64;
19297 	iocbq->iocb.un.xseq64.bdl.bdeSize = frame_len;
19298 
19299 	iocbq->context2 = pcmd;
19300 	iocbq->vport = vport;
19301 	iocbq->iocb_flag &= ~LPFC_FIP_ELS_ID_MASK;
19302 	iocbq->iocb_flag |= LPFC_USE_FCPWQIDX;
19303 
19304 	/*
19305 	 * Setup rest of the iocb as though it were a WQE
19306 	 * Build the SEND_FRAME WQE
19307 	 */
19308 	wqe = (union lpfc_wqe *)&iocbq->iocb;
19309 
19310 	wqe->send_frame.frame_len = frame_len;
19311 	wqe->send_frame.fc_hdr_wd0 = be32_to_cpu(*((uint32_t *)fc_hdr));
19312 	wqe->send_frame.fc_hdr_wd1 = be32_to_cpu(*((uint32_t *)fc_hdr + 1));
19313 	wqe->send_frame.fc_hdr_wd2 = be32_to_cpu(*((uint32_t *)fc_hdr + 2));
19314 	wqe->send_frame.fc_hdr_wd3 = be32_to_cpu(*((uint32_t *)fc_hdr + 3));
19315 	wqe->send_frame.fc_hdr_wd4 = be32_to_cpu(*((uint32_t *)fc_hdr + 4));
19316 	wqe->send_frame.fc_hdr_wd5 = be32_to_cpu(*((uint32_t *)fc_hdr + 5));
19317 
19318 	iocbq->iocb.ulpCommand = CMD_SEND_FRAME;
19319 	iocbq->iocb.ulpLe = 1;
19320 	iocbq->iocb_cmpl = lpfc_sli4_mds_loopback_cmpl;
19321 	rc = lpfc_sli_issue_iocb(phba, LPFC_ELS_RING, iocbq, 0);
19322 	if (rc == IOCB_ERROR)
19323 		goto exit;
19324 
19325 	lpfc_in_buf_free(phba, &dmabuf->dbuf);
19326 	return;
19327 
19328 exit:
19329 	lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
19330 			"2023 Unable to process MDS loopback frame\n");
19331 	if (pcmd && pcmd->virt)
19332 		dma_pool_free(phba->lpfc_drb_pool, pcmd->virt, pcmd->phys);
19333 	kfree(pcmd);
19334 	if (iocbq)
19335 		lpfc_sli_release_iocbq(phba, iocbq);
19336 	lpfc_in_buf_free(phba, &dmabuf->dbuf);
19337 }
19338 
19339 /**
19340  * lpfc_sli4_handle_received_buffer - Handle received buffers from firmware
19341  * @phba: Pointer to HBA context object.
19342  * @dmabuf: Pointer to a dmabuf that describes the FC sequence.
19343  *
19344  * This function is called with no lock held. This function processes all
19345  * the received buffers and gives it to upper layers when a received buffer
19346  * indicates that it is the final frame in the sequence. The interrupt
19347  * service routine processes received buffers at interrupt contexts.
19348  * Worker thread calls lpfc_sli4_handle_received_buffer, which will call the
19349  * appropriate receive function when the final frame in a sequence is received.
19350  **/
19351 void
19352 lpfc_sli4_handle_received_buffer(struct lpfc_hba *phba,
19353 				 struct hbq_dmabuf *dmabuf)
19354 {
19355 	struct hbq_dmabuf *seq_dmabuf;
19356 	struct fc_frame_header *fc_hdr;
19357 	struct lpfc_vport *vport;
19358 	uint32_t fcfi;
19359 	uint32_t did;
19360 
19361 	/* Process each received buffer */
19362 	fc_hdr = (struct fc_frame_header *)dmabuf->hbuf.virt;
19363 
19364 	if (fc_hdr->fh_r_ctl == FC_RCTL_MDS_DIAGS ||
19365 	    fc_hdr->fh_r_ctl == FC_RCTL_DD_UNSOL_DATA) {
19366 		vport = phba->pport;
19367 		/* Handle MDS Loopback frames */
19368 		if  (!(phba->pport->load_flag & FC_UNLOADING))
19369 			lpfc_sli4_handle_mds_loopback(vport, dmabuf);
19370 		else
19371 			lpfc_in_buf_free(phba, &dmabuf->dbuf);
19372 		return;
19373 	}
19374 
19375 	/* check to see if this a valid type of frame */
19376 	if (lpfc_fc_frame_check(phba, fc_hdr)) {
19377 		lpfc_in_buf_free(phba, &dmabuf->dbuf);
19378 		return;
19379 	}
19380 
19381 	if ((bf_get(lpfc_cqe_code,
19382 		    &dmabuf->cq_event.cqe.rcqe_cmpl) == CQE_CODE_RECEIVE_V1))
19383 		fcfi = bf_get(lpfc_rcqe_fcf_id_v1,
19384 			      &dmabuf->cq_event.cqe.rcqe_cmpl);
19385 	else
19386 		fcfi = bf_get(lpfc_rcqe_fcf_id,
19387 			      &dmabuf->cq_event.cqe.rcqe_cmpl);
19388 
19389 	if (fc_hdr->fh_r_ctl == 0xF4 && fc_hdr->fh_type == 0xFF) {
19390 		vport = phba->pport;
19391 		lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
19392 				"2023 MDS Loopback %d bytes\n",
19393 				bf_get(lpfc_rcqe_length,
19394 				       &dmabuf->cq_event.cqe.rcqe_cmpl));
19395 		/* Handle MDS Loopback frames */
19396 		lpfc_sli4_handle_mds_loopback(vport, dmabuf);
19397 		return;
19398 	}
19399 
19400 	/* d_id this frame is directed to */
19401 	did = sli4_did_from_fc_hdr(fc_hdr);
19402 
19403 	vport = lpfc_fc_frame_to_vport(phba, fc_hdr, fcfi, did);
19404 	if (!vport) {
19405 		/* throw out the frame */
19406 		lpfc_in_buf_free(phba, &dmabuf->dbuf);
19407 		return;
19408 	}
19409 
19410 	/* vport is registered unless we rcv a FLOGI directed to Fabric_DID */
19411 	if (!(vport->vpi_state & LPFC_VPI_REGISTERED) &&
19412 		(did != Fabric_DID)) {
19413 		/*
19414 		 * Throw out the frame if we are not pt2pt.
19415 		 * The pt2pt protocol allows for discovery frames
19416 		 * to be received without a registered VPI.
19417 		 */
19418 		if (!(vport->fc_flag & FC_PT2PT) ||
19419 			(phba->link_state == LPFC_HBA_READY)) {
19420 			lpfc_in_buf_free(phba, &dmabuf->dbuf);
19421 			return;
19422 		}
19423 	}
19424 
19425 	/* Handle the basic abort sequence (BA_ABTS) event */
19426 	if (fc_hdr->fh_r_ctl == FC_RCTL_BA_ABTS) {
19427 		lpfc_sli4_handle_unsol_abort(vport, dmabuf);
19428 		return;
19429 	}
19430 
19431 	/* Link this frame */
19432 	seq_dmabuf = lpfc_fc_frame_add(vport, dmabuf);
19433 	if (!seq_dmabuf) {
19434 		/* unable to add frame to vport - throw it out */
19435 		lpfc_in_buf_free(phba, &dmabuf->dbuf);
19436 		return;
19437 	}
19438 	/* If not last frame in sequence continue processing frames. */
19439 	if (!lpfc_seq_complete(seq_dmabuf))
19440 		return;
19441 
19442 	/* Send the complete sequence to the upper layer protocol */
19443 	lpfc_sli4_send_seq_to_ulp(vport, seq_dmabuf);
19444 }
19445 
19446 /**
19447  * lpfc_sli4_post_all_rpi_hdrs - Post the rpi header memory region to the port
19448  * @phba: pointer to lpfc hba data structure.
19449  *
19450  * This routine is invoked to post rpi header templates to the
19451  * HBA consistent with the SLI-4 interface spec.  This routine
19452  * posts a SLI4_PAGE_SIZE memory region to the port to hold up to
19453  * SLI4_PAGE_SIZE modulo 64 rpi context headers.
19454  *
19455  * This routine does not require any locks.  It's usage is expected
19456  * to be driver load or reset recovery when the driver is
19457  * sequential.
19458  *
19459  * Return codes
19460  * 	0 - successful
19461  *      -EIO - The mailbox failed to complete successfully.
19462  * 	When this error occurs, the driver is not guaranteed
19463  *	to have any rpi regions posted to the device and
19464  *	must either attempt to repost the regions or take a
19465  *	fatal error.
19466  **/
19467 int
19468 lpfc_sli4_post_all_rpi_hdrs(struct lpfc_hba *phba)
19469 {
19470 	struct lpfc_rpi_hdr *rpi_page;
19471 	uint32_t rc = 0;
19472 	uint16_t lrpi = 0;
19473 
19474 	/* SLI4 ports that support extents do not require RPI headers. */
19475 	if (!phba->sli4_hba.rpi_hdrs_in_use)
19476 		goto exit;
19477 	if (phba->sli4_hba.extents_in_use)
19478 		return -EIO;
19479 
19480 	list_for_each_entry(rpi_page, &phba->sli4_hba.lpfc_rpi_hdr_list, list) {
19481 		/*
19482 		 * Assign the rpi headers a physical rpi only if the driver
19483 		 * has not initialized those resources.  A port reset only
19484 		 * needs the headers posted.
19485 		 */
19486 		if (bf_get(lpfc_rpi_rsrc_rdy, &phba->sli4_hba.sli4_flags) !=
19487 		    LPFC_RPI_RSRC_RDY)
19488 			rpi_page->start_rpi = phba->sli4_hba.rpi_ids[lrpi];
19489 
19490 		rc = lpfc_sli4_post_rpi_hdr(phba, rpi_page);
19491 		if (rc != MBX_SUCCESS) {
19492 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
19493 					"2008 Error %d posting all rpi "
19494 					"headers\n", rc);
19495 			rc = -EIO;
19496 			break;
19497 		}
19498 	}
19499 
19500  exit:
19501 	bf_set(lpfc_rpi_rsrc_rdy, &phba->sli4_hba.sli4_flags,
19502 	       LPFC_RPI_RSRC_RDY);
19503 	return rc;
19504 }
19505 
19506 /**
19507  * lpfc_sli4_post_rpi_hdr - Post an rpi header memory region to the port
19508  * @phba: pointer to lpfc hba data structure.
19509  * @rpi_page:  pointer to the rpi memory region.
19510  *
19511  * This routine is invoked to post a single rpi header to the
19512  * HBA consistent with the SLI-4 interface spec.  This memory region
19513  * maps up to 64 rpi context regions.
19514  *
19515  * Return codes
19516  * 	0 - successful
19517  * 	-ENOMEM - No available memory
19518  *      -EIO - The mailbox failed to complete successfully.
19519  **/
19520 int
19521 lpfc_sli4_post_rpi_hdr(struct lpfc_hba *phba, struct lpfc_rpi_hdr *rpi_page)
19522 {
19523 	LPFC_MBOXQ_t *mboxq;
19524 	struct lpfc_mbx_post_hdr_tmpl *hdr_tmpl;
19525 	uint32_t rc = 0;
19526 	uint32_t shdr_status, shdr_add_status;
19527 	union lpfc_sli4_cfg_shdr *shdr;
19528 
19529 	/* SLI4 ports that support extents do not require RPI headers. */
19530 	if (!phba->sli4_hba.rpi_hdrs_in_use)
19531 		return rc;
19532 	if (phba->sli4_hba.extents_in_use)
19533 		return -EIO;
19534 
19535 	/* The port is notified of the header region via a mailbox command. */
19536 	mboxq = (LPFC_MBOXQ_t *) mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
19537 	if (!mboxq) {
19538 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
19539 				"2001 Unable to allocate memory for issuing "
19540 				"SLI_CONFIG_SPECIAL mailbox command\n");
19541 		return -ENOMEM;
19542 	}
19543 
19544 	/* Post all rpi memory regions to the port. */
19545 	hdr_tmpl = &mboxq->u.mqe.un.hdr_tmpl;
19546 	lpfc_sli4_config(phba, mboxq, LPFC_MBOX_SUBSYSTEM_FCOE,
19547 			 LPFC_MBOX_OPCODE_FCOE_POST_HDR_TEMPLATE,
19548 			 sizeof(struct lpfc_mbx_post_hdr_tmpl) -
19549 			 sizeof(struct lpfc_sli4_cfg_mhdr),
19550 			 LPFC_SLI4_MBX_EMBED);
19551 
19552 
19553 	/* Post the physical rpi to the port for this rpi header. */
19554 	bf_set(lpfc_mbx_post_hdr_tmpl_rpi_offset, hdr_tmpl,
19555 	       rpi_page->start_rpi);
19556 	bf_set(lpfc_mbx_post_hdr_tmpl_page_cnt,
19557 	       hdr_tmpl, rpi_page->page_count);
19558 
19559 	hdr_tmpl->rpi_paddr_lo = putPaddrLow(rpi_page->dmabuf->phys);
19560 	hdr_tmpl->rpi_paddr_hi = putPaddrHigh(rpi_page->dmabuf->phys);
19561 	rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
19562 	shdr = (union lpfc_sli4_cfg_shdr *) &hdr_tmpl->header.cfg_shdr;
19563 	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
19564 	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
19565 	mempool_free(mboxq, phba->mbox_mem_pool);
19566 	if (shdr_status || shdr_add_status || rc) {
19567 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
19568 				"2514 POST_RPI_HDR mailbox failed with "
19569 				"status x%x add_status x%x, mbx status x%x\n",
19570 				shdr_status, shdr_add_status, rc);
19571 		rc = -ENXIO;
19572 	} else {
19573 		/*
19574 		 * The next_rpi stores the next logical module-64 rpi value used
19575 		 * to post physical rpis in subsequent rpi postings.
19576 		 */
19577 		spin_lock_irq(&phba->hbalock);
19578 		phba->sli4_hba.next_rpi = rpi_page->next_rpi;
19579 		spin_unlock_irq(&phba->hbalock);
19580 	}
19581 	return rc;
19582 }
19583 
19584 /**
19585  * lpfc_sli4_alloc_rpi - Get an available rpi in the device's range
19586  * @phba: pointer to lpfc hba data structure.
19587  *
19588  * This routine is invoked to post rpi header templates to the
19589  * HBA consistent with the SLI-4 interface spec.  This routine
19590  * posts a SLI4_PAGE_SIZE memory region to the port to hold up to
19591  * SLI4_PAGE_SIZE modulo 64 rpi context headers.
19592  *
19593  * Returns
19594  * 	A nonzero rpi defined as rpi_base <= rpi < max_rpi if successful
19595  * 	LPFC_RPI_ALLOC_ERROR if no rpis are available.
19596  **/
19597 int
19598 lpfc_sli4_alloc_rpi(struct lpfc_hba *phba)
19599 {
19600 	unsigned long rpi;
19601 	uint16_t max_rpi, rpi_limit;
19602 	uint16_t rpi_remaining, lrpi = 0;
19603 	struct lpfc_rpi_hdr *rpi_hdr;
19604 	unsigned long iflag;
19605 
19606 	/*
19607 	 * Fetch the next logical rpi.  Because this index is logical,
19608 	 * the  driver starts at 0 each time.
19609 	 */
19610 	spin_lock_irqsave(&phba->hbalock, iflag);
19611 	max_rpi = phba->sli4_hba.max_cfg_param.max_rpi;
19612 	rpi_limit = phba->sli4_hba.next_rpi;
19613 
19614 	rpi = find_next_zero_bit(phba->sli4_hba.rpi_bmask, rpi_limit, 0);
19615 	if (rpi >= rpi_limit)
19616 		rpi = LPFC_RPI_ALLOC_ERROR;
19617 	else {
19618 		set_bit(rpi, phba->sli4_hba.rpi_bmask);
19619 		phba->sli4_hba.max_cfg_param.rpi_used++;
19620 		phba->sli4_hba.rpi_count++;
19621 	}
19622 	lpfc_printf_log(phba, KERN_INFO,
19623 			LOG_NODE | LOG_DISCOVERY,
19624 			"0001 Allocated rpi:x%x max:x%x lim:x%x\n",
19625 			(int) rpi, max_rpi, rpi_limit);
19626 
19627 	/*
19628 	 * Don't try to allocate more rpi header regions if the device limit
19629 	 * has been exhausted.
19630 	 */
19631 	if ((rpi == LPFC_RPI_ALLOC_ERROR) &&
19632 	    (phba->sli4_hba.rpi_count >= max_rpi)) {
19633 		spin_unlock_irqrestore(&phba->hbalock, iflag);
19634 		return rpi;
19635 	}
19636 
19637 	/*
19638 	 * RPI header postings are not required for SLI4 ports capable of
19639 	 * extents.
19640 	 */
19641 	if (!phba->sli4_hba.rpi_hdrs_in_use) {
19642 		spin_unlock_irqrestore(&phba->hbalock, iflag);
19643 		return rpi;
19644 	}
19645 
19646 	/*
19647 	 * If the driver is running low on rpi resources, allocate another
19648 	 * page now.  Note that the next_rpi value is used because
19649 	 * it represents how many are actually in use whereas max_rpi notes
19650 	 * how many are supported max by the device.
19651 	 */
19652 	rpi_remaining = phba->sli4_hba.next_rpi - phba->sli4_hba.rpi_count;
19653 	spin_unlock_irqrestore(&phba->hbalock, iflag);
19654 	if (rpi_remaining < LPFC_RPI_LOW_WATER_MARK) {
19655 		rpi_hdr = lpfc_sli4_create_rpi_hdr(phba);
19656 		if (!rpi_hdr) {
19657 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
19658 					"2002 Error Could not grow rpi "
19659 					"count\n");
19660 		} else {
19661 			lrpi = rpi_hdr->start_rpi;
19662 			rpi_hdr->start_rpi = phba->sli4_hba.rpi_ids[lrpi];
19663 			lpfc_sli4_post_rpi_hdr(phba, rpi_hdr);
19664 		}
19665 	}
19666 
19667 	return rpi;
19668 }
19669 
19670 /**
19671  * __lpfc_sli4_free_rpi - Release an rpi for reuse.
19672  * @phba: pointer to lpfc hba data structure.
19673  * @rpi: rpi to free
19674  *
19675  * This routine is invoked to release an rpi to the pool of
19676  * available rpis maintained by the driver.
19677  **/
19678 static void
19679 __lpfc_sli4_free_rpi(struct lpfc_hba *phba, int rpi)
19680 {
19681 	/*
19682 	 * if the rpi value indicates a prior unreg has already
19683 	 * been done, skip the unreg.
19684 	 */
19685 	if (rpi == LPFC_RPI_ALLOC_ERROR)
19686 		return;
19687 
19688 	if (test_and_clear_bit(rpi, phba->sli4_hba.rpi_bmask)) {
19689 		phba->sli4_hba.rpi_count--;
19690 		phba->sli4_hba.max_cfg_param.rpi_used--;
19691 	} else {
19692 		lpfc_printf_log(phba, KERN_INFO,
19693 				LOG_NODE | LOG_DISCOVERY,
19694 				"2016 rpi %x not inuse\n",
19695 				rpi);
19696 	}
19697 }
19698 
19699 /**
19700  * lpfc_sli4_free_rpi - Release an rpi for reuse.
19701  * @phba: pointer to lpfc hba data structure.
19702  * @rpi: rpi to free
19703  *
19704  * This routine is invoked to release an rpi to the pool of
19705  * available rpis maintained by the driver.
19706  **/
19707 void
19708 lpfc_sli4_free_rpi(struct lpfc_hba *phba, int rpi)
19709 {
19710 	spin_lock_irq(&phba->hbalock);
19711 	__lpfc_sli4_free_rpi(phba, rpi);
19712 	spin_unlock_irq(&phba->hbalock);
19713 }
19714 
19715 /**
19716  * lpfc_sli4_remove_rpis - Remove the rpi bitmask region
19717  * @phba: pointer to lpfc hba data structure.
19718  *
19719  * This routine is invoked to remove the memory region that
19720  * provided rpi via a bitmask.
19721  **/
19722 void
19723 lpfc_sli4_remove_rpis(struct lpfc_hba *phba)
19724 {
19725 	kfree(phba->sli4_hba.rpi_bmask);
19726 	kfree(phba->sli4_hba.rpi_ids);
19727 	bf_set(lpfc_rpi_rsrc_rdy, &phba->sli4_hba.sli4_flags, 0);
19728 }
19729 
19730 /**
19731  * lpfc_sli4_resume_rpi - Remove the rpi bitmask region
19732  * @ndlp: pointer to lpfc nodelist data structure.
19733  * @cmpl: completion call-back.
19734  * @arg: data to load as MBox 'caller buffer information'
19735  *
19736  * This routine is invoked to remove the memory region that
19737  * provided rpi via a bitmask.
19738  **/
19739 int
19740 lpfc_sli4_resume_rpi(struct lpfc_nodelist *ndlp,
19741 	void (*cmpl)(struct lpfc_hba *, LPFC_MBOXQ_t *), void *arg)
19742 {
19743 	LPFC_MBOXQ_t *mboxq;
19744 	struct lpfc_hba *phba = ndlp->phba;
19745 	int rc;
19746 
19747 	/* The port is notified of the header region via a mailbox command. */
19748 	mboxq = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
19749 	if (!mboxq)
19750 		return -ENOMEM;
19751 
19752 	/* If cmpl assigned, then this nlp_get pairs with
19753 	 * lpfc_mbx_cmpl_resume_rpi.
19754 	 *
19755 	 * Else cmpl is NULL, then this nlp_get pairs with
19756 	 * lpfc_sli_def_mbox_cmpl.
19757 	 */
19758 	if (!lpfc_nlp_get(ndlp)) {
19759 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
19760 				"2122 %s: Failed to get nlp ref\n",
19761 				__func__);
19762 		mempool_free(mboxq, phba->mbox_mem_pool);
19763 		return -EIO;
19764 	}
19765 
19766 	/* Post all rpi memory regions to the port. */
19767 	lpfc_resume_rpi(mboxq, ndlp);
19768 	if (cmpl) {
19769 		mboxq->mbox_cmpl = cmpl;
19770 		mboxq->ctx_buf = arg;
19771 	} else
19772 		mboxq->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
19773 	mboxq->ctx_ndlp = ndlp;
19774 	mboxq->vport = ndlp->vport;
19775 	rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_NOWAIT);
19776 	if (rc == MBX_NOT_FINISHED) {
19777 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
19778 				"2010 Resume RPI Mailbox failed "
19779 				"status %d, mbxStatus x%x\n", rc,
19780 				bf_get(lpfc_mqe_status, &mboxq->u.mqe));
19781 		lpfc_nlp_put(ndlp);
19782 		mempool_free(mboxq, phba->mbox_mem_pool);
19783 		return -EIO;
19784 	}
19785 	return 0;
19786 }
19787 
19788 /**
19789  * lpfc_sli4_init_vpi - Initialize a vpi with the port
19790  * @vport: Pointer to the vport for which the vpi is being initialized
19791  *
19792  * This routine is invoked to activate a vpi with the port.
19793  *
19794  * Returns:
19795  *    0 success
19796  *    -Evalue otherwise
19797  **/
19798 int
19799 lpfc_sli4_init_vpi(struct lpfc_vport *vport)
19800 {
19801 	LPFC_MBOXQ_t *mboxq;
19802 	int rc = 0;
19803 	int retval = MBX_SUCCESS;
19804 	uint32_t mbox_tmo;
19805 	struct lpfc_hba *phba = vport->phba;
19806 	mboxq = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
19807 	if (!mboxq)
19808 		return -ENOMEM;
19809 	lpfc_init_vpi(phba, mboxq, vport->vpi);
19810 	mbox_tmo = lpfc_mbox_tmo_val(phba, mboxq);
19811 	rc = lpfc_sli_issue_mbox_wait(phba, mboxq, mbox_tmo);
19812 	if (rc != MBX_SUCCESS) {
19813 		lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
19814 				"2022 INIT VPI Mailbox failed "
19815 				"status %d, mbxStatus x%x\n", rc,
19816 				bf_get(lpfc_mqe_status, &mboxq->u.mqe));
19817 		retval = -EIO;
19818 	}
19819 	if (rc != MBX_TIMEOUT)
19820 		mempool_free(mboxq, vport->phba->mbox_mem_pool);
19821 
19822 	return retval;
19823 }
19824 
19825 /**
19826  * lpfc_mbx_cmpl_add_fcf_record - add fcf mbox completion handler.
19827  * @phba: pointer to lpfc hba data structure.
19828  * @mboxq: Pointer to mailbox object.
19829  *
19830  * This routine is invoked to manually add a single FCF record. The caller
19831  * must pass a completely initialized FCF_Record.  This routine takes
19832  * care of the nonembedded mailbox operations.
19833  **/
19834 static void
19835 lpfc_mbx_cmpl_add_fcf_record(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq)
19836 {
19837 	void *virt_addr;
19838 	union lpfc_sli4_cfg_shdr *shdr;
19839 	uint32_t shdr_status, shdr_add_status;
19840 
19841 	virt_addr = mboxq->sge_array->addr[0];
19842 	/* The IOCTL status is embedded in the mailbox subheader. */
19843 	shdr = (union lpfc_sli4_cfg_shdr *) virt_addr;
19844 	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
19845 	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
19846 
19847 	if ((shdr_status || shdr_add_status) &&
19848 		(shdr_status != STATUS_FCF_IN_USE))
19849 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
19850 			"2558 ADD_FCF_RECORD mailbox failed with "
19851 			"status x%x add_status x%x\n",
19852 			shdr_status, shdr_add_status);
19853 
19854 	lpfc_sli4_mbox_cmd_free(phba, mboxq);
19855 }
19856 
19857 /**
19858  * lpfc_sli4_add_fcf_record - Manually add an FCF Record.
19859  * @phba: pointer to lpfc hba data structure.
19860  * @fcf_record:  pointer to the initialized fcf record to add.
19861  *
19862  * This routine is invoked to manually add a single FCF record. The caller
19863  * must pass a completely initialized FCF_Record.  This routine takes
19864  * care of the nonembedded mailbox operations.
19865  **/
19866 int
19867 lpfc_sli4_add_fcf_record(struct lpfc_hba *phba, struct fcf_record *fcf_record)
19868 {
19869 	int rc = 0;
19870 	LPFC_MBOXQ_t *mboxq;
19871 	uint8_t *bytep;
19872 	void *virt_addr;
19873 	struct lpfc_mbx_sge sge;
19874 	uint32_t alloc_len, req_len;
19875 	uint32_t fcfindex;
19876 
19877 	mboxq = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
19878 	if (!mboxq) {
19879 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
19880 			"2009 Failed to allocate mbox for ADD_FCF cmd\n");
19881 		return -ENOMEM;
19882 	}
19883 
19884 	req_len = sizeof(struct fcf_record) + sizeof(union lpfc_sli4_cfg_shdr) +
19885 		  sizeof(uint32_t);
19886 
19887 	/* Allocate DMA memory and set up the non-embedded mailbox command */
19888 	alloc_len = lpfc_sli4_config(phba, mboxq, LPFC_MBOX_SUBSYSTEM_FCOE,
19889 				     LPFC_MBOX_OPCODE_FCOE_ADD_FCF,
19890 				     req_len, LPFC_SLI4_MBX_NEMBED);
19891 	if (alloc_len < req_len) {
19892 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
19893 			"2523 Allocated DMA memory size (x%x) is "
19894 			"less than the requested DMA memory "
19895 			"size (x%x)\n", alloc_len, req_len);
19896 		lpfc_sli4_mbox_cmd_free(phba, mboxq);
19897 		return -ENOMEM;
19898 	}
19899 
19900 	/*
19901 	 * Get the first SGE entry from the non-embedded DMA memory.  This
19902 	 * routine only uses a single SGE.
19903 	 */
19904 	lpfc_sli4_mbx_sge_get(mboxq, 0, &sge);
19905 	virt_addr = mboxq->sge_array->addr[0];
19906 	/*
19907 	 * Configure the FCF record for FCFI 0.  This is the driver's
19908 	 * hardcoded default and gets used in nonFIP mode.
19909 	 */
19910 	fcfindex = bf_get(lpfc_fcf_record_fcf_index, fcf_record);
19911 	bytep = virt_addr + sizeof(union lpfc_sli4_cfg_shdr);
19912 	lpfc_sli_pcimem_bcopy(&fcfindex, bytep, sizeof(uint32_t));
19913 
19914 	/*
19915 	 * Copy the fcf_index and the FCF Record Data. The data starts after
19916 	 * the FCoE header plus word10. The data copy needs to be endian
19917 	 * correct.
19918 	 */
19919 	bytep += sizeof(uint32_t);
19920 	lpfc_sli_pcimem_bcopy(fcf_record, bytep, sizeof(struct fcf_record));
19921 	mboxq->vport = phba->pport;
19922 	mboxq->mbox_cmpl = lpfc_mbx_cmpl_add_fcf_record;
19923 	rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_NOWAIT);
19924 	if (rc == MBX_NOT_FINISHED) {
19925 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
19926 			"2515 ADD_FCF_RECORD mailbox failed with "
19927 			"status 0x%x\n", rc);
19928 		lpfc_sli4_mbox_cmd_free(phba, mboxq);
19929 		rc = -EIO;
19930 	} else
19931 		rc = 0;
19932 
19933 	return rc;
19934 }
19935 
19936 /**
19937  * lpfc_sli4_build_dflt_fcf_record - Build the driver's default FCF Record.
19938  * @phba: pointer to lpfc hba data structure.
19939  * @fcf_record:  pointer to the fcf record to write the default data.
19940  * @fcf_index: FCF table entry index.
19941  *
19942  * This routine is invoked to build the driver's default FCF record.  The
19943  * values used are hardcoded.  This routine handles memory initialization.
19944  *
19945  **/
19946 void
19947 lpfc_sli4_build_dflt_fcf_record(struct lpfc_hba *phba,
19948 				struct fcf_record *fcf_record,
19949 				uint16_t fcf_index)
19950 {
19951 	memset(fcf_record, 0, sizeof(struct fcf_record));
19952 	fcf_record->max_rcv_size = LPFC_FCOE_MAX_RCV_SIZE;
19953 	fcf_record->fka_adv_period = LPFC_FCOE_FKA_ADV_PER;
19954 	fcf_record->fip_priority = LPFC_FCOE_FIP_PRIORITY;
19955 	bf_set(lpfc_fcf_record_mac_0, fcf_record, phba->fc_map[0]);
19956 	bf_set(lpfc_fcf_record_mac_1, fcf_record, phba->fc_map[1]);
19957 	bf_set(lpfc_fcf_record_mac_2, fcf_record, phba->fc_map[2]);
19958 	bf_set(lpfc_fcf_record_mac_3, fcf_record, LPFC_FCOE_FCF_MAC3);
19959 	bf_set(lpfc_fcf_record_mac_4, fcf_record, LPFC_FCOE_FCF_MAC4);
19960 	bf_set(lpfc_fcf_record_mac_5, fcf_record, LPFC_FCOE_FCF_MAC5);
19961 	bf_set(lpfc_fcf_record_fc_map_0, fcf_record, phba->fc_map[0]);
19962 	bf_set(lpfc_fcf_record_fc_map_1, fcf_record, phba->fc_map[1]);
19963 	bf_set(lpfc_fcf_record_fc_map_2, fcf_record, phba->fc_map[2]);
19964 	bf_set(lpfc_fcf_record_fcf_valid, fcf_record, 1);
19965 	bf_set(lpfc_fcf_record_fcf_avail, fcf_record, 1);
19966 	bf_set(lpfc_fcf_record_fcf_index, fcf_record, fcf_index);
19967 	bf_set(lpfc_fcf_record_mac_addr_prov, fcf_record,
19968 		LPFC_FCF_FPMA | LPFC_FCF_SPMA);
19969 	/* Set the VLAN bit map */
19970 	if (phba->valid_vlan) {
19971 		fcf_record->vlan_bitmap[phba->vlan_id / 8]
19972 			= 1 << (phba->vlan_id % 8);
19973 	}
19974 }
19975 
19976 /**
19977  * lpfc_sli4_fcf_scan_read_fcf_rec - Read hba fcf record for fcf scan.
19978  * @phba: pointer to lpfc hba data structure.
19979  * @fcf_index: FCF table entry offset.
19980  *
19981  * This routine is invoked to scan the entire FCF table by reading FCF
19982  * record and processing it one at a time starting from the @fcf_index
19983  * for initial FCF discovery or fast FCF failover rediscovery.
19984  *
19985  * Return 0 if the mailbox command is submitted successfully, none 0
19986  * otherwise.
19987  **/
19988 int
19989 lpfc_sli4_fcf_scan_read_fcf_rec(struct lpfc_hba *phba, uint16_t fcf_index)
19990 {
19991 	int rc = 0, error;
19992 	LPFC_MBOXQ_t *mboxq;
19993 
19994 	phba->fcoe_eventtag_at_fcf_scan = phba->fcoe_eventtag;
19995 	phba->fcoe_cvl_eventtag_attn = phba->fcoe_cvl_eventtag;
19996 	mboxq = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
19997 	if (!mboxq) {
19998 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
19999 				"2000 Failed to allocate mbox for "
20000 				"READ_FCF cmd\n");
20001 		error = -ENOMEM;
20002 		goto fail_fcf_scan;
20003 	}
20004 	/* Construct the read FCF record mailbox command */
20005 	rc = lpfc_sli4_mbx_read_fcf_rec(phba, mboxq, fcf_index);
20006 	if (rc) {
20007 		error = -EINVAL;
20008 		goto fail_fcf_scan;
20009 	}
20010 	/* Issue the mailbox command asynchronously */
20011 	mboxq->vport = phba->pport;
20012 	mboxq->mbox_cmpl = lpfc_mbx_cmpl_fcf_scan_read_fcf_rec;
20013 
20014 	spin_lock_irq(&phba->hbalock);
20015 	phba->hba_flag |= FCF_TS_INPROG;
20016 	spin_unlock_irq(&phba->hbalock);
20017 
20018 	rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_NOWAIT);
20019 	if (rc == MBX_NOT_FINISHED)
20020 		error = -EIO;
20021 	else {
20022 		/* Reset eligible FCF count for new scan */
20023 		if (fcf_index == LPFC_FCOE_FCF_GET_FIRST)
20024 			phba->fcf.eligible_fcf_cnt = 0;
20025 		error = 0;
20026 	}
20027 fail_fcf_scan:
20028 	if (error) {
20029 		if (mboxq)
20030 			lpfc_sli4_mbox_cmd_free(phba, mboxq);
20031 		/* FCF scan failed, clear FCF_TS_INPROG flag */
20032 		spin_lock_irq(&phba->hbalock);
20033 		phba->hba_flag &= ~FCF_TS_INPROG;
20034 		spin_unlock_irq(&phba->hbalock);
20035 	}
20036 	return error;
20037 }
20038 
20039 /**
20040  * lpfc_sli4_fcf_rr_read_fcf_rec - Read hba fcf record for roundrobin fcf.
20041  * @phba: pointer to lpfc hba data structure.
20042  * @fcf_index: FCF table entry offset.
20043  *
20044  * This routine is invoked to read an FCF record indicated by @fcf_index
20045  * and to use it for FLOGI roundrobin FCF failover.
20046  *
20047  * Return 0 if the mailbox command is submitted successfully, none 0
20048  * otherwise.
20049  **/
20050 int
20051 lpfc_sli4_fcf_rr_read_fcf_rec(struct lpfc_hba *phba, uint16_t fcf_index)
20052 {
20053 	int rc = 0, error;
20054 	LPFC_MBOXQ_t *mboxq;
20055 
20056 	mboxq = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
20057 	if (!mboxq) {
20058 		lpfc_printf_log(phba, KERN_ERR, LOG_FIP | LOG_INIT,
20059 				"2763 Failed to allocate mbox for "
20060 				"READ_FCF cmd\n");
20061 		error = -ENOMEM;
20062 		goto fail_fcf_read;
20063 	}
20064 	/* Construct the read FCF record mailbox command */
20065 	rc = lpfc_sli4_mbx_read_fcf_rec(phba, mboxq, fcf_index);
20066 	if (rc) {
20067 		error = -EINVAL;
20068 		goto fail_fcf_read;
20069 	}
20070 	/* Issue the mailbox command asynchronously */
20071 	mboxq->vport = phba->pport;
20072 	mboxq->mbox_cmpl = lpfc_mbx_cmpl_fcf_rr_read_fcf_rec;
20073 	rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_NOWAIT);
20074 	if (rc == MBX_NOT_FINISHED)
20075 		error = -EIO;
20076 	else
20077 		error = 0;
20078 
20079 fail_fcf_read:
20080 	if (error && mboxq)
20081 		lpfc_sli4_mbox_cmd_free(phba, mboxq);
20082 	return error;
20083 }
20084 
20085 /**
20086  * lpfc_sli4_read_fcf_rec - Read hba fcf record for update eligible fcf bmask.
20087  * @phba: pointer to lpfc hba data structure.
20088  * @fcf_index: FCF table entry offset.
20089  *
20090  * This routine is invoked to read an FCF record indicated by @fcf_index to
20091  * determine whether it's eligible for FLOGI roundrobin failover list.
20092  *
20093  * Return 0 if the mailbox command is submitted successfully, none 0
20094  * otherwise.
20095  **/
20096 int
20097 lpfc_sli4_read_fcf_rec(struct lpfc_hba *phba, uint16_t fcf_index)
20098 {
20099 	int rc = 0, error;
20100 	LPFC_MBOXQ_t *mboxq;
20101 
20102 	mboxq = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
20103 	if (!mboxq) {
20104 		lpfc_printf_log(phba, KERN_ERR, LOG_FIP | LOG_INIT,
20105 				"2758 Failed to allocate mbox for "
20106 				"READ_FCF cmd\n");
20107 				error = -ENOMEM;
20108 				goto fail_fcf_read;
20109 	}
20110 	/* Construct the read FCF record mailbox command */
20111 	rc = lpfc_sli4_mbx_read_fcf_rec(phba, mboxq, fcf_index);
20112 	if (rc) {
20113 		error = -EINVAL;
20114 		goto fail_fcf_read;
20115 	}
20116 	/* Issue the mailbox command asynchronously */
20117 	mboxq->vport = phba->pport;
20118 	mboxq->mbox_cmpl = lpfc_mbx_cmpl_read_fcf_rec;
20119 	rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_NOWAIT);
20120 	if (rc == MBX_NOT_FINISHED)
20121 		error = -EIO;
20122 	else
20123 		error = 0;
20124 
20125 fail_fcf_read:
20126 	if (error && mboxq)
20127 		lpfc_sli4_mbox_cmd_free(phba, mboxq);
20128 	return error;
20129 }
20130 
20131 /**
20132  * lpfc_check_next_fcf_pri_level
20133  * @phba: pointer to the lpfc_hba struct for this port.
20134  * This routine is called from the lpfc_sli4_fcf_rr_next_index_get
20135  * routine when the rr_bmask is empty. The FCF indecies are put into the
20136  * rr_bmask based on their priority level. Starting from the highest priority
20137  * to the lowest. The most likely FCF candidate will be in the highest
20138  * priority group. When this routine is called it searches the fcf_pri list for
20139  * next lowest priority group and repopulates the rr_bmask with only those
20140  * fcf_indexes.
20141  * returns:
20142  * 1=success 0=failure
20143  **/
20144 static int
20145 lpfc_check_next_fcf_pri_level(struct lpfc_hba *phba)
20146 {
20147 	uint16_t next_fcf_pri;
20148 	uint16_t last_index;
20149 	struct lpfc_fcf_pri *fcf_pri;
20150 	int rc;
20151 	int ret = 0;
20152 
20153 	last_index = find_first_bit(phba->fcf.fcf_rr_bmask,
20154 			LPFC_SLI4_FCF_TBL_INDX_MAX);
20155 	lpfc_printf_log(phba, KERN_INFO, LOG_FIP,
20156 			"3060 Last IDX %d\n", last_index);
20157 
20158 	/* Verify the priority list has 2 or more entries */
20159 	spin_lock_irq(&phba->hbalock);
20160 	if (list_empty(&phba->fcf.fcf_pri_list) ||
20161 	    list_is_singular(&phba->fcf.fcf_pri_list)) {
20162 		spin_unlock_irq(&phba->hbalock);
20163 		lpfc_printf_log(phba, KERN_ERR, LOG_FIP,
20164 			"3061 Last IDX %d\n", last_index);
20165 		return 0; /* Empty rr list */
20166 	}
20167 	spin_unlock_irq(&phba->hbalock);
20168 
20169 	next_fcf_pri = 0;
20170 	/*
20171 	 * Clear the rr_bmask and set all of the bits that are at this
20172 	 * priority.
20173 	 */
20174 	memset(phba->fcf.fcf_rr_bmask, 0,
20175 			sizeof(*phba->fcf.fcf_rr_bmask));
20176 	spin_lock_irq(&phba->hbalock);
20177 	list_for_each_entry(fcf_pri, &phba->fcf.fcf_pri_list, list) {
20178 		if (fcf_pri->fcf_rec.flag & LPFC_FCF_FLOGI_FAILED)
20179 			continue;
20180 		/*
20181 		 * the 1st priority that has not FLOGI failed
20182 		 * will be the highest.
20183 		 */
20184 		if (!next_fcf_pri)
20185 			next_fcf_pri = fcf_pri->fcf_rec.priority;
20186 		spin_unlock_irq(&phba->hbalock);
20187 		if (fcf_pri->fcf_rec.priority == next_fcf_pri) {
20188 			rc = lpfc_sli4_fcf_rr_index_set(phba,
20189 						fcf_pri->fcf_rec.fcf_index);
20190 			if (rc)
20191 				return 0;
20192 		}
20193 		spin_lock_irq(&phba->hbalock);
20194 	}
20195 	/*
20196 	 * if next_fcf_pri was not set above and the list is not empty then
20197 	 * we have failed flogis on all of them. So reset flogi failed
20198 	 * and start at the beginning.
20199 	 */
20200 	if (!next_fcf_pri && !list_empty(&phba->fcf.fcf_pri_list)) {
20201 		list_for_each_entry(fcf_pri, &phba->fcf.fcf_pri_list, list) {
20202 			fcf_pri->fcf_rec.flag &= ~LPFC_FCF_FLOGI_FAILED;
20203 			/*
20204 			 * the 1st priority that has not FLOGI failed
20205 			 * will be the highest.
20206 			 */
20207 			if (!next_fcf_pri)
20208 				next_fcf_pri = fcf_pri->fcf_rec.priority;
20209 			spin_unlock_irq(&phba->hbalock);
20210 			if (fcf_pri->fcf_rec.priority == next_fcf_pri) {
20211 				rc = lpfc_sli4_fcf_rr_index_set(phba,
20212 						fcf_pri->fcf_rec.fcf_index);
20213 				if (rc)
20214 					return 0;
20215 			}
20216 			spin_lock_irq(&phba->hbalock);
20217 		}
20218 	} else
20219 		ret = 1;
20220 	spin_unlock_irq(&phba->hbalock);
20221 
20222 	return ret;
20223 }
20224 /**
20225  * lpfc_sli4_fcf_rr_next_index_get - Get next eligible fcf record index
20226  * @phba: pointer to lpfc hba data structure.
20227  *
20228  * This routine is to get the next eligible FCF record index in a round
20229  * robin fashion. If the next eligible FCF record index equals to the
20230  * initial roundrobin FCF record index, LPFC_FCOE_FCF_NEXT_NONE (0xFFFF)
20231  * shall be returned, otherwise, the next eligible FCF record's index
20232  * shall be returned.
20233  **/
20234 uint16_t
20235 lpfc_sli4_fcf_rr_next_index_get(struct lpfc_hba *phba)
20236 {
20237 	uint16_t next_fcf_index;
20238 
20239 initial_priority:
20240 	/* Search start from next bit of currently registered FCF index */
20241 	next_fcf_index = phba->fcf.current_rec.fcf_indx;
20242 
20243 next_priority:
20244 	/* Determine the next fcf index to check */
20245 	next_fcf_index = (next_fcf_index + 1) % LPFC_SLI4_FCF_TBL_INDX_MAX;
20246 	next_fcf_index = find_next_bit(phba->fcf.fcf_rr_bmask,
20247 				       LPFC_SLI4_FCF_TBL_INDX_MAX,
20248 				       next_fcf_index);
20249 
20250 	/* Wrap around condition on phba->fcf.fcf_rr_bmask */
20251 	if (next_fcf_index >= LPFC_SLI4_FCF_TBL_INDX_MAX) {
20252 		/*
20253 		 * If we have wrapped then we need to clear the bits that
20254 		 * have been tested so that we can detect when we should
20255 		 * change the priority level.
20256 		 */
20257 		next_fcf_index = find_next_bit(phba->fcf.fcf_rr_bmask,
20258 					       LPFC_SLI4_FCF_TBL_INDX_MAX, 0);
20259 	}
20260 
20261 
20262 	/* Check roundrobin failover list empty condition */
20263 	if (next_fcf_index >= LPFC_SLI4_FCF_TBL_INDX_MAX ||
20264 		next_fcf_index == phba->fcf.current_rec.fcf_indx) {
20265 		/*
20266 		 * If next fcf index is not found check if there are lower
20267 		 * Priority level fcf's in the fcf_priority list.
20268 		 * Set up the rr_bmask with all of the avaiable fcf bits
20269 		 * at that level and continue the selection process.
20270 		 */
20271 		if (lpfc_check_next_fcf_pri_level(phba))
20272 			goto initial_priority;
20273 		lpfc_printf_log(phba, KERN_WARNING, LOG_FIP,
20274 				"2844 No roundrobin failover FCF available\n");
20275 
20276 		return LPFC_FCOE_FCF_NEXT_NONE;
20277 	}
20278 
20279 	if (next_fcf_index < LPFC_SLI4_FCF_TBL_INDX_MAX &&
20280 		phba->fcf.fcf_pri[next_fcf_index].fcf_rec.flag &
20281 		LPFC_FCF_FLOGI_FAILED) {
20282 		if (list_is_singular(&phba->fcf.fcf_pri_list))
20283 			return LPFC_FCOE_FCF_NEXT_NONE;
20284 
20285 		goto next_priority;
20286 	}
20287 
20288 	lpfc_printf_log(phba, KERN_INFO, LOG_FIP,
20289 			"2845 Get next roundrobin failover FCF (x%x)\n",
20290 			next_fcf_index);
20291 
20292 	return next_fcf_index;
20293 }
20294 
20295 /**
20296  * lpfc_sli4_fcf_rr_index_set - Set bmask with eligible fcf record index
20297  * @phba: pointer to lpfc hba data structure.
20298  * @fcf_index: index into the FCF table to 'set'
20299  *
20300  * This routine sets the FCF record index in to the eligible bmask for
20301  * roundrobin failover search. It checks to make sure that the index
20302  * does not go beyond the range of the driver allocated bmask dimension
20303  * before setting the bit.
20304  *
20305  * Returns 0 if the index bit successfully set, otherwise, it returns
20306  * -EINVAL.
20307  **/
20308 int
20309 lpfc_sli4_fcf_rr_index_set(struct lpfc_hba *phba, uint16_t fcf_index)
20310 {
20311 	if (fcf_index >= LPFC_SLI4_FCF_TBL_INDX_MAX) {
20312 		lpfc_printf_log(phba, KERN_ERR, LOG_FIP,
20313 				"2610 FCF (x%x) reached driver's book "
20314 				"keeping dimension:x%x\n",
20315 				fcf_index, LPFC_SLI4_FCF_TBL_INDX_MAX);
20316 		return -EINVAL;
20317 	}
20318 	/* Set the eligible FCF record index bmask */
20319 	set_bit(fcf_index, phba->fcf.fcf_rr_bmask);
20320 
20321 	lpfc_printf_log(phba, KERN_INFO, LOG_FIP,
20322 			"2790 Set FCF (x%x) to roundrobin FCF failover "
20323 			"bmask\n", fcf_index);
20324 
20325 	return 0;
20326 }
20327 
20328 /**
20329  * lpfc_sli4_fcf_rr_index_clear - Clear bmask from eligible fcf record index
20330  * @phba: pointer to lpfc hba data structure.
20331  * @fcf_index: index into the FCF table to 'clear'
20332  *
20333  * This routine clears the FCF record index from the eligible bmask for
20334  * roundrobin failover search. It checks to make sure that the index
20335  * does not go beyond the range of the driver allocated bmask dimension
20336  * before clearing the bit.
20337  **/
20338 void
20339 lpfc_sli4_fcf_rr_index_clear(struct lpfc_hba *phba, uint16_t fcf_index)
20340 {
20341 	struct lpfc_fcf_pri *fcf_pri, *fcf_pri_next;
20342 	if (fcf_index >= LPFC_SLI4_FCF_TBL_INDX_MAX) {
20343 		lpfc_printf_log(phba, KERN_ERR, LOG_FIP,
20344 				"2762 FCF (x%x) reached driver's book "
20345 				"keeping dimension:x%x\n",
20346 				fcf_index, LPFC_SLI4_FCF_TBL_INDX_MAX);
20347 		return;
20348 	}
20349 	/* Clear the eligible FCF record index bmask */
20350 	spin_lock_irq(&phba->hbalock);
20351 	list_for_each_entry_safe(fcf_pri, fcf_pri_next, &phba->fcf.fcf_pri_list,
20352 				 list) {
20353 		if (fcf_pri->fcf_rec.fcf_index == fcf_index) {
20354 			list_del_init(&fcf_pri->list);
20355 			break;
20356 		}
20357 	}
20358 	spin_unlock_irq(&phba->hbalock);
20359 	clear_bit(fcf_index, phba->fcf.fcf_rr_bmask);
20360 
20361 	lpfc_printf_log(phba, KERN_INFO, LOG_FIP,
20362 			"2791 Clear FCF (x%x) from roundrobin failover "
20363 			"bmask\n", fcf_index);
20364 }
20365 
20366 /**
20367  * lpfc_mbx_cmpl_redisc_fcf_table - completion routine for rediscover FCF table
20368  * @phba: pointer to lpfc hba data structure.
20369  * @mbox: An allocated pointer to type LPFC_MBOXQ_t
20370  *
20371  * This routine is the completion routine for the rediscover FCF table mailbox
20372  * command. If the mailbox command returned failure, it will try to stop the
20373  * FCF rediscover wait timer.
20374  **/
20375 static void
20376 lpfc_mbx_cmpl_redisc_fcf_table(struct lpfc_hba *phba, LPFC_MBOXQ_t *mbox)
20377 {
20378 	struct lpfc_mbx_redisc_fcf_tbl *redisc_fcf;
20379 	uint32_t shdr_status, shdr_add_status;
20380 
20381 	redisc_fcf = &mbox->u.mqe.un.redisc_fcf_tbl;
20382 
20383 	shdr_status = bf_get(lpfc_mbox_hdr_status,
20384 			     &redisc_fcf->header.cfg_shdr.response);
20385 	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status,
20386 			     &redisc_fcf->header.cfg_shdr.response);
20387 	if (shdr_status || shdr_add_status) {
20388 		lpfc_printf_log(phba, KERN_ERR, LOG_FIP,
20389 				"2746 Requesting for FCF rediscovery failed "
20390 				"status x%x add_status x%x\n",
20391 				shdr_status, shdr_add_status);
20392 		if (phba->fcf.fcf_flag & FCF_ACVL_DISC) {
20393 			spin_lock_irq(&phba->hbalock);
20394 			phba->fcf.fcf_flag &= ~FCF_ACVL_DISC;
20395 			spin_unlock_irq(&phba->hbalock);
20396 			/*
20397 			 * CVL event triggered FCF rediscover request failed,
20398 			 * last resort to re-try current registered FCF entry.
20399 			 */
20400 			lpfc_retry_pport_discovery(phba);
20401 		} else {
20402 			spin_lock_irq(&phba->hbalock);
20403 			phba->fcf.fcf_flag &= ~FCF_DEAD_DISC;
20404 			spin_unlock_irq(&phba->hbalock);
20405 			/*
20406 			 * DEAD FCF event triggered FCF rediscover request
20407 			 * failed, last resort to fail over as a link down
20408 			 * to FCF registration.
20409 			 */
20410 			lpfc_sli4_fcf_dead_failthrough(phba);
20411 		}
20412 	} else {
20413 		lpfc_printf_log(phba, KERN_INFO, LOG_FIP,
20414 				"2775 Start FCF rediscover quiescent timer\n");
20415 		/*
20416 		 * Start FCF rediscovery wait timer for pending FCF
20417 		 * before rescan FCF record table.
20418 		 */
20419 		lpfc_fcf_redisc_wait_start_timer(phba);
20420 	}
20421 
20422 	mempool_free(mbox, phba->mbox_mem_pool);
20423 }
20424 
20425 /**
20426  * lpfc_sli4_redisc_fcf_table - Request to rediscover entire FCF table by port.
20427  * @phba: pointer to lpfc hba data structure.
20428  *
20429  * This routine is invoked to request for rediscovery of the entire FCF table
20430  * by the port.
20431  **/
20432 int
20433 lpfc_sli4_redisc_fcf_table(struct lpfc_hba *phba)
20434 {
20435 	LPFC_MBOXQ_t *mbox;
20436 	struct lpfc_mbx_redisc_fcf_tbl *redisc_fcf;
20437 	int rc, length;
20438 
20439 	/* Cancel retry delay timers to all vports before FCF rediscover */
20440 	lpfc_cancel_all_vport_retry_delay_timer(phba);
20441 
20442 	mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
20443 	if (!mbox) {
20444 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
20445 				"2745 Failed to allocate mbox for "
20446 				"requesting FCF rediscover.\n");
20447 		return -ENOMEM;
20448 	}
20449 
20450 	length = (sizeof(struct lpfc_mbx_redisc_fcf_tbl) -
20451 		  sizeof(struct lpfc_sli4_cfg_mhdr));
20452 	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_FCOE,
20453 			 LPFC_MBOX_OPCODE_FCOE_REDISCOVER_FCF,
20454 			 length, LPFC_SLI4_MBX_EMBED);
20455 
20456 	redisc_fcf = &mbox->u.mqe.un.redisc_fcf_tbl;
20457 	/* Set count to 0 for invalidating the entire FCF database */
20458 	bf_set(lpfc_mbx_redisc_fcf_count, redisc_fcf, 0);
20459 
20460 	/* Issue the mailbox command asynchronously */
20461 	mbox->vport = phba->pport;
20462 	mbox->mbox_cmpl = lpfc_mbx_cmpl_redisc_fcf_table;
20463 	rc = lpfc_sli_issue_mbox(phba, mbox, MBX_NOWAIT);
20464 
20465 	if (rc == MBX_NOT_FINISHED) {
20466 		mempool_free(mbox, phba->mbox_mem_pool);
20467 		return -EIO;
20468 	}
20469 	return 0;
20470 }
20471 
20472 /**
20473  * lpfc_sli4_fcf_dead_failthrough - Failthrough routine to fcf dead event
20474  * @phba: pointer to lpfc hba data structure.
20475  *
20476  * This function is the failover routine as a last resort to the FCF DEAD
20477  * event when driver failed to perform fast FCF failover.
20478  **/
20479 void
20480 lpfc_sli4_fcf_dead_failthrough(struct lpfc_hba *phba)
20481 {
20482 	uint32_t link_state;
20483 
20484 	/*
20485 	 * Last resort as FCF DEAD event failover will treat this as
20486 	 * a link down, but save the link state because we don't want
20487 	 * it to be changed to Link Down unless it is already down.
20488 	 */
20489 	link_state = phba->link_state;
20490 	lpfc_linkdown(phba);
20491 	phba->link_state = link_state;
20492 
20493 	/* Unregister FCF if no devices connected to it */
20494 	lpfc_unregister_unused_fcf(phba);
20495 }
20496 
20497 /**
20498  * lpfc_sli_get_config_region23 - Get sli3 port region 23 data.
20499  * @phba: pointer to lpfc hba data structure.
20500  * @rgn23_data: pointer to configure region 23 data.
20501  *
20502  * This function gets SLI3 port configure region 23 data through memory dump
20503  * mailbox command. When it successfully retrieves data, the size of the data
20504  * will be returned, otherwise, 0 will be returned.
20505  **/
20506 static uint32_t
20507 lpfc_sli_get_config_region23(struct lpfc_hba *phba, char *rgn23_data)
20508 {
20509 	LPFC_MBOXQ_t *pmb = NULL;
20510 	MAILBOX_t *mb;
20511 	uint32_t offset = 0;
20512 	int rc;
20513 
20514 	if (!rgn23_data)
20515 		return 0;
20516 
20517 	pmb = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
20518 	if (!pmb) {
20519 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
20520 				"2600 failed to allocate mailbox memory\n");
20521 		return 0;
20522 	}
20523 	mb = &pmb->u.mb;
20524 
20525 	do {
20526 		lpfc_dump_mem(phba, pmb, offset, DMP_REGION_23);
20527 		rc = lpfc_sli_issue_mbox(phba, pmb, MBX_POLL);
20528 
20529 		if (rc != MBX_SUCCESS) {
20530 			lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
20531 					"2601 failed to read config "
20532 					"region 23, rc 0x%x Status 0x%x\n",
20533 					rc, mb->mbxStatus);
20534 			mb->un.varDmp.word_cnt = 0;
20535 		}
20536 		/*
20537 		 * dump mem may return a zero when finished or we got a
20538 		 * mailbox error, either way we are done.
20539 		 */
20540 		if (mb->un.varDmp.word_cnt == 0)
20541 			break;
20542 
20543 		if (mb->un.varDmp.word_cnt > DMP_RGN23_SIZE - offset)
20544 			mb->un.varDmp.word_cnt = DMP_RGN23_SIZE - offset;
20545 
20546 		lpfc_sli_pcimem_bcopy(((uint8_t *)mb) + DMP_RSP_OFFSET,
20547 				       rgn23_data + offset,
20548 				       mb->un.varDmp.word_cnt);
20549 		offset += mb->un.varDmp.word_cnt;
20550 	} while (mb->un.varDmp.word_cnt && offset < DMP_RGN23_SIZE);
20551 
20552 	mempool_free(pmb, phba->mbox_mem_pool);
20553 	return offset;
20554 }
20555 
20556 /**
20557  * lpfc_sli4_get_config_region23 - Get sli4 port region 23 data.
20558  * @phba: pointer to lpfc hba data structure.
20559  * @rgn23_data: pointer to configure region 23 data.
20560  *
20561  * This function gets SLI4 port configure region 23 data through memory dump
20562  * mailbox command. When it successfully retrieves data, the size of the data
20563  * will be returned, otherwise, 0 will be returned.
20564  **/
20565 static uint32_t
20566 lpfc_sli4_get_config_region23(struct lpfc_hba *phba, char *rgn23_data)
20567 {
20568 	LPFC_MBOXQ_t *mboxq = NULL;
20569 	struct lpfc_dmabuf *mp = NULL;
20570 	struct lpfc_mqe *mqe;
20571 	uint32_t data_length = 0;
20572 	int rc;
20573 
20574 	if (!rgn23_data)
20575 		return 0;
20576 
20577 	mboxq = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
20578 	if (!mboxq) {
20579 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
20580 				"3105 failed to allocate mailbox memory\n");
20581 		return 0;
20582 	}
20583 
20584 	if (lpfc_sli4_dump_cfg_rg23(phba, mboxq))
20585 		goto out;
20586 	mqe = &mboxq->u.mqe;
20587 	mp = (struct lpfc_dmabuf *)mboxq->ctx_buf;
20588 	rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
20589 	if (rc)
20590 		goto out;
20591 	data_length = mqe->un.mb_words[5];
20592 	if (data_length == 0)
20593 		goto out;
20594 	if (data_length > DMP_RGN23_SIZE) {
20595 		data_length = 0;
20596 		goto out;
20597 	}
20598 	lpfc_sli_pcimem_bcopy((char *)mp->virt, rgn23_data, data_length);
20599 out:
20600 	mempool_free(mboxq, phba->mbox_mem_pool);
20601 	if (mp) {
20602 		lpfc_mbuf_free(phba, mp->virt, mp->phys);
20603 		kfree(mp);
20604 	}
20605 	return data_length;
20606 }
20607 
20608 /**
20609  * lpfc_sli_read_link_ste - Read region 23 to decide if link is disabled.
20610  * @phba: pointer to lpfc hba data structure.
20611  *
20612  * This function read region 23 and parse TLV for port status to
20613  * decide if the user disaled the port. If the TLV indicates the
20614  * port is disabled, the hba_flag is set accordingly.
20615  **/
20616 void
20617 lpfc_sli_read_link_ste(struct lpfc_hba *phba)
20618 {
20619 	uint8_t *rgn23_data = NULL;
20620 	uint32_t if_type, data_size, sub_tlv_len, tlv_offset;
20621 	uint32_t offset = 0;
20622 
20623 	/* Get adapter Region 23 data */
20624 	rgn23_data = kzalloc(DMP_RGN23_SIZE, GFP_KERNEL);
20625 	if (!rgn23_data)
20626 		goto out;
20627 
20628 	if (phba->sli_rev < LPFC_SLI_REV4)
20629 		data_size = lpfc_sli_get_config_region23(phba, rgn23_data);
20630 	else {
20631 		if_type = bf_get(lpfc_sli_intf_if_type,
20632 				 &phba->sli4_hba.sli_intf);
20633 		if (if_type == LPFC_SLI_INTF_IF_TYPE_0)
20634 			goto out;
20635 		data_size = lpfc_sli4_get_config_region23(phba, rgn23_data);
20636 	}
20637 
20638 	if (!data_size)
20639 		goto out;
20640 
20641 	/* Check the region signature first */
20642 	if (memcmp(&rgn23_data[offset], LPFC_REGION23_SIGNATURE, 4)) {
20643 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
20644 			"2619 Config region 23 has bad signature\n");
20645 			goto out;
20646 	}
20647 	offset += 4;
20648 
20649 	/* Check the data structure version */
20650 	if (rgn23_data[offset] != LPFC_REGION23_VERSION) {
20651 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
20652 			"2620 Config region 23 has bad version\n");
20653 		goto out;
20654 	}
20655 	offset += 4;
20656 
20657 	/* Parse TLV entries in the region */
20658 	while (offset < data_size) {
20659 		if (rgn23_data[offset] == LPFC_REGION23_LAST_REC)
20660 			break;
20661 		/*
20662 		 * If the TLV is not driver specific TLV or driver id is
20663 		 * not linux driver id, skip the record.
20664 		 */
20665 		if ((rgn23_data[offset] != DRIVER_SPECIFIC_TYPE) ||
20666 		    (rgn23_data[offset + 2] != LINUX_DRIVER_ID) ||
20667 		    (rgn23_data[offset + 3] != 0)) {
20668 			offset += rgn23_data[offset + 1] * 4 + 4;
20669 			continue;
20670 		}
20671 
20672 		/* Driver found a driver specific TLV in the config region */
20673 		sub_tlv_len = rgn23_data[offset + 1] * 4;
20674 		offset += 4;
20675 		tlv_offset = 0;
20676 
20677 		/*
20678 		 * Search for configured port state sub-TLV.
20679 		 */
20680 		while ((offset < data_size) &&
20681 			(tlv_offset < sub_tlv_len)) {
20682 			if (rgn23_data[offset] == LPFC_REGION23_LAST_REC) {
20683 				offset += 4;
20684 				tlv_offset += 4;
20685 				break;
20686 			}
20687 			if (rgn23_data[offset] != PORT_STE_TYPE) {
20688 				offset += rgn23_data[offset + 1] * 4 + 4;
20689 				tlv_offset += rgn23_data[offset + 1] * 4 + 4;
20690 				continue;
20691 			}
20692 
20693 			/* This HBA contains PORT_STE configured */
20694 			if (!rgn23_data[offset + 2])
20695 				phba->hba_flag |= LINK_DISABLED;
20696 
20697 			goto out;
20698 		}
20699 	}
20700 
20701 out:
20702 	kfree(rgn23_data);
20703 	return;
20704 }
20705 
20706 /**
20707  * lpfc_log_fw_write_cmpl - logs firmware write completion status
20708  * @phba: pointer to lpfc hba data structure
20709  * @shdr_status: wr_object rsp's status field
20710  * @shdr_add_status: wr_object rsp's add_status field
20711  * @shdr_add_status_2: wr_object rsp's add_status_2 field
20712  * @shdr_change_status: wr_object rsp's change_status field
20713  * @shdr_csf: wr_object rsp's csf bit
20714  *
20715  * This routine is intended to be called after a firmware write completes.
20716  * It will log next action items to be performed by the user to instantiate
20717  * the newly downloaded firmware or reason for incompatibility.
20718  **/
20719 static void
20720 lpfc_log_fw_write_cmpl(struct lpfc_hba *phba, u32 shdr_status,
20721 		       u32 shdr_add_status, u32 shdr_add_status_2,
20722 		       u32 shdr_change_status, u32 shdr_csf)
20723 {
20724 	lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI,
20725 			"4198 %s: flash_id x%02x, asic_rev x%02x, "
20726 			"status x%02x, add_status x%02x, add_status_2 x%02x, "
20727 			"change_status x%02x, csf %01x\n", __func__,
20728 			phba->sli4_hba.flash_id, phba->sli4_hba.asic_rev,
20729 			shdr_status, shdr_add_status, shdr_add_status_2,
20730 			shdr_change_status, shdr_csf);
20731 
20732 	if (shdr_add_status == LPFC_ADD_STATUS_INCOMPAT_OBJ) {
20733 		switch (shdr_add_status_2) {
20734 		case LPFC_ADD_STATUS_2_INCOMPAT_FLASH:
20735 			lpfc_printf_log(phba, KERN_WARNING, LOG_MBOX | LOG_SLI,
20736 					"4199 Firmware write failed: "
20737 					"image incompatible with flash x%02x\n",
20738 					phba->sli4_hba.flash_id);
20739 			break;
20740 		case LPFC_ADD_STATUS_2_INCORRECT_ASIC:
20741 			lpfc_printf_log(phba, KERN_WARNING, LOG_MBOX | LOG_SLI,
20742 					"4200 Firmware write failed: "
20743 					"image incompatible with ASIC "
20744 					"architecture x%02x\n",
20745 					phba->sli4_hba.asic_rev);
20746 			break;
20747 		default:
20748 			lpfc_printf_log(phba, KERN_WARNING, LOG_MBOX | LOG_SLI,
20749 					"4210 Firmware write failed: "
20750 					"add_status_2 x%02x\n",
20751 					shdr_add_status_2);
20752 			break;
20753 		}
20754 	} else if (!shdr_status && !shdr_add_status) {
20755 		if (shdr_change_status == LPFC_CHANGE_STATUS_FW_RESET ||
20756 		    shdr_change_status == LPFC_CHANGE_STATUS_PORT_MIGRATION) {
20757 			if (shdr_csf)
20758 				shdr_change_status =
20759 						   LPFC_CHANGE_STATUS_PCI_RESET;
20760 		}
20761 
20762 		switch (shdr_change_status) {
20763 		case (LPFC_CHANGE_STATUS_PHYS_DEV_RESET):
20764 			lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI,
20765 					"3198 Firmware write complete: System "
20766 					"reboot required to instantiate\n");
20767 			break;
20768 		case (LPFC_CHANGE_STATUS_FW_RESET):
20769 			lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI,
20770 					"3199 Firmware write complete: "
20771 					"Firmware reset required to "
20772 					"instantiate\n");
20773 			break;
20774 		case (LPFC_CHANGE_STATUS_PORT_MIGRATION):
20775 			lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI,
20776 					"3200 Firmware write complete: Port "
20777 					"Migration or PCI Reset required to "
20778 					"instantiate\n");
20779 			break;
20780 		case (LPFC_CHANGE_STATUS_PCI_RESET):
20781 			lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI,
20782 					"3201 Firmware write complete: PCI "
20783 					"Reset required to instantiate\n");
20784 			break;
20785 		default:
20786 			break;
20787 		}
20788 	}
20789 }
20790 
20791 /**
20792  * lpfc_wr_object - write an object to the firmware
20793  * @phba: HBA structure that indicates port to create a queue on.
20794  * @dmabuf_list: list of dmabufs to write to the port.
20795  * @size: the total byte value of the objects to write to the port.
20796  * @offset: the current offset to be used to start the transfer.
20797  *
20798  * This routine will create a wr_object mailbox command to send to the port.
20799  * the mailbox command will be constructed using the dma buffers described in
20800  * @dmabuf_list to create a list of BDEs. This routine will fill in as many
20801  * BDEs that the imbedded mailbox can support. The @offset variable will be
20802  * used to indicate the starting offset of the transfer and will also return
20803  * the offset after the write object mailbox has completed. @size is used to
20804  * determine the end of the object and whether the eof bit should be set.
20805  *
20806  * Return 0 is successful and offset will contain the the new offset to use
20807  * for the next write.
20808  * Return negative value for error cases.
20809  **/
20810 int
20811 lpfc_wr_object(struct lpfc_hba *phba, struct list_head *dmabuf_list,
20812 	       uint32_t size, uint32_t *offset)
20813 {
20814 	struct lpfc_mbx_wr_object *wr_object;
20815 	LPFC_MBOXQ_t *mbox;
20816 	int rc = 0, i = 0;
20817 	uint32_t shdr_status, shdr_add_status, shdr_add_status_2;
20818 	uint32_t shdr_change_status = 0, shdr_csf = 0;
20819 	uint32_t mbox_tmo;
20820 	struct lpfc_dmabuf *dmabuf;
20821 	uint32_t written = 0;
20822 	bool check_change_status = false;
20823 
20824 	mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
20825 	if (!mbox)
20826 		return -ENOMEM;
20827 
20828 	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
20829 			LPFC_MBOX_OPCODE_WRITE_OBJECT,
20830 			sizeof(struct lpfc_mbx_wr_object) -
20831 			sizeof(struct lpfc_sli4_cfg_mhdr), LPFC_SLI4_MBX_EMBED);
20832 
20833 	wr_object = (struct lpfc_mbx_wr_object *)&mbox->u.mqe.un.wr_object;
20834 	wr_object->u.request.write_offset = *offset;
20835 	sprintf((uint8_t *)wr_object->u.request.object_name, "/");
20836 	wr_object->u.request.object_name[0] =
20837 		cpu_to_le32(wr_object->u.request.object_name[0]);
20838 	bf_set(lpfc_wr_object_eof, &wr_object->u.request, 0);
20839 	list_for_each_entry(dmabuf, dmabuf_list, list) {
20840 		if (i >= LPFC_MBX_WR_CONFIG_MAX_BDE || written >= size)
20841 			break;
20842 		wr_object->u.request.bde[i].addrLow = putPaddrLow(dmabuf->phys);
20843 		wr_object->u.request.bde[i].addrHigh =
20844 			putPaddrHigh(dmabuf->phys);
20845 		if (written + SLI4_PAGE_SIZE >= size) {
20846 			wr_object->u.request.bde[i].tus.f.bdeSize =
20847 				(size - written);
20848 			written += (size - written);
20849 			bf_set(lpfc_wr_object_eof, &wr_object->u.request, 1);
20850 			bf_set(lpfc_wr_object_eas, &wr_object->u.request, 1);
20851 			check_change_status = true;
20852 		} else {
20853 			wr_object->u.request.bde[i].tus.f.bdeSize =
20854 				SLI4_PAGE_SIZE;
20855 			written += SLI4_PAGE_SIZE;
20856 		}
20857 		i++;
20858 	}
20859 	wr_object->u.request.bde_count = i;
20860 	bf_set(lpfc_wr_object_write_length, &wr_object->u.request, written);
20861 	if (!phba->sli4_hba.intr_enable)
20862 		rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL);
20863 	else {
20864 		mbox_tmo = lpfc_mbox_tmo_val(phba, mbox);
20865 		rc = lpfc_sli_issue_mbox_wait(phba, mbox, mbox_tmo);
20866 	}
20867 	/* The IOCTL status is embedded in the mailbox subheader. */
20868 	shdr_status = bf_get(lpfc_mbox_hdr_status,
20869 			     &wr_object->header.cfg_shdr.response);
20870 	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status,
20871 				 &wr_object->header.cfg_shdr.response);
20872 	shdr_add_status_2 = bf_get(lpfc_mbox_hdr_add_status_2,
20873 				   &wr_object->header.cfg_shdr.response);
20874 	if (check_change_status) {
20875 		shdr_change_status = bf_get(lpfc_wr_object_change_status,
20876 					    &wr_object->u.response);
20877 		shdr_csf = bf_get(lpfc_wr_object_csf,
20878 				  &wr_object->u.response);
20879 	}
20880 
20881 	if (!phba->sli4_hba.intr_enable)
20882 		mempool_free(mbox, phba->mbox_mem_pool);
20883 	else if (rc != MBX_TIMEOUT)
20884 		mempool_free(mbox, phba->mbox_mem_pool);
20885 	if (shdr_status || shdr_add_status || shdr_add_status_2 || rc) {
20886 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
20887 				"3025 Write Object mailbox failed with "
20888 				"status x%x add_status x%x, add_status_2 x%x, "
20889 				"mbx status x%x\n",
20890 				shdr_status, shdr_add_status, shdr_add_status_2,
20891 				rc);
20892 		rc = -ENXIO;
20893 		*offset = shdr_add_status;
20894 	} else {
20895 		*offset += wr_object->u.response.actual_write_length;
20896 	}
20897 
20898 	if (rc || check_change_status)
20899 		lpfc_log_fw_write_cmpl(phba, shdr_status, shdr_add_status,
20900 				       shdr_add_status_2, shdr_change_status,
20901 				       shdr_csf);
20902 	return rc;
20903 }
20904 
20905 /**
20906  * lpfc_cleanup_pending_mbox - Free up vport discovery mailbox commands.
20907  * @vport: pointer to vport data structure.
20908  *
20909  * This function iterate through the mailboxq and clean up all REG_LOGIN
20910  * and REG_VPI mailbox commands associated with the vport. This function
20911  * is called when driver want to restart discovery of the vport due to
20912  * a Clear Virtual Link event.
20913  **/
20914 void
20915 lpfc_cleanup_pending_mbox(struct lpfc_vport *vport)
20916 {
20917 	struct lpfc_hba *phba = vport->phba;
20918 	LPFC_MBOXQ_t *mb, *nextmb;
20919 	struct lpfc_dmabuf *mp;
20920 	struct lpfc_nodelist *ndlp;
20921 	struct lpfc_nodelist *act_mbx_ndlp = NULL;
20922 	LIST_HEAD(mbox_cmd_list);
20923 	uint8_t restart_loop;
20924 
20925 	/* Clean up internally queued mailbox commands with the vport */
20926 	spin_lock_irq(&phba->hbalock);
20927 	list_for_each_entry_safe(mb, nextmb, &phba->sli.mboxq, list) {
20928 		if (mb->vport != vport)
20929 			continue;
20930 
20931 		if ((mb->u.mb.mbxCommand != MBX_REG_LOGIN64) &&
20932 			(mb->u.mb.mbxCommand != MBX_REG_VPI))
20933 			continue;
20934 
20935 		list_move_tail(&mb->list, &mbox_cmd_list);
20936 	}
20937 	/* Clean up active mailbox command with the vport */
20938 	mb = phba->sli.mbox_active;
20939 	if (mb && (mb->vport == vport)) {
20940 		if ((mb->u.mb.mbxCommand == MBX_REG_LOGIN64) ||
20941 			(mb->u.mb.mbxCommand == MBX_REG_VPI))
20942 			mb->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
20943 		if (mb->u.mb.mbxCommand == MBX_REG_LOGIN64) {
20944 			act_mbx_ndlp = (struct lpfc_nodelist *)mb->ctx_ndlp;
20945 			/* Put reference count for delayed processing */
20946 			act_mbx_ndlp = lpfc_nlp_get(act_mbx_ndlp);
20947 			/* Unregister the RPI when mailbox complete */
20948 			mb->mbox_flag |= LPFC_MBX_IMED_UNREG;
20949 		}
20950 	}
20951 	/* Cleanup any mailbox completions which are not yet processed */
20952 	do {
20953 		restart_loop = 0;
20954 		list_for_each_entry(mb, &phba->sli.mboxq_cmpl, list) {
20955 			/*
20956 			 * If this mailox is already processed or it is
20957 			 * for another vport ignore it.
20958 			 */
20959 			if ((mb->vport != vport) ||
20960 				(mb->mbox_flag & LPFC_MBX_IMED_UNREG))
20961 				continue;
20962 
20963 			if ((mb->u.mb.mbxCommand != MBX_REG_LOGIN64) &&
20964 				(mb->u.mb.mbxCommand != MBX_REG_VPI))
20965 				continue;
20966 
20967 			mb->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
20968 			if (mb->u.mb.mbxCommand == MBX_REG_LOGIN64) {
20969 				ndlp = (struct lpfc_nodelist *)mb->ctx_ndlp;
20970 				/* Unregister the RPI when mailbox complete */
20971 				mb->mbox_flag |= LPFC_MBX_IMED_UNREG;
20972 				restart_loop = 1;
20973 				spin_unlock_irq(&phba->hbalock);
20974 				spin_lock(&ndlp->lock);
20975 				ndlp->nlp_flag &= ~NLP_IGNR_REG_CMPL;
20976 				spin_unlock(&ndlp->lock);
20977 				spin_lock_irq(&phba->hbalock);
20978 				break;
20979 			}
20980 		}
20981 	} while (restart_loop);
20982 
20983 	spin_unlock_irq(&phba->hbalock);
20984 
20985 	/* Release the cleaned-up mailbox commands */
20986 	while (!list_empty(&mbox_cmd_list)) {
20987 		list_remove_head(&mbox_cmd_list, mb, LPFC_MBOXQ_t, list);
20988 		if (mb->u.mb.mbxCommand == MBX_REG_LOGIN64) {
20989 			mp = (struct lpfc_dmabuf *)(mb->ctx_buf);
20990 			if (mp) {
20991 				__lpfc_mbuf_free(phba, mp->virt, mp->phys);
20992 				kfree(mp);
20993 			}
20994 			mb->ctx_buf = NULL;
20995 			ndlp = (struct lpfc_nodelist *)mb->ctx_ndlp;
20996 			mb->ctx_ndlp = NULL;
20997 			if (ndlp) {
20998 				spin_lock(&ndlp->lock);
20999 				ndlp->nlp_flag &= ~NLP_IGNR_REG_CMPL;
21000 				spin_unlock(&ndlp->lock);
21001 				lpfc_nlp_put(ndlp);
21002 			}
21003 		}
21004 		mempool_free(mb, phba->mbox_mem_pool);
21005 	}
21006 
21007 	/* Release the ndlp with the cleaned-up active mailbox command */
21008 	if (act_mbx_ndlp) {
21009 		spin_lock(&act_mbx_ndlp->lock);
21010 		act_mbx_ndlp->nlp_flag &= ~NLP_IGNR_REG_CMPL;
21011 		spin_unlock(&act_mbx_ndlp->lock);
21012 		lpfc_nlp_put(act_mbx_ndlp);
21013 	}
21014 }
21015 
21016 /**
21017  * lpfc_drain_txq - Drain the txq
21018  * @phba: Pointer to HBA context object.
21019  *
21020  * This function attempt to submit IOCBs on the txq
21021  * to the adapter.  For SLI4 adapters, the txq contains
21022  * ELS IOCBs that have been deferred because the there
21023  * are no SGLs.  This congestion can occur with large
21024  * vport counts during node discovery.
21025  **/
21026 
21027 uint32_t
21028 lpfc_drain_txq(struct lpfc_hba *phba)
21029 {
21030 	LIST_HEAD(completions);
21031 	struct lpfc_sli_ring *pring;
21032 	struct lpfc_iocbq *piocbq = NULL;
21033 	unsigned long iflags = 0;
21034 	char *fail_msg = NULL;
21035 	struct lpfc_sglq *sglq;
21036 	union lpfc_wqe128 wqe;
21037 	uint32_t txq_cnt = 0;
21038 	struct lpfc_queue *wq;
21039 
21040 	if (phba->link_flag & LS_MDS_LOOPBACK) {
21041 		/* MDS WQE are posted only to first WQ*/
21042 		wq = phba->sli4_hba.hdwq[0].io_wq;
21043 		if (unlikely(!wq))
21044 			return 0;
21045 		pring = wq->pring;
21046 	} else {
21047 		wq = phba->sli4_hba.els_wq;
21048 		if (unlikely(!wq))
21049 			return 0;
21050 		pring = lpfc_phba_elsring(phba);
21051 	}
21052 
21053 	if (unlikely(!pring) || list_empty(&pring->txq))
21054 		return 0;
21055 
21056 	spin_lock_irqsave(&pring->ring_lock, iflags);
21057 	list_for_each_entry(piocbq, &pring->txq, list) {
21058 		txq_cnt++;
21059 	}
21060 
21061 	if (txq_cnt > pring->txq_max)
21062 		pring->txq_max = txq_cnt;
21063 
21064 	spin_unlock_irqrestore(&pring->ring_lock, iflags);
21065 
21066 	while (!list_empty(&pring->txq)) {
21067 		spin_lock_irqsave(&pring->ring_lock, iflags);
21068 
21069 		piocbq = lpfc_sli_ringtx_get(phba, pring);
21070 		if (!piocbq) {
21071 			spin_unlock_irqrestore(&pring->ring_lock, iflags);
21072 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
21073 				"2823 txq empty and txq_cnt is %d\n ",
21074 				txq_cnt);
21075 			break;
21076 		}
21077 		sglq = __lpfc_sli_get_els_sglq(phba, piocbq);
21078 		if (!sglq) {
21079 			__lpfc_sli_ringtx_put(phba, pring, piocbq);
21080 			spin_unlock_irqrestore(&pring->ring_lock, iflags);
21081 			break;
21082 		}
21083 		txq_cnt--;
21084 
21085 		/* The xri and iocb resources secured,
21086 		 * attempt to issue request
21087 		 */
21088 		piocbq->sli4_lxritag = sglq->sli4_lxritag;
21089 		piocbq->sli4_xritag = sglq->sli4_xritag;
21090 		if (NO_XRI == lpfc_sli4_bpl2sgl(phba, piocbq, sglq))
21091 			fail_msg = "to convert bpl to sgl";
21092 		else if (lpfc_sli4_iocb2wqe(phba, piocbq, &wqe))
21093 			fail_msg = "to convert iocb to wqe";
21094 		else if (lpfc_sli4_wq_put(wq, &wqe))
21095 			fail_msg = " - Wq is full";
21096 		else
21097 			lpfc_sli_ringtxcmpl_put(phba, pring, piocbq);
21098 
21099 		if (fail_msg) {
21100 			/* Failed means we can't issue and need to cancel */
21101 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
21102 					"2822 IOCB failed %s iotag 0x%x "
21103 					"xri 0x%x\n",
21104 					fail_msg,
21105 					piocbq->iotag, piocbq->sli4_xritag);
21106 			list_add_tail(&piocbq->list, &completions);
21107 		}
21108 		spin_unlock_irqrestore(&pring->ring_lock, iflags);
21109 	}
21110 
21111 	/* Cancel all the IOCBs that cannot be issued */
21112 	lpfc_sli_cancel_iocbs(phba, &completions, IOSTAT_LOCAL_REJECT,
21113 				IOERR_SLI_ABORTED);
21114 
21115 	return txq_cnt;
21116 }
21117 
21118 /**
21119  * lpfc_wqe_bpl2sgl - Convert the bpl/bde to a sgl.
21120  * @phba: Pointer to HBA context object.
21121  * @pwqeq: Pointer to command WQE.
21122  * @sglq: Pointer to the scatter gather queue object.
21123  *
21124  * This routine converts the bpl or bde that is in the WQE
21125  * to a sgl list for the sli4 hardware. The physical address
21126  * of the bpl/bde is converted back to a virtual address.
21127  * If the WQE contains a BPL then the list of BDE's is
21128  * converted to sli4_sge's. If the WQE contains a single
21129  * BDE then it is converted to a single sli_sge.
21130  * The WQE is still in cpu endianness so the contents of
21131  * the bpl can be used without byte swapping.
21132  *
21133  * Returns valid XRI = Success, NO_XRI = Failure.
21134  */
21135 static uint16_t
21136 lpfc_wqe_bpl2sgl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeq,
21137 		 struct lpfc_sglq *sglq)
21138 {
21139 	uint16_t xritag = NO_XRI;
21140 	struct ulp_bde64 *bpl = NULL;
21141 	struct ulp_bde64 bde;
21142 	struct sli4_sge *sgl  = NULL;
21143 	struct lpfc_dmabuf *dmabuf;
21144 	union lpfc_wqe128 *wqe;
21145 	int numBdes = 0;
21146 	int i = 0;
21147 	uint32_t offset = 0; /* accumulated offset in the sg request list */
21148 	int inbound = 0; /* number of sg reply entries inbound from firmware */
21149 	uint32_t cmd;
21150 
21151 	if (!pwqeq || !sglq)
21152 		return xritag;
21153 
21154 	sgl  = (struct sli4_sge *)sglq->sgl;
21155 	wqe = &pwqeq->wqe;
21156 	pwqeq->iocb.ulpIoTag = pwqeq->iotag;
21157 
21158 	cmd = bf_get(wqe_cmnd, &wqe->generic.wqe_com);
21159 	if (cmd == CMD_XMIT_BLS_RSP64_WQE)
21160 		return sglq->sli4_xritag;
21161 	numBdes = pwqeq->rsvd2;
21162 	if (numBdes) {
21163 		/* The addrHigh and addrLow fields within the WQE
21164 		 * have not been byteswapped yet so there is no
21165 		 * need to swap them back.
21166 		 */
21167 		if (pwqeq->context3)
21168 			dmabuf = (struct lpfc_dmabuf *)pwqeq->context3;
21169 		else
21170 			return xritag;
21171 
21172 		bpl  = (struct ulp_bde64 *)dmabuf->virt;
21173 		if (!bpl)
21174 			return xritag;
21175 
21176 		for (i = 0; i < numBdes; i++) {
21177 			/* Should already be byte swapped. */
21178 			sgl->addr_hi = bpl->addrHigh;
21179 			sgl->addr_lo = bpl->addrLow;
21180 
21181 			sgl->word2 = le32_to_cpu(sgl->word2);
21182 			if ((i+1) == numBdes)
21183 				bf_set(lpfc_sli4_sge_last, sgl, 1);
21184 			else
21185 				bf_set(lpfc_sli4_sge_last, sgl, 0);
21186 			/* swap the size field back to the cpu so we
21187 			 * can assign it to the sgl.
21188 			 */
21189 			bde.tus.w = le32_to_cpu(bpl->tus.w);
21190 			sgl->sge_len = cpu_to_le32(bde.tus.f.bdeSize);
21191 			/* The offsets in the sgl need to be accumulated
21192 			 * separately for the request and reply lists.
21193 			 * The request is always first, the reply follows.
21194 			 */
21195 			switch (cmd) {
21196 			case CMD_GEN_REQUEST64_WQE:
21197 				/* add up the reply sg entries */
21198 				if (bpl->tus.f.bdeFlags == BUFF_TYPE_BDE_64I)
21199 					inbound++;
21200 				/* first inbound? reset the offset */
21201 				if (inbound == 1)
21202 					offset = 0;
21203 				bf_set(lpfc_sli4_sge_offset, sgl, offset);
21204 				bf_set(lpfc_sli4_sge_type, sgl,
21205 					LPFC_SGE_TYPE_DATA);
21206 				offset += bde.tus.f.bdeSize;
21207 				break;
21208 			case CMD_FCP_TRSP64_WQE:
21209 				bf_set(lpfc_sli4_sge_offset, sgl, 0);
21210 				bf_set(lpfc_sli4_sge_type, sgl,
21211 					LPFC_SGE_TYPE_DATA);
21212 				break;
21213 			case CMD_FCP_TSEND64_WQE:
21214 			case CMD_FCP_TRECEIVE64_WQE:
21215 				bf_set(lpfc_sli4_sge_type, sgl,
21216 					bpl->tus.f.bdeFlags);
21217 				if (i < 3)
21218 					offset = 0;
21219 				else
21220 					offset += bde.tus.f.bdeSize;
21221 				bf_set(lpfc_sli4_sge_offset, sgl, offset);
21222 				break;
21223 			}
21224 			sgl->word2 = cpu_to_le32(sgl->word2);
21225 			bpl++;
21226 			sgl++;
21227 		}
21228 	} else if (wqe->gen_req.bde.tus.f.bdeFlags == BUFF_TYPE_BDE_64) {
21229 		/* The addrHigh and addrLow fields of the BDE have not
21230 		 * been byteswapped yet so they need to be swapped
21231 		 * before putting them in the sgl.
21232 		 */
21233 		sgl->addr_hi = cpu_to_le32(wqe->gen_req.bde.addrHigh);
21234 		sgl->addr_lo = cpu_to_le32(wqe->gen_req.bde.addrLow);
21235 		sgl->word2 = le32_to_cpu(sgl->word2);
21236 		bf_set(lpfc_sli4_sge_last, sgl, 1);
21237 		sgl->word2 = cpu_to_le32(sgl->word2);
21238 		sgl->sge_len = cpu_to_le32(wqe->gen_req.bde.tus.f.bdeSize);
21239 	}
21240 	return sglq->sli4_xritag;
21241 }
21242 
21243 /**
21244  * lpfc_sli4_issue_wqe - Issue an SLI4 Work Queue Entry (WQE)
21245  * @phba: Pointer to HBA context object.
21246  * @qp: Pointer to HDW queue.
21247  * @pwqe: Pointer to command WQE.
21248  **/
21249 int
21250 lpfc_sli4_issue_wqe(struct lpfc_hba *phba, struct lpfc_sli4_hdw_queue *qp,
21251 		    struct lpfc_iocbq *pwqe)
21252 {
21253 	union lpfc_wqe128 *wqe = &pwqe->wqe;
21254 	struct lpfc_async_xchg_ctx *ctxp;
21255 	struct lpfc_queue *wq;
21256 	struct lpfc_sglq *sglq;
21257 	struct lpfc_sli_ring *pring;
21258 	unsigned long iflags;
21259 	uint32_t ret = 0;
21260 
21261 	/* NVME_LS and NVME_LS ABTS requests. */
21262 	if (pwqe->iocb_flag & LPFC_IO_NVME_LS) {
21263 		pring =  phba->sli4_hba.nvmels_wq->pring;
21264 		lpfc_qp_spin_lock_irqsave(&pring->ring_lock, iflags,
21265 					  qp, wq_access);
21266 		sglq = __lpfc_sli_get_els_sglq(phba, pwqe);
21267 		if (!sglq) {
21268 			spin_unlock_irqrestore(&pring->ring_lock, iflags);
21269 			return WQE_BUSY;
21270 		}
21271 		pwqe->sli4_lxritag = sglq->sli4_lxritag;
21272 		pwqe->sli4_xritag = sglq->sli4_xritag;
21273 		if (lpfc_wqe_bpl2sgl(phba, pwqe, sglq) == NO_XRI) {
21274 			spin_unlock_irqrestore(&pring->ring_lock, iflags);
21275 			return WQE_ERROR;
21276 		}
21277 		bf_set(wqe_xri_tag, &pwqe->wqe.xmit_bls_rsp.wqe_com,
21278 		       pwqe->sli4_xritag);
21279 		ret = lpfc_sli4_wq_put(phba->sli4_hba.nvmels_wq, wqe);
21280 		if (ret) {
21281 			spin_unlock_irqrestore(&pring->ring_lock, iflags);
21282 			return ret;
21283 		}
21284 
21285 		lpfc_sli_ringtxcmpl_put(phba, pring, pwqe);
21286 		spin_unlock_irqrestore(&pring->ring_lock, iflags);
21287 
21288 		lpfc_sli4_poll_eq(qp->hba_eq, LPFC_POLL_FASTPATH);
21289 		return 0;
21290 	}
21291 
21292 	/* NVME_FCREQ and NVME_ABTS requests */
21293 	if (pwqe->iocb_flag & (LPFC_IO_NVME | LPFC_IO_FCP | LPFC_IO_CMF)) {
21294 		/* Get the IO distribution (hba_wqidx) for WQ assignment. */
21295 		wq = qp->io_wq;
21296 		pring = wq->pring;
21297 
21298 		bf_set(wqe_cqid, &wqe->generic.wqe_com, qp->io_cq_map);
21299 
21300 		lpfc_qp_spin_lock_irqsave(&pring->ring_lock, iflags,
21301 					  qp, wq_access);
21302 		ret = lpfc_sli4_wq_put(wq, wqe);
21303 		if (ret) {
21304 			spin_unlock_irqrestore(&pring->ring_lock, iflags);
21305 			return ret;
21306 		}
21307 		lpfc_sli_ringtxcmpl_put(phba, pring, pwqe);
21308 		spin_unlock_irqrestore(&pring->ring_lock, iflags);
21309 
21310 		lpfc_sli4_poll_eq(qp->hba_eq, LPFC_POLL_FASTPATH);
21311 		return 0;
21312 	}
21313 
21314 	/* NVMET requests */
21315 	if (pwqe->iocb_flag & LPFC_IO_NVMET) {
21316 		/* Get the IO distribution (hba_wqidx) for WQ assignment. */
21317 		wq = qp->io_wq;
21318 		pring = wq->pring;
21319 
21320 		ctxp = pwqe->context2;
21321 		sglq = ctxp->ctxbuf->sglq;
21322 		if (pwqe->sli4_xritag ==  NO_XRI) {
21323 			pwqe->sli4_lxritag = sglq->sli4_lxritag;
21324 			pwqe->sli4_xritag = sglq->sli4_xritag;
21325 		}
21326 		bf_set(wqe_xri_tag, &pwqe->wqe.xmit_bls_rsp.wqe_com,
21327 		       pwqe->sli4_xritag);
21328 		bf_set(wqe_cqid, &wqe->generic.wqe_com, qp->io_cq_map);
21329 
21330 		lpfc_qp_spin_lock_irqsave(&pring->ring_lock, iflags,
21331 					  qp, wq_access);
21332 		ret = lpfc_sli4_wq_put(wq, wqe);
21333 		if (ret) {
21334 			spin_unlock_irqrestore(&pring->ring_lock, iflags);
21335 			return ret;
21336 		}
21337 		lpfc_sli_ringtxcmpl_put(phba, pring, pwqe);
21338 		spin_unlock_irqrestore(&pring->ring_lock, iflags);
21339 
21340 		lpfc_sli4_poll_eq(qp->hba_eq, LPFC_POLL_FASTPATH);
21341 		return 0;
21342 	}
21343 	return WQE_ERROR;
21344 }
21345 
21346 /**
21347  * lpfc_sli4_issue_abort_iotag - SLI-4 WQE init & issue for the Abort
21348  * @phba: Pointer to HBA context object.
21349  * @cmdiocb: Pointer to driver command iocb object.
21350  * @cmpl: completion function.
21351  *
21352  * Fill the appropriate fields for the abort WQE and call
21353  * internal routine lpfc_sli4_issue_wqe to send the WQE
21354  * This function is called with hbalock held and no ring_lock held.
21355  *
21356  * RETURNS 0 - SUCCESS
21357  **/
21358 
21359 int
21360 lpfc_sli4_issue_abort_iotag(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
21361 			    void *cmpl)
21362 {
21363 	struct lpfc_vport *vport = cmdiocb->vport;
21364 	struct lpfc_iocbq *abtsiocb = NULL;
21365 	union lpfc_wqe128 *abtswqe;
21366 	struct lpfc_io_buf *lpfc_cmd;
21367 	int retval = IOCB_ERROR;
21368 	u16 xritag = cmdiocb->sli4_xritag;
21369 
21370 	/*
21371 	 * The scsi command can not be in txq and it is in flight because the
21372 	 * pCmd is still pointing at the SCSI command we have to abort. There
21373 	 * is no need to search the txcmplq. Just send an abort to the FW.
21374 	 */
21375 
21376 	abtsiocb = __lpfc_sli_get_iocbq(phba);
21377 	if (!abtsiocb)
21378 		return WQE_NORESOURCE;
21379 
21380 	/* Indicate the IO is being aborted by the driver. */
21381 	cmdiocb->iocb_flag |= LPFC_DRIVER_ABORTED;
21382 
21383 	abtswqe = &abtsiocb->wqe;
21384 	memset(abtswqe, 0, sizeof(*abtswqe));
21385 
21386 	if (!lpfc_is_link_up(phba))
21387 		bf_set(abort_cmd_ia, &abtswqe->abort_cmd, 1);
21388 	bf_set(abort_cmd_criteria, &abtswqe->abort_cmd, T_XRI_TAG);
21389 	abtswqe->abort_cmd.rsrvd5 = 0;
21390 	abtswqe->abort_cmd.wqe_com.abort_tag = xritag;
21391 	bf_set(wqe_reqtag, &abtswqe->abort_cmd.wqe_com, abtsiocb->iotag);
21392 	bf_set(wqe_cmnd, &abtswqe->abort_cmd.wqe_com, CMD_ABORT_XRI_CX);
21393 	bf_set(wqe_xri_tag, &abtswqe->generic.wqe_com, 0);
21394 	bf_set(wqe_qosd, &abtswqe->abort_cmd.wqe_com, 1);
21395 	bf_set(wqe_lenloc, &abtswqe->abort_cmd.wqe_com, LPFC_WQE_LENLOC_NONE);
21396 	bf_set(wqe_cmd_type, &abtswqe->abort_cmd.wqe_com, OTHER_COMMAND);
21397 
21398 	/* ABTS WQE must go to the same WQ as the WQE to be aborted */
21399 	abtsiocb->hba_wqidx = cmdiocb->hba_wqidx;
21400 	abtsiocb->iocb_flag |= LPFC_USE_FCPWQIDX;
21401 	if (cmdiocb->iocb_flag & LPFC_IO_FCP)
21402 		abtsiocb->iocb_flag |= LPFC_IO_FCP;
21403 	if (cmdiocb->iocb_flag & LPFC_IO_NVME)
21404 		abtsiocb->iocb_flag |= LPFC_IO_NVME;
21405 	if (cmdiocb->iocb_flag & LPFC_IO_FOF)
21406 		abtsiocb->iocb_flag |= LPFC_IO_FOF;
21407 	abtsiocb->vport = vport;
21408 	abtsiocb->wqe_cmpl = cmpl;
21409 
21410 	lpfc_cmd = container_of(cmdiocb, struct lpfc_io_buf, cur_iocbq);
21411 	retval = lpfc_sli4_issue_wqe(phba, lpfc_cmd->hdwq, abtsiocb);
21412 
21413 	lpfc_printf_vlog(vport, KERN_INFO, LOG_SLI | LOG_NVME_ABTS | LOG_FCP,
21414 			 "0359 Abort xri x%x, original iotag x%x, "
21415 			 "abort cmd iotag x%x retval x%x\n",
21416 			 xritag, cmdiocb->iotag, abtsiocb->iotag, retval);
21417 
21418 	if (retval) {
21419 		cmdiocb->iocb_flag &= ~LPFC_DRIVER_ABORTED;
21420 		__lpfc_sli_release_iocbq(phba, abtsiocb);
21421 	}
21422 
21423 	return retval;
21424 }
21425 
21426 #ifdef LPFC_MXP_STAT
21427 /**
21428  * lpfc_snapshot_mxp - Snapshot pbl, pvt and busy count
21429  * @phba: pointer to lpfc hba data structure.
21430  * @hwqid: belong to which HWQ.
21431  *
21432  * The purpose of this routine is to take a snapshot of pbl, pvt and busy count
21433  * 15 seconds after a test case is running.
21434  *
21435  * The user should call lpfc_debugfs_multixripools_write before running a test
21436  * case to clear stat_snapshot_taken. Then the user starts a test case. During
21437  * test case is running, stat_snapshot_taken is incremented by 1 every time when
21438  * this routine is called from heartbeat timer. When stat_snapshot_taken is
21439  * equal to LPFC_MXP_SNAPSHOT_TAKEN, a snapshot is taken.
21440  **/
21441 void lpfc_snapshot_mxp(struct lpfc_hba *phba, u32 hwqid)
21442 {
21443 	struct lpfc_sli4_hdw_queue *qp;
21444 	struct lpfc_multixri_pool *multixri_pool;
21445 	struct lpfc_pvt_pool *pvt_pool;
21446 	struct lpfc_pbl_pool *pbl_pool;
21447 	u32 txcmplq_cnt;
21448 
21449 	qp = &phba->sli4_hba.hdwq[hwqid];
21450 	multixri_pool = qp->p_multixri_pool;
21451 	if (!multixri_pool)
21452 		return;
21453 
21454 	if (multixri_pool->stat_snapshot_taken == LPFC_MXP_SNAPSHOT_TAKEN) {
21455 		pvt_pool = &qp->p_multixri_pool->pvt_pool;
21456 		pbl_pool = &qp->p_multixri_pool->pbl_pool;
21457 		txcmplq_cnt = qp->io_wq->pring->txcmplq_cnt;
21458 
21459 		multixri_pool->stat_pbl_count = pbl_pool->count;
21460 		multixri_pool->stat_pvt_count = pvt_pool->count;
21461 		multixri_pool->stat_busy_count = txcmplq_cnt;
21462 	}
21463 
21464 	multixri_pool->stat_snapshot_taken++;
21465 }
21466 #endif
21467 
21468 /**
21469  * lpfc_adjust_pvt_pool_count - Adjust private pool count
21470  * @phba: pointer to lpfc hba data structure.
21471  * @hwqid: belong to which HWQ.
21472  *
21473  * This routine moves some XRIs from private to public pool when private pool
21474  * is not busy.
21475  **/
21476 void lpfc_adjust_pvt_pool_count(struct lpfc_hba *phba, u32 hwqid)
21477 {
21478 	struct lpfc_multixri_pool *multixri_pool;
21479 	u32 io_req_count;
21480 	u32 prev_io_req_count;
21481 
21482 	multixri_pool = phba->sli4_hba.hdwq[hwqid].p_multixri_pool;
21483 	if (!multixri_pool)
21484 		return;
21485 	io_req_count = multixri_pool->io_req_count;
21486 	prev_io_req_count = multixri_pool->prev_io_req_count;
21487 
21488 	if (prev_io_req_count != io_req_count) {
21489 		/* Private pool is busy */
21490 		multixri_pool->prev_io_req_count = io_req_count;
21491 	} else {
21492 		/* Private pool is not busy.
21493 		 * Move XRIs from private to public pool.
21494 		 */
21495 		lpfc_move_xri_pvt_to_pbl(phba, hwqid);
21496 	}
21497 }
21498 
21499 /**
21500  * lpfc_adjust_high_watermark - Adjust high watermark
21501  * @phba: pointer to lpfc hba data structure.
21502  * @hwqid: belong to which HWQ.
21503  *
21504  * This routine sets high watermark as number of outstanding XRIs,
21505  * but make sure the new value is between xri_limit/2 and xri_limit.
21506  **/
21507 void lpfc_adjust_high_watermark(struct lpfc_hba *phba, u32 hwqid)
21508 {
21509 	u32 new_watermark;
21510 	u32 watermark_max;
21511 	u32 watermark_min;
21512 	u32 xri_limit;
21513 	u32 txcmplq_cnt;
21514 	u32 abts_io_bufs;
21515 	struct lpfc_multixri_pool *multixri_pool;
21516 	struct lpfc_sli4_hdw_queue *qp;
21517 
21518 	qp = &phba->sli4_hba.hdwq[hwqid];
21519 	multixri_pool = qp->p_multixri_pool;
21520 	if (!multixri_pool)
21521 		return;
21522 	xri_limit = multixri_pool->xri_limit;
21523 
21524 	watermark_max = xri_limit;
21525 	watermark_min = xri_limit / 2;
21526 
21527 	txcmplq_cnt = qp->io_wq->pring->txcmplq_cnt;
21528 	abts_io_bufs = qp->abts_scsi_io_bufs;
21529 	abts_io_bufs += qp->abts_nvme_io_bufs;
21530 
21531 	new_watermark = txcmplq_cnt + abts_io_bufs;
21532 	new_watermark = min(watermark_max, new_watermark);
21533 	new_watermark = max(watermark_min, new_watermark);
21534 	multixri_pool->pvt_pool.high_watermark = new_watermark;
21535 
21536 #ifdef LPFC_MXP_STAT
21537 	multixri_pool->stat_max_hwm = max(multixri_pool->stat_max_hwm,
21538 					  new_watermark);
21539 #endif
21540 }
21541 
21542 /**
21543  * lpfc_move_xri_pvt_to_pbl - Move some XRIs from private to public pool
21544  * @phba: pointer to lpfc hba data structure.
21545  * @hwqid: belong to which HWQ.
21546  *
21547  * This routine is called from hearbeat timer when pvt_pool is idle.
21548  * All free XRIs are moved from private to public pool on hwqid with 2 steps.
21549  * The first step moves (all - low_watermark) amount of XRIs.
21550  * The second step moves the rest of XRIs.
21551  **/
21552 void lpfc_move_xri_pvt_to_pbl(struct lpfc_hba *phba, u32 hwqid)
21553 {
21554 	struct lpfc_pbl_pool *pbl_pool;
21555 	struct lpfc_pvt_pool *pvt_pool;
21556 	struct lpfc_sli4_hdw_queue *qp;
21557 	struct lpfc_io_buf *lpfc_ncmd;
21558 	struct lpfc_io_buf *lpfc_ncmd_next;
21559 	unsigned long iflag;
21560 	struct list_head tmp_list;
21561 	u32 tmp_count;
21562 
21563 	qp = &phba->sli4_hba.hdwq[hwqid];
21564 	pbl_pool = &qp->p_multixri_pool->pbl_pool;
21565 	pvt_pool = &qp->p_multixri_pool->pvt_pool;
21566 	tmp_count = 0;
21567 
21568 	lpfc_qp_spin_lock_irqsave(&pbl_pool->lock, iflag, qp, mv_to_pub_pool);
21569 	lpfc_qp_spin_lock(&pvt_pool->lock, qp, mv_from_pvt_pool);
21570 
21571 	if (pvt_pool->count > pvt_pool->low_watermark) {
21572 		/* Step 1: move (all - low_watermark) from pvt_pool
21573 		 * to pbl_pool
21574 		 */
21575 
21576 		/* Move low watermark of bufs from pvt_pool to tmp_list */
21577 		INIT_LIST_HEAD(&tmp_list);
21578 		list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next,
21579 					 &pvt_pool->list, list) {
21580 			list_move_tail(&lpfc_ncmd->list, &tmp_list);
21581 			tmp_count++;
21582 			if (tmp_count >= pvt_pool->low_watermark)
21583 				break;
21584 		}
21585 
21586 		/* Move all bufs from pvt_pool to pbl_pool */
21587 		list_splice_init(&pvt_pool->list, &pbl_pool->list);
21588 
21589 		/* Move all bufs from tmp_list to pvt_pool */
21590 		list_splice(&tmp_list, &pvt_pool->list);
21591 
21592 		pbl_pool->count += (pvt_pool->count - tmp_count);
21593 		pvt_pool->count = tmp_count;
21594 	} else {
21595 		/* Step 2: move the rest from pvt_pool to pbl_pool */
21596 		list_splice_init(&pvt_pool->list, &pbl_pool->list);
21597 		pbl_pool->count += pvt_pool->count;
21598 		pvt_pool->count = 0;
21599 	}
21600 
21601 	spin_unlock(&pvt_pool->lock);
21602 	spin_unlock_irqrestore(&pbl_pool->lock, iflag);
21603 }
21604 
21605 /**
21606  * _lpfc_move_xri_pbl_to_pvt - Move some XRIs from public to private pool
21607  * @phba: pointer to lpfc hba data structure
21608  * @qp: pointer to HDW queue
21609  * @pbl_pool: specified public free XRI pool
21610  * @pvt_pool: specified private free XRI pool
21611  * @count: number of XRIs to move
21612  *
21613  * This routine tries to move some free common bufs from the specified pbl_pool
21614  * to the specified pvt_pool. It might move less than count XRIs if there's not
21615  * enough in public pool.
21616  *
21617  * Return:
21618  *   true - if XRIs are successfully moved from the specified pbl_pool to the
21619  *          specified pvt_pool
21620  *   false - if the specified pbl_pool is empty or locked by someone else
21621  **/
21622 static bool
21623 _lpfc_move_xri_pbl_to_pvt(struct lpfc_hba *phba, struct lpfc_sli4_hdw_queue *qp,
21624 			  struct lpfc_pbl_pool *pbl_pool,
21625 			  struct lpfc_pvt_pool *pvt_pool, u32 count)
21626 {
21627 	struct lpfc_io_buf *lpfc_ncmd;
21628 	struct lpfc_io_buf *lpfc_ncmd_next;
21629 	unsigned long iflag;
21630 	int ret;
21631 
21632 	ret = spin_trylock_irqsave(&pbl_pool->lock, iflag);
21633 	if (ret) {
21634 		if (pbl_pool->count) {
21635 			/* Move a batch of XRIs from public to private pool */
21636 			lpfc_qp_spin_lock(&pvt_pool->lock, qp, mv_to_pvt_pool);
21637 			list_for_each_entry_safe(lpfc_ncmd,
21638 						 lpfc_ncmd_next,
21639 						 &pbl_pool->list,
21640 						 list) {
21641 				list_move_tail(&lpfc_ncmd->list,
21642 					       &pvt_pool->list);
21643 				pvt_pool->count++;
21644 				pbl_pool->count--;
21645 				count--;
21646 				if (count == 0)
21647 					break;
21648 			}
21649 
21650 			spin_unlock(&pvt_pool->lock);
21651 			spin_unlock_irqrestore(&pbl_pool->lock, iflag);
21652 			return true;
21653 		}
21654 		spin_unlock_irqrestore(&pbl_pool->lock, iflag);
21655 	}
21656 
21657 	return false;
21658 }
21659 
21660 /**
21661  * lpfc_move_xri_pbl_to_pvt - Move some XRIs from public to private pool
21662  * @phba: pointer to lpfc hba data structure.
21663  * @hwqid: belong to which HWQ.
21664  * @count: number of XRIs to move
21665  *
21666  * This routine tries to find some free common bufs in one of public pools with
21667  * Round Robin method. The search always starts from local hwqid, then the next
21668  * HWQ which was found last time (rrb_next_hwqid). Once a public pool is found,
21669  * a batch of free common bufs are moved to private pool on hwqid.
21670  * It might move less than count XRIs if there's not enough in public pool.
21671  **/
21672 void lpfc_move_xri_pbl_to_pvt(struct lpfc_hba *phba, u32 hwqid, u32 count)
21673 {
21674 	struct lpfc_multixri_pool *multixri_pool;
21675 	struct lpfc_multixri_pool *next_multixri_pool;
21676 	struct lpfc_pvt_pool *pvt_pool;
21677 	struct lpfc_pbl_pool *pbl_pool;
21678 	struct lpfc_sli4_hdw_queue *qp;
21679 	u32 next_hwqid;
21680 	u32 hwq_count;
21681 	int ret;
21682 
21683 	qp = &phba->sli4_hba.hdwq[hwqid];
21684 	multixri_pool = qp->p_multixri_pool;
21685 	pvt_pool = &multixri_pool->pvt_pool;
21686 	pbl_pool = &multixri_pool->pbl_pool;
21687 
21688 	/* Check if local pbl_pool is available */
21689 	ret = _lpfc_move_xri_pbl_to_pvt(phba, qp, pbl_pool, pvt_pool, count);
21690 	if (ret) {
21691 #ifdef LPFC_MXP_STAT
21692 		multixri_pool->local_pbl_hit_count++;
21693 #endif
21694 		return;
21695 	}
21696 
21697 	hwq_count = phba->cfg_hdw_queue;
21698 
21699 	/* Get the next hwqid which was found last time */
21700 	next_hwqid = multixri_pool->rrb_next_hwqid;
21701 
21702 	do {
21703 		/* Go to next hwq */
21704 		next_hwqid = (next_hwqid + 1) % hwq_count;
21705 
21706 		next_multixri_pool =
21707 			phba->sli4_hba.hdwq[next_hwqid].p_multixri_pool;
21708 		pbl_pool = &next_multixri_pool->pbl_pool;
21709 
21710 		/* Check if the public free xri pool is available */
21711 		ret = _lpfc_move_xri_pbl_to_pvt(
21712 			phba, qp, pbl_pool, pvt_pool, count);
21713 
21714 		/* Exit while-loop if success or all hwqid are checked */
21715 	} while (!ret && next_hwqid != multixri_pool->rrb_next_hwqid);
21716 
21717 	/* Starting point for the next time */
21718 	multixri_pool->rrb_next_hwqid = next_hwqid;
21719 
21720 	if (!ret) {
21721 		/* stats: all public pools are empty*/
21722 		multixri_pool->pbl_empty_count++;
21723 	}
21724 
21725 #ifdef LPFC_MXP_STAT
21726 	if (ret) {
21727 		if (next_hwqid == hwqid)
21728 			multixri_pool->local_pbl_hit_count++;
21729 		else
21730 			multixri_pool->other_pbl_hit_count++;
21731 	}
21732 #endif
21733 }
21734 
21735 /**
21736  * lpfc_keep_pvt_pool_above_lowwm - Keep pvt_pool above low watermark
21737  * @phba: pointer to lpfc hba data structure.
21738  * @hwqid: belong to which HWQ.
21739  *
21740  * This routine get a batch of XRIs from pbl_pool if pvt_pool is less than
21741  * low watermark.
21742  **/
21743 void lpfc_keep_pvt_pool_above_lowwm(struct lpfc_hba *phba, u32 hwqid)
21744 {
21745 	struct lpfc_multixri_pool *multixri_pool;
21746 	struct lpfc_pvt_pool *pvt_pool;
21747 
21748 	multixri_pool = phba->sli4_hba.hdwq[hwqid].p_multixri_pool;
21749 	pvt_pool = &multixri_pool->pvt_pool;
21750 
21751 	if (pvt_pool->count < pvt_pool->low_watermark)
21752 		lpfc_move_xri_pbl_to_pvt(phba, hwqid, XRI_BATCH);
21753 }
21754 
21755 /**
21756  * lpfc_release_io_buf - Return one IO buf back to free pool
21757  * @phba: pointer to lpfc hba data structure.
21758  * @lpfc_ncmd: IO buf to be returned.
21759  * @qp: belong to which HWQ.
21760  *
21761  * This routine returns one IO buf back to free pool. If this is an urgent IO,
21762  * the IO buf is returned to expedite pool. If cfg_xri_rebalancing==1,
21763  * the IO buf is returned to pbl_pool or pvt_pool based on watermark and
21764  * xri_limit.  If cfg_xri_rebalancing==0, the IO buf is returned to
21765  * lpfc_io_buf_list_put.
21766  **/
21767 void lpfc_release_io_buf(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_ncmd,
21768 			 struct lpfc_sli4_hdw_queue *qp)
21769 {
21770 	unsigned long iflag;
21771 	struct lpfc_pbl_pool *pbl_pool;
21772 	struct lpfc_pvt_pool *pvt_pool;
21773 	struct lpfc_epd_pool *epd_pool;
21774 	u32 txcmplq_cnt;
21775 	u32 xri_owned;
21776 	u32 xri_limit;
21777 	u32 abts_io_bufs;
21778 
21779 	/* MUST zero fields if buffer is reused by another protocol */
21780 	lpfc_ncmd->nvmeCmd = NULL;
21781 	lpfc_ncmd->cur_iocbq.wqe_cmpl = NULL;
21782 	lpfc_ncmd->cur_iocbq.iocb_cmpl = NULL;
21783 
21784 	if (phba->cfg_xpsgl && !phba->nvmet_support &&
21785 	    !list_empty(&lpfc_ncmd->dma_sgl_xtra_list))
21786 		lpfc_put_sgl_per_hdwq(phba, lpfc_ncmd);
21787 
21788 	if (!list_empty(&lpfc_ncmd->dma_cmd_rsp_list))
21789 		lpfc_put_cmd_rsp_buf_per_hdwq(phba, lpfc_ncmd);
21790 
21791 	if (phba->cfg_xri_rebalancing) {
21792 		if (lpfc_ncmd->expedite) {
21793 			/* Return to expedite pool */
21794 			epd_pool = &phba->epd_pool;
21795 			spin_lock_irqsave(&epd_pool->lock, iflag);
21796 			list_add_tail(&lpfc_ncmd->list, &epd_pool->list);
21797 			epd_pool->count++;
21798 			spin_unlock_irqrestore(&epd_pool->lock, iflag);
21799 			return;
21800 		}
21801 
21802 		/* Avoid invalid access if an IO sneaks in and is being rejected
21803 		 * just _after_ xri pools are destroyed in lpfc_offline.
21804 		 * Nothing much can be done at this point.
21805 		 */
21806 		if (!qp->p_multixri_pool)
21807 			return;
21808 
21809 		pbl_pool = &qp->p_multixri_pool->pbl_pool;
21810 		pvt_pool = &qp->p_multixri_pool->pvt_pool;
21811 
21812 		txcmplq_cnt = qp->io_wq->pring->txcmplq_cnt;
21813 		abts_io_bufs = qp->abts_scsi_io_bufs;
21814 		abts_io_bufs += qp->abts_nvme_io_bufs;
21815 
21816 		xri_owned = pvt_pool->count + txcmplq_cnt + abts_io_bufs;
21817 		xri_limit = qp->p_multixri_pool->xri_limit;
21818 
21819 #ifdef LPFC_MXP_STAT
21820 		if (xri_owned <= xri_limit)
21821 			qp->p_multixri_pool->below_limit_count++;
21822 		else
21823 			qp->p_multixri_pool->above_limit_count++;
21824 #endif
21825 
21826 		/* XRI goes to either public or private free xri pool
21827 		 *     based on watermark and xri_limit
21828 		 */
21829 		if ((pvt_pool->count < pvt_pool->low_watermark) ||
21830 		    (xri_owned < xri_limit &&
21831 		     pvt_pool->count < pvt_pool->high_watermark)) {
21832 			lpfc_qp_spin_lock_irqsave(&pvt_pool->lock, iflag,
21833 						  qp, free_pvt_pool);
21834 			list_add_tail(&lpfc_ncmd->list,
21835 				      &pvt_pool->list);
21836 			pvt_pool->count++;
21837 			spin_unlock_irqrestore(&pvt_pool->lock, iflag);
21838 		} else {
21839 			lpfc_qp_spin_lock_irqsave(&pbl_pool->lock, iflag,
21840 						  qp, free_pub_pool);
21841 			list_add_tail(&lpfc_ncmd->list,
21842 				      &pbl_pool->list);
21843 			pbl_pool->count++;
21844 			spin_unlock_irqrestore(&pbl_pool->lock, iflag);
21845 		}
21846 	} else {
21847 		lpfc_qp_spin_lock_irqsave(&qp->io_buf_list_put_lock, iflag,
21848 					  qp, free_xri);
21849 		list_add_tail(&lpfc_ncmd->list,
21850 			      &qp->lpfc_io_buf_list_put);
21851 		qp->put_io_bufs++;
21852 		spin_unlock_irqrestore(&qp->io_buf_list_put_lock,
21853 				       iflag);
21854 	}
21855 }
21856 
21857 /**
21858  * lpfc_get_io_buf_from_private_pool - Get one free IO buf from private pool
21859  * @phba: pointer to lpfc hba data structure.
21860  * @qp: pointer to HDW queue
21861  * @pvt_pool: pointer to private pool data structure.
21862  * @ndlp: pointer to lpfc nodelist data structure.
21863  *
21864  * This routine tries to get one free IO buf from private pool.
21865  *
21866  * Return:
21867  *   pointer to one free IO buf - if private pool is not empty
21868  *   NULL - if private pool is empty
21869  **/
21870 static struct lpfc_io_buf *
21871 lpfc_get_io_buf_from_private_pool(struct lpfc_hba *phba,
21872 				  struct lpfc_sli4_hdw_queue *qp,
21873 				  struct lpfc_pvt_pool *pvt_pool,
21874 				  struct lpfc_nodelist *ndlp)
21875 {
21876 	struct lpfc_io_buf *lpfc_ncmd;
21877 	struct lpfc_io_buf *lpfc_ncmd_next;
21878 	unsigned long iflag;
21879 
21880 	lpfc_qp_spin_lock_irqsave(&pvt_pool->lock, iflag, qp, alloc_pvt_pool);
21881 	list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next,
21882 				 &pvt_pool->list, list) {
21883 		if (lpfc_test_rrq_active(
21884 			phba, ndlp, lpfc_ncmd->cur_iocbq.sli4_lxritag))
21885 			continue;
21886 		list_del(&lpfc_ncmd->list);
21887 		pvt_pool->count--;
21888 		spin_unlock_irqrestore(&pvt_pool->lock, iflag);
21889 		return lpfc_ncmd;
21890 	}
21891 	spin_unlock_irqrestore(&pvt_pool->lock, iflag);
21892 
21893 	return NULL;
21894 }
21895 
21896 /**
21897  * lpfc_get_io_buf_from_expedite_pool - Get one free IO buf from expedite pool
21898  * @phba: pointer to lpfc hba data structure.
21899  *
21900  * This routine tries to get one free IO buf from expedite pool.
21901  *
21902  * Return:
21903  *   pointer to one free IO buf - if expedite pool is not empty
21904  *   NULL - if expedite pool is empty
21905  **/
21906 static struct lpfc_io_buf *
21907 lpfc_get_io_buf_from_expedite_pool(struct lpfc_hba *phba)
21908 {
21909 	struct lpfc_io_buf *lpfc_ncmd;
21910 	struct lpfc_io_buf *lpfc_ncmd_next;
21911 	unsigned long iflag;
21912 	struct lpfc_epd_pool *epd_pool;
21913 
21914 	epd_pool = &phba->epd_pool;
21915 	lpfc_ncmd = NULL;
21916 
21917 	spin_lock_irqsave(&epd_pool->lock, iflag);
21918 	if (epd_pool->count > 0) {
21919 		list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next,
21920 					 &epd_pool->list, list) {
21921 			list_del(&lpfc_ncmd->list);
21922 			epd_pool->count--;
21923 			break;
21924 		}
21925 	}
21926 	spin_unlock_irqrestore(&epd_pool->lock, iflag);
21927 
21928 	return lpfc_ncmd;
21929 }
21930 
21931 /**
21932  * lpfc_get_io_buf_from_multixri_pools - Get one free IO bufs
21933  * @phba: pointer to lpfc hba data structure.
21934  * @ndlp: pointer to lpfc nodelist data structure.
21935  * @hwqid: belong to which HWQ
21936  * @expedite: 1 means this request is urgent.
21937  *
21938  * This routine will do the following actions and then return a pointer to
21939  * one free IO buf.
21940  *
21941  * 1. If private free xri count is empty, move some XRIs from public to
21942  *    private pool.
21943  * 2. Get one XRI from private free xri pool.
21944  * 3. If we fail to get one from pvt_pool and this is an expedite request,
21945  *    get one free xri from expedite pool.
21946  *
21947  * Note: ndlp is only used on SCSI side for RRQ testing.
21948  *       The caller should pass NULL for ndlp on NVME side.
21949  *
21950  * Return:
21951  *   pointer to one free IO buf - if private pool is not empty
21952  *   NULL - if private pool is empty
21953  **/
21954 static struct lpfc_io_buf *
21955 lpfc_get_io_buf_from_multixri_pools(struct lpfc_hba *phba,
21956 				    struct lpfc_nodelist *ndlp,
21957 				    int hwqid, int expedite)
21958 {
21959 	struct lpfc_sli4_hdw_queue *qp;
21960 	struct lpfc_multixri_pool *multixri_pool;
21961 	struct lpfc_pvt_pool *pvt_pool;
21962 	struct lpfc_io_buf *lpfc_ncmd;
21963 
21964 	qp = &phba->sli4_hba.hdwq[hwqid];
21965 	lpfc_ncmd = NULL;
21966 	multixri_pool = qp->p_multixri_pool;
21967 	pvt_pool = &multixri_pool->pvt_pool;
21968 	multixri_pool->io_req_count++;
21969 
21970 	/* If pvt_pool is empty, move some XRIs from public to private pool */
21971 	if (pvt_pool->count == 0)
21972 		lpfc_move_xri_pbl_to_pvt(phba, hwqid, XRI_BATCH);
21973 
21974 	/* Get one XRI from private free xri pool */
21975 	lpfc_ncmd = lpfc_get_io_buf_from_private_pool(phba, qp, pvt_pool, ndlp);
21976 
21977 	if (lpfc_ncmd) {
21978 		lpfc_ncmd->hdwq = qp;
21979 		lpfc_ncmd->hdwq_no = hwqid;
21980 	} else if (expedite) {
21981 		/* If we fail to get one from pvt_pool and this is an expedite
21982 		 * request, get one free xri from expedite pool.
21983 		 */
21984 		lpfc_ncmd = lpfc_get_io_buf_from_expedite_pool(phba);
21985 	}
21986 
21987 	return lpfc_ncmd;
21988 }
21989 
21990 static inline struct lpfc_io_buf *
21991 lpfc_io_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, int idx)
21992 {
21993 	struct lpfc_sli4_hdw_queue *qp;
21994 	struct lpfc_io_buf *lpfc_cmd, *lpfc_cmd_next;
21995 
21996 	qp = &phba->sli4_hba.hdwq[idx];
21997 	list_for_each_entry_safe(lpfc_cmd, lpfc_cmd_next,
21998 				 &qp->lpfc_io_buf_list_get, list) {
21999 		if (lpfc_test_rrq_active(phba, ndlp,
22000 					 lpfc_cmd->cur_iocbq.sli4_lxritag))
22001 			continue;
22002 
22003 		if (lpfc_cmd->flags & LPFC_SBUF_NOT_POSTED)
22004 			continue;
22005 
22006 		list_del_init(&lpfc_cmd->list);
22007 		qp->get_io_bufs--;
22008 		lpfc_cmd->hdwq = qp;
22009 		lpfc_cmd->hdwq_no = idx;
22010 		return lpfc_cmd;
22011 	}
22012 	return NULL;
22013 }
22014 
22015 /**
22016  * lpfc_get_io_buf - Get one IO buffer from free pool
22017  * @phba: The HBA for which this call is being executed.
22018  * @ndlp: pointer to lpfc nodelist data structure.
22019  * @hwqid: belong to which HWQ
22020  * @expedite: 1 means this request is urgent.
22021  *
22022  * This routine gets one IO buffer from free pool. If cfg_xri_rebalancing==1,
22023  * removes a IO buffer from multiXRI pools. If cfg_xri_rebalancing==0, removes
22024  * a IO buffer from head of @hdwq io_buf_list and returns to caller.
22025  *
22026  * Note: ndlp is only used on SCSI side for RRQ testing.
22027  *       The caller should pass NULL for ndlp on NVME side.
22028  *
22029  * Return codes:
22030  *   NULL - Error
22031  *   Pointer to lpfc_io_buf - Success
22032  **/
22033 struct lpfc_io_buf *lpfc_get_io_buf(struct lpfc_hba *phba,
22034 				    struct lpfc_nodelist *ndlp,
22035 				    u32 hwqid, int expedite)
22036 {
22037 	struct lpfc_sli4_hdw_queue *qp;
22038 	unsigned long iflag;
22039 	struct lpfc_io_buf *lpfc_cmd;
22040 
22041 	qp = &phba->sli4_hba.hdwq[hwqid];
22042 	lpfc_cmd = NULL;
22043 
22044 	if (phba->cfg_xri_rebalancing)
22045 		lpfc_cmd = lpfc_get_io_buf_from_multixri_pools(
22046 			phba, ndlp, hwqid, expedite);
22047 	else {
22048 		lpfc_qp_spin_lock_irqsave(&qp->io_buf_list_get_lock, iflag,
22049 					  qp, alloc_xri_get);
22050 		if (qp->get_io_bufs > LPFC_NVME_EXPEDITE_XRICNT || expedite)
22051 			lpfc_cmd = lpfc_io_buf(phba, ndlp, hwqid);
22052 		if (!lpfc_cmd) {
22053 			lpfc_qp_spin_lock(&qp->io_buf_list_put_lock,
22054 					  qp, alloc_xri_put);
22055 			list_splice(&qp->lpfc_io_buf_list_put,
22056 				    &qp->lpfc_io_buf_list_get);
22057 			qp->get_io_bufs += qp->put_io_bufs;
22058 			INIT_LIST_HEAD(&qp->lpfc_io_buf_list_put);
22059 			qp->put_io_bufs = 0;
22060 			spin_unlock(&qp->io_buf_list_put_lock);
22061 			if (qp->get_io_bufs > LPFC_NVME_EXPEDITE_XRICNT ||
22062 			    expedite)
22063 				lpfc_cmd = lpfc_io_buf(phba, ndlp, hwqid);
22064 		}
22065 		spin_unlock_irqrestore(&qp->io_buf_list_get_lock, iflag);
22066 	}
22067 
22068 	return lpfc_cmd;
22069 }
22070 
22071 /**
22072  * lpfc_read_object - Retrieve object data from HBA
22073  * @phba: The HBA for which this call is being executed.
22074  * @rdobject: Pathname of object data we want to read.
22075  * @datap: Pointer to where data will be copied to.
22076  * @datasz: size of data area
22077  *
22078  * This routine is limited to object sizes of LPFC_BPL_SIZE (1024) or less.
22079  * The data will be truncated if datasz is not large enough.
22080  * Version 1 is not supported with Embedded mbox cmd, so we must use version 0.
22081  * Returns the actual bytes read from the object.
22082  */
22083 int
22084 lpfc_read_object(struct lpfc_hba *phba, char *rdobject, uint32_t *datap,
22085 		 uint32_t datasz)
22086 {
22087 	struct lpfc_mbx_read_object *read_object;
22088 	LPFC_MBOXQ_t *mbox;
22089 	int rc, length, eof, j, byte_cnt = 0;
22090 	uint32_t shdr_status, shdr_add_status;
22091 	union lpfc_sli4_cfg_shdr *shdr;
22092 	struct lpfc_dmabuf *pcmd;
22093 	u32 rd_object_name[LPFC_MBX_OBJECT_NAME_LEN_DW] = {0};
22094 
22095 	/* sanity check on queue memory */
22096 	if (!datap)
22097 		return -ENODEV;
22098 
22099 	mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
22100 	if (!mbox)
22101 		return -ENOMEM;
22102 	length = (sizeof(struct lpfc_mbx_read_object) -
22103 		  sizeof(struct lpfc_sli4_cfg_mhdr));
22104 	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
22105 			 LPFC_MBOX_OPCODE_READ_OBJECT,
22106 			 length, LPFC_SLI4_MBX_EMBED);
22107 	read_object = &mbox->u.mqe.un.read_object;
22108 	shdr = (union lpfc_sli4_cfg_shdr *)&read_object->header.cfg_shdr;
22109 
22110 	bf_set(lpfc_mbox_hdr_version, &shdr->request, LPFC_Q_CREATE_VERSION_0);
22111 	bf_set(lpfc_mbx_rd_object_rlen, &read_object->u.request, datasz);
22112 	read_object->u.request.rd_object_offset = 0;
22113 	read_object->u.request.rd_object_cnt = 1;
22114 
22115 	memset((void *)read_object->u.request.rd_object_name, 0,
22116 	       LPFC_OBJ_NAME_SZ);
22117 	scnprintf((char *)rd_object_name, sizeof(rd_object_name), rdobject);
22118 	for (j = 0; j < strlen(rdobject); j++)
22119 		read_object->u.request.rd_object_name[j] =
22120 			cpu_to_le32(rd_object_name[j]);
22121 
22122 	pcmd = kmalloc(sizeof(*pcmd), GFP_KERNEL);
22123 	if (pcmd)
22124 		pcmd->virt = lpfc_mbuf_alloc(phba, MEM_PRI, &pcmd->phys);
22125 	if (!pcmd || !pcmd->virt) {
22126 		kfree(pcmd);
22127 		mempool_free(mbox, phba->mbox_mem_pool);
22128 		return -ENOMEM;
22129 	}
22130 	memset((void *)pcmd->virt, 0, LPFC_BPL_SIZE);
22131 	read_object->u.request.rd_object_hbuf[0].pa_lo =
22132 		putPaddrLow(pcmd->phys);
22133 	read_object->u.request.rd_object_hbuf[0].pa_hi =
22134 		putPaddrHigh(pcmd->phys);
22135 	read_object->u.request.rd_object_hbuf[0].length = LPFC_BPL_SIZE;
22136 
22137 	mbox->vport = phba->pport;
22138 	mbox->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
22139 	mbox->ctx_buf = NULL;
22140 	mbox->ctx_ndlp = NULL;
22141 
22142 	rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL);
22143 	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
22144 	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
22145 
22146 	if (shdr_status == STATUS_FAILED &&
22147 	    shdr_add_status == ADD_STATUS_INVALID_OBJECT_NAME) {
22148 		lpfc_printf_log(phba, KERN_ERR, LOG_INIT | LOG_CGN_MGMT,
22149 				"4674 No port cfg file in FW.\n");
22150 		byte_cnt = -ENOENT;
22151 	} else if (shdr_status || shdr_add_status || rc) {
22152 		lpfc_printf_log(phba, KERN_ERR, LOG_INIT | LOG_CGN_MGMT,
22153 				"2625 READ_OBJECT mailbox failed with "
22154 				"status x%x add_status x%x, mbx status x%x\n",
22155 				shdr_status, shdr_add_status, rc);
22156 		byte_cnt = -ENXIO;
22157 	} else {
22158 		/* Success */
22159 		length = read_object->u.response.rd_object_actual_rlen;
22160 		eof = bf_get(lpfc_mbx_rd_object_eof, &read_object->u.response);
22161 		lpfc_printf_log(phba, KERN_INFO, LOG_INIT | LOG_CGN_MGMT,
22162 				"2626 READ_OBJECT Success len %d:%d, EOF %d\n",
22163 				length, datasz, eof);
22164 
22165 		/* Detect the port config file exists but is empty */
22166 		if (!length && eof) {
22167 			byte_cnt = 0;
22168 			goto exit;
22169 		}
22170 
22171 		byte_cnt = length;
22172 		lpfc_sli_pcimem_bcopy(pcmd->virt, datap, byte_cnt);
22173 	}
22174 
22175  exit:
22176 	lpfc_mbuf_free(phba, pcmd->virt, pcmd->phys);
22177 	kfree(pcmd);
22178 	mempool_free(mbox, phba->mbox_mem_pool);
22179 	return byte_cnt;
22180 }
22181 
22182 /**
22183  * lpfc_get_sgl_per_hdwq - Get one SGL chunk from hdwq's pool
22184  * @phba: The HBA for which this call is being executed.
22185  * @lpfc_buf: IO buf structure to append the SGL chunk
22186  *
22187  * This routine gets one SGL chunk buffer from hdwq's SGL chunk pool,
22188  * and will allocate an SGL chunk if the pool is empty.
22189  *
22190  * Return codes:
22191  *   NULL - Error
22192  *   Pointer to sli4_hybrid_sgl - Success
22193  **/
22194 struct sli4_hybrid_sgl *
22195 lpfc_get_sgl_per_hdwq(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_buf)
22196 {
22197 	struct sli4_hybrid_sgl *list_entry = NULL;
22198 	struct sli4_hybrid_sgl *tmp = NULL;
22199 	struct sli4_hybrid_sgl *allocated_sgl = NULL;
22200 	struct lpfc_sli4_hdw_queue *hdwq = lpfc_buf->hdwq;
22201 	struct list_head *buf_list = &hdwq->sgl_list;
22202 	unsigned long iflags;
22203 
22204 	spin_lock_irqsave(&hdwq->hdwq_lock, iflags);
22205 
22206 	if (likely(!list_empty(buf_list))) {
22207 		/* break off 1 chunk from the sgl_list */
22208 		list_for_each_entry_safe(list_entry, tmp,
22209 					 buf_list, list_node) {
22210 			list_move_tail(&list_entry->list_node,
22211 				       &lpfc_buf->dma_sgl_xtra_list);
22212 			break;
22213 		}
22214 	} else {
22215 		/* allocate more */
22216 		spin_unlock_irqrestore(&hdwq->hdwq_lock, iflags);
22217 		tmp = kmalloc_node(sizeof(*tmp), GFP_ATOMIC,
22218 				   cpu_to_node(hdwq->io_wq->chann));
22219 		if (!tmp) {
22220 			lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
22221 					"8353 error kmalloc memory for HDWQ "
22222 					"%d %s\n",
22223 					lpfc_buf->hdwq_no, __func__);
22224 			return NULL;
22225 		}
22226 
22227 		tmp->dma_sgl = dma_pool_alloc(phba->lpfc_sg_dma_buf_pool,
22228 					      GFP_ATOMIC, &tmp->dma_phys_sgl);
22229 		if (!tmp->dma_sgl) {
22230 			lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
22231 					"8354 error pool_alloc memory for HDWQ "
22232 					"%d %s\n",
22233 					lpfc_buf->hdwq_no, __func__);
22234 			kfree(tmp);
22235 			return NULL;
22236 		}
22237 
22238 		spin_lock_irqsave(&hdwq->hdwq_lock, iflags);
22239 		list_add_tail(&tmp->list_node, &lpfc_buf->dma_sgl_xtra_list);
22240 	}
22241 
22242 	allocated_sgl = list_last_entry(&lpfc_buf->dma_sgl_xtra_list,
22243 					struct sli4_hybrid_sgl,
22244 					list_node);
22245 
22246 	spin_unlock_irqrestore(&hdwq->hdwq_lock, iflags);
22247 
22248 	return allocated_sgl;
22249 }
22250 
22251 /**
22252  * lpfc_put_sgl_per_hdwq - Put one SGL chunk into hdwq pool
22253  * @phba: The HBA for which this call is being executed.
22254  * @lpfc_buf: IO buf structure with the SGL chunk
22255  *
22256  * This routine puts one SGL chunk buffer into hdwq's SGL chunk pool.
22257  *
22258  * Return codes:
22259  *   0 - Success
22260  *   -EINVAL - Error
22261  **/
22262 int
22263 lpfc_put_sgl_per_hdwq(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_buf)
22264 {
22265 	int rc = 0;
22266 	struct sli4_hybrid_sgl *list_entry = NULL;
22267 	struct sli4_hybrid_sgl *tmp = NULL;
22268 	struct lpfc_sli4_hdw_queue *hdwq = lpfc_buf->hdwq;
22269 	struct list_head *buf_list = &hdwq->sgl_list;
22270 	unsigned long iflags;
22271 
22272 	spin_lock_irqsave(&hdwq->hdwq_lock, iflags);
22273 
22274 	if (likely(!list_empty(&lpfc_buf->dma_sgl_xtra_list))) {
22275 		list_for_each_entry_safe(list_entry, tmp,
22276 					 &lpfc_buf->dma_sgl_xtra_list,
22277 					 list_node) {
22278 			list_move_tail(&list_entry->list_node,
22279 				       buf_list);
22280 		}
22281 	} else {
22282 		rc = -EINVAL;
22283 	}
22284 
22285 	spin_unlock_irqrestore(&hdwq->hdwq_lock, iflags);
22286 	return rc;
22287 }
22288 
22289 /**
22290  * lpfc_free_sgl_per_hdwq - Free all SGL chunks of hdwq pool
22291  * @phba: phba object
22292  * @hdwq: hdwq to cleanup sgl buff resources on
22293  *
22294  * This routine frees all SGL chunks of hdwq SGL chunk pool.
22295  *
22296  * Return codes:
22297  *   None
22298  **/
22299 void
22300 lpfc_free_sgl_per_hdwq(struct lpfc_hba *phba,
22301 		       struct lpfc_sli4_hdw_queue *hdwq)
22302 {
22303 	struct list_head *buf_list = &hdwq->sgl_list;
22304 	struct sli4_hybrid_sgl *list_entry = NULL;
22305 	struct sli4_hybrid_sgl *tmp = NULL;
22306 	unsigned long iflags;
22307 
22308 	spin_lock_irqsave(&hdwq->hdwq_lock, iflags);
22309 
22310 	/* Free sgl pool */
22311 	list_for_each_entry_safe(list_entry, tmp,
22312 				 buf_list, list_node) {
22313 		dma_pool_free(phba->lpfc_sg_dma_buf_pool,
22314 			      list_entry->dma_sgl,
22315 			      list_entry->dma_phys_sgl);
22316 		list_del(&list_entry->list_node);
22317 		kfree(list_entry);
22318 	}
22319 
22320 	spin_unlock_irqrestore(&hdwq->hdwq_lock, iflags);
22321 }
22322 
22323 /**
22324  * lpfc_get_cmd_rsp_buf_per_hdwq - Get one CMD/RSP buffer from hdwq
22325  * @phba: The HBA for which this call is being executed.
22326  * @lpfc_buf: IO buf structure to attach the CMD/RSP buffer
22327  *
22328  * This routine gets one CMD/RSP buffer from hdwq's CMD/RSP pool,
22329  * and will allocate an CMD/RSP buffer if the pool is empty.
22330  *
22331  * Return codes:
22332  *   NULL - Error
22333  *   Pointer to fcp_cmd_rsp_buf - Success
22334  **/
22335 struct fcp_cmd_rsp_buf *
22336 lpfc_get_cmd_rsp_buf_per_hdwq(struct lpfc_hba *phba,
22337 			      struct lpfc_io_buf *lpfc_buf)
22338 {
22339 	struct fcp_cmd_rsp_buf *list_entry = NULL;
22340 	struct fcp_cmd_rsp_buf *tmp = NULL;
22341 	struct fcp_cmd_rsp_buf *allocated_buf = NULL;
22342 	struct lpfc_sli4_hdw_queue *hdwq = lpfc_buf->hdwq;
22343 	struct list_head *buf_list = &hdwq->cmd_rsp_buf_list;
22344 	unsigned long iflags;
22345 
22346 	spin_lock_irqsave(&hdwq->hdwq_lock, iflags);
22347 
22348 	if (likely(!list_empty(buf_list))) {
22349 		/* break off 1 chunk from the list */
22350 		list_for_each_entry_safe(list_entry, tmp,
22351 					 buf_list,
22352 					 list_node) {
22353 			list_move_tail(&list_entry->list_node,
22354 				       &lpfc_buf->dma_cmd_rsp_list);
22355 			break;
22356 		}
22357 	} else {
22358 		/* allocate more */
22359 		spin_unlock_irqrestore(&hdwq->hdwq_lock, iflags);
22360 		tmp = kmalloc_node(sizeof(*tmp), GFP_ATOMIC,
22361 				   cpu_to_node(hdwq->io_wq->chann));
22362 		if (!tmp) {
22363 			lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
22364 					"8355 error kmalloc memory for HDWQ "
22365 					"%d %s\n",
22366 					lpfc_buf->hdwq_no, __func__);
22367 			return NULL;
22368 		}
22369 
22370 		tmp->fcp_cmnd = dma_pool_alloc(phba->lpfc_cmd_rsp_buf_pool,
22371 						GFP_ATOMIC,
22372 						&tmp->fcp_cmd_rsp_dma_handle);
22373 
22374 		if (!tmp->fcp_cmnd) {
22375 			lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
22376 					"8356 error pool_alloc memory for HDWQ "
22377 					"%d %s\n",
22378 					lpfc_buf->hdwq_no, __func__);
22379 			kfree(tmp);
22380 			return NULL;
22381 		}
22382 
22383 		tmp->fcp_rsp = (struct fcp_rsp *)((uint8_t *)tmp->fcp_cmnd +
22384 				sizeof(struct fcp_cmnd));
22385 
22386 		spin_lock_irqsave(&hdwq->hdwq_lock, iflags);
22387 		list_add_tail(&tmp->list_node, &lpfc_buf->dma_cmd_rsp_list);
22388 	}
22389 
22390 	allocated_buf = list_last_entry(&lpfc_buf->dma_cmd_rsp_list,
22391 					struct fcp_cmd_rsp_buf,
22392 					list_node);
22393 
22394 	spin_unlock_irqrestore(&hdwq->hdwq_lock, iflags);
22395 
22396 	return allocated_buf;
22397 }
22398 
22399 /**
22400  * lpfc_put_cmd_rsp_buf_per_hdwq - Put one CMD/RSP buffer into hdwq pool
22401  * @phba: The HBA for which this call is being executed.
22402  * @lpfc_buf: IO buf structure with the CMD/RSP buf
22403  *
22404  * This routine puts one CMD/RSP buffer into executing CPU's CMD/RSP pool.
22405  *
22406  * Return codes:
22407  *   0 - Success
22408  *   -EINVAL - Error
22409  **/
22410 int
22411 lpfc_put_cmd_rsp_buf_per_hdwq(struct lpfc_hba *phba,
22412 			      struct lpfc_io_buf *lpfc_buf)
22413 {
22414 	int rc = 0;
22415 	struct fcp_cmd_rsp_buf *list_entry = NULL;
22416 	struct fcp_cmd_rsp_buf *tmp = NULL;
22417 	struct lpfc_sli4_hdw_queue *hdwq = lpfc_buf->hdwq;
22418 	struct list_head *buf_list = &hdwq->cmd_rsp_buf_list;
22419 	unsigned long iflags;
22420 
22421 	spin_lock_irqsave(&hdwq->hdwq_lock, iflags);
22422 
22423 	if (likely(!list_empty(&lpfc_buf->dma_cmd_rsp_list))) {
22424 		list_for_each_entry_safe(list_entry, tmp,
22425 					 &lpfc_buf->dma_cmd_rsp_list,
22426 					 list_node) {
22427 			list_move_tail(&list_entry->list_node,
22428 				       buf_list);
22429 		}
22430 	} else {
22431 		rc = -EINVAL;
22432 	}
22433 
22434 	spin_unlock_irqrestore(&hdwq->hdwq_lock, iflags);
22435 	return rc;
22436 }
22437 
22438 /**
22439  * lpfc_free_cmd_rsp_buf_per_hdwq - Free all CMD/RSP chunks of hdwq pool
22440  * @phba: phba object
22441  * @hdwq: hdwq to cleanup cmd rsp buff resources on
22442  *
22443  * This routine frees all CMD/RSP buffers of hdwq's CMD/RSP buf pool.
22444  *
22445  * Return codes:
22446  *   None
22447  **/
22448 void
22449 lpfc_free_cmd_rsp_buf_per_hdwq(struct lpfc_hba *phba,
22450 			       struct lpfc_sli4_hdw_queue *hdwq)
22451 {
22452 	struct list_head *buf_list = &hdwq->cmd_rsp_buf_list;
22453 	struct fcp_cmd_rsp_buf *list_entry = NULL;
22454 	struct fcp_cmd_rsp_buf *tmp = NULL;
22455 	unsigned long iflags;
22456 
22457 	spin_lock_irqsave(&hdwq->hdwq_lock, iflags);
22458 
22459 	/* Free cmd_rsp buf pool */
22460 	list_for_each_entry_safe(list_entry, tmp,
22461 				 buf_list,
22462 				 list_node) {
22463 		dma_pool_free(phba->lpfc_cmd_rsp_buf_pool,
22464 			      list_entry->fcp_cmnd,
22465 			      list_entry->fcp_cmd_rsp_dma_handle);
22466 		list_del(&list_entry->list_node);
22467 		kfree(list_entry);
22468 	}
22469 
22470 	spin_unlock_irqrestore(&hdwq->hdwq_lock, iflags);
22471 }
22472