xref: /openbmc/linux/drivers/scsi/csiostor/csio_hw.c (revision 12eb4683)
1 /*
2  * This file is part of the Chelsio FCoE driver for Linux.
3  *
4  * Copyright (c) 2008-2012 Chelsio Communications, Inc. All rights reserved.
5  *
6  * This software is available to you under a choice of one of two
7  * licenses.  You may choose to be licensed under the terms of the GNU
8  * General Public License (GPL) Version 2, available from the file
9  * COPYING in the main directory of this source tree, or the
10  * OpenIB.org BSD license below:
11  *
12  *     Redistribution and use in source and binary forms, with or
13  *     without modification, are permitted provided that the following
14  *     conditions are met:
15  *
16  *      - Redistributions of source code must retain the above
17  *        copyright notice, this list of conditions and the following
18  *        disclaimer.
19  *
20  *      - Redistributions in binary form must reproduce the above
21  *        copyright notice, this list of conditions and the following
22  *        disclaimer in the documentation and/or other materials
23  *        provided with the distribution.
24  *
25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32  * SOFTWARE.
33  */
34 
35 #include <linux/pci.h>
36 #include <linux/pci_regs.h>
37 #include <linux/firmware.h>
38 #include <linux/stddef.h>
39 #include <linux/delay.h>
40 #include <linux/string.h>
41 #include <linux/compiler.h>
42 #include <linux/jiffies.h>
43 #include <linux/kernel.h>
44 #include <linux/log2.h>
45 
46 #include "csio_hw.h"
47 #include "csio_lnode.h"
48 #include "csio_rnode.h"
49 
50 int csio_force_master;
51 int csio_dbg_level = 0xFEFF;
52 unsigned int csio_port_mask = 0xf;
53 
54 /* Default FW event queue entries. */
55 static uint32_t csio_evtq_sz = CSIO_EVTQ_SIZE;
56 
57 /* Default MSI param level */
58 int csio_msi = 2;
59 
60 /* FCoE function instances */
61 static int dev_num;
62 
63 /* FCoE Adapter types & its description */
64 static const struct csio_adap_desc csio_t4_fcoe_adapters[] = {
65 	{"T440-Dbg 10G", "Chelsio T440-Dbg 10G [FCoE]"},
66 	{"T420-CR 10G", "Chelsio T420-CR 10G [FCoE]"},
67 	{"T422-CR 10G/1G", "Chelsio T422-CR 10G/1G [FCoE]"},
68 	{"T440-CR 10G", "Chelsio T440-CR 10G [FCoE]"},
69 	{"T420-BCH 10G", "Chelsio T420-BCH 10G [FCoE]"},
70 	{"T440-BCH 10G", "Chelsio T440-BCH 10G [FCoE]"},
71 	{"T440-CH 10G", "Chelsio T440-CH 10G [FCoE]"},
72 	{"T420-SO 10G", "Chelsio T420-SO 10G [FCoE]"},
73 	{"T420-CX4 10G", "Chelsio T420-CX4 10G [FCoE]"},
74 	{"T420-BT 10G", "Chelsio T420-BT 10G [FCoE]"},
75 	{"T404-BT 1G", "Chelsio T404-BT 1G [FCoE]"},
76 	{"B420-SR 10G", "Chelsio B420-SR 10G [FCoE]"},
77 	{"B404-BT 1G", "Chelsio B404-BT 1G [FCoE]"},
78 	{"T480-CR 10G", "Chelsio T480-CR 10G [FCoE]"},
79 	{"T440-LP-CR 10G", "Chelsio T440-LP-CR 10G [FCoE]"},
80 	{"AMSTERDAM 10G", "Chelsio AMSTERDAM 10G [FCoE]"},
81 	{"HUAWEI T480 10G", "Chelsio HUAWEI T480 10G [FCoE]"},
82 	{"HUAWEI T440 10G", "Chelsio HUAWEI T440 10G [FCoE]"},
83 	{"HUAWEI STG 10G", "Chelsio HUAWEI STG 10G [FCoE]"},
84 	{"ACROMAG XAUI 10G", "Chelsio ACROMAG XAUI 10G [FCoE]"},
85 	{"ACROMAG SFP+ 10G", "Chelsio ACROMAG SFP+ 10G [FCoE]"},
86 	{"QUANTA SFP+ 10G", "Chelsio QUANTA SFP+ 10G [FCoE]"},
87 	{"HUAWEI 10Gbase-T", "Chelsio HUAWEI 10Gbase-T [FCoE]"},
88 	{"HUAWEI T4TOE 10G", "Chelsio HUAWEI T4TOE 10G [FCoE]"}
89 };
90 
91 static const struct csio_adap_desc csio_t5_fcoe_adapters[] = {
92 	{"T580-Dbg 10G", "Chelsio T580-Dbg 10G [FCoE]"},
93 	{"T520-CR 10G", "Chelsio T520-CR 10G [FCoE]"},
94 	{"T522-CR 10G/1G", "Chelsio T452-CR 10G/1G [FCoE]"},
95 	{"T540-CR 10G", "Chelsio T540-CR 10G [FCoE]"},
96 	{"T520-BCH 10G", "Chelsio T520-BCH 10G [FCoE]"},
97 	{"T540-BCH 10G", "Chelsio T540-BCH 10G [FCoE]"},
98 	{"T540-CH 10G", "Chelsio T540-CH 10G [FCoE]"},
99 	{"T520-SO 10G", "Chelsio T520-SO 10G [FCoE]"},
100 	{"T520-CX4 10G", "Chelsio T520-CX4 10G [FCoE]"},
101 	{"T520-BT 10G", "Chelsio T520-BT 10G [FCoE]"},
102 	{"T504-BT 1G", "Chelsio T504-BT 1G [FCoE]"},
103 	{"B520-SR 10G", "Chelsio B520-SR 10G [FCoE]"},
104 	{"B504-BT 1G", "Chelsio B504-BT 1G [FCoE]"},
105 	{"T580-CR 10G", "Chelsio T580-CR 10G [FCoE]"},
106 	{"T540-LP-CR 10G", "Chelsio T540-LP-CR 10G [FCoE]"},
107 	{"AMSTERDAM 10G", "Chelsio AMSTERDAM 10G [FCoE]"},
108 	{"T580-LP-CR 40G", "Chelsio T580-LP-CR 40G [FCoE]"},
109 	{"T520-LL-CR 10G", "Chelsio T520-LL-CR 10G [FCoE]"},
110 	{"T560-CR 40G", "Chelsio T560-CR 40G [FCoE]"},
111 	{"T580-CR 40G", "Chelsio T580-CR 40G [FCoE]"}
112 };
113 
114 static void csio_mgmtm_cleanup(struct csio_mgmtm *);
115 static void csio_hw_mbm_cleanup(struct csio_hw *);
116 
117 /* State machine forward declarations */
118 static void csio_hws_uninit(struct csio_hw *, enum csio_hw_ev);
119 static void csio_hws_configuring(struct csio_hw *, enum csio_hw_ev);
120 static void csio_hws_initializing(struct csio_hw *, enum csio_hw_ev);
121 static void csio_hws_ready(struct csio_hw *, enum csio_hw_ev);
122 static void csio_hws_quiescing(struct csio_hw *, enum csio_hw_ev);
123 static void csio_hws_quiesced(struct csio_hw *, enum csio_hw_ev);
124 static void csio_hws_resetting(struct csio_hw *, enum csio_hw_ev);
125 static void csio_hws_removing(struct csio_hw *, enum csio_hw_ev);
126 static void csio_hws_pcierr(struct csio_hw *, enum csio_hw_ev);
127 
128 static void csio_hw_initialize(struct csio_hw *hw);
129 static void csio_evtq_stop(struct csio_hw *hw);
130 static void csio_evtq_start(struct csio_hw *hw);
131 
132 int csio_is_hw_ready(struct csio_hw *hw)
133 {
134 	return csio_match_state(hw, csio_hws_ready);
135 }
136 
137 int csio_is_hw_removing(struct csio_hw *hw)
138 {
139 	return csio_match_state(hw, csio_hws_removing);
140 }
141 
142 
143 /*
144  *	csio_hw_wait_op_done_val - wait until an operation is completed
145  *	@hw: the HW module
146  *	@reg: the register to check for completion
147  *	@mask: a single-bit field within @reg that indicates completion
148  *	@polarity: the value of the field when the operation is completed
149  *	@attempts: number of check iterations
150  *	@delay: delay in usecs between iterations
151  *	@valp: where to store the value of the register at completion time
152  *
153  *	Wait until an operation is completed by checking a bit in a register
154  *	up to @attempts times.  If @valp is not NULL the value of the register
155  *	at the time it indicated completion is stored there.  Returns 0 if the
156  *	operation completes and	-EAGAIN	otherwise.
157  */
158 int
159 csio_hw_wait_op_done_val(struct csio_hw *hw, int reg, uint32_t mask,
160 			 int polarity, int attempts, int delay, uint32_t *valp)
161 {
162 	uint32_t val;
163 	while (1) {
164 		val = csio_rd_reg32(hw, reg);
165 
166 		if (!!(val & mask) == polarity) {
167 			if (valp)
168 				*valp = val;
169 			return 0;
170 		}
171 
172 		if (--attempts == 0)
173 			return -EAGAIN;
174 		if (delay)
175 			udelay(delay);
176 	}
177 }
178 
179 /*
180  *	csio_hw_tp_wr_bits_indirect - set/clear bits in an indirect TP register
181  *	@hw: the adapter
182  *	@addr: the indirect TP register address
183  *	@mask: specifies the field within the register to modify
184  *	@val: new value for the field
185  *
186  *	Sets a field of an indirect TP register to the given value.
187  */
188 void
189 csio_hw_tp_wr_bits_indirect(struct csio_hw *hw, unsigned int addr,
190 			unsigned int mask, unsigned int val)
191 {
192 	csio_wr_reg32(hw, addr, TP_PIO_ADDR);
193 	val |= csio_rd_reg32(hw, TP_PIO_DATA) & ~mask;
194 	csio_wr_reg32(hw, val, TP_PIO_DATA);
195 }
196 
197 void
198 csio_set_reg_field(struct csio_hw *hw, uint32_t reg, uint32_t mask,
199 		   uint32_t value)
200 {
201 	uint32_t val = csio_rd_reg32(hw, reg) & ~mask;
202 
203 	csio_wr_reg32(hw, val | value, reg);
204 	/* Flush */
205 	csio_rd_reg32(hw, reg);
206 
207 }
208 
209 static int
210 csio_memory_write(struct csio_hw *hw, int mtype, u32 addr, u32 len, u32 *buf)
211 {
212 	return hw->chip_ops->chip_memory_rw(hw, MEMWIN_CSIOSTOR, mtype,
213 					    addr, len, buf, 0);
214 }
215 
216 /*
217  * EEPROM reads take a few tens of us while writes can take a bit over 5 ms.
218  */
219 #define EEPROM_MAX_RD_POLL	40
220 #define EEPROM_MAX_WR_POLL	6
221 #define EEPROM_STAT_ADDR	0x7bfc
222 #define VPD_BASE		0x400
223 #define VPD_BASE_OLD		0
224 #define VPD_LEN			1024
225 #define VPD_INFO_FLD_HDR_SIZE	3
226 
227 /*
228  *	csio_hw_seeprom_read - read a serial EEPROM location
229  *	@hw: hw to read
230  *	@addr: EEPROM virtual address
231  *	@data: where to store the read data
232  *
233  *	Read a 32-bit word from a location in serial EEPROM using the card's PCI
234  *	VPD capability.  Note that this function must be called with a virtual
235  *	address.
236  */
237 static int
238 csio_hw_seeprom_read(struct csio_hw *hw, uint32_t addr, uint32_t *data)
239 {
240 	uint16_t val = 0;
241 	int attempts = EEPROM_MAX_RD_POLL;
242 	uint32_t base = hw->params.pci.vpd_cap_addr;
243 
244 	if (addr >= EEPROMVSIZE || (addr & 3))
245 		return -EINVAL;
246 
247 	pci_write_config_word(hw->pdev, base + PCI_VPD_ADDR, (uint16_t)addr);
248 
249 	do {
250 		udelay(10);
251 		pci_read_config_word(hw->pdev, base + PCI_VPD_ADDR, &val);
252 	} while (!(val & PCI_VPD_ADDR_F) && --attempts);
253 
254 	if (!(val & PCI_VPD_ADDR_F)) {
255 		csio_err(hw, "reading EEPROM address 0x%x failed\n", addr);
256 		return -EINVAL;
257 	}
258 
259 	pci_read_config_dword(hw->pdev, base + PCI_VPD_DATA, data);
260 	*data = le32_to_cpu(*data);
261 
262 	return 0;
263 }
264 
265 /*
266  * Partial EEPROM Vital Product Data structure.  Includes only the ID and
267  * VPD-R sections.
268  */
269 struct t4_vpd_hdr {
270 	u8  id_tag;
271 	u8  id_len[2];
272 	u8  id_data[ID_LEN];
273 	u8  vpdr_tag;
274 	u8  vpdr_len[2];
275 };
276 
277 /*
278  *	csio_hw_get_vpd_keyword_val - Locates an information field keyword in
279  *				      the VPD
280  *	@v: Pointer to buffered vpd data structure
281  *	@kw: The keyword to search for
282  *
283  *	Returns the value of the information field keyword or
284  *	-EINVAL otherwise.
285  */
286 static int
287 csio_hw_get_vpd_keyword_val(const struct t4_vpd_hdr *v, const char *kw)
288 {
289 	int32_t i;
290 	int32_t offset , len;
291 	const uint8_t *buf = &v->id_tag;
292 	const uint8_t *vpdr_len = &v->vpdr_tag;
293 	offset = sizeof(struct t4_vpd_hdr);
294 	len =  (uint16_t)vpdr_len[1] + ((uint16_t)vpdr_len[2] << 8);
295 
296 	if (len + sizeof(struct t4_vpd_hdr) > VPD_LEN)
297 		return -EINVAL;
298 
299 	for (i = offset; (i + VPD_INFO_FLD_HDR_SIZE) <= (offset + len);) {
300 		if (memcmp(buf + i , kw, 2) == 0) {
301 			i += VPD_INFO_FLD_HDR_SIZE;
302 			return i;
303 		}
304 
305 		i += VPD_INFO_FLD_HDR_SIZE + buf[i+2];
306 	}
307 
308 	return -EINVAL;
309 }
310 
311 static int
312 csio_pci_capability(struct pci_dev *pdev, int cap, int *pos)
313 {
314 	*pos = pci_find_capability(pdev, cap);
315 	if (*pos)
316 		return 0;
317 
318 	return -1;
319 }
320 
321 /*
322  *	csio_hw_get_vpd_params - read VPD parameters from VPD EEPROM
323  *	@hw: HW module
324  *	@p: where to store the parameters
325  *
326  *	Reads card parameters stored in VPD EEPROM.
327  */
328 static int
329 csio_hw_get_vpd_params(struct csio_hw *hw, struct csio_vpd *p)
330 {
331 	int i, ret, ec, sn, addr;
332 	uint8_t *vpd, csum;
333 	const struct t4_vpd_hdr *v;
334 	/* To get around compilation warning from strstrip */
335 	char *s;
336 
337 	if (csio_is_valid_vpd(hw))
338 		return 0;
339 
340 	ret = csio_pci_capability(hw->pdev, PCI_CAP_ID_VPD,
341 				  &hw->params.pci.vpd_cap_addr);
342 	if (ret)
343 		return -EINVAL;
344 
345 	vpd = kzalloc(VPD_LEN, GFP_ATOMIC);
346 	if (vpd == NULL)
347 		return -ENOMEM;
348 
349 	/*
350 	 * Card information normally starts at VPD_BASE but early cards had
351 	 * it at 0.
352 	 */
353 	ret = csio_hw_seeprom_read(hw, VPD_BASE, (uint32_t *)(vpd));
354 	addr = *vpd == 0x82 ? VPD_BASE : VPD_BASE_OLD;
355 
356 	for (i = 0; i < VPD_LEN; i += 4) {
357 		ret = csio_hw_seeprom_read(hw, addr + i, (uint32_t *)(vpd + i));
358 		if (ret) {
359 			kfree(vpd);
360 			return ret;
361 		}
362 	}
363 
364 	/* Reset the VPD flag! */
365 	hw->flags &= (~CSIO_HWF_VPD_VALID);
366 
367 	v = (const struct t4_vpd_hdr *)vpd;
368 
369 #define FIND_VPD_KW(var, name) do { \
370 	var = csio_hw_get_vpd_keyword_val(v, name); \
371 	if (var < 0) { \
372 		csio_err(hw, "missing VPD keyword " name "\n"); \
373 		kfree(vpd); \
374 		return -EINVAL; \
375 	} \
376 } while (0)
377 
378 	FIND_VPD_KW(i, "RV");
379 	for (csum = 0; i >= 0; i--)
380 		csum += vpd[i];
381 
382 	if (csum) {
383 		csio_err(hw, "corrupted VPD EEPROM, actual csum %u\n", csum);
384 		kfree(vpd);
385 		return -EINVAL;
386 	}
387 	FIND_VPD_KW(ec, "EC");
388 	FIND_VPD_KW(sn, "SN");
389 #undef FIND_VPD_KW
390 
391 	memcpy(p->id, v->id_data, ID_LEN);
392 	s = strstrip(p->id);
393 	memcpy(p->ec, vpd + ec, EC_LEN);
394 	s = strstrip(p->ec);
395 	i = vpd[sn - VPD_INFO_FLD_HDR_SIZE + 2];
396 	memcpy(p->sn, vpd + sn, min(i, SERNUM_LEN));
397 	s = strstrip(p->sn);
398 
399 	csio_valid_vpd_copied(hw);
400 
401 	kfree(vpd);
402 	return 0;
403 }
404 
405 /*
406  *	csio_hw_sf1_read - read data from the serial flash
407  *	@hw: the HW module
408  *	@byte_cnt: number of bytes to read
409  *	@cont: whether another operation will be chained
410  *      @lock: whether to lock SF for PL access only
411  *	@valp: where to store the read data
412  *
413  *	Reads up to 4 bytes of data from the serial flash.  The location of
414  *	the read needs to be specified prior to calling this by issuing the
415  *	appropriate commands to the serial flash.
416  */
417 static int
418 csio_hw_sf1_read(struct csio_hw *hw, uint32_t byte_cnt, int32_t cont,
419 		 int32_t lock, uint32_t *valp)
420 {
421 	int ret;
422 
423 	if (!byte_cnt || byte_cnt > 4)
424 		return -EINVAL;
425 	if (csio_rd_reg32(hw, SF_OP) & SF_BUSY)
426 		return -EBUSY;
427 
428 	cont = cont ? SF_CONT : 0;
429 	lock = lock ? SF_LOCK : 0;
430 
431 	csio_wr_reg32(hw, lock | cont | BYTECNT(byte_cnt - 1), SF_OP);
432 	ret = csio_hw_wait_op_done_val(hw, SF_OP, SF_BUSY, 0, SF_ATTEMPTS,
433 					 10, NULL);
434 	if (!ret)
435 		*valp = csio_rd_reg32(hw, SF_DATA);
436 	return ret;
437 }
438 
439 /*
440  *	csio_hw_sf1_write - write data to the serial flash
441  *	@hw: the HW module
442  *	@byte_cnt: number of bytes to write
443  *	@cont: whether another operation will be chained
444  *      @lock: whether to lock SF for PL access only
445  *	@val: value to write
446  *
447  *	Writes up to 4 bytes of data to the serial flash.  The location of
448  *	the write needs to be specified prior to calling this by issuing the
449  *	appropriate commands to the serial flash.
450  */
451 static int
452 csio_hw_sf1_write(struct csio_hw *hw, uint32_t byte_cnt, uint32_t cont,
453 		  int32_t lock, uint32_t val)
454 {
455 	if (!byte_cnt || byte_cnt > 4)
456 		return -EINVAL;
457 	if (csio_rd_reg32(hw, SF_OP) & SF_BUSY)
458 		return -EBUSY;
459 
460 	cont = cont ? SF_CONT : 0;
461 	lock = lock ? SF_LOCK : 0;
462 
463 	csio_wr_reg32(hw, val, SF_DATA);
464 	csio_wr_reg32(hw, cont | BYTECNT(byte_cnt - 1) | OP_WR | lock, SF_OP);
465 
466 	return csio_hw_wait_op_done_val(hw, SF_OP, SF_BUSY, 0, SF_ATTEMPTS,
467 					10, NULL);
468 }
469 
470 /*
471  *	csio_hw_flash_wait_op - wait for a flash operation to complete
472  *	@hw: the HW module
473  *	@attempts: max number of polls of the status register
474  *	@delay: delay between polls in ms
475  *
476  *	Wait for a flash operation to complete by polling the status register.
477  */
478 static int
479 csio_hw_flash_wait_op(struct csio_hw *hw, int32_t attempts, int32_t delay)
480 {
481 	int ret;
482 	uint32_t status;
483 
484 	while (1) {
485 		ret = csio_hw_sf1_write(hw, 1, 1, 1, SF_RD_STATUS);
486 		if (ret != 0)
487 			return ret;
488 
489 		ret = csio_hw_sf1_read(hw, 1, 0, 1, &status);
490 		if (ret != 0)
491 			return ret;
492 
493 		if (!(status & 1))
494 			return 0;
495 		if (--attempts == 0)
496 			return -EAGAIN;
497 		if (delay)
498 			msleep(delay);
499 	}
500 }
501 
502 /*
503  *	csio_hw_read_flash - read words from serial flash
504  *	@hw: the HW module
505  *	@addr: the start address for the read
506  *	@nwords: how many 32-bit words to read
507  *	@data: where to store the read data
508  *	@byte_oriented: whether to store data as bytes or as words
509  *
510  *	Read the specified number of 32-bit words from the serial flash.
511  *	If @byte_oriented is set the read data is stored as a byte array
512  *	(i.e., big-endian), otherwise as 32-bit words in the platform's
513  *	natural endianess.
514  */
515 static int
516 csio_hw_read_flash(struct csio_hw *hw, uint32_t addr, uint32_t nwords,
517 		  uint32_t *data, int32_t byte_oriented)
518 {
519 	int ret;
520 
521 	if (addr + nwords * sizeof(uint32_t) > hw->params.sf_size || (addr & 3))
522 		return -EINVAL;
523 
524 	addr = swab32(addr) | SF_RD_DATA_FAST;
525 
526 	ret = csio_hw_sf1_write(hw, 4, 1, 0, addr);
527 	if (ret != 0)
528 		return ret;
529 
530 	ret = csio_hw_sf1_read(hw, 1, 1, 0, data);
531 	if (ret != 0)
532 		return ret;
533 
534 	for ( ; nwords; nwords--, data++) {
535 		ret = csio_hw_sf1_read(hw, 4, nwords > 1, nwords == 1, data);
536 		if (nwords == 1)
537 			csio_wr_reg32(hw, 0, SF_OP);    /* unlock SF */
538 		if (ret)
539 			return ret;
540 		if (byte_oriented)
541 			*data = htonl(*data);
542 	}
543 	return 0;
544 }
545 
546 /*
547  *	csio_hw_write_flash - write up to a page of data to the serial flash
548  *	@hw: the hw
549  *	@addr: the start address to write
550  *	@n: length of data to write in bytes
551  *	@data: the data to write
552  *
553  *	Writes up to a page of data (256 bytes) to the serial flash starting
554  *	at the given address.  All the data must be written to the same page.
555  */
556 static int
557 csio_hw_write_flash(struct csio_hw *hw, uint32_t addr,
558 		    uint32_t n, const uint8_t *data)
559 {
560 	int ret = -EINVAL;
561 	uint32_t buf[64];
562 	uint32_t i, c, left, val, offset = addr & 0xff;
563 
564 	if (addr >= hw->params.sf_size || offset + n > SF_PAGE_SIZE)
565 		return -EINVAL;
566 
567 	val = swab32(addr) | SF_PROG_PAGE;
568 
569 	ret = csio_hw_sf1_write(hw, 1, 0, 1, SF_WR_ENABLE);
570 	if (ret != 0)
571 		goto unlock;
572 
573 	ret = csio_hw_sf1_write(hw, 4, 1, 1, val);
574 	if (ret != 0)
575 		goto unlock;
576 
577 	for (left = n; left; left -= c) {
578 		c = min(left, 4U);
579 		for (val = 0, i = 0; i < c; ++i)
580 			val = (val << 8) + *data++;
581 
582 		ret = csio_hw_sf1_write(hw, c, c != left, 1, val);
583 		if (ret)
584 			goto unlock;
585 	}
586 	ret = csio_hw_flash_wait_op(hw, 8, 1);
587 	if (ret)
588 		goto unlock;
589 
590 	csio_wr_reg32(hw, 0, SF_OP);    /* unlock SF */
591 
592 	/* Read the page to verify the write succeeded */
593 	ret = csio_hw_read_flash(hw, addr & ~0xff, ARRAY_SIZE(buf), buf, 1);
594 	if (ret)
595 		return ret;
596 
597 	if (memcmp(data - n, (uint8_t *)buf + offset, n)) {
598 		csio_err(hw,
599 			 "failed to correctly write the flash page at %#x\n",
600 			 addr);
601 		return -EINVAL;
602 	}
603 
604 	return 0;
605 
606 unlock:
607 	csio_wr_reg32(hw, 0, SF_OP);    /* unlock SF */
608 	return ret;
609 }
610 
611 /*
612  *	csio_hw_flash_erase_sectors - erase a range of flash sectors
613  *	@hw: the HW module
614  *	@start: the first sector to erase
615  *	@end: the last sector to erase
616  *
617  *	Erases the sectors in the given inclusive range.
618  */
619 static int
620 csio_hw_flash_erase_sectors(struct csio_hw *hw, int32_t start, int32_t end)
621 {
622 	int ret = 0;
623 
624 	while (start <= end) {
625 
626 		ret = csio_hw_sf1_write(hw, 1, 0, 1, SF_WR_ENABLE);
627 		if (ret != 0)
628 			goto out;
629 
630 		ret = csio_hw_sf1_write(hw, 4, 0, 1,
631 					SF_ERASE_SECTOR | (start << 8));
632 		if (ret != 0)
633 			goto out;
634 
635 		ret = csio_hw_flash_wait_op(hw, 14, 500);
636 		if (ret != 0)
637 			goto out;
638 
639 		start++;
640 	}
641 out:
642 	if (ret)
643 		csio_err(hw, "erase of flash sector %d failed, error %d\n",
644 			 start, ret);
645 	csio_wr_reg32(hw, 0, SF_OP);    /* unlock SF */
646 	return 0;
647 }
648 
649 static void
650 csio_hw_print_fw_version(struct csio_hw *hw, char *str)
651 {
652 	csio_info(hw, "%s: %u.%u.%u.%u\n", str,
653 		    FW_HDR_FW_VER_MAJOR_GET(hw->fwrev),
654 		    FW_HDR_FW_VER_MINOR_GET(hw->fwrev),
655 		    FW_HDR_FW_VER_MICRO_GET(hw->fwrev),
656 		    FW_HDR_FW_VER_BUILD_GET(hw->fwrev));
657 }
658 
659 /*
660  * csio_hw_get_fw_version - read the firmware version
661  * @hw: HW module
662  * @vers: where to place the version
663  *
664  * Reads the FW version from flash.
665  */
666 static int
667 csio_hw_get_fw_version(struct csio_hw *hw, uint32_t *vers)
668 {
669 	return csio_hw_read_flash(hw, FW_IMG_START +
670 				  offsetof(struct fw_hdr, fw_ver), 1,
671 				  vers, 0);
672 }
673 
674 /*
675  *	csio_hw_get_tp_version - read the TP microcode version
676  *	@hw: HW module
677  *	@vers: where to place the version
678  *
679  *	Reads the TP microcode version from flash.
680  */
681 static int
682 csio_hw_get_tp_version(struct csio_hw *hw, u32 *vers)
683 {
684 	return csio_hw_read_flash(hw, FLASH_FW_START +
685 			offsetof(struct fw_hdr, tp_microcode_ver), 1,
686 			vers, 0);
687 }
688 
689 /*
690  *	csio_hw_check_fw_version - check if the FW is compatible with
691  *				   this driver
692  *	@hw: HW module
693  *
694  *	Checks if an adapter's FW is compatible with the driver.  Returns 0
695  *	if there's exact match, a negative error if the version could not be
696  *	read or there's a major/minor version mismatch/minor.
697  */
698 static int
699 csio_hw_check_fw_version(struct csio_hw *hw)
700 {
701 	int ret, major, minor, micro;
702 
703 	ret = csio_hw_get_fw_version(hw, &hw->fwrev);
704 	if (!ret)
705 		ret = csio_hw_get_tp_version(hw, &hw->tp_vers);
706 	if (ret)
707 		return ret;
708 
709 	major = FW_HDR_FW_VER_MAJOR_GET(hw->fwrev);
710 	minor = FW_HDR_FW_VER_MINOR_GET(hw->fwrev);
711 	micro = FW_HDR_FW_VER_MICRO_GET(hw->fwrev);
712 
713 	if (major != FW_VERSION_MAJOR(hw)) {	/* major mismatch - fail */
714 		csio_err(hw, "card FW has major version %u, driver wants %u\n",
715 			 major, FW_VERSION_MAJOR(hw));
716 		return -EINVAL;
717 	}
718 
719 	if (minor == FW_VERSION_MINOR(hw) && micro == FW_VERSION_MICRO(hw))
720 		return 0;        /* perfect match */
721 
722 	/* Minor/micro version mismatch */
723 	return -EINVAL;
724 }
725 
726 /*
727  * csio_hw_fw_dload - download firmware.
728  * @hw: HW module
729  * @fw_data: firmware image to write.
730  * @size: image size
731  *
732  * Write the supplied firmware image to the card's serial flash.
733  */
734 static int
735 csio_hw_fw_dload(struct csio_hw *hw, uint8_t *fw_data, uint32_t size)
736 {
737 	uint32_t csum;
738 	int32_t addr;
739 	int ret;
740 	uint32_t i;
741 	uint8_t first_page[SF_PAGE_SIZE];
742 	const __be32 *p = (const __be32 *)fw_data;
743 	struct fw_hdr *hdr = (struct fw_hdr *)fw_data;
744 	uint32_t sf_sec_size;
745 
746 	if ((!hw->params.sf_size) || (!hw->params.sf_nsec)) {
747 		csio_err(hw, "Serial Flash data invalid\n");
748 		return -EINVAL;
749 	}
750 
751 	if (!size) {
752 		csio_err(hw, "FW image has no data\n");
753 		return -EINVAL;
754 	}
755 
756 	if (size & 511) {
757 		csio_err(hw, "FW image size not multiple of 512 bytes\n");
758 		return -EINVAL;
759 	}
760 
761 	if (ntohs(hdr->len512) * 512 != size) {
762 		csio_err(hw, "FW image size differs from size in FW header\n");
763 		return -EINVAL;
764 	}
765 
766 	if (size > FW_MAX_SIZE) {
767 		csio_err(hw, "FW image too large, max is %u bytes\n",
768 			    FW_MAX_SIZE);
769 		return -EINVAL;
770 	}
771 
772 	for (csum = 0, i = 0; i < size / sizeof(csum); i++)
773 		csum += ntohl(p[i]);
774 
775 	if (csum != 0xffffffff) {
776 		csio_err(hw, "corrupted firmware image, checksum %#x\n", csum);
777 		return -EINVAL;
778 	}
779 
780 	sf_sec_size = hw->params.sf_size / hw->params.sf_nsec;
781 	i = DIV_ROUND_UP(size, sf_sec_size);        /* # of sectors spanned */
782 
783 	csio_dbg(hw, "Erasing sectors... start:%d end:%d\n",
784 			  FW_START_SEC, FW_START_SEC + i - 1);
785 
786 	ret = csio_hw_flash_erase_sectors(hw, FW_START_SEC,
787 					  FW_START_SEC + i - 1);
788 	if (ret) {
789 		csio_err(hw, "Flash Erase failed\n");
790 		goto out;
791 	}
792 
793 	/*
794 	 * We write the correct version at the end so the driver can see a bad
795 	 * version if the FW write fails.  Start by writing a copy of the
796 	 * first page with a bad version.
797 	 */
798 	memcpy(first_page, fw_data, SF_PAGE_SIZE);
799 	((struct fw_hdr *)first_page)->fw_ver = htonl(0xffffffff);
800 	ret = csio_hw_write_flash(hw, FW_IMG_START, SF_PAGE_SIZE, first_page);
801 	if (ret)
802 		goto out;
803 
804 	csio_dbg(hw, "Writing Flash .. start:%d end:%d\n",
805 		    FW_IMG_START, FW_IMG_START + size);
806 
807 	addr = FW_IMG_START;
808 	for (size -= SF_PAGE_SIZE; size; size -= SF_PAGE_SIZE) {
809 		addr += SF_PAGE_SIZE;
810 		fw_data += SF_PAGE_SIZE;
811 		ret = csio_hw_write_flash(hw, addr, SF_PAGE_SIZE, fw_data);
812 		if (ret)
813 			goto out;
814 	}
815 
816 	ret = csio_hw_write_flash(hw,
817 				  FW_IMG_START +
818 					offsetof(struct fw_hdr, fw_ver),
819 				  sizeof(hdr->fw_ver),
820 				  (const uint8_t *)&hdr->fw_ver);
821 
822 out:
823 	if (ret)
824 		csio_err(hw, "firmware download failed, error %d\n", ret);
825 	return ret;
826 }
827 
828 static int
829 csio_hw_get_flash_params(struct csio_hw *hw)
830 {
831 	int ret;
832 	uint32_t info = 0;
833 
834 	ret = csio_hw_sf1_write(hw, 1, 1, 0, SF_RD_ID);
835 	if (!ret)
836 		ret = csio_hw_sf1_read(hw, 3, 0, 1, &info);
837 	csio_wr_reg32(hw, 0, SF_OP);    /* unlock SF */
838 	if (ret != 0)
839 		return ret;
840 
841 	if ((info & 0xff) != 0x20)		/* not a Numonix flash */
842 		return -EINVAL;
843 	info >>= 16;				/* log2 of size */
844 	if (info >= 0x14 && info < 0x18)
845 		hw->params.sf_nsec = 1 << (info - 16);
846 	else if (info == 0x18)
847 		hw->params.sf_nsec = 64;
848 	else
849 		return -EINVAL;
850 	hw->params.sf_size = 1 << info;
851 
852 	return 0;
853 }
854 
855 /*****************************************************************************/
856 /* HW State machine assists                                                  */
857 /*****************************************************************************/
858 
859 static int
860 csio_hw_dev_ready(struct csio_hw *hw)
861 {
862 	uint32_t reg;
863 	int cnt = 6;
864 
865 	while (((reg = csio_rd_reg32(hw, PL_WHOAMI)) == 0xFFFFFFFF) &&
866 								(--cnt != 0))
867 		mdelay(100);
868 
869 	if ((cnt == 0) && (((int32_t)(SOURCEPF_GET(reg)) < 0) ||
870 			    (SOURCEPF_GET(reg) >= CSIO_MAX_PFN))) {
871 		csio_err(hw, "PL_WHOAMI returned 0x%x, cnt:%d\n", reg, cnt);
872 		return -EIO;
873 	}
874 
875 	hw->pfn = SOURCEPF_GET(reg);
876 
877 	return 0;
878 }
879 
880 /*
881  * csio_do_hello - Perform the HELLO FW Mailbox command and process response.
882  * @hw: HW module
883  * @state: Device state
884  *
885  * FW_HELLO_CMD has to be polled for completion.
886  */
887 static int
888 csio_do_hello(struct csio_hw *hw, enum csio_dev_state *state)
889 {
890 	struct csio_mb	*mbp;
891 	int	rv = 0;
892 	enum csio_dev_master master;
893 	enum fw_retval retval;
894 	uint8_t mpfn;
895 	char state_str[16];
896 	int retries = FW_CMD_HELLO_RETRIES;
897 
898 	memset(state_str, 0, sizeof(state_str));
899 
900 	mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC);
901 	if (!mbp) {
902 		rv = -ENOMEM;
903 		CSIO_INC_STATS(hw, n_err_nomem);
904 		goto out;
905 	}
906 
907 	master = csio_force_master ? CSIO_MASTER_MUST : CSIO_MASTER_MAY;
908 
909 retry:
910 	csio_mb_hello(hw, mbp, CSIO_MB_DEFAULT_TMO, hw->pfn,
911 		      hw->pfn, master, NULL);
912 
913 	rv = csio_mb_issue(hw, mbp);
914 	if (rv) {
915 		csio_err(hw, "failed to issue HELLO cmd. ret:%d.\n", rv);
916 		goto out_free_mb;
917 	}
918 
919 	csio_mb_process_hello_rsp(hw, mbp, &retval, state, &mpfn);
920 	if (retval != FW_SUCCESS) {
921 		csio_err(hw, "HELLO cmd failed with ret: %d\n", retval);
922 		rv = -EINVAL;
923 		goto out_free_mb;
924 	}
925 
926 	/* Firmware has designated us to be master */
927 	if (hw->pfn == mpfn) {
928 		hw->flags |= CSIO_HWF_MASTER;
929 	} else if (*state == CSIO_DEV_STATE_UNINIT) {
930 		/*
931 		 * If we're not the Master PF then we need to wait around for
932 		 * the Master PF Driver to finish setting up the adapter.
933 		 *
934 		 * Note that we also do this wait if we're a non-Master-capable
935 		 * PF and there is no current Master PF; a Master PF may show up
936 		 * momentarily and we wouldn't want to fail pointlessly.  (This
937 		 * can happen when an OS loads lots of different drivers rapidly
938 		 * at the same time). In this case, the Master PF returned by
939 		 * the firmware will be PCIE_FW_MASTER_MASK so the test below
940 		 * will work ...
941 		 */
942 
943 		int waiting = FW_CMD_HELLO_TIMEOUT;
944 
945 		/*
946 		 * Wait for the firmware to either indicate an error or
947 		 * initialized state.  If we see either of these we bail out
948 		 * and report the issue to the caller.  If we exhaust the
949 		 * "hello timeout" and we haven't exhausted our retries, try
950 		 * again.  Otherwise bail with a timeout error.
951 		 */
952 		for (;;) {
953 			uint32_t pcie_fw;
954 
955 			spin_unlock_irq(&hw->lock);
956 			msleep(50);
957 			spin_lock_irq(&hw->lock);
958 			waiting -= 50;
959 
960 			/*
961 			 * If neither Error nor Initialialized are indicated
962 			 * by the firmware keep waiting till we exaust our
963 			 * timeout ... and then retry if we haven't exhausted
964 			 * our retries ...
965 			 */
966 			pcie_fw = csio_rd_reg32(hw, PCIE_FW);
967 			if (!(pcie_fw & (PCIE_FW_ERR|PCIE_FW_INIT))) {
968 				if (waiting <= 0) {
969 					if (retries-- > 0)
970 						goto retry;
971 
972 					rv = -ETIMEDOUT;
973 					break;
974 				}
975 				continue;
976 			}
977 
978 			/*
979 			 * We either have an Error or Initialized condition
980 			 * report errors preferentially.
981 			 */
982 			if (state) {
983 				if (pcie_fw & PCIE_FW_ERR) {
984 					*state = CSIO_DEV_STATE_ERR;
985 					rv = -ETIMEDOUT;
986 				} else if (pcie_fw & PCIE_FW_INIT)
987 					*state = CSIO_DEV_STATE_INIT;
988 			}
989 
990 			/*
991 			 * If we arrived before a Master PF was selected and
992 			 * there's not a valid Master PF, grab its identity
993 			 * for our caller.
994 			 */
995 			if (mpfn == PCIE_FW_MASTER_MASK &&
996 			    (pcie_fw & PCIE_FW_MASTER_VLD))
997 				mpfn = PCIE_FW_MASTER_GET(pcie_fw);
998 			break;
999 		}
1000 		hw->flags &= ~CSIO_HWF_MASTER;
1001 	}
1002 
1003 	switch (*state) {
1004 	case CSIO_DEV_STATE_UNINIT:
1005 		strcpy(state_str, "Initializing");
1006 		break;
1007 	case CSIO_DEV_STATE_INIT:
1008 		strcpy(state_str, "Initialized");
1009 		break;
1010 	case CSIO_DEV_STATE_ERR:
1011 		strcpy(state_str, "Error");
1012 		break;
1013 	default:
1014 		strcpy(state_str, "Unknown");
1015 		break;
1016 	}
1017 
1018 	if (hw->pfn == mpfn)
1019 		csio_info(hw, "PF: %d, Coming up as MASTER, HW state: %s\n",
1020 			hw->pfn, state_str);
1021 	else
1022 		csio_info(hw,
1023 		    "PF: %d, Coming up as SLAVE, Master PF: %d, HW state: %s\n",
1024 		    hw->pfn, mpfn, state_str);
1025 
1026 out_free_mb:
1027 	mempool_free(mbp, hw->mb_mempool);
1028 out:
1029 	return rv;
1030 }
1031 
1032 /*
1033  * csio_do_bye - Perform the BYE FW Mailbox command and process response.
1034  * @hw: HW module
1035  *
1036  */
1037 static int
1038 csio_do_bye(struct csio_hw *hw)
1039 {
1040 	struct csio_mb	*mbp;
1041 	enum fw_retval retval;
1042 
1043 	mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC);
1044 	if (!mbp) {
1045 		CSIO_INC_STATS(hw, n_err_nomem);
1046 		return -ENOMEM;
1047 	}
1048 
1049 	csio_mb_bye(hw, mbp, CSIO_MB_DEFAULT_TMO, NULL);
1050 
1051 	if (csio_mb_issue(hw, mbp)) {
1052 		csio_err(hw, "Issue of BYE command failed\n");
1053 		mempool_free(mbp, hw->mb_mempool);
1054 		return -EINVAL;
1055 	}
1056 
1057 	retval = csio_mb_fw_retval(mbp);
1058 	if (retval != FW_SUCCESS) {
1059 		mempool_free(mbp, hw->mb_mempool);
1060 		return -EINVAL;
1061 	}
1062 
1063 	mempool_free(mbp, hw->mb_mempool);
1064 
1065 	return 0;
1066 }
1067 
1068 /*
1069  * csio_do_reset- Perform the device reset.
1070  * @hw: HW module
1071  * @fw_rst: FW reset
1072  *
1073  * If fw_rst is set, issues FW reset mbox cmd otherwise
1074  * does PIO reset.
1075  * Performs reset of the function.
1076  */
1077 static int
1078 csio_do_reset(struct csio_hw *hw, bool fw_rst)
1079 {
1080 	struct csio_mb	*mbp;
1081 	enum fw_retval retval;
1082 
1083 	if (!fw_rst) {
1084 		/* PIO reset */
1085 		csio_wr_reg32(hw, PIORSTMODE | PIORST, PL_RST);
1086 		mdelay(2000);
1087 		return 0;
1088 	}
1089 
1090 	mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC);
1091 	if (!mbp) {
1092 		CSIO_INC_STATS(hw, n_err_nomem);
1093 		return -ENOMEM;
1094 	}
1095 
1096 	csio_mb_reset(hw, mbp, CSIO_MB_DEFAULT_TMO,
1097 		      PIORSTMODE | PIORST, 0, NULL);
1098 
1099 	if (csio_mb_issue(hw, mbp)) {
1100 		csio_err(hw, "Issue of RESET command failed.n");
1101 		mempool_free(mbp, hw->mb_mempool);
1102 		return -EINVAL;
1103 	}
1104 
1105 	retval = csio_mb_fw_retval(mbp);
1106 	if (retval != FW_SUCCESS) {
1107 		csio_err(hw, "RESET cmd failed with ret:0x%x.\n", retval);
1108 		mempool_free(mbp, hw->mb_mempool);
1109 		return -EINVAL;
1110 	}
1111 
1112 	mempool_free(mbp, hw->mb_mempool);
1113 
1114 	return 0;
1115 }
1116 
1117 static int
1118 csio_hw_validate_caps(struct csio_hw *hw, struct csio_mb *mbp)
1119 {
1120 	struct fw_caps_config_cmd *rsp = (struct fw_caps_config_cmd *)mbp->mb;
1121 	uint16_t caps;
1122 
1123 	caps = ntohs(rsp->fcoecaps);
1124 
1125 	if (!(caps & FW_CAPS_CONFIG_FCOE_INITIATOR)) {
1126 		csio_err(hw, "No FCoE Initiator capability in the firmware.\n");
1127 		return -EINVAL;
1128 	}
1129 
1130 	if (!(caps & FW_CAPS_CONFIG_FCOE_CTRL_OFLD)) {
1131 		csio_err(hw, "No FCoE Control Offload capability\n");
1132 		return -EINVAL;
1133 	}
1134 
1135 	return 0;
1136 }
1137 
1138 /*
1139  *	csio_hw_fw_halt - issue a reset/halt to FW and put uP into RESET
1140  *	@hw: the HW module
1141  *	@mbox: mailbox to use for the FW RESET command (if desired)
1142  *	@force: force uP into RESET even if FW RESET command fails
1143  *
1144  *	Issues a RESET command to firmware (if desired) with a HALT indication
1145  *	and then puts the microprocessor into RESET state.  The RESET command
1146  *	will only be issued if a legitimate mailbox is provided (mbox <=
1147  *	PCIE_FW_MASTER_MASK).
1148  *
1149  *	This is generally used in order for the host to safely manipulate the
1150  *	adapter without fear of conflicting with whatever the firmware might
1151  *	be doing.  The only way out of this state is to RESTART the firmware
1152  *	...
1153  */
1154 static int
1155 csio_hw_fw_halt(struct csio_hw *hw, uint32_t mbox, int32_t force)
1156 {
1157 	enum fw_retval retval = 0;
1158 
1159 	/*
1160 	 * If a legitimate mailbox is provided, issue a RESET command
1161 	 * with a HALT indication.
1162 	 */
1163 	if (mbox <= PCIE_FW_MASTER_MASK) {
1164 		struct csio_mb	*mbp;
1165 
1166 		mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC);
1167 		if (!mbp) {
1168 			CSIO_INC_STATS(hw, n_err_nomem);
1169 			return -ENOMEM;
1170 		}
1171 
1172 		csio_mb_reset(hw, mbp, CSIO_MB_DEFAULT_TMO,
1173 			      PIORSTMODE | PIORST, FW_RESET_CMD_HALT(1),
1174 			      NULL);
1175 
1176 		if (csio_mb_issue(hw, mbp)) {
1177 			csio_err(hw, "Issue of RESET command failed!\n");
1178 			mempool_free(mbp, hw->mb_mempool);
1179 			return -EINVAL;
1180 		}
1181 
1182 		retval = csio_mb_fw_retval(mbp);
1183 		mempool_free(mbp, hw->mb_mempool);
1184 	}
1185 
1186 	/*
1187 	 * Normally we won't complete the operation if the firmware RESET
1188 	 * command fails but if our caller insists we'll go ahead and put the
1189 	 * uP into RESET.  This can be useful if the firmware is hung or even
1190 	 * missing ...  We'll have to take the risk of putting the uP into
1191 	 * RESET without the cooperation of firmware in that case.
1192 	 *
1193 	 * We also force the firmware's HALT flag to be on in case we bypassed
1194 	 * the firmware RESET command above or we're dealing with old firmware
1195 	 * which doesn't have the HALT capability.  This will serve as a flag
1196 	 * for the incoming firmware to know that it's coming out of a HALT
1197 	 * rather than a RESET ... if it's new enough to understand that ...
1198 	 */
1199 	if (retval == 0 || force) {
1200 		csio_set_reg_field(hw, CIM_BOOT_CFG, UPCRST, UPCRST);
1201 		csio_set_reg_field(hw, PCIE_FW, PCIE_FW_HALT, PCIE_FW_HALT);
1202 	}
1203 
1204 	/*
1205 	 * And we always return the result of the firmware RESET command
1206 	 * even when we force the uP into RESET ...
1207 	 */
1208 	return retval ? -EINVAL : 0;
1209 }
1210 
1211 /*
1212  *	csio_hw_fw_restart - restart the firmware by taking the uP out of RESET
1213  *	@hw: the HW module
1214  *	@reset: if we want to do a RESET to restart things
1215  *
1216  *	Restart firmware previously halted by csio_hw_fw_halt().  On successful
1217  *	return the previous PF Master remains as the new PF Master and there
1218  *	is no need to issue a new HELLO command, etc.
1219  *
1220  *	We do this in two ways:
1221  *
1222  *	 1. If we're dealing with newer firmware we'll simply want to take
1223  *	    the chip's microprocessor out of RESET.  This will cause the
1224  *	    firmware to start up from its start vector.  And then we'll loop
1225  *	    until the firmware indicates it's started again (PCIE_FW.HALT
1226  *	    reset to 0) or we timeout.
1227  *
1228  *	 2. If we're dealing with older firmware then we'll need to RESET
1229  *	    the chip since older firmware won't recognize the PCIE_FW.HALT
1230  *	    flag and automatically RESET itself on startup.
1231  */
1232 static int
1233 csio_hw_fw_restart(struct csio_hw *hw, uint32_t mbox, int32_t reset)
1234 {
1235 	if (reset) {
1236 		/*
1237 		 * Since we're directing the RESET instead of the firmware
1238 		 * doing it automatically, we need to clear the PCIE_FW.HALT
1239 		 * bit.
1240 		 */
1241 		csio_set_reg_field(hw, PCIE_FW, PCIE_FW_HALT, 0);
1242 
1243 		/*
1244 		 * If we've been given a valid mailbox, first try to get the
1245 		 * firmware to do the RESET.  If that works, great and we can
1246 		 * return success.  Otherwise, if we haven't been given a
1247 		 * valid mailbox or the RESET command failed, fall back to
1248 		 * hitting the chip with a hammer.
1249 		 */
1250 		if (mbox <= PCIE_FW_MASTER_MASK) {
1251 			csio_set_reg_field(hw, CIM_BOOT_CFG, UPCRST, 0);
1252 			msleep(100);
1253 			if (csio_do_reset(hw, true) == 0)
1254 				return 0;
1255 		}
1256 
1257 		csio_wr_reg32(hw, PIORSTMODE | PIORST, PL_RST);
1258 		msleep(2000);
1259 	} else {
1260 		int ms;
1261 
1262 		csio_set_reg_field(hw, CIM_BOOT_CFG, UPCRST, 0);
1263 		for (ms = 0; ms < FW_CMD_MAX_TIMEOUT; ) {
1264 			if (!(csio_rd_reg32(hw, PCIE_FW) & PCIE_FW_HALT))
1265 				return 0;
1266 			msleep(100);
1267 			ms += 100;
1268 		}
1269 		return -ETIMEDOUT;
1270 	}
1271 	return 0;
1272 }
1273 
1274 /*
1275  *	csio_hw_fw_upgrade - perform all of the steps necessary to upgrade FW
1276  *	@hw: the HW module
1277  *	@mbox: mailbox to use for the FW RESET command (if desired)
1278  *	@fw_data: the firmware image to write
1279  *	@size: image size
1280  *	@force: force upgrade even if firmware doesn't cooperate
1281  *
1282  *	Perform all of the steps necessary for upgrading an adapter's
1283  *	firmware image.  Normally this requires the cooperation of the
1284  *	existing firmware in order to halt all existing activities
1285  *	but if an invalid mailbox token is passed in we skip that step
1286  *	(though we'll still put the adapter microprocessor into RESET in
1287  *	that case).
1288  *
1289  *	On successful return the new firmware will have been loaded and
1290  *	the adapter will have been fully RESET losing all previous setup
1291  *	state.  On unsuccessful return the adapter may be completely hosed ...
1292  *	positive errno indicates that the adapter is ~probably~ intact, a
1293  *	negative errno indicates that things are looking bad ...
1294  */
1295 static int
1296 csio_hw_fw_upgrade(struct csio_hw *hw, uint32_t mbox,
1297 		  const u8 *fw_data, uint32_t size, int32_t force)
1298 {
1299 	const struct fw_hdr *fw_hdr = (const struct fw_hdr *)fw_data;
1300 	int reset, ret;
1301 
1302 	ret = csio_hw_fw_halt(hw, mbox, force);
1303 	if (ret != 0 && !force)
1304 		return ret;
1305 
1306 	ret = csio_hw_fw_dload(hw, (uint8_t *) fw_data, size);
1307 	if (ret != 0)
1308 		return ret;
1309 
1310 	/*
1311 	 * Older versions of the firmware don't understand the new
1312 	 * PCIE_FW.HALT flag and so won't know to perform a RESET when they
1313 	 * restart.  So for newly loaded older firmware we'll have to do the
1314 	 * RESET for it so it starts up on a clean slate.  We can tell if
1315 	 * the newly loaded firmware will handle this right by checking
1316 	 * its header flags to see if it advertises the capability.
1317 	 */
1318 	reset = ((ntohl(fw_hdr->flags) & FW_HDR_FLAGS_RESET_HALT) == 0);
1319 	return csio_hw_fw_restart(hw, mbox, reset);
1320 }
1321 
1322 
1323 /*
1324  *	csio_hw_fw_config_file - setup an adapter via a Configuration File
1325  *	@hw: the HW module
1326  *	@mbox: mailbox to use for the FW command
1327  *	@mtype: the memory type where the Configuration File is located
1328  *	@maddr: the memory address where the Configuration File is located
1329  *	@finiver: return value for CF [fini] version
1330  *	@finicsum: return value for CF [fini] checksum
1331  *	@cfcsum: return value for CF computed checksum
1332  *
1333  *	Issue a command to get the firmware to process the Configuration
1334  *	File located at the specified mtype/maddress.  If the Configuration
1335  *	File is processed successfully and return value pointers are
1336  *	provided, the Configuration File "[fini] section version and
1337  *	checksum values will be returned along with the computed checksum.
1338  *	It's up to the caller to decide how it wants to respond to the
1339  *	checksums not matching but it recommended that a prominant warning
1340  *	be emitted in order to help people rapidly identify changed or
1341  *	corrupted Configuration Files.
1342  *
1343  *	Also note that it's possible to modify things like "niccaps",
1344  *	"toecaps",etc. between processing the Configuration File and telling
1345  *	the firmware to use the new configuration.  Callers which want to
1346  *	do this will need to "hand-roll" their own CAPS_CONFIGS commands for
1347  *	Configuration Files if they want to do this.
1348  */
1349 static int
1350 csio_hw_fw_config_file(struct csio_hw *hw,
1351 		      unsigned int mtype, unsigned int maddr,
1352 		      uint32_t *finiver, uint32_t *finicsum, uint32_t *cfcsum)
1353 {
1354 	struct csio_mb	*mbp;
1355 	struct fw_caps_config_cmd *caps_cmd;
1356 	int rv = -EINVAL;
1357 	enum fw_retval ret;
1358 
1359 	mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC);
1360 	if (!mbp) {
1361 		CSIO_INC_STATS(hw, n_err_nomem);
1362 		return -ENOMEM;
1363 	}
1364 	/*
1365 	 * Tell the firmware to process the indicated Configuration File.
1366 	 * If there are no errors and the caller has provided return value
1367 	 * pointers for the [fini] section version, checksum and computed
1368 	 * checksum, pass those back to the caller.
1369 	 */
1370 	caps_cmd = (struct fw_caps_config_cmd *)(mbp->mb);
1371 	CSIO_INIT_MBP(mbp, caps_cmd, CSIO_MB_DEFAULT_TMO, hw, NULL, 1);
1372 	caps_cmd->op_to_write =
1373 		htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
1374 		      FW_CMD_REQUEST |
1375 		      FW_CMD_READ);
1376 	caps_cmd->cfvalid_to_len16 =
1377 		htonl(FW_CAPS_CONFIG_CMD_CFVALID |
1378 		      FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
1379 		      FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(maddr >> 16) |
1380 		      FW_LEN16(*caps_cmd));
1381 
1382 	if (csio_mb_issue(hw, mbp)) {
1383 		csio_err(hw, "Issue of FW_CAPS_CONFIG_CMD failed!\n");
1384 		goto out;
1385 	}
1386 
1387 	ret = csio_mb_fw_retval(mbp);
1388 	if (ret != FW_SUCCESS) {
1389 		csio_dbg(hw, "FW_CAPS_CONFIG_CMD returned %d!\n", rv);
1390 		goto out;
1391 	}
1392 
1393 	if (finiver)
1394 		*finiver = ntohl(caps_cmd->finiver);
1395 	if (finicsum)
1396 		*finicsum = ntohl(caps_cmd->finicsum);
1397 	if (cfcsum)
1398 		*cfcsum = ntohl(caps_cmd->cfcsum);
1399 
1400 	/* Validate device capabilities */
1401 	if (csio_hw_validate_caps(hw, mbp)) {
1402 		rv = -ENOENT;
1403 		goto out;
1404 	}
1405 
1406 	/*
1407 	 * And now tell the firmware to use the configuration we just loaded.
1408 	 */
1409 	caps_cmd->op_to_write =
1410 		htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
1411 		      FW_CMD_REQUEST |
1412 		      FW_CMD_WRITE);
1413 	caps_cmd->cfvalid_to_len16 = htonl(FW_LEN16(*caps_cmd));
1414 
1415 	if (csio_mb_issue(hw, mbp)) {
1416 		csio_err(hw, "Issue of FW_CAPS_CONFIG_CMD failed!\n");
1417 		goto out;
1418 	}
1419 
1420 	ret = csio_mb_fw_retval(mbp);
1421 	if (ret != FW_SUCCESS) {
1422 		csio_dbg(hw, "FW_CAPS_CONFIG_CMD returned %d!\n", rv);
1423 		goto out;
1424 	}
1425 
1426 	rv = 0;
1427 out:
1428 	mempool_free(mbp, hw->mb_mempool);
1429 	return rv;
1430 }
1431 
1432 /*
1433  * csio_get_device_params - Get device parameters.
1434  * @hw: HW module
1435  *
1436  */
1437 static int
1438 csio_get_device_params(struct csio_hw *hw)
1439 {
1440 	struct csio_wrm *wrm	= csio_hw_to_wrm(hw);
1441 	struct csio_mb	*mbp;
1442 	enum fw_retval retval;
1443 	u32 param[6];
1444 	int i, j = 0;
1445 
1446 	/* Initialize portids to -1 */
1447 	for (i = 0; i < CSIO_MAX_PPORTS; i++)
1448 		hw->pport[i].portid = -1;
1449 
1450 	mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC);
1451 	if (!mbp) {
1452 		CSIO_INC_STATS(hw, n_err_nomem);
1453 		return -ENOMEM;
1454 	}
1455 
1456 	/* Get port vec information. */
1457 	param[0] = FW_PARAM_DEV(PORTVEC);
1458 
1459 	/* Get Core clock. */
1460 	param[1] = FW_PARAM_DEV(CCLK);
1461 
1462 	/* Get EQ id start and end. */
1463 	param[2] = FW_PARAM_PFVF(EQ_START);
1464 	param[3] = FW_PARAM_PFVF(EQ_END);
1465 
1466 	/* Get IQ id start and end. */
1467 	param[4] = FW_PARAM_PFVF(IQFLINT_START);
1468 	param[5] = FW_PARAM_PFVF(IQFLINT_END);
1469 
1470 	csio_mb_params(hw, mbp, CSIO_MB_DEFAULT_TMO, hw->pfn, 0,
1471 		       ARRAY_SIZE(param), param, NULL, false, NULL);
1472 	if (csio_mb_issue(hw, mbp)) {
1473 		csio_err(hw, "Issue of FW_PARAMS_CMD(read) failed!\n");
1474 		mempool_free(mbp, hw->mb_mempool);
1475 		return -EINVAL;
1476 	}
1477 
1478 	csio_mb_process_read_params_rsp(hw, mbp, &retval,
1479 			ARRAY_SIZE(param), param);
1480 	if (retval != FW_SUCCESS) {
1481 		csio_err(hw, "FW_PARAMS_CMD(read) failed with ret:0x%x!\n",
1482 				retval);
1483 		mempool_free(mbp, hw->mb_mempool);
1484 		return -EINVAL;
1485 	}
1486 
1487 	/* cache the information. */
1488 	hw->port_vec = param[0];
1489 	hw->vpd.cclk = param[1];
1490 	wrm->fw_eq_start = param[2];
1491 	wrm->fw_iq_start = param[4];
1492 
1493 	/* Using FW configured max iqs & eqs */
1494 	if ((hw->flags & CSIO_HWF_USING_SOFT_PARAMS) ||
1495 		!csio_is_hw_master(hw)) {
1496 		hw->cfg_niq = param[5] - param[4] + 1;
1497 		hw->cfg_neq = param[3] - param[2] + 1;
1498 		csio_dbg(hw, "Using fwconfig max niqs %d neqs %d\n",
1499 			hw->cfg_niq, hw->cfg_neq);
1500 	}
1501 
1502 	hw->port_vec &= csio_port_mask;
1503 
1504 	hw->num_pports	= hweight32(hw->port_vec);
1505 
1506 	csio_dbg(hw, "Port vector: 0x%x, #ports: %d\n",
1507 		    hw->port_vec, hw->num_pports);
1508 
1509 	for (i = 0; i < hw->num_pports; i++) {
1510 		while ((hw->port_vec & (1 << j)) == 0)
1511 			j++;
1512 		hw->pport[i].portid = j++;
1513 		csio_dbg(hw, "Found Port:%d\n", hw->pport[i].portid);
1514 	}
1515 	mempool_free(mbp, hw->mb_mempool);
1516 
1517 	return 0;
1518 }
1519 
1520 
1521 /*
1522  * csio_config_device_caps - Get and set device capabilities.
1523  * @hw: HW module
1524  *
1525  */
1526 static int
1527 csio_config_device_caps(struct csio_hw *hw)
1528 {
1529 	struct csio_mb	*mbp;
1530 	enum fw_retval retval;
1531 	int rv = -EINVAL;
1532 
1533 	mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC);
1534 	if (!mbp) {
1535 		CSIO_INC_STATS(hw, n_err_nomem);
1536 		return -ENOMEM;
1537 	}
1538 
1539 	/* Get device capabilities */
1540 	csio_mb_caps_config(hw, mbp, CSIO_MB_DEFAULT_TMO, 0, 0, 0, 0, NULL);
1541 
1542 	if (csio_mb_issue(hw, mbp)) {
1543 		csio_err(hw, "Issue of FW_CAPS_CONFIG_CMD(r) failed!\n");
1544 		goto out;
1545 	}
1546 
1547 	retval = csio_mb_fw_retval(mbp);
1548 	if (retval != FW_SUCCESS) {
1549 		csio_err(hw, "FW_CAPS_CONFIG_CMD(r) returned %d!\n", retval);
1550 		goto out;
1551 	}
1552 
1553 	/* Validate device capabilities */
1554 	if (csio_hw_validate_caps(hw, mbp))
1555 		goto out;
1556 
1557 	/* Don't config device capabilities if already configured */
1558 	if (hw->fw_state == CSIO_DEV_STATE_INIT) {
1559 		rv = 0;
1560 		goto out;
1561 	}
1562 
1563 	/* Write back desired device capabilities */
1564 	csio_mb_caps_config(hw, mbp, CSIO_MB_DEFAULT_TMO, true, true,
1565 			    false, true, NULL);
1566 
1567 	if (csio_mb_issue(hw, mbp)) {
1568 		csio_err(hw, "Issue of FW_CAPS_CONFIG_CMD(w) failed!\n");
1569 		goto out;
1570 	}
1571 
1572 	retval = csio_mb_fw_retval(mbp);
1573 	if (retval != FW_SUCCESS) {
1574 		csio_err(hw, "FW_CAPS_CONFIG_CMD(w) returned %d!\n", retval);
1575 		goto out;
1576 	}
1577 
1578 	rv = 0;
1579 out:
1580 	mempool_free(mbp, hw->mb_mempool);
1581 	return rv;
1582 }
1583 
1584 /*
1585  * csio_enable_ports - Bring up all available ports.
1586  * @hw: HW module.
1587  *
1588  */
1589 static int
1590 csio_enable_ports(struct csio_hw *hw)
1591 {
1592 	struct csio_mb  *mbp;
1593 	enum fw_retval retval;
1594 	uint8_t portid;
1595 	int i;
1596 
1597 	mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC);
1598 	if (!mbp) {
1599 		CSIO_INC_STATS(hw, n_err_nomem);
1600 		return -ENOMEM;
1601 	}
1602 
1603 	for (i = 0; i < hw->num_pports; i++) {
1604 		portid = hw->pport[i].portid;
1605 
1606 		/* Read PORT information */
1607 		csio_mb_port(hw, mbp, CSIO_MB_DEFAULT_TMO, portid,
1608 			     false, 0, 0, NULL);
1609 
1610 		if (csio_mb_issue(hw, mbp)) {
1611 			csio_err(hw, "failed to issue FW_PORT_CMD(r) port:%d\n",
1612 				 portid);
1613 			mempool_free(mbp, hw->mb_mempool);
1614 			return -EINVAL;
1615 		}
1616 
1617 		csio_mb_process_read_port_rsp(hw, mbp, &retval,
1618 					      &hw->pport[i].pcap);
1619 		if (retval != FW_SUCCESS) {
1620 			csio_err(hw, "FW_PORT_CMD(r) port:%d failed: 0x%x\n",
1621 				 portid, retval);
1622 			mempool_free(mbp, hw->mb_mempool);
1623 			return -EINVAL;
1624 		}
1625 
1626 		/* Write back PORT information */
1627 		csio_mb_port(hw, mbp, CSIO_MB_DEFAULT_TMO, portid, true,
1628 			     (PAUSE_RX | PAUSE_TX), hw->pport[i].pcap, NULL);
1629 
1630 		if (csio_mb_issue(hw, mbp)) {
1631 			csio_err(hw, "failed to issue FW_PORT_CMD(w) port:%d\n",
1632 				 portid);
1633 			mempool_free(mbp, hw->mb_mempool);
1634 			return -EINVAL;
1635 		}
1636 
1637 		retval = csio_mb_fw_retval(mbp);
1638 		if (retval != FW_SUCCESS) {
1639 			csio_err(hw, "FW_PORT_CMD(w) port:%d failed :0x%x\n",
1640 				 portid, retval);
1641 			mempool_free(mbp, hw->mb_mempool);
1642 			return -EINVAL;
1643 		}
1644 
1645 	} /* For all ports */
1646 
1647 	mempool_free(mbp, hw->mb_mempool);
1648 
1649 	return 0;
1650 }
1651 
1652 /*
1653  * csio_get_fcoe_resinfo - Read fcoe fw resource info.
1654  * @hw: HW module
1655  * Issued with lock held.
1656  */
1657 static int
1658 csio_get_fcoe_resinfo(struct csio_hw *hw)
1659 {
1660 	struct csio_fcoe_res_info *res_info = &hw->fres_info;
1661 	struct fw_fcoe_res_info_cmd *rsp;
1662 	struct csio_mb  *mbp;
1663 	enum fw_retval retval;
1664 
1665 	mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC);
1666 	if (!mbp) {
1667 		CSIO_INC_STATS(hw, n_err_nomem);
1668 		return -ENOMEM;
1669 	}
1670 
1671 	/* Get FCoE FW resource information */
1672 	csio_fcoe_read_res_info_init_mb(hw, mbp, CSIO_MB_DEFAULT_TMO, NULL);
1673 
1674 	if (csio_mb_issue(hw, mbp)) {
1675 		csio_err(hw, "failed to issue FW_FCOE_RES_INFO_CMD\n");
1676 		mempool_free(mbp, hw->mb_mempool);
1677 		return -EINVAL;
1678 	}
1679 
1680 	rsp = (struct fw_fcoe_res_info_cmd *)(mbp->mb);
1681 	retval = FW_CMD_RETVAL_GET(ntohl(rsp->retval_len16));
1682 	if (retval != FW_SUCCESS) {
1683 		csio_err(hw, "FW_FCOE_RES_INFO_CMD failed with ret x%x\n",
1684 			 retval);
1685 		mempool_free(mbp, hw->mb_mempool);
1686 		return -EINVAL;
1687 	}
1688 
1689 	res_info->e_d_tov = ntohs(rsp->e_d_tov);
1690 	res_info->r_a_tov_seq = ntohs(rsp->r_a_tov_seq);
1691 	res_info->r_a_tov_els = ntohs(rsp->r_a_tov_els);
1692 	res_info->r_r_tov = ntohs(rsp->r_r_tov);
1693 	res_info->max_xchgs = ntohl(rsp->max_xchgs);
1694 	res_info->max_ssns = ntohl(rsp->max_ssns);
1695 	res_info->used_xchgs = ntohl(rsp->used_xchgs);
1696 	res_info->used_ssns = ntohl(rsp->used_ssns);
1697 	res_info->max_fcfs = ntohl(rsp->max_fcfs);
1698 	res_info->max_vnps = ntohl(rsp->max_vnps);
1699 	res_info->used_fcfs = ntohl(rsp->used_fcfs);
1700 	res_info->used_vnps = ntohl(rsp->used_vnps);
1701 
1702 	csio_dbg(hw, "max ssns:%d max xchgs:%d\n", res_info->max_ssns,
1703 						  res_info->max_xchgs);
1704 	mempool_free(mbp, hw->mb_mempool);
1705 
1706 	return 0;
1707 }
1708 
1709 static int
1710 csio_hw_check_fwconfig(struct csio_hw *hw, u32 *param)
1711 {
1712 	struct csio_mb	*mbp;
1713 	enum fw_retval retval;
1714 	u32 _param[1];
1715 
1716 	mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC);
1717 	if (!mbp) {
1718 		CSIO_INC_STATS(hw, n_err_nomem);
1719 		return -ENOMEM;
1720 	}
1721 
1722 	/*
1723 	 * Find out whether we're dealing with a version of
1724 	 * the firmware which has configuration file support.
1725 	 */
1726 	_param[0] = (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
1727 		     FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_CF));
1728 
1729 	csio_mb_params(hw, mbp, CSIO_MB_DEFAULT_TMO, hw->pfn, 0,
1730 		       ARRAY_SIZE(_param), _param, NULL, false, NULL);
1731 	if (csio_mb_issue(hw, mbp)) {
1732 		csio_err(hw, "Issue of FW_PARAMS_CMD(read) failed!\n");
1733 		mempool_free(mbp, hw->mb_mempool);
1734 		return -EINVAL;
1735 	}
1736 
1737 	csio_mb_process_read_params_rsp(hw, mbp, &retval,
1738 			ARRAY_SIZE(_param), _param);
1739 	if (retval != FW_SUCCESS) {
1740 		csio_err(hw, "FW_PARAMS_CMD(read) failed with ret:0x%x!\n",
1741 				retval);
1742 		mempool_free(mbp, hw->mb_mempool);
1743 		return -EINVAL;
1744 	}
1745 
1746 	mempool_free(mbp, hw->mb_mempool);
1747 	*param = _param[0];
1748 
1749 	return 0;
1750 }
1751 
1752 static int
1753 csio_hw_flash_config(struct csio_hw *hw, u32 *fw_cfg_param, char *path)
1754 {
1755 	int ret = 0;
1756 	const struct firmware *cf;
1757 	struct pci_dev *pci_dev = hw->pdev;
1758 	struct device *dev = &pci_dev->dev;
1759 	unsigned int mtype = 0, maddr = 0;
1760 	uint32_t *cfg_data;
1761 	int value_to_add = 0;
1762 
1763 	if (request_firmware(&cf, CSIO_CF_FNAME(hw), dev) < 0) {
1764 		csio_err(hw, "could not find config file %s, err: %d\n",
1765 			 CSIO_CF_FNAME(hw), ret);
1766 		return -ENOENT;
1767 	}
1768 
1769 	if (cf->size%4 != 0)
1770 		value_to_add = 4 - (cf->size % 4);
1771 
1772 	cfg_data = kzalloc(cf->size+value_to_add, GFP_KERNEL);
1773 	if (cfg_data == NULL) {
1774 		ret = -ENOMEM;
1775 		goto leave;
1776 	}
1777 
1778 	memcpy((void *)cfg_data, (const void *)cf->data, cf->size);
1779 	if (csio_hw_check_fwconfig(hw, fw_cfg_param) != 0) {
1780 		ret = -EINVAL;
1781 		goto leave;
1782 	}
1783 
1784 	mtype = FW_PARAMS_PARAM_Y_GET(*fw_cfg_param);
1785 	maddr = FW_PARAMS_PARAM_Z_GET(*fw_cfg_param) << 16;
1786 
1787 	ret = csio_memory_write(hw, mtype, maddr,
1788 				cf->size + value_to_add, cfg_data);
1789 
1790 	if ((ret == 0) && (value_to_add != 0)) {
1791 		union {
1792 			u32 word;
1793 			char buf[4];
1794 		} last;
1795 		size_t size = cf->size & ~0x3;
1796 		int i;
1797 
1798 		last.word = cfg_data[size >> 2];
1799 		for (i = value_to_add; i < 4; i++)
1800 			last.buf[i] = 0;
1801 		ret = csio_memory_write(hw, mtype, maddr + size, 4, &last.word);
1802 	}
1803 	if (ret == 0) {
1804 		csio_info(hw, "config file upgraded to %s\n",
1805 			  CSIO_CF_FNAME(hw));
1806 		snprintf(path, 64, "%s%s", "/lib/firmware/", CSIO_CF_FNAME(hw));
1807 	}
1808 
1809 leave:
1810 	kfree(cfg_data);
1811 	release_firmware(cf);
1812 	return ret;
1813 }
1814 
1815 /*
1816  * HW initialization: contact FW, obtain config, perform basic init.
1817  *
1818  * If the firmware we're dealing with has Configuration File support, then
1819  * we use that to perform all configuration -- either using the configuration
1820  * file stored in flash on the adapter or using a filesystem-local file
1821  * if available.
1822  *
1823  * If we don't have configuration file support in the firmware, then we'll
1824  * have to set things up the old fashioned way with hard-coded register
1825  * writes and firmware commands ...
1826  */
1827 
1828 /*
1829  * Attempt to initialize the HW via a Firmware Configuration File.
1830  */
1831 static int
1832 csio_hw_use_fwconfig(struct csio_hw *hw, int reset, u32 *fw_cfg_param)
1833 {
1834 	unsigned int mtype, maddr;
1835 	int rv;
1836 	uint32_t finiver = 0, finicsum = 0, cfcsum = 0;
1837 	int using_flash;
1838 	char path[64];
1839 
1840 	/*
1841 	 * Reset device if necessary
1842 	 */
1843 	if (reset) {
1844 		rv = csio_do_reset(hw, true);
1845 		if (rv != 0)
1846 			goto bye;
1847 	}
1848 
1849 	/*
1850 	 * If we have a configuration file in host ,
1851 	 * then use that.  Otherwise, use the configuration file stored
1852 	 * in the HW flash ...
1853 	 */
1854 	spin_unlock_irq(&hw->lock);
1855 	rv = csio_hw_flash_config(hw, fw_cfg_param, path);
1856 	spin_lock_irq(&hw->lock);
1857 	if (rv != 0) {
1858 		if (rv == -ENOENT) {
1859 			/*
1860 			 * config file was not found. Use default
1861 			 * config file from flash.
1862 			 */
1863 			mtype = FW_MEMTYPE_CF_FLASH;
1864 			maddr = hw->chip_ops->chip_flash_cfg_addr(hw);
1865 			using_flash = 1;
1866 		} else {
1867 			/*
1868 			 * we revert back to the hardwired config if
1869 			 * flashing failed.
1870 			 */
1871 			goto bye;
1872 		}
1873 	} else {
1874 		mtype = FW_PARAMS_PARAM_Y_GET(*fw_cfg_param);
1875 		maddr = FW_PARAMS_PARAM_Z_GET(*fw_cfg_param) << 16;
1876 		using_flash = 0;
1877 	}
1878 
1879 	hw->cfg_store = (uint8_t)mtype;
1880 
1881 	/*
1882 	 * Issue a Capability Configuration command to the firmware to get it
1883 	 * to parse the Configuration File.
1884 	 */
1885 	rv = csio_hw_fw_config_file(hw, mtype, maddr, &finiver,
1886 		&finicsum, &cfcsum);
1887 	if (rv != 0)
1888 		goto bye;
1889 
1890 	hw->cfg_finiver		= finiver;
1891 	hw->cfg_finicsum	= finicsum;
1892 	hw->cfg_cfcsum		= cfcsum;
1893 	hw->cfg_csum_status	= true;
1894 
1895 	if (finicsum != cfcsum) {
1896 		csio_warn(hw,
1897 		      "Config File checksum mismatch: csum=%#x, computed=%#x\n",
1898 		      finicsum, cfcsum);
1899 
1900 		hw->cfg_csum_status = false;
1901 	}
1902 
1903 	/*
1904 	 * Note that we're operating with parameters
1905 	 * not supplied by the driver, rather than from hard-wired
1906 	 * initialization constants buried in the driver.
1907 	 */
1908 	hw->flags |= CSIO_HWF_USING_SOFT_PARAMS;
1909 
1910 	/* device parameters */
1911 	rv = csio_get_device_params(hw);
1912 	if (rv != 0)
1913 		goto bye;
1914 
1915 	/* Configure SGE */
1916 	csio_wr_sge_init(hw);
1917 
1918 	/*
1919 	 * And finally tell the firmware to initialize itself using the
1920 	 * parameters from the Configuration File.
1921 	 */
1922 	/* Post event to notify completion of configuration */
1923 	csio_post_event(&hw->sm, CSIO_HWE_INIT);
1924 
1925 	csio_info(hw,
1926 	 "Firmware Configuration File %s, version %#x, computed checksum %#x\n",
1927 		  (using_flash ? "in device FLASH" : path), finiver, cfcsum);
1928 
1929 	return 0;
1930 
1931 	/*
1932 	 * Something bad happened.  Return the error ...
1933 	 */
1934 bye:
1935 	hw->flags &= ~CSIO_HWF_USING_SOFT_PARAMS;
1936 	csio_dbg(hw, "Configuration file error %d\n", rv);
1937 	return rv;
1938 }
1939 
1940 /*
1941  * Attempt to initialize the adapter via hard-coded, driver supplied
1942  * parameters ...
1943  */
1944 static int
1945 csio_hw_no_fwconfig(struct csio_hw *hw, int reset)
1946 {
1947 	int		rv;
1948 	/*
1949 	 * Reset device if necessary
1950 	 */
1951 	if (reset) {
1952 		rv = csio_do_reset(hw, true);
1953 		if (rv != 0)
1954 			goto out;
1955 	}
1956 
1957 	/* Get and set device capabilities */
1958 	rv = csio_config_device_caps(hw);
1959 	if (rv != 0)
1960 		goto out;
1961 
1962 	/* device parameters */
1963 	rv = csio_get_device_params(hw);
1964 	if (rv != 0)
1965 		goto out;
1966 
1967 	/* Configure SGE */
1968 	csio_wr_sge_init(hw);
1969 
1970 	/* Post event to notify completion of configuration */
1971 	csio_post_event(&hw->sm, CSIO_HWE_INIT);
1972 
1973 out:
1974 	return rv;
1975 }
1976 
1977 /*
1978  * Returns -EINVAL if attempts to flash the firmware failed
1979  * else returns 0,
1980  * if flashing was not attempted because the card had the
1981  * latest firmware ECANCELED is returned
1982  */
1983 static int
1984 csio_hw_flash_fw(struct csio_hw *hw)
1985 {
1986 	int ret = -ECANCELED;
1987 	const struct firmware *fw;
1988 	const struct fw_hdr *hdr;
1989 	u32 fw_ver;
1990 	struct pci_dev *pci_dev = hw->pdev;
1991 	struct device *dev = &pci_dev->dev ;
1992 
1993 	if (request_firmware(&fw, CSIO_FW_FNAME(hw), dev) < 0) {
1994 		csio_err(hw, "could not find firmware image %s, err: %d\n",
1995 			 CSIO_FW_FNAME(hw), ret);
1996 		return -EINVAL;
1997 	}
1998 
1999 	hdr = (const struct fw_hdr *)fw->data;
2000 	fw_ver = ntohl(hdr->fw_ver);
2001 	if (FW_HDR_FW_VER_MAJOR_GET(fw_ver) != FW_VERSION_MAJOR(hw))
2002 		return -EINVAL;      /* wrong major version, won't do */
2003 
2004 	/*
2005 	 * If the flash FW is unusable or we found something newer, load it.
2006 	 */
2007 	if (FW_HDR_FW_VER_MAJOR_GET(hw->fwrev) != FW_VERSION_MAJOR(hw) ||
2008 	    fw_ver > hw->fwrev) {
2009 		ret = csio_hw_fw_upgrade(hw, hw->pfn, fw->data, fw->size,
2010 				    /*force=*/false);
2011 		if (!ret)
2012 			csio_info(hw,
2013 				  "firmware upgraded to version %pI4 from %s\n",
2014 				  &hdr->fw_ver, CSIO_FW_FNAME(hw));
2015 		else
2016 			csio_err(hw, "firmware upgrade failed! err=%d\n", ret);
2017 	} else
2018 		ret = -EINVAL;
2019 
2020 	release_firmware(fw);
2021 
2022 	return ret;
2023 }
2024 
2025 
2026 /*
2027  * csio_hw_configure - Configure HW
2028  * @hw - HW module
2029  *
2030  */
2031 static void
2032 csio_hw_configure(struct csio_hw *hw)
2033 {
2034 	int reset = 1;
2035 	int rv;
2036 	u32 param[1];
2037 
2038 	rv = csio_hw_dev_ready(hw);
2039 	if (rv != 0) {
2040 		CSIO_INC_STATS(hw, n_err_fatal);
2041 		csio_post_event(&hw->sm, CSIO_HWE_FATAL);
2042 		goto out;
2043 	}
2044 
2045 	/* HW version */
2046 	hw->chip_ver = (char)csio_rd_reg32(hw, PL_REV);
2047 
2048 	/* Needed for FW download */
2049 	rv = csio_hw_get_flash_params(hw);
2050 	if (rv != 0) {
2051 		csio_err(hw, "Failed to get serial flash params rv:%d\n", rv);
2052 		csio_post_event(&hw->sm, CSIO_HWE_FATAL);
2053 		goto out;
2054 	}
2055 
2056 	/* Set PCIe completion timeout to 4 seconds */
2057 	if (pci_is_pcie(hw->pdev))
2058 		pcie_capability_clear_and_set_word(hw->pdev, PCI_EXP_DEVCTL2,
2059 				PCI_EXP_DEVCTL2_COMP_TIMEOUT, 0xd);
2060 
2061 	hw->chip_ops->chip_set_mem_win(hw, MEMWIN_CSIOSTOR);
2062 
2063 	rv = csio_hw_get_fw_version(hw, &hw->fwrev);
2064 	if (rv != 0)
2065 		goto out;
2066 
2067 	csio_hw_print_fw_version(hw, "Firmware revision");
2068 
2069 	rv = csio_do_hello(hw, &hw->fw_state);
2070 	if (rv != 0) {
2071 		CSIO_INC_STATS(hw, n_err_fatal);
2072 		csio_post_event(&hw->sm, CSIO_HWE_FATAL);
2073 		goto out;
2074 	}
2075 
2076 	/* Read vpd */
2077 	rv = csio_hw_get_vpd_params(hw, &hw->vpd);
2078 	if (rv != 0)
2079 		goto out;
2080 
2081 	if (csio_is_hw_master(hw) && hw->fw_state != CSIO_DEV_STATE_INIT) {
2082 		rv = csio_hw_check_fw_version(hw);
2083 		if (rv == -EINVAL) {
2084 
2085 			/* Do firmware update */
2086 			spin_unlock_irq(&hw->lock);
2087 			rv = csio_hw_flash_fw(hw);
2088 			spin_lock_irq(&hw->lock);
2089 
2090 			if (rv == 0) {
2091 				reset = 0;
2092 				/*
2093 				 * Note that the chip was reset as part of the
2094 				 * firmware upgrade so we don't reset it again
2095 				 * below and grab the new firmware version.
2096 				 */
2097 				rv = csio_hw_check_fw_version(hw);
2098 			}
2099 		}
2100 		/*
2101 		 * If the firmware doesn't support Configuration
2102 		 * Files, use the old Driver-based, hard-wired
2103 		 * initialization.  Otherwise, try using the
2104 		 * Configuration File support and fall back to the
2105 		 * Driver-based initialization if there's no
2106 		 * Configuration File found.
2107 		 */
2108 		if (csio_hw_check_fwconfig(hw, param) == 0) {
2109 			rv = csio_hw_use_fwconfig(hw, reset, param);
2110 			if (rv == -ENOENT)
2111 				goto out;
2112 			if (rv != 0) {
2113 				csio_info(hw,
2114 				    "No Configuration File present "
2115 				    "on adapter.  Using hard-wired "
2116 				    "configuration parameters.\n");
2117 				rv = csio_hw_no_fwconfig(hw, reset);
2118 			}
2119 		} else {
2120 			rv = csio_hw_no_fwconfig(hw, reset);
2121 		}
2122 
2123 		if (rv != 0)
2124 			goto out;
2125 
2126 	} else {
2127 		if (hw->fw_state == CSIO_DEV_STATE_INIT) {
2128 
2129 			hw->flags |= CSIO_HWF_USING_SOFT_PARAMS;
2130 
2131 			/* device parameters */
2132 			rv = csio_get_device_params(hw);
2133 			if (rv != 0)
2134 				goto out;
2135 
2136 			/* Get device capabilities */
2137 			rv = csio_config_device_caps(hw);
2138 			if (rv != 0)
2139 				goto out;
2140 
2141 			/* Configure SGE */
2142 			csio_wr_sge_init(hw);
2143 
2144 			/* Post event to notify completion of configuration */
2145 			csio_post_event(&hw->sm, CSIO_HWE_INIT);
2146 			goto out;
2147 		}
2148 	} /* if not master */
2149 
2150 out:
2151 	return;
2152 }
2153 
2154 /*
2155  * csio_hw_initialize - Initialize HW
2156  * @hw - HW module
2157  *
2158  */
2159 static void
2160 csio_hw_initialize(struct csio_hw *hw)
2161 {
2162 	struct csio_mb	*mbp;
2163 	enum fw_retval retval;
2164 	int rv;
2165 	int i;
2166 
2167 	if (csio_is_hw_master(hw) && hw->fw_state != CSIO_DEV_STATE_INIT) {
2168 		mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC);
2169 		if (!mbp)
2170 			goto out;
2171 
2172 		csio_mb_initialize(hw, mbp, CSIO_MB_DEFAULT_TMO, NULL);
2173 
2174 		if (csio_mb_issue(hw, mbp)) {
2175 			csio_err(hw, "Issue of FW_INITIALIZE_CMD failed!\n");
2176 			goto free_and_out;
2177 		}
2178 
2179 		retval = csio_mb_fw_retval(mbp);
2180 		if (retval != FW_SUCCESS) {
2181 			csio_err(hw, "FW_INITIALIZE_CMD returned 0x%x!\n",
2182 				 retval);
2183 			goto free_and_out;
2184 		}
2185 
2186 		mempool_free(mbp, hw->mb_mempool);
2187 	}
2188 
2189 	rv = csio_get_fcoe_resinfo(hw);
2190 	if (rv != 0) {
2191 		csio_err(hw, "Failed to read fcoe resource info: %d\n", rv);
2192 		goto out;
2193 	}
2194 
2195 	spin_unlock_irq(&hw->lock);
2196 	rv = csio_config_queues(hw);
2197 	spin_lock_irq(&hw->lock);
2198 
2199 	if (rv != 0) {
2200 		csio_err(hw, "Config of queues failed!: %d\n", rv);
2201 		goto out;
2202 	}
2203 
2204 	for (i = 0; i < hw->num_pports; i++)
2205 		hw->pport[i].mod_type = FW_PORT_MOD_TYPE_NA;
2206 
2207 	if (csio_is_hw_master(hw) && hw->fw_state != CSIO_DEV_STATE_INIT) {
2208 		rv = csio_enable_ports(hw);
2209 		if (rv != 0) {
2210 			csio_err(hw, "Failed to enable ports: %d\n", rv);
2211 			goto out;
2212 		}
2213 	}
2214 
2215 	csio_post_event(&hw->sm, CSIO_HWE_INIT_DONE);
2216 	return;
2217 
2218 free_and_out:
2219 	mempool_free(mbp, hw->mb_mempool);
2220 out:
2221 	return;
2222 }
2223 
2224 #define PF_INTR_MASK (PFSW | PFCIM)
2225 
2226 /*
2227  * csio_hw_intr_enable - Enable HW interrupts
2228  * @hw: Pointer to HW module.
2229  *
2230  * Enable interrupts in HW registers.
2231  */
2232 static void
2233 csio_hw_intr_enable(struct csio_hw *hw)
2234 {
2235 	uint16_t vec = (uint16_t)csio_get_mb_intr_idx(csio_hw_to_mbm(hw));
2236 	uint32_t pf = SOURCEPF_GET(csio_rd_reg32(hw, PL_WHOAMI));
2237 	uint32_t pl = csio_rd_reg32(hw, PL_INT_ENABLE);
2238 
2239 	/*
2240 	 * Set aivec for MSI/MSIX. PCIE_PF_CFG.INTXType is set up
2241 	 * by FW, so do nothing for INTX.
2242 	 */
2243 	if (hw->intr_mode == CSIO_IM_MSIX)
2244 		csio_set_reg_field(hw, MYPF_REG(PCIE_PF_CFG),
2245 				   AIVEC(AIVEC_MASK), vec);
2246 	else if (hw->intr_mode == CSIO_IM_MSI)
2247 		csio_set_reg_field(hw, MYPF_REG(PCIE_PF_CFG),
2248 				   AIVEC(AIVEC_MASK), 0);
2249 
2250 	csio_wr_reg32(hw, PF_INTR_MASK, MYPF_REG(PL_PF_INT_ENABLE));
2251 
2252 	/* Turn on MB interrupts - this will internally flush PIO as well */
2253 	csio_mb_intr_enable(hw);
2254 
2255 	/* These are common registers - only a master can modify them */
2256 	if (csio_is_hw_master(hw)) {
2257 		/*
2258 		 * Disable the Serial FLASH interrupt, if enabled!
2259 		 */
2260 		pl &= (~SF);
2261 		csio_wr_reg32(hw, pl, PL_INT_ENABLE);
2262 
2263 		csio_wr_reg32(hw, ERR_CPL_EXCEED_IQE_SIZE |
2264 			      EGRESS_SIZE_ERR | ERR_INVALID_CIDX_INC |
2265 			      ERR_CPL_OPCODE_0 | ERR_DROPPED_DB |
2266 			      ERR_DATA_CPL_ON_HIGH_QID1 |
2267 			      ERR_DATA_CPL_ON_HIGH_QID0 | ERR_BAD_DB_PIDX3 |
2268 			      ERR_BAD_DB_PIDX2 | ERR_BAD_DB_PIDX1 |
2269 			      ERR_BAD_DB_PIDX0 | ERR_ING_CTXT_PRIO |
2270 			      ERR_EGR_CTXT_PRIO | INGRESS_SIZE_ERR,
2271 			      SGE_INT_ENABLE3);
2272 		csio_set_reg_field(hw, PL_INT_MAP0, 0, 1 << pf);
2273 	}
2274 
2275 	hw->flags |= CSIO_HWF_HW_INTR_ENABLED;
2276 
2277 }
2278 
2279 /*
2280  * csio_hw_intr_disable - Disable HW interrupts
2281  * @hw: Pointer to HW module.
2282  *
2283  * Turn off Mailbox and PCI_PF_CFG interrupts.
2284  */
2285 void
2286 csio_hw_intr_disable(struct csio_hw *hw)
2287 {
2288 	uint32_t pf = SOURCEPF_GET(csio_rd_reg32(hw, PL_WHOAMI));
2289 
2290 	if (!(hw->flags & CSIO_HWF_HW_INTR_ENABLED))
2291 		return;
2292 
2293 	hw->flags &= ~CSIO_HWF_HW_INTR_ENABLED;
2294 
2295 	csio_wr_reg32(hw, 0, MYPF_REG(PL_PF_INT_ENABLE));
2296 	if (csio_is_hw_master(hw))
2297 		csio_set_reg_field(hw, PL_INT_MAP0, 1 << pf, 0);
2298 
2299 	/* Turn off MB interrupts */
2300 	csio_mb_intr_disable(hw);
2301 
2302 }
2303 
2304 void
2305 csio_hw_fatal_err(struct csio_hw *hw)
2306 {
2307 	csio_set_reg_field(hw, SGE_CONTROL, GLOBALENABLE, 0);
2308 	csio_hw_intr_disable(hw);
2309 
2310 	/* Do not reset HW, we may need FW state for debugging */
2311 	csio_fatal(hw, "HW Fatal error encountered!\n");
2312 }
2313 
2314 /*****************************************************************************/
2315 /* START: HW SM                                                              */
2316 /*****************************************************************************/
2317 /*
2318  * csio_hws_uninit - Uninit state
2319  * @hw - HW module
2320  * @evt - Event
2321  *
2322  */
2323 static void
2324 csio_hws_uninit(struct csio_hw *hw, enum csio_hw_ev evt)
2325 {
2326 	hw->prev_evt = hw->cur_evt;
2327 	hw->cur_evt = evt;
2328 	CSIO_INC_STATS(hw, n_evt_sm[evt]);
2329 
2330 	switch (evt) {
2331 	case CSIO_HWE_CFG:
2332 		csio_set_state(&hw->sm, csio_hws_configuring);
2333 		csio_hw_configure(hw);
2334 		break;
2335 
2336 	default:
2337 		CSIO_INC_STATS(hw, n_evt_unexp);
2338 		break;
2339 	}
2340 }
2341 
2342 /*
2343  * csio_hws_configuring - Configuring state
2344  * @hw - HW module
2345  * @evt - Event
2346  *
2347  */
2348 static void
2349 csio_hws_configuring(struct csio_hw *hw, enum csio_hw_ev evt)
2350 {
2351 	hw->prev_evt = hw->cur_evt;
2352 	hw->cur_evt = evt;
2353 	CSIO_INC_STATS(hw, n_evt_sm[evt]);
2354 
2355 	switch (evt) {
2356 	case CSIO_HWE_INIT:
2357 		csio_set_state(&hw->sm, csio_hws_initializing);
2358 		csio_hw_initialize(hw);
2359 		break;
2360 
2361 	case CSIO_HWE_INIT_DONE:
2362 		csio_set_state(&hw->sm, csio_hws_ready);
2363 		/* Fan out event to all lnode SMs */
2364 		csio_notify_lnodes(hw, CSIO_LN_NOTIFY_HWREADY);
2365 		break;
2366 
2367 	case CSIO_HWE_FATAL:
2368 		csio_set_state(&hw->sm, csio_hws_uninit);
2369 		break;
2370 
2371 	case CSIO_HWE_PCI_REMOVE:
2372 		csio_do_bye(hw);
2373 		break;
2374 	default:
2375 		CSIO_INC_STATS(hw, n_evt_unexp);
2376 		break;
2377 	}
2378 }
2379 
2380 /*
2381  * csio_hws_initializing - Initialiazing state
2382  * @hw - HW module
2383  * @evt - Event
2384  *
2385  */
2386 static void
2387 csio_hws_initializing(struct csio_hw *hw, enum csio_hw_ev evt)
2388 {
2389 	hw->prev_evt = hw->cur_evt;
2390 	hw->cur_evt = evt;
2391 	CSIO_INC_STATS(hw, n_evt_sm[evt]);
2392 
2393 	switch (evt) {
2394 	case CSIO_HWE_INIT_DONE:
2395 		csio_set_state(&hw->sm, csio_hws_ready);
2396 
2397 		/* Fan out event to all lnode SMs */
2398 		csio_notify_lnodes(hw, CSIO_LN_NOTIFY_HWREADY);
2399 
2400 		/* Enable interrupts */
2401 		csio_hw_intr_enable(hw);
2402 		break;
2403 
2404 	case CSIO_HWE_FATAL:
2405 		csio_set_state(&hw->sm, csio_hws_uninit);
2406 		break;
2407 
2408 	case CSIO_HWE_PCI_REMOVE:
2409 		csio_do_bye(hw);
2410 		break;
2411 
2412 	default:
2413 		CSIO_INC_STATS(hw, n_evt_unexp);
2414 		break;
2415 	}
2416 }
2417 
2418 /*
2419  * csio_hws_ready - Ready state
2420  * @hw - HW module
2421  * @evt - Event
2422  *
2423  */
2424 static void
2425 csio_hws_ready(struct csio_hw *hw, enum csio_hw_ev evt)
2426 {
2427 	/* Remember the event */
2428 	hw->evtflag = evt;
2429 
2430 	hw->prev_evt = hw->cur_evt;
2431 	hw->cur_evt = evt;
2432 	CSIO_INC_STATS(hw, n_evt_sm[evt]);
2433 
2434 	switch (evt) {
2435 	case CSIO_HWE_HBA_RESET:
2436 	case CSIO_HWE_FW_DLOAD:
2437 	case CSIO_HWE_SUSPEND:
2438 	case CSIO_HWE_PCI_REMOVE:
2439 	case CSIO_HWE_PCIERR_DETECTED:
2440 		csio_set_state(&hw->sm, csio_hws_quiescing);
2441 		/* cleanup all outstanding cmds */
2442 		if (evt == CSIO_HWE_HBA_RESET ||
2443 		    evt == CSIO_HWE_PCIERR_DETECTED)
2444 			csio_scsim_cleanup_io(csio_hw_to_scsim(hw), false);
2445 		else
2446 			csio_scsim_cleanup_io(csio_hw_to_scsim(hw), true);
2447 
2448 		csio_hw_intr_disable(hw);
2449 		csio_hw_mbm_cleanup(hw);
2450 		csio_evtq_stop(hw);
2451 		csio_notify_lnodes(hw, CSIO_LN_NOTIFY_HWSTOP);
2452 		csio_evtq_flush(hw);
2453 		csio_mgmtm_cleanup(csio_hw_to_mgmtm(hw));
2454 		csio_post_event(&hw->sm, CSIO_HWE_QUIESCED);
2455 		break;
2456 
2457 	case CSIO_HWE_FATAL:
2458 		csio_set_state(&hw->sm, csio_hws_uninit);
2459 		break;
2460 
2461 	default:
2462 		CSIO_INC_STATS(hw, n_evt_unexp);
2463 		break;
2464 	}
2465 }
2466 
2467 /*
2468  * csio_hws_quiescing - Quiescing state
2469  * @hw - HW module
2470  * @evt - Event
2471  *
2472  */
2473 static void
2474 csio_hws_quiescing(struct csio_hw *hw, enum csio_hw_ev evt)
2475 {
2476 	hw->prev_evt = hw->cur_evt;
2477 	hw->cur_evt = evt;
2478 	CSIO_INC_STATS(hw, n_evt_sm[evt]);
2479 
2480 	switch (evt) {
2481 	case CSIO_HWE_QUIESCED:
2482 		switch (hw->evtflag) {
2483 		case CSIO_HWE_FW_DLOAD:
2484 			csio_set_state(&hw->sm, csio_hws_resetting);
2485 			/* Download firmware */
2486 			/* Fall through */
2487 
2488 		case CSIO_HWE_HBA_RESET:
2489 			csio_set_state(&hw->sm, csio_hws_resetting);
2490 			/* Start reset of the HBA */
2491 			csio_notify_lnodes(hw, CSIO_LN_NOTIFY_HWRESET);
2492 			csio_wr_destroy_queues(hw, false);
2493 			csio_do_reset(hw, false);
2494 			csio_post_event(&hw->sm, CSIO_HWE_HBA_RESET_DONE);
2495 			break;
2496 
2497 		case CSIO_HWE_PCI_REMOVE:
2498 			csio_set_state(&hw->sm, csio_hws_removing);
2499 			csio_notify_lnodes(hw, CSIO_LN_NOTIFY_HWREMOVE);
2500 			csio_wr_destroy_queues(hw, true);
2501 			/* Now send the bye command */
2502 			csio_do_bye(hw);
2503 			break;
2504 
2505 		case CSIO_HWE_SUSPEND:
2506 			csio_set_state(&hw->sm, csio_hws_quiesced);
2507 			break;
2508 
2509 		case CSIO_HWE_PCIERR_DETECTED:
2510 			csio_set_state(&hw->sm, csio_hws_pcierr);
2511 			csio_wr_destroy_queues(hw, false);
2512 			break;
2513 
2514 		default:
2515 			CSIO_INC_STATS(hw, n_evt_unexp);
2516 			break;
2517 
2518 		}
2519 		break;
2520 
2521 	default:
2522 		CSIO_INC_STATS(hw, n_evt_unexp);
2523 		break;
2524 	}
2525 }
2526 
2527 /*
2528  * csio_hws_quiesced - Quiesced state
2529  * @hw - HW module
2530  * @evt - Event
2531  *
2532  */
2533 static void
2534 csio_hws_quiesced(struct csio_hw *hw, enum csio_hw_ev evt)
2535 {
2536 	hw->prev_evt = hw->cur_evt;
2537 	hw->cur_evt = evt;
2538 	CSIO_INC_STATS(hw, n_evt_sm[evt]);
2539 
2540 	switch (evt) {
2541 	case CSIO_HWE_RESUME:
2542 		csio_set_state(&hw->sm, csio_hws_configuring);
2543 		csio_hw_configure(hw);
2544 		break;
2545 
2546 	default:
2547 		CSIO_INC_STATS(hw, n_evt_unexp);
2548 		break;
2549 	}
2550 }
2551 
2552 /*
2553  * csio_hws_resetting - HW Resetting state
2554  * @hw - HW module
2555  * @evt - Event
2556  *
2557  */
2558 static void
2559 csio_hws_resetting(struct csio_hw *hw, enum csio_hw_ev evt)
2560 {
2561 	hw->prev_evt = hw->cur_evt;
2562 	hw->cur_evt = evt;
2563 	CSIO_INC_STATS(hw, n_evt_sm[evt]);
2564 
2565 	switch (evt) {
2566 	case CSIO_HWE_HBA_RESET_DONE:
2567 		csio_evtq_start(hw);
2568 		csio_set_state(&hw->sm, csio_hws_configuring);
2569 		csio_hw_configure(hw);
2570 		break;
2571 
2572 	default:
2573 		CSIO_INC_STATS(hw, n_evt_unexp);
2574 		break;
2575 	}
2576 }
2577 
2578 /*
2579  * csio_hws_removing - PCI Hotplug removing state
2580  * @hw - HW module
2581  * @evt - Event
2582  *
2583  */
2584 static void
2585 csio_hws_removing(struct csio_hw *hw, enum csio_hw_ev evt)
2586 {
2587 	hw->prev_evt = hw->cur_evt;
2588 	hw->cur_evt = evt;
2589 	CSIO_INC_STATS(hw, n_evt_sm[evt]);
2590 
2591 	switch (evt) {
2592 	case CSIO_HWE_HBA_RESET:
2593 		if (!csio_is_hw_master(hw))
2594 			break;
2595 		/*
2596 		 * The BYE should have alerady been issued, so we cant
2597 		 * use the mailbox interface. Hence we use the PL_RST
2598 		 * register directly.
2599 		 */
2600 		csio_err(hw, "Resetting HW and waiting 2 seconds...\n");
2601 		csio_wr_reg32(hw, PIORSTMODE | PIORST, PL_RST);
2602 		mdelay(2000);
2603 		break;
2604 
2605 	/* Should never receive any new events */
2606 	default:
2607 		CSIO_INC_STATS(hw, n_evt_unexp);
2608 		break;
2609 
2610 	}
2611 }
2612 
2613 /*
2614  * csio_hws_pcierr - PCI Error state
2615  * @hw - HW module
2616  * @evt - Event
2617  *
2618  */
2619 static void
2620 csio_hws_pcierr(struct csio_hw *hw, enum csio_hw_ev evt)
2621 {
2622 	hw->prev_evt = hw->cur_evt;
2623 	hw->cur_evt = evt;
2624 	CSIO_INC_STATS(hw, n_evt_sm[evt]);
2625 
2626 	switch (evt) {
2627 	case CSIO_HWE_PCIERR_SLOT_RESET:
2628 		csio_evtq_start(hw);
2629 		csio_set_state(&hw->sm, csio_hws_configuring);
2630 		csio_hw_configure(hw);
2631 		break;
2632 
2633 	default:
2634 		CSIO_INC_STATS(hw, n_evt_unexp);
2635 		break;
2636 	}
2637 }
2638 
2639 /*****************************************************************************/
2640 /* END: HW SM                                                                */
2641 /*****************************************************************************/
2642 
2643 /*
2644  *	csio_handle_intr_status - table driven interrupt handler
2645  *	@hw: HW instance
2646  *	@reg: the interrupt status register to process
2647  *	@acts: table of interrupt actions
2648  *
2649  *	A table driven interrupt handler that applies a set of masks to an
2650  *	interrupt status word and performs the corresponding actions if the
2651  *	interrupts described by the mask have occured.  The actions include
2652  *	optionally emitting a warning or alert message. The table is terminated
2653  *	by an entry specifying mask 0.  Returns the number of fatal interrupt
2654  *	conditions.
2655  */
2656 int
2657 csio_handle_intr_status(struct csio_hw *hw, unsigned int reg,
2658 				 const struct intr_info *acts)
2659 {
2660 	int fatal = 0;
2661 	unsigned int mask = 0;
2662 	unsigned int status = csio_rd_reg32(hw, reg);
2663 
2664 	for ( ; acts->mask; ++acts) {
2665 		if (!(status & acts->mask))
2666 			continue;
2667 		if (acts->fatal) {
2668 			fatal++;
2669 			csio_fatal(hw, "Fatal %s (0x%x)\n",
2670 				    acts->msg, status & acts->mask);
2671 		} else if (acts->msg)
2672 			csio_info(hw, "%s (0x%x)\n",
2673 				    acts->msg, status & acts->mask);
2674 		mask |= acts->mask;
2675 	}
2676 	status &= mask;
2677 	if (status)                           /* clear processed interrupts */
2678 		csio_wr_reg32(hw, status, reg);
2679 	return fatal;
2680 }
2681 
2682 /*
2683  * TP interrupt handler.
2684  */
2685 static void csio_tp_intr_handler(struct csio_hw *hw)
2686 {
2687 	static struct intr_info tp_intr_info[] = {
2688 		{ 0x3fffffff, "TP parity error", -1, 1 },
2689 		{ FLMTXFLSTEMPTY, "TP out of Tx pages", -1, 1 },
2690 		{ 0, NULL, 0, 0 }
2691 	};
2692 
2693 	if (csio_handle_intr_status(hw, TP_INT_CAUSE, tp_intr_info))
2694 		csio_hw_fatal_err(hw);
2695 }
2696 
2697 /*
2698  * SGE interrupt handler.
2699  */
2700 static void csio_sge_intr_handler(struct csio_hw *hw)
2701 {
2702 	uint64_t v;
2703 
2704 	static struct intr_info sge_intr_info[] = {
2705 		{ ERR_CPL_EXCEED_IQE_SIZE,
2706 		  "SGE received CPL exceeding IQE size", -1, 1 },
2707 		{ ERR_INVALID_CIDX_INC,
2708 		  "SGE GTS CIDX increment too large", -1, 0 },
2709 		{ ERR_CPL_OPCODE_0, "SGE received 0-length CPL", -1, 0 },
2710 		{ ERR_DROPPED_DB, "SGE doorbell dropped", -1, 0 },
2711 		{ ERR_DATA_CPL_ON_HIGH_QID1 | ERR_DATA_CPL_ON_HIGH_QID0,
2712 		  "SGE IQID > 1023 received CPL for FL", -1, 0 },
2713 		{ ERR_BAD_DB_PIDX3, "SGE DBP 3 pidx increment too large", -1,
2714 		  0 },
2715 		{ ERR_BAD_DB_PIDX2, "SGE DBP 2 pidx increment too large", -1,
2716 		  0 },
2717 		{ ERR_BAD_DB_PIDX1, "SGE DBP 1 pidx increment too large", -1,
2718 		  0 },
2719 		{ ERR_BAD_DB_PIDX0, "SGE DBP 0 pidx increment too large", -1,
2720 		  0 },
2721 		{ ERR_ING_CTXT_PRIO,
2722 		  "SGE too many priority ingress contexts", -1, 0 },
2723 		{ ERR_EGR_CTXT_PRIO,
2724 		  "SGE too many priority egress contexts", -1, 0 },
2725 		{ INGRESS_SIZE_ERR, "SGE illegal ingress QID", -1, 0 },
2726 		{ EGRESS_SIZE_ERR, "SGE illegal egress QID", -1, 0 },
2727 		{ 0, NULL, 0, 0 }
2728 	};
2729 
2730 	v = (uint64_t)csio_rd_reg32(hw, SGE_INT_CAUSE1) |
2731 	    ((uint64_t)csio_rd_reg32(hw, SGE_INT_CAUSE2) << 32);
2732 	if (v) {
2733 		csio_fatal(hw, "SGE parity error (%#llx)\n",
2734 			    (unsigned long long)v);
2735 		csio_wr_reg32(hw, (uint32_t)(v & 0xFFFFFFFF),
2736 						SGE_INT_CAUSE1);
2737 		csio_wr_reg32(hw, (uint32_t)(v >> 32), SGE_INT_CAUSE2);
2738 	}
2739 
2740 	v |= csio_handle_intr_status(hw, SGE_INT_CAUSE3, sge_intr_info);
2741 
2742 	if (csio_handle_intr_status(hw, SGE_INT_CAUSE3, sge_intr_info) ||
2743 	    v != 0)
2744 		csio_hw_fatal_err(hw);
2745 }
2746 
2747 #define CIM_OBQ_INTR (OBQULP0PARERR | OBQULP1PARERR | OBQULP2PARERR |\
2748 		      OBQULP3PARERR | OBQSGEPARERR | OBQNCSIPARERR)
2749 #define CIM_IBQ_INTR (IBQTP0PARERR | IBQTP1PARERR | IBQULPPARERR |\
2750 		      IBQSGEHIPARERR | IBQSGELOPARERR | IBQNCSIPARERR)
2751 
2752 /*
2753  * CIM interrupt handler.
2754  */
2755 static void csio_cim_intr_handler(struct csio_hw *hw)
2756 {
2757 	static struct intr_info cim_intr_info[] = {
2758 		{ PREFDROPINT, "CIM control register prefetch drop", -1, 1 },
2759 		{ CIM_OBQ_INTR, "CIM OBQ parity error", -1, 1 },
2760 		{ CIM_IBQ_INTR, "CIM IBQ parity error", -1, 1 },
2761 		{ MBUPPARERR, "CIM mailbox uP parity error", -1, 1 },
2762 		{ MBHOSTPARERR, "CIM mailbox host parity error", -1, 1 },
2763 		{ TIEQINPARERRINT, "CIM TIEQ outgoing parity error", -1, 1 },
2764 		{ TIEQOUTPARERRINT, "CIM TIEQ incoming parity error", -1, 1 },
2765 		{ 0, NULL, 0, 0 }
2766 	};
2767 	static struct intr_info cim_upintr_info[] = {
2768 		{ RSVDSPACEINT, "CIM reserved space access", -1, 1 },
2769 		{ ILLTRANSINT, "CIM illegal transaction", -1, 1 },
2770 		{ ILLWRINT, "CIM illegal write", -1, 1 },
2771 		{ ILLRDINT, "CIM illegal read", -1, 1 },
2772 		{ ILLRDBEINT, "CIM illegal read BE", -1, 1 },
2773 		{ ILLWRBEINT, "CIM illegal write BE", -1, 1 },
2774 		{ SGLRDBOOTINT, "CIM single read from boot space", -1, 1 },
2775 		{ SGLWRBOOTINT, "CIM single write to boot space", -1, 1 },
2776 		{ BLKWRBOOTINT, "CIM block write to boot space", -1, 1 },
2777 		{ SGLRDFLASHINT, "CIM single read from flash space", -1, 1 },
2778 		{ SGLWRFLASHINT, "CIM single write to flash space", -1, 1 },
2779 		{ BLKWRFLASHINT, "CIM block write to flash space", -1, 1 },
2780 		{ SGLRDEEPROMINT, "CIM single EEPROM read", -1, 1 },
2781 		{ SGLWREEPROMINT, "CIM single EEPROM write", -1, 1 },
2782 		{ BLKRDEEPROMINT, "CIM block EEPROM read", -1, 1 },
2783 		{ BLKWREEPROMINT, "CIM block EEPROM write", -1, 1 },
2784 		{ SGLRDCTLINT , "CIM single read from CTL space", -1, 1 },
2785 		{ SGLWRCTLINT , "CIM single write to CTL space", -1, 1 },
2786 		{ BLKRDCTLINT , "CIM block read from CTL space", -1, 1 },
2787 		{ BLKWRCTLINT , "CIM block write to CTL space", -1, 1 },
2788 		{ SGLRDPLINT , "CIM single read from PL space", -1, 1 },
2789 		{ SGLWRPLINT , "CIM single write to PL space", -1, 1 },
2790 		{ BLKRDPLINT , "CIM block read from PL space", -1, 1 },
2791 		{ BLKWRPLINT , "CIM block write to PL space", -1, 1 },
2792 		{ REQOVRLOOKUPINT , "CIM request FIFO overwrite", -1, 1 },
2793 		{ RSPOVRLOOKUPINT , "CIM response FIFO overwrite", -1, 1 },
2794 		{ TIMEOUTINT , "CIM PIF timeout", -1, 1 },
2795 		{ TIMEOUTMAINT , "CIM PIF MA timeout", -1, 1 },
2796 		{ 0, NULL, 0, 0 }
2797 	};
2798 
2799 	int fat;
2800 
2801 	fat = csio_handle_intr_status(hw, CIM_HOST_INT_CAUSE,
2802 				    cim_intr_info) +
2803 	      csio_handle_intr_status(hw, CIM_HOST_UPACC_INT_CAUSE,
2804 				    cim_upintr_info);
2805 	if (fat)
2806 		csio_hw_fatal_err(hw);
2807 }
2808 
2809 /*
2810  * ULP RX interrupt handler.
2811  */
2812 static void csio_ulprx_intr_handler(struct csio_hw *hw)
2813 {
2814 	static struct intr_info ulprx_intr_info[] = {
2815 		{ 0x1800000, "ULPRX context error", -1, 1 },
2816 		{ 0x7fffff, "ULPRX parity error", -1, 1 },
2817 		{ 0, NULL, 0, 0 }
2818 	};
2819 
2820 	if (csio_handle_intr_status(hw, ULP_RX_INT_CAUSE, ulprx_intr_info))
2821 		csio_hw_fatal_err(hw);
2822 }
2823 
2824 /*
2825  * ULP TX interrupt handler.
2826  */
2827 static void csio_ulptx_intr_handler(struct csio_hw *hw)
2828 {
2829 	static struct intr_info ulptx_intr_info[] = {
2830 		{ PBL_BOUND_ERR_CH3, "ULPTX channel 3 PBL out of bounds", -1,
2831 		  0 },
2832 		{ PBL_BOUND_ERR_CH2, "ULPTX channel 2 PBL out of bounds", -1,
2833 		  0 },
2834 		{ PBL_BOUND_ERR_CH1, "ULPTX channel 1 PBL out of bounds", -1,
2835 		  0 },
2836 		{ PBL_BOUND_ERR_CH0, "ULPTX channel 0 PBL out of bounds", -1,
2837 		  0 },
2838 		{ 0xfffffff, "ULPTX parity error", -1, 1 },
2839 		{ 0, NULL, 0, 0 }
2840 	};
2841 
2842 	if (csio_handle_intr_status(hw, ULP_TX_INT_CAUSE, ulptx_intr_info))
2843 		csio_hw_fatal_err(hw);
2844 }
2845 
2846 /*
2847  * PM TX interrupt handler.
2848  */
2849 static void csio_pmtx_intr_handler(struct csio_hw *hw)
2850 {
2851 	static struct intr_info pmtx_intr_info[] = {
2852 		{ PCMD_LEN_OVFL0, "PMTX channel 0 pcmd too large", -1, 1 },
2853 		{ PCMD_LEN_OVFL1, "PMTX channel 1 pcmd too large", -1, 1 },
2854 		{ PCMD_LEN_OVFL2, "PMTX channel 2 pcmd too large", -1, 1 },
2855 		{ ZERO_C_CMD_ERROR, "PMTX 0-length pcmd", -1, 1 },
2856 		{ 0xffffff0, "PMTX framing error", -1, 1 },
2857 		{ OESPI_PAR_ERROR, "PMTX oespi parity error", -1, 1 },
2858 		{ DB_OPTIONS_PAR_ERROR, "PMTX db_options parity error", -1,
2859 		  1 },
2860 		{ ICSPI_PAR_ERROR, "PMTX icspi parity error", -1, 1 },
2861 		{ C_PCMD_PAR_ERROR, "PMTX c_pcmd parity error", -1, 1},
2862 		{ 0, NULL, 0, 0 }
2863 	};
2864 
2865 	if (csio_handle_intr_status(hw, PM_TX_INT_CAUSE, pmtx_intr_info))
2866 		csio_hw_fatal_err(hw);
2867 }
2868 
2869 /*
2870  * PM RX interrupt handler.
2871  */
2872 static void csio_pmrx_intr_handler(struct csio_hw *hw)
2873 {
2874 	static struct intr_info pmrx_intr_info[] = {
2875 		{ ZERO_E_CMD_ERROR, "PMRX 0-length pcmd", -1, 1 },
2876 		{ 0x3ffff0, "PMRX framing error", -1, 1 },
2877 		{ OCSPI_PAR_ERROR, "PMRX ocspi parity error", -1, 1 },
2878 		{ DB_OPTIONS_PAR_ERROR, "PMRX db_options parity error", -1,
2879 		  1 },
2880 		{ IESPI_PAR_ERROR, "PMRX iespi parity error", -1, 1 },
2881 		{ E_PCMD_PAR_ERROR, "PMRX e_pcmd parity error", -1, 1},
2882 		{ 0, NULL, 0, 0 }
2883 	};
2884 
2885 	if (csio_handle_intr_status(hw, PM_RX_INT_CAUSE, pmrx_intr_info))
2886 		csio_hw_fatal_err(hw);
2887 }
2888 
2889 /*
2890  * CPL switch interrupt handler.
2891  */
2892 static void csio_cplsw_intr_handler(struct csio_hw *hw)
2893 {
2894 	static struct intr_info cplsw_intr_info[] = {
2895 		{ CIM_OP_MAP_PERR, "CPLSW CIM op_map parity error", -1, 1 },
2896 		{ CIM_OVFL_ERROR, "CPLSW CIM overflow", -1, 1 },
2897 		{ TP_FRAMING_ERROR, "CPLSW TP framing error", -1, 1 },
2898 		{ SGE_FRAMING_ERROR, "CPLSW SGE framing error", -1, 1 },
2899 		{ CIM_FRAMING_ERROR, "CPLSW CIM framing error", -1, 1 },
2900 		{ ZERO_SWITCH_ERROR, "CPLSW no-switch error", -1, 1 },
2901 		{ 0, NULL, 0, 0 }
2902 	};
2903 
2904 	if (csio_handle_intr_status(hw, CPL_INTR_CAUSE, cplsw_intr_info))
2905 		csio_hw_fatal_err(hw);
2906 }
2907 
2908 /*
2909  * LE interrupt handler.
2910  */
2911 static void csio_le_intr_handler(struct csio_hw *hw)
2912 {
2913 	static struct intr_info le_intr_info[] = {
2914 		{ LIPMISS, "LE LIP miss", -1, 0 },
2915 		{ LIP0, "LE 0 LIP error", -1, 0 },
2916 		{ PARITYERR, "LE parity error", -1, 1 },
2917 		{ UNKNOWNCMD, "LE unknown command", -1, 1 },
2918 		{ REQQPARERR, "LE request queue parity error", -1, 1 },
2919 		{ 0, NULL, 0, 0 }
2920 	};
2921 
2922 	if (csio_handle_intr_status(hw, LE_DB_INT_CAUSE, le_intr_info))
2923 		csio_hw_fatal_err(hw);
2924 }
2925 
2926 /*
2927  * MPS interrupt handler.
2928  */
2929 static void csio_mps_intr_handler(struct csio_hw *hw)
2930 {
2931 	static struct intr_info mps_rx_intr_info[] = {
2932 		{ 0xffffff, "MPS Rx parity error", -1, 1 },
2933 		{ 0, NULL, 0, 0 }
2934 	};
2935 	static struct intr_info mps_tx_intr_info[] = {
2936 		{ TPFIFO, "MPS Tx TP FIFO parity error", -1, 1 },
2937 		{ NCSIFIFO, "MPS Tx NC-SI FIFO parity error", -1, 1 },
2938 		{ TXDATAFIFO, "MPS Tx data FIFO parity error", -1, 1 },
2939 		{ TXDESCFIFO, "MPS Tx desc FIFO parity error", -1, 1 },
2940 		{ BUBBLE, "MPS Tx underflow", -1, 1 },
2941 		{ SECNTERR, "MPS Tx SOP/EOP error", -1, 1 },
2942 		{ FRMERR, "MPS Tx framing error", -1, 1 },
2943 		{ 0, NULL, 0, 0 }
2944 	};
2945 	static struct intr_info mps_trc_intr_info[] = {
2946 		{ FILTMEM, "MPS TRC filter parity error", -1, 1 },
2947 		{ PKTFIFO, "MPS TRC packet FIFO parity error", -1, 1 },
2948 		{ MISCPERR, "MPS TRC misc parity error", -1, 1 },
2949 		{ 0, NULL, 0, 0 }
2950 	};
2951 	static struct intr_info mps_stat_sram_intr_info[] = {
2952 		{ 0x1fffff, "MPS statistics SRAM parity error", -1, 1 },
2953 		{ 0, NULL, 0, 0 }
2954 	};
2955 	static struct intr_info mps_stat_tx_intr_info[] = {
2956 		{ 0xfffff, "MPS statistics Tx FIFO parity error", -1, 1 },
2957 		{ 0, NULL, 0, 0 }
2958 	};
2959 	static struct intr_info mps_stat_rx_intr_info[] = {
2960 		{ 0xffffff, "MPS statistics Rx FIFO parity error", -1, 1 },
2961 		{ 0, NULL, 0, 0 }
2962 	};
2963 	static struct intr_info mps_cls_intr_info[] = {
2964 		{ MATCHSRAM, "MPS match SRAM parity error", -1, 1 },
2965 		{ MATCHTCAM, "MPS match TCAM parity error", -1, 1 },
2966 		{ HASHSRAM, "MPS hash SRAM parity error", -1, 1 },
2967 		{ 0, NULL, 0, 0 }
2968 	};
2969 
2970 	int fat;
2971 
2972 	fat = csio_handle_intr_status(hw, MPS_RX_PERR_INT_CAUSE,
2973 				    mps_rx_intr_info) +
2974 	      csio_handle_intr_status(hw, MPS_TX_INT_CAUSE,
2975 				    mps_tx_intr_info) +
2976 	      csio_handle_intr_status(hw, MPS_TRC_INT_CAUSE,
2977 				    mps_trc_intr_info) +
2978 	      csio_handle_intr_status(hw, MPS_STAT_PERR_INT_CAUSE_SRAM,
2979 				    mps_stat_sram_intr_info) +
2980 	      csio_handle_intr_status(hw, MPS_STAT_PERR_INT_CAUSE_TX_FIFO,
2981 				    mps_stat_tx_intr_info) +
2982 	      csio_handle_intr_status(hw, MPS_STAT_PERR_INT_CAUSE_RX_FIFO,
2983 				    mps_stat_rx_intr_info) +
2984 	      csio_handle_intr_status(hw, MPS_CLS_INT_CAUSE,
2985 				    mps_cls_intr_info);
2986 
2987 	csio_wr_reg32(hw, 0, MPS_INT_CAUSE);
2988 	csio_rd_reg32(hw, MPS_INT_CAUSE);                    /* flush */
2989 	if (fat)
2990 		csio_hw_fatal_err(hw);
2991 }
2992 
2993 #define MEM_INT_MASK (PERR_INT_CAUSE | ECC_CE_INT_CAUSE | ECC_UE_INT_CAUSE)
2994 
2995 /*
2996  * EDC/MC interrupt handler.
2997  */
2998 static void csio_mem_intr_handler(struct csio_hw *hw, int idx)
2999 {
3000 	static const char name[3][5] = { "EDC0", "EDC1", "MC" };
3001 
3002 	unsigned int addr, cnt_addr, v;
3003 
3004 	if (idx <= MEM_EDC1) {
3005 		addr = EDC_REG(EDC_INT_CAUSE, idx);
3006 		cnt_addr = EDC_REG(EDC_ECC_STATUS, idx);
3007 	} else {
3008 		addr = MC_INT_CAUSE;
3009 		cnt_addr = MC_ECC_STATUS;
3010 	}
3011 
3012 	v = csio_rd_reg32(hw, addr) & MEM_INT_MASK;
3013 	if (v & PERR_INT_CAUSE)
3014 		csio_fatal(hw, "%s FIFO parity error\n", name[idx]);
3015 	if (v & ECC_CE_INT_CAUSE) {
3016 		uint32_t cnt = ECC_CECNT_GET(csio_rd_reg32(hw, cnt_addr));
3017 
3018 		csio_wr_reg32(hw, ECC_CECNT_MASK, cnt_addr);
3019 		csio_warn(hw, "%u %s correctable ECC data error%s\n",
3020 			    cnt, name[idx], cnt > 1 ? "s" : "");
3021 	}
3022 	if (v & ECC_UE_INT_CAUSE)
3023 		csio_fatal(hw, "%s uncorrectable ECC data error\n", name[idx]);
3024 
3025 	csio_wr_reg32(hw, v, addr);
3026 	if (v & (PERR_INT_CAUSE | ECC_UE_INT_CAUSE))
3027 		csio_hw_fatal_err(hw);
3028 }
3029 
3030 /*
3031  * MA interrupt handler.
3032  */
3033 static void csio_ma_intr_handler(struct csio_hw *hw)
3034 {
3035 	uint32_t v, status = csio_rd_reg32(hw, MA_INT_CAUSE);
3036 
3037 	if (status & MEM_PERR_INT_CAUSE)
3038 		csio_fatal(hw, "MA parity error, parity status %#x\n",
3039 			    csio_rd_reg32(hw, MA_PARITY_ERROR_STATUS));
3040 	if (status & MEM_WRAP_INT_CAUSE) {
3041 		v = csio_rd_reg32(hw, MA_INT_WRAP_STATUS);
3042 		csio_fatal(hw,
3043 		   "MA address wrap-around error by client %u to address %#x\n",
3044 		   MEM_WRAP_CLIENT_NUM_GET(v), MEM_WRAP_ADDRESS_GET(v) << 4);
3045 	}
3046 	csio_wr_reg32(hw, status, MA_INT_CAUSE);
3047 	csio_hw_fatal_err(hw);
3048 }
3049 
3050 /*
3051  * SMB interrupt handler.
3052  */
3053 static void csio_smb_intr_handler(struct csio_hw *hw)
3054 {
3055 	static struct intr_info smb_intr_info[] = {
3056 		{ MSTTXFIFOPARINT, "SMB master Tx FIFO parity error", -1, 1 },
3057 		{ MSTRXFIFOPARINT, "SMB master Rx FIFO parity error", -1, 1 },
3058 		{ SLVFIFOPARINT, "SMB slave FIFO parity error", -1, 1 },
3059 		{ 0, NULL, 0, 0 }
3060 	};
3061 
3062 	if (csio_handle_intr_status(hw, SMB_INT_CAUSE, smb_intr_info))
3063 		csio_hw_fatal_err(hw);
3064 }
3065 
3066 /*
3067  * NC-SI interrupt handler.
3068  */
3069 static void csio_ncsi_intr_handler(struct csio_hw *hw)
3070 {
3071 	static struct intr_info ncsi_intr_info[] = {
3072 		{ CIM_DM_PRTY_ERR, "NC-SI CIM parity error", -1, 1 },
3073 		{ MPS_DM_PRTY_ERR, "NC-SI MPS parity error", -1, 1 },
3074 		{ TXFIFO_PRTY_ERR, "NC-SI Tx FIFO parity error", -1, 1 },
3075 		{ RXFIFO_PRTY_ERR, "NC-SI Rx FIFO parity error", -1, 1 },
3076 		{ 0, NULL, 0, 0 }
3077 	};
3078 
3079 	if (csio_handle_intr_status(hw, NCSI_INT_CAUSE, ncsi_intr_info))
3080 		csio_hw_fatal_err(hw);
3081 }
3082 
3083 /*
3084  * XGMAC interrupt handler.
3085  */
3086 static void csio_xgmac_intr_handler(struct csio_hw *hw, int port)
3087 {
3088 	uint32_t v = csio_rd_reg32(hw, CSIO_MAC_INT_CAUSE_REG(hw, port));
3089 
3090 	v &= TXFIFO_PRTY_ERR | RXFIFO_PRTY_ERR;
3091 	if (!v)
3092 		return;
3093 
3094 	if (v & TXFIFO_PRTY_ERR)
3095 		csio_fatal(hw, "XGMAC %d Tx FIFO parity error\n", port);
3096 	if (v & RXFIFO_PRTY_ERR)
3097 		csio_fatal(hw, "XGMAC %d Rx FIFO parity error\n", port);
3098 	csio_wr_reg32(hw, v, CSIO_MAC_INT_CAUSE_REG(hw, port));
3099 	csio_hw_fatal_err(hw);
3100 }
3101 
3102 /*
3103  * PL interrupt handler.
3104  */
3105 static void csio_pl_intr_handler(struct csio_hw *hw)
3106 {
3107 	static struct intr_info pl_intr_info[] = {
3108 		{ FATALPERR, "T4 fatal parity error", -1, 1 },
3109 		{ PERRVFID, "PL VFID_MAP parity error", -1, 1 },
3110 		{ 0, NULL, 0, 0 }
3111 	};
3112 
3113 	if (csio_handle_intr_status(hw, PL_PL_INT_CAUSE, pl_intr_info))
3114 		csio_hw_fatal_err(hw);
3115 }
3116 
3117 /*
3118  *	csio_hw_slow_intr_handler - control path interrupt handler
3119  *	@hw: HW module
3120  *
3121  *	Interrupt handler for non-data global interrupt events, e.g., errors.
3122  *	The designation 'slow' is because it involves register reads, while
3123  *	data interrupts typically don't involve any MMIOs.
3124  */
3125 int
3126 csio_hw_slow_intr_handler(struct csio_hw *hw)
3127 {
3128 	uint32_t cause = csio_rd_reg32(hw, PL_INT_CAUSE);
3129 
3130 	if (!(cause & CSIO_GLBL_INTR_MASK)) {
3131 		CSIO_INC_STATS(hw, n_plint_unexp);
3132 		return 0;
3133 	}
3134 
3135 	csio_dbg(hw, "Slow interrupt! cause: 0x%x\n", cause);
3136 
3137 	CSIO_INC_STATS(hw, n_plint_cnt);
3138 
3139 	if (cause & CIM)
3140 		csio_cim_intr_handler(hw);
3141 
3142 	if (cause & MPS)
3143 		csio_mps_intr_handler(hw);
3144 
3145 	if (cause & NCSI)
3146 		csio_ncsi_intr_handler(hw);
3147 
3148 	if (cause & PL)
3149 		csio_pl_intr_handler(hw);
3150 
3151 	if (cause & SMB)
3152 		csio_smb_intr_handler(hw);
3153 
3154 	if (cause & XGMAC0)
3155 		csio_xgmac_intr_handler(hw, 0);
3156 
3157 	if (cause & XGMAC1)
3158 		csio_xgmac_intr_handler(hw, 1);
3159 
3160 	if (cause & XGMAC_KR0)
3161 		csio_xgmac_intr_handler(hw, 2);
3162 
3163 	if (cause & XGMAC_KR1)
3164 		csio_xgmac_intr_handler(hw, 3);
3165 
3166 	if (cause & PCIE)
3167 		hw->chip_ops->chip_pcie_intr_handler(hw);
3168 
3169 	if (cause & MC)
3170 		csio_mem_intr_handler(hw, MEM_MC);
3171 
3172 	if (cause & EDC0)
3173 		csio_mem_intr_handler(hw, MEM_EDC0);
3174 
3175 	if (cause & EDC1)
3176 		csio_mem_intr_handler(hw, MEM_EDC1);
3177 
3178 	if (cause & LE)
3179 		csio_le_intr_handler(hw);
3180 
3181 	if (cause & TP)
3182 		csio_tp_intr_handler(hw);
3183 
3184 	if (cause & MA)
3185 		csio_ma_intr_handler(hw);
3186 
3187 	if (cause & PM_TX)
3188 		csio_pmtx_intr_handler(hw);
3189 
3190 	if (cause & PM_RX)
3191 		csio_pmrx_intr_handler(hw);
3192 
3193 	if (cause & ULP_RX)
3194 		csio_ulprx_intr_handler(hw);
3195 
3196 	if (cause & CPL_SWITCH)
3197 		csio_cplsw_intr_handler(hw);
3198 
3199 	if (cause & SGE)
3200 		csio_sge_intr_handler(hw);
3201 
3202 	if (cause & ULP_TX)
3203 		csio_ulptx_intr_handler(hw);
3204 
3205 	/* Clear the interrupts just processed for which we are the master. */
3206 	csio_wr_reg32(hw, cause & CSIO_GLBL_INTR_MASK, PL_INT_CAUSE);
3207 	csio_rd_reg32(hw, PL_INT_CAUSE); /* flush */
3208 
3209 	return 1;
3210 }
3211 
3212 /*****************************************************************************
3213  * HW <--> mailbox interfacing routines.
3214  ****************************************************************************/
3215 /*
3216  * csio_mberr_worker - Worker thread (dpc) for mailbox/error completions
3217  *
3218  * @data: Private data pointer.
3219  *
3220  * Called from worker thread context.
3221  */
3222 static void
3223 csio_mberr_worker(void *data)
3224 {
3225 	struct csio_hw *hw = (struct csio_hw *)data;
3226 	struct csio_mbm *mbm = &hw->mbm;
3227 	LIST_HEAD(cbfn_q);
3228 	struct csio_mb *mbp_next;
3229 	int rv;
3230 
3231 	del_timer_sync(&mbm->timer);
3232 
3233 	spin_lock_irq(&hw->lock);
3234 	if (list_empty(&mbm->cbfn_q)) {
3235 		spin_unlock_irq(&hw->lock);
3236 		return;
3237 	}
3238 
3239 	list_splice_tail_init(&mbm->cbfn_q, &cbfn_q);
3240 	mbm->stats.n_cbfnq = 0;
3241 
3242 	/* Try to start waiting mailboxes */
3243 	if (!list_empty(&mbm->req_q)) {
3244 		mbp_next = list_first_entry(&mbm->req_q, struct csio_mb, list);
3245 		list_del_init(&mbp_next->list);
3246 
3247 		rv = csio_mb_issue(hw, mbp_next);
3248 		if (rv != 0)
3249 			list_add_tail(&mbp_next->list, &mbm->req_q);
3250 		else
3251 			CSIO_DEC_STATS(mbm, n_activeq);
3252 	}
3253 	spin_unlock_irq(&hw->lock);
3254 
3255 	/* Now callback completions */
3256 	csio_mb_completions(hw, &cbfn_q);
3257 }
3258 
3259 /*
3260  * csio_hw_mb_timer - Top-level Mailbox timeout handler.
3261  *
3262  * @data: private data pointer
3263  *
3264  **/
3265 static void
3266 csio_hw_mb_timer(uintptr_t data)
3267 {
3268 	struct csio_hw *hw = (struct csio_hw *)data;
3269 	struct csio_mb *mbp = NULL;
3270 
3271 	spin_lock_irq(&hw->lock);
3272 	mbp = csio_mb_tmo_handler(hw);
3273 	spin_unlock_irq(&hw->lock);
3274 
3275 	/* Call back the function for the timed-out Mailbox */
3276 	if (mbp)
3277 		mbp->mb_cbfn(hw, mbp);
3278 
3279 }
3280 
3281 /*
3282  * csio_hw_mbm_cleanup - Cleanup Mailbox module.
3283  * @hw: HW module
3284  *
3285  * Called with lock held, should exit with lock held.
3286  * Cancels outstanding mailboxes (waiting, in-flight) and gathers them
3287  * into a local queue. Drops lock and calls the completions. Holds
3288  * lock and returns.
3289  */
3290 static void
3291 csio_hw_mbm_cleanup(struct csio_hw *hw)
3292 {
3293 	LIST_HEAD(cbfn_q);
3294 
3295 	csio_mb_cancel_all(hw, &cbfn_q);
3296 
3297 	spin_unlock_irq(&hw->lock);
3298 	csio_mb_completions(hw, &cbfn_q);
3299 	spin_lock_irq(&hw->lock);
3300 }
3301 
3302 /*****************************************************************************
3303  * Event handling
3304  ****************************************************************************/
3305 int
3306 csio_enqueue_evt(struct csio_hw *hw, enum csio_evt type, void *evt_msg,
3307 			uint16_t len)
3308 {
3309 	struct csio_evt_msg *evt_entry = NULL;
3310 
3311 	if (type >= CSIO_EVT_MAX)
3312 		return -EINVAL;
3313 
3314 	if (len > CSIO_EVT_MSG_SIZE)
3315 		return -EINVAL;
3316 
3317 	if (hw->flags & CSIO_HWF_FWEVT_STOP)
3318 		return -EINVAL;
3319 
3320 	if (list_empty(&hw->evt_free_q)) {
3321 		csio_err(hw, "Failed to alloc evt entry, msg type %d len %d\n",
3322 			 type, len);
3323 		return -ENOMEM;
3324 	}
3325 
3326 	evt_entry = list_first_entry(&hw->evt_free_q,
3327 				     struct csio_evt_msg, list);
3328 	list_del_init(&evt_entry->list);
3329 
3330 	/* copy event msg and queue the event */
3331 	evt_entry->type = type;
3332 	memcpy((void *)evt_entry->data, evt_msg, len);
3333 	list_add_tail(&evt_entry->list, &hw->evt_active_q);
3334 
3335 	CSIO_DEC_STATS(hw, n_evt_freeq);
3336 	CSIO_INC_STATS(hw, n_evt_activeq);
3337 
3338 	return 0;
3339 }
3340 
3341 static int
3342 csio_enqueue_evt_lock(struct csio_hw *hw, enum csio_evt type, void *evt_msg,
3343 			uint16_t len, bool msg_sg)
3344 {
3345 	struct csio_evt_msg *evt_entry = NULL;
3346 	struct csio_fl_dma_buf *fl_sg;
3347 	uint32_t off = 0;
3348 	unsigned long flags;
3349 	int n, ret = 0;
3350 
3351 	if (type >= CSIO_EVT_MAX)
3352 		return -EINVAL;
3353 
3354 	if (len > CSIO_EVT_MSG_SIZE)
3355 		return -EINVAL;
3356 
3357 	spin_lock_irqsave(&hw->lock, flags);
3358 	if (hw->flags & CSIO_HWF_FWEVT_STOP) {
3359 		ret = -EINVAL;
3360 		goto out;
3361 	}
3362 
3363 	if (list_empty(&hw->evt_free_q)) {
3364 		csio_err(hw, "Failed to alloc evt entry, msg type %d len %d\n",
3365 			 type, len);
3366 		ret = -ENOMEM;
3367 		goto out;
3368 	}
3369 
3370 	evt_entry = list_first_entry(&hw->evt_free_q,
3371 				     struct csio_evt_msg, list);
3372 	list_del_init(&evt_entry->list);
3373 
3374 	/* copy event msg and queue the event */
3375 	evt_entry->type = type;
3376 
3377 	/* If Payload in SG list*/
3378 	if (msg_sg) {
3379 		fl_sg = (struct csio_fl_dma_buf *) evt_msg;
3380 		for (n = 0; (n < CSIO_MAX_FLBUF_PER_IQWR && off < len); n++) {
3381 			memcpy((void *)((uintptr_t)evt_entry->data + off),
3382 				fl_sg->flbufs[n].vaddr,
3383 				fl_sg->flbufs[n].len);
3384 			off += fl_sg->flbufs[n].len;
3385 		}
3386 	} else
3387 		memcpy((void *)evt_entry->data, evt_msg, len);
3388 
3389 	list_add_tail(&evt_entry->list, &hw->evt_active_q);
3390 	CSIO_DEC_STATS(hw, n_evt_freeq);
3391 	CSIO_INC_STATS(hw, n_evt_activeq);
3392 out:
3393 	spin_unlock_irqrestore(&hw->lock, flags);
3394 	return ret;
3395 }
3396 
3397 static void
3398 csio_free_evt(struct csio_hw *hw, struct csio_evt_msg *evt_entry)
3399 {
3400 	if (evt_entry) {
3401 		spin_lock_irq(&hw->lock);
3402 		list_del_init(&evt_entry->list);
3403 		list_add_tail(&evt_entry->list, &hw->evt_free_q);
3404 		CSIO_DEC_STATS(hw, n_evt_activeq);
3405 		CSIO_INC_STATS(hw, n_evt_freeq);
3406 		spin_unlock_irq(&hw->lock);
3407 	}
3408 }
3409 
3410 void
3411 csio_evtq_flush(struct csio_hw *hw)
3412 {
3413 	uint32_t count;
3414 	count = 30;
3415 	while (hw->flags & CSIO_HWF_FWEVT_PENDING && count--) {
3416 		spin_unlock_irq(&hw->lock);
3417 		msleep(2000);
3418 		spin_lock_irq(&hw->lock);
3419 	}
3420 
3421 	CSIO_DB_ASSERT(!(hw->flags & CSIO_HWF_FWEVT_PENDING));
3422 }
3423 
3424 static void
3425 csio_evtq_stop(struct csio_hw *hw)
3426 {
3427 	hw->flags |= CSIO_HWF_FWEVT_STOP;
3428 }
3429 
3430 static void
3431 csio_evtq_start(struct csio_hw *hw)
3432 {
3433 	hw->flags &= ~CSIO_HWF_FWEVT_STOP;
3434 }
3435 
3436 static void
3437 csio_evtq_cleanup(struct csio_hw *hw)
3438 {
3439 	struct list_head *evt_entry, *next_entry;
3440 
3441 	/* Release outstanding events from activeq to freeq*/
3442 	if (!list_empty(&hw->evt_active_q))
3443 		list_splice_tail_init(&hw->evt_active_q, &hw->evt_free_q);
3444 
3445 	hw->stats.n_evt_activeq = 0;
3446 	hw->flags &= ~CSIO_HWF_FWEVT_PENDING;
3447 
3448 	/* Freeup event entry */
3449 	list_for_each_safe(evt_entry, next_entry, &hw->evt_free_q) {
3450 		kfree(evt_entry);
3451 		CSIO_DEC_STATS(hw, n_evt_freeq);
3452 	}
3453 
3454 	hw->stats.n_evt_freeq = 0;
3455 }
3456 
3457 
3458 static void
3459 csio_process_fwevtq_entry(struct csio_hw *hw, void *wr, uint32_t len,
3460 			  struct csio_fl_dma_buf *flb, void *priv)
3461 {
3462 	__u8 op;
3463 	void *msg = NULL;
3464 	uint32_t msg_len = 0;
3465 	bool msg_sg = 0;
3466 
3467 	op = ((struct rss_header *) wr)->opcode;
3468 	if (op == CPL_FW6_PLD) {
3469 		CSIO_INC_STATS(hw, n_cpl_fw6_pld);
3470 		if (!flb || !flb->totlen) {
3471 			CSIO_INC_STATS(hw, n_cpl_unexp);
3472 			return;
3473 		}
3474 
3475 		msg = (void *) flb;
3476 		msg_len = flb->totlen;
3477 		msg_sg = 1;
3478 	} else if (op == CPL_FW6_MSG || op == CPL_FW4_MSG) {
3479 
3480 		CSIO_INC_STATS(hw, n_cpl_fw6_msg);
3481 		/* skip RSS header */
3482 		msg = (void *)((uintptr_t)wr + sizeof(__be64));
3483 		msg_len = (op == CPL_FW6_MSG) ? sizeof(struct cpl_fw6_msg) :
3484 			   sizeof(struct cpl_fw4_msg);
3485 	} else {
3486 		csio_warn(hw, "unexpected CPL %#x on FW event queue\n", op);
3487 		CSIO_INC_STATS(hw, n_cpl_unexp);
3488 		return;
3489 	}
3490 
3491 	/*
3492 	 * Enqueue event to EventQ. Events processing happens
3493 	 * in Event worker thread context
3494 	 */
3495 	if (csio_enqueue_evt_lock(hw, CSIO_EVT_FW, msg,
3496 				  (uint16_t)msg_len, msg_sg))
3497 		CSIO_INC_STATS(hw, n_evt_drop);
3498 }
3499 
3500 void
3501 csio_evtq_worker(struct work_struct *work)
3502 {
3503 	struct csio_hw *hw = container_of(work, struct csio_hw, evtq_work);
3504 	struct list_head *evt_entry, *next_entry;
3505 	LIST_HEAD(evt_q);
3506 	struct csio_evt_msg	*evt_msg;
3507 	struct cpl_fw6_msg *msg;
3508 	struct csio_rnode *rn;
3509 	int rv = 0;
3510 	uint8_t evtq_stop = 0;
3511 
3512 	csio_dbg(hw, "event worker thread active evts#%d\n",
3513 		 hw->stats.n_evt_activeq);
3514 
3515 	spin_lock_irq(&hw->lock);
3516 	while (!list_empty(&hw->evt_active_q)) {
3517 		list_splice_tail_init(&hw->evt_active_q, &evt_q);
3518 		spin_unlock_irq(&hw->lock);
3519 
3520 		list_for_each_safe(evt_entry, next_entry, &evt_q) {
3521 			evt_msg = (struct csio_evt_msg *) evt_entry;
3522 
3523 			/* Drop events if queue is STOPPED */
3524 			spin_lock_irq(&hw->lock);
3525 			if (hw->flags & CSIO_HWF_FWEVT_STOP)
3526 				evtq_stop = 1;
3527 			spin_unlock_irq(&hw->lock);
3528 			if (evtq_stop) {
3529 				CSIO_INC_STATS(hw, n_evt_drop);
3530 				goto free_evt;
3531 			}
3532 
3533 			switch (evt_msg->type) {
3534 			case CSIO_EVT_FW:
3535 				msg = (struct cpl_fw6_msg *)(evt_msg->data);
3536 
3537 				if ((msg->opcode == CPL_FW6_MSG ||
3538 				     msg->opcode == CPL_FW4_MSG) &&
3539 				    !msg->type) {
3540 					rv = csio_mb_fwevt_handler(hw,
3541 								msg->data);
3542 					if (!rv)
3543 						break;
3544 					/* Handle any remaining fw events */
3545 					csio_fcoe_fwevt_handler(hw,
3546 							msg->opcode, msg->data);
3547 				} else if (msg->opcode == CPL_FW6_PLD) {
3548 
3549 					csio_fcoe_fwevt_handler(hw,
3550 							msg->opcode, msg->data);
3551 				} else {
3552 					csio_warn(hw,
3553 					     "Unhandled FW msg op %x type %x\n",
3554 						  msg->opcode, msg->type);
3555 					CSIO_INC_STATS(hw, n_evt_drop);
3556 				}
3557 				break;
3558 
3559 			case CSIO_EVT_MBX:
3560 				csio_mberr_worker(hw);
3561 				break;
3562 
3563 			case CSIO_EVT_DEV_LOSS:
3564 				memcpy(&rn, evt_msg->data, sizeof(rn));
3565 				csio_rnode_devloss_handler(rn);
3566 				break;
3567 
3568 			default:
3569 				csio_warn(hw, "Unhandled event %x on evtq\n",
3570 					  evt_msg->type);
3571 				CSIO_INC_STATS(hw, n_evt_unexp);
3572 				break;
3573 			}
3574 free_evt:
3575 			csio_free_evt(hw, evt_msg);
3576 		}
3577 
3578 		spin_lock_irq(&hw->lock);
3579 	}
3580 	hw->flags &= ~CSIO_HWF_FWEVT_PENDING;
3581 	spin_unlock_irq(&hw->lock);
3582 }
3583 
3584 int
3585 csio_fwevtq_handler(struct csio_hw *hw)
3586 {
3587 	int rv;
3588 
3589 	if (csio_q_iqid(hw, hw->fwevt_iq_idx) == CSIO_MAX_QID) {
3590 		CSIO_INC_STATS(hw, n_int_stray);
3591 		return -EINVAL;
3592 	}
3593 
3594 	rv = csio_wr_process_iq_idx(hw, hw->fwevt_iq_idx,
3595 			   csio_process_fwevtq_entry, NULL);
3596 	return rv;
3597 }
3598 
3599 /****************************************************************************
3600  * Entry points
3601  ****************************************************************************/
3602 
3603 /* Management module */
3604 /*
3605  * csio_mgmt_req_lookup - Lookup the given IO req exist in Active Q.
3606  * mgmt - mgmt module
3607  * @io_req - io request
3608  *
3609  * Return - 0:if given IO Req exists in active Q.
3610  *          -EINVAL  :if lookup fails.
3611  */
3612 int
3613 csio_mgmt_req_lookup(struct csio_mgmtm *mgmtm, struct csio_ioreq *io_req)
3614 {
3615 	struct list_head *tmp;
3616 
3617 	/* Lookup ioreq in the ACTIVEQ */
3618 	list_for_each(tmp, &mgmtm->active_q) {
3619 		if (io_req == (struct csio_ioreq *)tmp)
3620 			return 0;
3621 	}
3622 	return -EINVAL;
3623 }
3624 
3625 #define	ECM_MIN_TMO	1000	/* Minimum timeout value for req */
3626 
3627 /*
3628  * csio_mgmts_tmo_handler - MGMT IO Timeout handler.
3629  * @data - Event data.
3630  *
3631  * Return - none.
3632  */
3633 static void
3634 csio_mgmt_tmo_handler(uintptr_t data)
3635 {
3636 	struct csio_mgmtm *mgmtm = (struct csio_mgmtm *) data;
3637 	struct list_head *tmp;
3638 	struct csio_ioreq *io_req;
3639 
3640 	csio_dbg(mgmtm->hw, "Mgmt timer invoked!\n");
3641 
3642 	spin_lock_irq(&mgmtm->hw->lock);
3643 
3644 	list_for_each(tmp, &mgmtm->active_q) {
3645 		io_req = (struct csio_ioreq *) tmp;
3646 		io_req->tmo -= min_t(uint32_t, io_req->tmo, ECM_MIN_TMO);
3647 
3648 		if (!io_req->tmo) {
3649 			/* Dequeue the request from retry Q. */
3650 			tmp = csio_list_prev(tmp);
3651 			list_del_init(&io_req->sm.sm_list);
3652 			if (io_req->io_cbfn) {
3653 				/* io_req will be freed by completion handler */
3654 				io_req->wr_status = -ETIMEDOUT;
3655 				io_req->io_cbfn(mgmtm->hw, io_req);
3656 			} else {
3657 				CSIO_DB_ASSERT(0);
3658 			}
3659 		}
3660 	}
3661 
3662 	/* If retry queue is not empty, re-arm timer */
3663 	if (!list_empty(&mgmtm->active_q))
3664 		mod_timer(&mgmtm->mgmt_timer,
3665 			  jiffies + msecs_to_jiffies(ECM_MIN_TMO));
3666 	spin_unlock_irq(&mgmtm->hw->lock);
3667 }
3668 
3669 static void
3670 csio_mgmtm_cleanup(struct csio_mgmtm *mgmtm)
3671 {
3672 	struct csio_hw *hw = mgmtm->hw;
3673 	struct csio_ioreq *io_req;
3674 	struct list_head *tmp;
3675 	uint32_t count;
3676 
3677 	count = 30;
3678 	/* Wait for all outstanding req to complete gracefully */
3679 	while ((!list_empty(&mgmtm->active_q)) && count--) {
3680 		spin_unlock_irq(&hw->lock);
3681 		msleep(2000);
3682 		spin_lock_irq(&hw->lock);
3683 	}
3684 
3685 	/* release outstanding req from ACTIVEQ */
3686 	list_for_each(tmp, &mgmtm->active_q) {
3687 		io_req = (struct csio_ioreq *) tmp;
3688 		tmp = csio_list_prev(tmp);
3689 		list_del_init(&io_req->sm.sm_list);
3690 		mgmtm->stats.n_active--;
3691 		if (io_req->io_cbfn) {
3692 			/* io_req will be freed by completion handler */
3693 			io_req->wr_status = -ETIMEDOUT;
3694 			io_req->io_cbfn(mgmtm->hw, io_req);
3695 		}
3696 	}
3697 }
3698 
3699 /*
3700  * csio_mgmt_init - Mgmt module init entry point
3701  * @mgmtsm - mgmt module
3702  * @hw	 - HW module
3703  *
3704  * Initialize mgmt timer, resource wait queue, active queue,
3705  * completion q. Allocate Egress and Ingress
3706  * WR queues and save off the queue index returned by the WR
3707  * module for future use. Allocate and save off mgmt reqs in the
3708  * mgmt_req_freelist for future use. Make sure their SM is initialized
3709  * to uninit state.
3710  * Returns: 0 - on success
3711  *          -ENOMEM   - on error.
3712  */
3713 static int
3714 csio_mgmtm_init(struct csio_mgmtm *mgmtm, struct csio_hw *hw)
3715 {
3716 	struct timer_list *timer = &mgmtm->mgmt_timer;
3717 
3718 	init_timer(timer);
3719 	timer->function = csio_mgmt_tmo_handler;
3720 	timer->data = (unsigned long)mgmtm;
3721 
3722 	INIT_LIST_HEAD(&mgmtm->active_q);
3723 	INIT_LIST_HEAD(&mgmtm->cbfn_q);
3724 
3725 	mgmtm->hw = hw;
3726 	/*mgmtm->iq_idx = hw->fwevt_iq_idx;*/
3727 
3728 	return 0;
3729 }
3730 
3731 /*
3732  * csio_mgmtm_exit - MGMT module exit entry point
3733  * @mgmtsm - mgmt module
3734  *
3735  * This function called during MGMT module uninit.
3736  * Stop timers, free ioreqs allocated.
3737  * Returns: None
3738  *
3739  */
3740 static void
3741 csio_mgmtm_exit(struct csio_mgmtm *mgmtm)
3742 {
3743 	del_timer_sync(&mgmtm->mgmt_timer);
3744 }
3745 
3746 
3747 /**
3748  * csio_hw_start - Kicks off the HW State machine
3749  * @hw:		Pointer to HW module.
3750  *
3751  * It is assumed that the initialization is a synchronous operation.
3752  * So when we return afer posting the event, the HW SM should be in
3753  * the ready state, if there were no errors during init.
3754  */
3755 int
3756 csio_hw_start(struct csio_hw *hw)
3757 {
3758 	spin_lock_irq(&hw->lock);
3759 	csio_post_event(&hw->sm, CSIO_HWE_CFG);
3760 	spin_unlock_irq(&hw->lock);
3761 
3762 	if (csio_is_hw_ready(hw))
3763 		return 0;
3764 	else
3765 		return -EINVAL;
3766 }
3767 
3768 int
3769 csio_hw_stop(struct csio_hw *hw)
3770 {
3771 	csio_post_event(&hw->sm, CSIO_HWE_PCI_REMOVE);
3772 
3773 	if (csio_is_hw_removing(hw))
3774 		return 0;
3775 	else
3776 		return -EINVAL;
3777 }
3778 
3779 /* Max reset retries */
3780 #define CSIO_MAX_RESET_RETRIES	3
3781 
3782 /**
3783  * csio_hw_reset - Reset the hardware
3784  * @hw:		HW module.
3785  *
3786  * Caller should hold lock across this function.
3787  */
3788 int
3789 csio_hw_reset(struct csio_hw *hw)
3790 {
3791 	if (!csio_is_hw_master(hw))
3792 		return -EPERM;
3793 
3794 	if (hw->rst_retries >= CSIO_MAX_RESET_RETRIES) {
3795 		csio_dbg(hw, "Max hw reset attempts reached..");
3796 		return -EINVAL;
3797 	}
3798 
3799 	hw->rst_retries++;
3800 	csio_post_event(&hw->sm, CSIO_HWE_HBA_RESET);
3801 
3802 	if (csio_is_hw_ready(hw)) {
3803 		hw->rst_retries = 0;
3804 		hw->stats.n_reset_start = jiffies_to_msecs(jiffies);
3805 		return 0;
3806 	} else
3807 		return -EINVAL;
3808 }
3809 
3810 /*
3811  * csio_hw_get_device_id - Caches the Adapter's vendor & device id.
3812  * @hw: HW module.
3813  */
3814 static void
3815 csio_hw_get_device_id(struct csio_hw *hw)
3816 {
3817 	/* Is the adapter device id cached already ?*/
3818 	if (csio_is_dev_id_cached(hw))
3819 		return;
3820 
3821 	/* Get the PCI vendor & device id */
3822 	pci_read_config_word(hw->pdev, PCI_VENDOR_ID,
3823 			     &hw->params.pci.vendor_id);
3824 	pci_read_config_word(hw->pdev, PCI_DEVICE_ID,
3825 			     &hw->params.pci.device_id);
3826 
3827 	csio_dev_id_cached(hw);
3828 	hw->chip_id = (hw->params.pci.device_id & CSIO_HW_CHIP_MASK);
3829 
3830 } /* csio_hw_get_device_id */
3831 
3832 /*
3833  * csio_hw_set_description - Set the model, description of the hw.
3834  * @hw: HW module.
3835  * @ven_id: PCI Vendor ID
3836  * @dev_id: PCI Device ID
3837  */
3838 static void
3839 csio_hw_set_description(struct csio_hw *hw, uint16_t ven_id, uint16_t dev_id)
3840 {
3841 	uint32_t adap_type, prot_type;
3842 
3843 	if (ven_id == CSIO_VENDOR_ID) {
3844 		prot_type = (dev_id & CSIO_ASIC_DEVID_PROTO_MASK);
3845 		adap_type = (dev_id & CSIO_ASIC_DEVID_TYPE_MASK);
3846 
3847 		if (prot_type == CSIO_T4_FCOE_ASIC) {
3848 			memcpy(hw->hw_ver,
3849 			       csio_t4_fcoe_adapters[adap_type].model_no, 16);
3850 			memcpy(hw->model_desc,
3851 			       csio_t4_fcoe_adapters[adap_type].description,
3852 			       32);
3853 		} else if (prot_type == CSIO_T5_FCOE_ASIC) {
3854 			memcpy(hw->hw_ver,
3855 			       csio_t5_fcoe_adapters[adap_type].model_no, 16);
3856 			memcpy(hw->model_desc,
3857 			       csio_t5_fcoe_adapters[adap_type].description,
3858 			       32);
3859 		} else {
3860 			char tempName[32] = "Chelsio FCoE Controller";
3861 			memcpy(hw->model_desc, tempName, 32);
3862 		}
3863 	}
3864 } /* csio_hw_set_description */
3865 
3866 /**
3867  * csio_hw_init - Initialize HW module.
3868  * @hw:		Pointer to HW module.
3869  *
3870  * Initialize the members of the HW module.
3871  */
3872 int
3873 csio_hw_init(struct csio_hw *hw)
3874 {
3875 	int rv = -EINVAL;
3876 	uint32_t i;
3877 	uint16_t ven_id, dev_id;
3878 	struct csio_evt_msg	*evt_entry;
3879 
3880 	INIT_LIST_HEAD(&hw->sm.sm_list);
3881 	csio_init_state(&hw->sm, csio_hws_uninit);
3882 	spin_lock_init(&hw->lock);
3883 	INIT_LIST_HEAD(&hw->sln_head);
3884 
3885 	/* Get the PCI vendor & device id */
3886 	csio_hw_get_device_id(hw);
3887 
3888 	strcpy(hw->name, CSIO_HW_NAME);
3889 
3890 	/* Initialize the HW chip ops with T4/T5 specific ops */
3891 	hw->chip_ops = csio_is_t4(hw->chip_id) ? &t4_ops : &t5_ops;
3892 
3893 	/* Set the model & its description */
3894 
3895 	ven_id = hw->params.pci.vendor_id;
3896 	dev_id = hw->params.pci.device_id;
3897 
3898 	csio_hw_set_description(hw, ven_id, dev_id);
3899 
3900 	/* Initialize default log level */
3901 	hw->params.log_level = (uint32_t) csio_dbg_level;
3902 
3903 	csio_set_fwevt_intr_idx(hw, -1);
3904 	csio_set_nondata_intr_idx(hw, -1);
3905 
3906 	/* Init all the modules: Mailbox, WorkRequest and Transport */
3907 	if (csio_mbm_init(csio_hw_to_mbm(hw), hw, csio_hw_mb_timer))
3908 		goto err;
3909 
3910 	rv = csio_wrm_init(csio_hw_to_wrm(hw), hw);
3911 	if (rv)
3912 		goto err_mbm_exit;
3913 
3914 	rv = csio_scsim_init(csio_hw_to_scsim(hw), hw);
3915 	if (rv)
3916 		goto err_wrm_exit;
3917 
3918 	rv = csio_mgmtm_init(csio_hw_to_mgmtm(hw), hw);
3919 	if (rv)
3920 		goto err_scsim_exit;
3921 	/* Pre-allocate evtq and initialize them */
3922 	INIT_LIST_HEAD(&hw->evt_active_q);
3923 	INIT_LIST_HEAD(&hw->evt_free_q);
3924 	for (i = 0; i < csio_evtq_sz; i++) {
3925 
3926 		evt_entry = kzalloc(sizeof(struct csio_evt_msg), GFP_KERNEL);
3927 		if (!evt_entry) {
3928 			csio_err(hw, "Failed to initialize eventq");
3929 			goto err_evtq_cleanup;
3930 		}
3931 
3932 		list_add_tail(&evt_entry->list, &hw->evt_free_q);
3933 		CSIO_INC_STATS(hw, n_evt_freeq);
3934 	}
3935 
3936 	hw->dev_num = dev_num;
3937 	dev_num++;
3938 
3939 	return 0;
3940 
3941 err_evtq_cleanup:
3942 	csio_evtq_cleanup(hw);
3943 	csio_mgmtm_exit(csio_hw_to_mgmtm(hw));
3944 err_scsim_exit:
3945 	csio_scsim_exit(csio_hw_to_scsim(hw));
3946 err_wrm_exit:
3947 	csio_wrm_exit(csio_hw_to_wrm(hw), hw);
3948 err_mbm_exit:
3949 	csio_mbm_exit(csio_hw_to_mbm(hw));
3950 err:
3951 	return rv;
3952 }
3953 
3954 /**
3955  * csio_hw_exit - Un-initialize HW module.
3956  * @hw:		Pointer to HW module.
3957  *
3958  */
3959 void
3960 csio_hw_exit(struct csio_hw *hw)
3961 {
3962 	csio_evtq_cleanup(hw);
3963 	csio_mgmtm_exit(csio_hw_to_mgmtm(hw));
3964 	csio_scsim_exit(csio_hw_to_scsim(hw));
3965 	csio_wrm_exit(csio_hw_to_wrm(hw), hw);
3966 	csio_mbm_exit(csio_hw_to_mbm(hw));
3967 }
3968