1 /* bnx2x_cmn.c: QLogic Everest network driver.
2  *
3  * Copyright (c) 2007-2013 Broadcom Corporation
4  * Copyright (c) 2014 QLogic Corporation
5  * All rights reserved
6  *
7  * This program is free software; you can redistribute it and/or modify
8  * it under the terms of the GNU General Public License as published by
9  * the Free Software Foundation.
10  *
11  * Maintained by: Ariel Elior <ariel.elior@qlogic.com>
12  * Written by: Eliezer Tamir
13  * Based on code from Michael Chan's bnx2 driver
14  * UDP CSUM errata workaround by Arik Gendelman
15  * Slowpath and fastpath rework by Vladislav Zolotarov
16  * Statistics and Link management by Yitchak Gertner
17  *
18  */
19 
20 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
21 
22 #include <linux/etherdevice.h>
23 #include <linux/if_vlan.h>
24 #include <linux/interrupt.h>
25 #include <linux/ip.h>
26 #include <linux/crash_dump.h>
27 #include <net/tcp.h>
28 #include <net/ipv6.h>
29 #include <net/ip6_checksum.h>
30 #include <linux/prefetch.h>
31 #include "bnx2x_cmn.h"
32 #include "bnx2x_init.h"
33 #include "bnx2x_sp.h"
34 
35 static void bnx2x_free_fp_mem_cnic(struct bnx2x *bp);
36 static int bnx2x_alloc_fp_mem_cnic(struct bnx2x *bp);
37 static int bnx2x_alloc_fp_mem(struct bnx2x *bp);
38 static int bnx2x_poll(struct napi_struct *napi, int budget);
39 
40 static void bnx2x_add_all_napi_cnic(struct bnx2x *bp)
41 {
42 	int i;
43 
44 	/* Add NAPI objects */
45 	for_each_rx_queue_cnic(bp, i) {
46 		netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi),
47 			       bnx2x_poll, NAPI_POLL_WEIGHT);
48 	}
49 }
50 
51 static void bnx2x_add_all_napi(struct bnx2x *bp)
52 {
53 	int i;
54 
55 	/* Add NAPI objects */
56 	for_each_eth_queue(bp, i) {
57 		netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi),
58 			       bnx2x_poll, NAPI_POLL_WEIGHT);
59 	}
60 }
61 
62 static int bnx2x_calc_num_queues(struct bnx2x *bp)
63 {
64 	int nq = bnx2x_num_queues ? : netif_get_num_default_rss_queues();
65 
66 	/* Reduce memory usage in kdump environment by using only one queue */
67 	if (is_kdump_kernel())
68 		nq = 1;
69 
70 	nq = clamp(nq, 1, BNX2X_MAX_QUEUES(bp));
71 	return nq;
72 }
73 
74 /**
75  * bnx2x_move_fp - move content of the fastpath structure.
76  *
77  * @bp:		driver handle
78  * @from:	source FP index
79  * @to:		destination FP index
80  *
81  * Makes sure the contents of the bp->fp[to].napi is kept
82  * intact. This is done by first copying the napi struct from
83  * the target to the source, and then mem copying the entire
84  * source onto the target. Update txdata pointers and related
85  * content.
86  */
87 static inline void bnx2x_move_fp(struct bnx2x *bp, int from, int to)
88 {
89 	struct bnx2x_fastpath *from_fp = &bp->fp[from];
90 	struct bnx2x_fastpath *to_fp = &bp->fp[to];
91 	struct bnx2x_sp_objs *from_sp_objs = &bp->sp_objs[from];
92 	struct bnx2x_sp_objs *to_sp_objs = &bp->sp_objs[to];
93 	struct bnx2x_fp_stats *from_fp_stats = &bp->fp_stats[from];
94 	struct bnx2x_fp_stats *to_fp_stats = &bp->fp_stats[to];
95 	int old_max_eth_txqs, new_max_eth_txqs;
96 	int old_txdata_index = 0, new_txdata_index = 0;
97 	struct bnx2x_agg_info *old_tpa_info = to_fp->tpa_info;
98 
99 	/* Copy the NAPI object as it has been already initialized */
100 	from_fp->napi = to_fp->napi;
101 
102 	/* Move bnx2x_fastpath contents */
103 	memcpy(to_fp, from_fp, sizeof(*to_fp));
104 	to_fp->index = to;
105 
106 	/* Retain the tpa_info of the original `to' version as we don't want
107 	 * 2 FPs to contain the same tpa_info pointer.
108 	 */
109 	to_fp->tpa_info = old_tpa_info;
110 
111 	/* move sp_objs contents as well, as their indices match fp ones */
112 	memcpy(to_sp_objs, from_sp_objs, sizeof(*to_sp_objs));
113 
114 	/* move fp_stats contents as well, as their indices match fp ones */
115 	memcpy(to_fp_stats, from_fp_stats, sizeof(*to_fp_stats));
116 
117 	/* Update txdata pointers in fp and move txdata content accordingly:
118 	 * Each fp consumes 'max_cos' txdata structures, so the index should be
119 	 * decremented by max_cos x delta.
120 	 */
121 
122 	old_max_eth_txqs = BNX2X_NUM_ETH_QUEUES(bp) * (bp)->max_cos;
123 	new_max_eth_txqs = (BNX2X_NUM_ETH_QUEUES(bp) - from + to) *
124 				(bp)->max_cos;
125 	if (from == FCOE_IDX(bp)) {
126 		old_txdata_index = old_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
127 		new_txdata_index = new_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
128 	}
129 
130 	memcpy(&bp->bnx2x_txq[new_txdata_index],
131 	       &bp->bnx2x_txq[old_txdata_index],
132 	       sizeof(struct bnx2x_fp_txdata));
133 	to_fp->txdata_ptr[0] = &bp->bnx2x_txq[new_txdata_index];
134 }
135 
136 /**
137  * bnx2x_fill_fw_str - Fill buffer with FW version string.
138  *
139  * @bp:        driver handle
140  * @buf:       character buffer to fill with the fw name
141  * @buf_len:   length of the above buffer
142  *
143  */
144 void bnx2x_fill_fw_str(struct bnx2x *bp, char *buf, size_t buf_len)
145 {
146 	if (IS_PF(bp)) {
147 		u8 phy_fw_ver[PHY_FW_VER_LEN];
148 
149 		phy_fw_ver[0] = '\0';
150 		bnx2x_get_ext_phy_fw_version(&bp->link_params,
151 					     phy_fw_ver, PHY_FW_VER_LEN);
152 		strlcpy(buf, bp->fw_ver, buf_len);
153 		snprintf(buf + strlen(bp->fw_ver), 32 - strlen(bp->fw_ver),
154 			 "bc %d.%d.%d%s%s",
155 			 (bp->common.bc_ver & 0xff0000) >> 16,
156 			 (bp->common.bc_ver & 0xff00) >> 8,
157 			 (bp->common.bc_ver & 0xff),
158 			 ((phy_fw_ver[0] != '\0') ? " phy " : ""), phy_fw_ver);
159 	} else {
160 		bnx2x_vf_fill_fw_str(bp, buf, buf_len);
161 	}
162 }
163 
164 /**
165  * bnx2x_shrink_eth_fp - guarantees fastpath structures stay intact
166  *
167  * @bp:	driver handle
168  * @delta:	number of eth queues which were not allocated
169  */
170 static void bnx2x_shrink_eth_fp(struct bnx2x *bp, int delta)
171 {
172 	int i, cos, old_eth_num = BNX2X_NUM_ETH_QUEUES(bp);
173 
174 	/* Queue pointer cannot be re-set on an fp-basis, as moving pointer
175 	 * backward along the array could cause memory to be overridden
176 	 */
177 	for (cos = 1; cos < bp->max_cos; cos++) {
178 		for (i = 0; i < old_eth_num - delta; i++) {
179 			struct bnx2x_fastpath *fp = &bp->fp[i];
180 			int new_idx = cos * (old_eth_num - delta) + i;
181 
182 			memcpy(&bp->bnx2x_txq[new_idx], fp->txdata_ptr[cos],
183 			       sizeof(struct bnx2x_fp_txdata));
184 			fp->txdata_ptr[cos] = &bp->bnx2x_txq[new_idx];
185 		}
186 	}
187 }
188 
189 int bnx2x_load_count[2][3] = { {0} }; /* per-path: 0-common, 1-port0, 2-port1 */
190 
191 /* free skb in the packet ring at pos idx
192  * return idx of last bd freed
193  */
194 static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata,
195 			     u16 idx, unsigned int *pkts_compl,
196 			     unsigned int *bytes_compl)
197 {
198 	struct sw_tx_bd *tx_buf = &txdata->tx_buf_ring[idx];
199 	struct eth_tx_start_bd *tx_start_bd;
200 	struct eth_tx_bd *tx_data_bd;
201 	struct sk_buff *skb = tx_buf->skb;
202 	u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
203 	int nbd;
204 	u16 split_bd_len = 0;
205 
206 	/* prefetch skb end pointer to speedup dev_kfree_skb() */
207 	prefetch(&skb->end);
208 
209 	DP(NETIF_MSG_TX_DONE, "fp[%d]: pkt_idx %d  buff @(%p)->skb %p\n",
210 	   txdata->txq_index, idx, tx_buf, skb);
211 
212 	tx_start_bd = &txdata->tx_desc_ring[bd_idx].start_bd;
213 
214 	nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
215 #ifdef BNX2X_STOP_ON_ERROR
216 	if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
217 		BNX2X_ERR("BAD nbd!\n");
218 		bnx2x_panic();
219 	}
220 #endif
221 	new_cons = nbd + tx_buf->first_bd;
222 
223 	/* Get the next bd */
224 	bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
225 
226 	/* Skip a parse bd... */
227 	--nbd;
228 	bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
229 
230 	if (tx_buf->flags & BNX2X_HAS_SECOND_PBD) {
231 		/* Skip second parse bd... */
232 		--nbd;
233 		bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
234 	}
235 
236 	/* TSO headers+data bds share a common mapping. See bnx2x_tx_split() */
237 	if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
238 		tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
239 		split_bd_len = BD_UNMAP_LEN(tx_data_bd);
240 		--nbd;
241 		bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
242 	}
243 
244 	/* unmap first bd */
245 	dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
246 			 BD_UNMAP_LEN(tx_start_bd) + split_bd_len,
247 			 DMA_TO_DEVICE);
248 
249 	/* now free frags */
250 	while (nbd > 0) {
251 
252 		tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
253 		dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
254 			       BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
255 		if (--nbd)
256 			bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
257 	}
258 
259 	/* release skb */
260 	WARN_ON(!skb);
261 	if (likely(skb)) {
262 		(*pkts_compl)++;
263 		(*bytes_compl) += skb->len;
264 		dev_kfree_skb_any(skb);
265 	}
266 
267 	tx_buf->first_bd = 0;
268 	tx_buf->skb = NULL;
269 
270 	return new_cons;
271 }
272 
273 int bnx2x_tx_int(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata)
274 {
275 	struct netdev_queue *txq;
276 	u16 hw_cons, sw_cons, bd_cons = txdata->tx_bd_cons;
277 	unsigned int pkts_compl = 0, bytes_compl = 0;
278 
279 #ifdef BNX2X_STOP_ON_ERROR
280 	if (unlikely(bp->panic))
281 		return -1;
282 #endif
283 
284 	txq = netdev_get_tx_queue(bp->dev, txdata->txq_index);
285 	hw_cons = le16_to_cpu(*txdata->tx_cons_sb);
286 	sw_cons = txdata->tx_pkt_cons;
287 
288 	/* Ensure subsequent loads occur after hw_cons */
289 	smp_rmb();
290 
291 	while (sw_cons != hw_cons) {
292 		u16 pkt_cons;
293 
294 		pkt_cons = TX_BD(sw_cons);
295 
296 		DP(NETIF_MSG_TX_DONE,
297 		   "queue[%d]: hw_cons %u  sw_cons %u  pkt_cons %u\n",
298 		   txdata->txq_index, hw_cons, sw_cons, pkt_cons);
299 
300 		bd_cons = bnx2x_free_tx_pkt(bp, txdata, pkt_cons,
301 					    &pkts_compl, &bytes_compl);
302 
303 		sw_cons++;
304 	}
305 
306 	netdev_tx_completed_queue(txq, pkts_compl, bytes_compl);
307 
308 	txdata->tx_pkt_cons = sw_cons;
309 	txdata->tx_bd_cons = bd_cons;
310 
311 	/* Need to make the tx_bd_cons update visible to start_xmit()
312 	 * before checking for netif_tx_queue_stopped().  Without the
313 	 * memory barrier, there is a small possibility that
314 	 * start_xmit() will miss it and cause the queue to be stopped
315 	 * forever.
316 	 * On the other hand we need an rmb() here to ensure the proper
317 	 * ordering of bit testing in the following
318 	 * netif_tx_queue_stopped(txq) call.
319 	 */
320 	smp_mb();
321 
322 	if (unlikely(netif_tx_queue_stopped(txq))) {
323 		/* Taking tx_lock() is needed to prevent re-enabling the queue
324 		 * while it's empty. This could have happen if rx_action() gets
325 		 * suspended in bnx2x_tx_int() after the condition before
326 		 * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
327 		 *
328 		 * stops the queue->sees fresh tx_bd_cons->releases the queue->
329 		 * sends some packets consuming the whole queue again->
330 		 * stops the queue
331 		 */
332 
333 		__netif_tx_lock(txq, smp_processor_id());
334 
335 		if ((netif_tx_queue_stopped(txq)) &&
336 		    (bp->state == BNX2X_STATE_OPEN) &&
337 		    (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT))
338 			netif_tx_wake_queue(txq);
339 
340 		__netif_tx_unlock(txq);
341 	}
342 	return 0;
343 }
344 
345 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
346 					     u16 idx)
347 {
348 	u16 last_max = fp->last_max_sge;
349 
350 	if (SUB_S16(idx, last_max) > 0)
351 		fp->last_max_sge = idx;
352 }
353 
354 static inline void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
355 					 u16 sge_len,
356 					 struct eth_end_agg_rx_cqe *cqe)
357 {
358 	struct bnx2x *bp = fp->bp;
359 	u16 last_max, last_elem, first_elem;
360 	u16 delta = 0;
361 	u16 i;
362 
363 	if (!sge_len)
364 		return;
365 
366 	/* First mark all used pages */
367 	for (i = 0; i < sge_len; i++)
368 		BIT_VEC64_CLEAR_BIT(fp->sge_mask,
369 			RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[i])));
370 
371 	DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
372 	   sge_len - 1, le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
373 
374 	/* Here we assume that the last SGE index is the biggest */
375 	prefetch((void *)(fp->sge_mask));
376 	bnx2x_update_last_max_sge(fp,
377 		le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
378 
379 	last_max = RX_SGE(fp->last_max_sge);
380 	last_elem = last_max >> BIT_VEC64_ELEM_SHIFT;
381 	first_elem = RX_SGE(fp->rx_sge_prod) >> BIT_VEC64_ELEM_SHIFT;
382 
383 	/* If ring is not full */
384 	if (last_elem + 1 != first_elem)
385 		last_elem++;
386 
387 	/* Now update the prod */
388 	for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
389 		if (likely(fp->sge_mask[i]))
390 			break;
391 
392 		fp->sge_mask[i] = BIT_VEC64_ELEM_ONE_MASK;
393 		delta += BIT_VEC64_ELEM_SZ;
394 	}
395 
396 	if (delta > 0) {
397 		fp->rx_sge_prod += delta;
398 		/* clear page-end entries */
399 		bnx2x_clear_sge_mask_next_elems(fp);
400 	}
401 
402 	DP(NETIF_MSG_RX_STATUS,
403 	   "fp->last_max_sge = %d  fp->rx_sge_prod = %d\n",
404 	   fp->last_max_sge, fp->rx_sge_prod);
405 }
406 
407 /* Get Toeplitz hash value in the skb using the value from the
408  * CQE (calculated by HW).
409  */
410 static u32 bnx2x_get_rxhash(const struct bnx2x *bp,
411 			    const struct eth_fast_path_rx_cqe *cqe,
412 			    enum pkt_hash_types *rxhash_type)
413 {
414 	/* Get Toeplitz hash from CQE */
415 	if ((bp->dev->features & NETIF_F_RXHASH) &&
416 	    (cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG)) {
417 		enum eth_rss_hash_type htype;
418 
419 		htype = cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_TYPE;
420 		*rxhash_type = ((htype == TCP_IPV4_HASH_TYPE) ||
421 				(htype == TCP_IPV6_HASH_TYPE)) ?
422 			       PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3;
423 
424 		return le32_to_cpu(cqe->rss_hash_result);
425 	}
426 	*rxhash_type = PKT_HASH_TYPE_NONE;
427 	return 0;
428 }
429 
430 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
431 			    u16 cons, u16 prod,
432 			    struct eth_fast_path_rx_cqe *cqe)
433 {
434 	struct bnx2x *bp = fp->bp;
435 	struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
436 	struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
437 	struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
438 	dma_addr_t mapping;
439 	struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
440 	struct sw_rx_bd *first_buf = &tpa_info->first_buf;
441 
442 	/* print error if current state != stop */
443 	if (tpa_info->tpa_state != BNX2X_TPA_STOP)
444 		BNX2X_ERR("start of bin not in stop [%d]\n", queue);
445 
446 	/* Try to map an empty data buffer from the aggregation info  */
447 	mapping = dma_map_single(&bp->pdev->dev,
448 				 first_buf->data + NET_SKB_PAD,
449 				 fp->rx_buf_size, DMA_FROM_DEVICE);
450 	/*
451 	 *  ...if it fails - move the skb from the consumer to the producer
452 	 *  and set the current aggregation state as ERROR to drop it
453 	 *  when TPA_STOP arrives.
454 	 */
455 
456 	if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
457 		/* Move the BD from the consumer to the producer */
458 		bnx2x_reuse_rx_data(fp, cons, prod);
459 		tpa_info->tpa_state = BNX2X_TPA_ERROR;
460 		return;
461 	}
462 
463 	/* move empty data from pool to prod */
464 	prod_rx_buf->data = first_buf->data;
465 	dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
466 	/* point prod_bd to new data */
467 	prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
468 	prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
469 
470 	/* move partial skb from cons to pool (don't unmap yet) */
471 	*first_buf = *cons_rx_buf;
472 
473 	/* mark bin state as START */
474 	tpa_info->parsing_flags =
475 		le16_to_cpu(cqe->pars_flags.flags);
476 	tpa_info->vlan_tag = le16_to_cpu(cqe->vlan_tag);
477 	tpa_info->tpa_state = BNX2X_TPA_START;
478 	tpa_info->len_on_bd = le16_to_cpu(cqe->len_on_bd);
479 	tpa_info->placement_offset = cqe->placement_offset;
480 	tpa_info->rxhash = bnx2x_get_rxhash(bp, cqe, &tpa_info->rxhash_type);
481 	if (fp->mode == TPA_MODE_GRO) {
482 		u16 gro_size = le16_to_cpu(cqe->pkt_len_or_gro_seg_len);
483 		tpa_info->full_page = SGE_PAGES / gro_size * gro_size;
484 		tpa_info->gro_size = gro_size;
485 	}
486 
487 #ifdef BNX2X_STOP_ON_ERROR
488 	fp->tpa_queue_used |= (1 << queue);
489 	DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
490 	   fp->tpa_queue_used);
491 #endif
492 }
493 
494 /* Timestamp option length allowed for TPA aggregation:
495  *
496  *		nop nop kind length echo val
497  */
498 #define TPA_TSTAMP_OPT_LEN	12
499 /**
500  * bnx2x_set_gro_params - compute GRO values
501  *
502  * @skb:		packet skb
503  * @parsing_flags:	parsing flags from the START CQE
504  * @len_on_bd:		total length of the first packet for the
505  *			aggregation.
506  * @pkt_len:		length of all segments
507  * @num_of_coalesced_segs: count of segments
508  *
509  * Approximate value of the MSS for this aggregation calculated using
510  * the first packet of it.
511  * Compute number of aggregated segments, and gso_type.
512  */
513 static void bnx2x_set_gro_params(struct sk_buff *skb, u16 parsing_flags,
514 				 u16 len_on_bd, unsigned int pkt_len,
515 				 u16 num_of_coalesced_segs)
516 {
517 	/* TPA aggregation won't have either IP options or TCP options
518 	 * other than timestamp or IPv6 extension headers.
519 	 */
520 	u16 hdrs_len = ETH_HLEN + sizeof(struct tcphdr);
521 
522 	if (GET_FLAG(parsing_flags, PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
523 	    PRS_FLAG_OVERETH_IPV6) {
524 		hdrs_len += sizeof(struct ipv6hdr);
525 		skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
526 	} else {
527 		hdrs_len += sizeof(struct iphdr);
528 		skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
529 	}
530 
531 	/* Check if there was a TCP timestamp, if there is it's will
532 	 * always be 12 bytes length: nop nop kind length echo val.
533 	 *
534 	 * Otherwise FW would close the aggregation.
535 	 */
536 	if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG)
537 		hdrs_len += TPA_TSTAMP_OPT_LEN;
538 
539 	skb_shinfo(skb)->gso_size = len_on_bd - hdrs_len;
540 
541 	/* tcp_gro_complete() will copy NAPI_GRO_CB(skb)->count
542 	 * to skb_shinfo(skb)->gso_segs
543 	 */
544 	NAPI_GRO_CB(skb)->count = num_of_coalesced_segs;
545 }
546 
547 static int bnx2x_alloc_rx_sge(struct bnx2x *bp, struct bnx2x_fastpath *fp,
548 			      u16 index, gfp_t gfp_mask)
549 {
550 	struct sw_rx_page *sw_buf = &fp->rx_page_ring[index];
551 	struct eth_rx_sge *sge = &fp->rx_sge_ring[index];
552 	struct bnx2x_alloc_pool *pool = &fp->page_pool;
553 	dma_addr_t mapping;
554 
555 	if (!pool->page) {
556 		pool->page = alloc_pages(gfp_mask, PAGES_PER_SGE_SHIFT);
557 		if (unlikely(!pool->page))
558 			return -ENOMEM;
559 
560 		pool->offset = 0;
561 	}
562 
563 	mapping = dma_map_page(&bp->pdev->dev, pool->page,
564 			       pool->offset, SGE_PAGE_SIZE, DMA_FROM_DEVICE);
565 	if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
566 		BNX2X_ERR("Can't map sge\n");
567 		return -ENOMEM;
568 	}
569 
570 	sw_buf->page = pool->page;
571 	sw_buf->offset = pool->offset;
572 
573 	dma_unmap_addr_set(sw_buf, mapping, mapping);
574 
575 	sge->addr_hi = cpu_to_le32(U64_HI(mapping));
576 	sge->addr_lo = cpu_to_le32(U64_LO(mapping));
577 
578 	pool->offset += SGE_PAGE_SIZE;
579 	if (PAGE_SIZE - pool->offset >= SGE_PAGE_SIZE)
580 		get_page(pool->page);
581 	else
582 		pool->page = NULL;
583 	return 0;
584 }
585 
586 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
587 			       struct bnx2x_agg_info *tpa_info,
588 			       u16 pages,
589 			       struct sk_buff *skb,
590 			       struct eth_end_agg_rx_cqe *cqe,
591 			       u16 cqe_idx)
592 {
593 	struct sw_rx_page *rx_pg, old_rx_pg;
594 	u32 i, frag_len, frag_size;
595 	int err, j, frag_id = 0;
596 	u16 len_on_bd = tpa_info->len_on_bd;
597 	u16 full_page = 0, gro_size = 0;
598 
599 	frag_size = le16_to_cpu(cqe->pkt_len) - len_on_bd;
600 
601 	if (fp->mode == TPA_MODE_GRO) {
602 		gro_size = tpa_info->gro_size;
603 		full_page = tpa_info->full_page;
604 	}
605 
606 	/* This is needed in order to enable forwarding support */
607 	if (frag_size)
608 		bnx2x_set_gro_params(skb, tpa_info->parsing_flags, len_on_bd,
609 				     le16_to_cpu(cqe->pkt_len),
610 				     le16_to_cpu(cqe->num_of_coalesced_segs));
611 
612 #ifdef BNX2X_STOP_ON_ERROR
613 	if (pages > min_t(u32, 8, MAX_SKB_FRAGS) * SGE_PAGES) {
614 		BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
615 			  pages, cqe_idx);
616 		BNX2X_ERR("cqe->pkt_len = %d\n", cqe->pkt_len);
617 		bnx2x_panic();
618 		return -EINVAL;
619 	}
620 #endif
621 
622 	/* Run through the SGL and compose the fragmented skb */
623 	for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
624 		u16 sge_idx = RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[j]));
625 
626 		/* FW gives the indices of the SGE as if the ring is an array
627 		   (meaning that "next" element will consume 2 indices) */
628 		if (fp->mode == TPA_MODE_GRO)
629 			frag_len = min_t(u32, frag_size, (u32)full_page);
630 		else /* LRO */
631 			frag_len = min_t(u32, frag_size, (u32)SGE_PAGES);
632 
633 		rx_pg = &fp->rx_page_ring[sge_idx];
634 		old_rx_pg = *rx_pg;
635 
636 		/* If we fail to allocate a substitute page, we simply stop
637 		   where we are and drop the whole packet */
638 		err = bnx2x_alloc_rx_sge(bp, fp, sge_idx, GFP_ATOMIC);
639 		if (unlikely(err)) {
640 			bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
641 			return err;
642 		}
643 
644 		dma_unmap_page(&bp->pdev->dev,
645 			       dma_unmap_addr(&old_rx_pg, mapping),
646 			       SGE_PAGE_SIZE, DMA_FROM_DEVICE);
647 		/* Add one frag and update the appropriate fields in the skb */
648 		if (fp->mode == TPA_MODE_LRO)
649 			skb_fill_page_desc(skb, j, old_rx_pg.page,
650 					   old_rx_pg.offset, frag_len);
651 		else { /* GRO */
652 			int rem;
653 			int offset = 0;
654 			for (rem = frag_len; rem > 0; rem -= gro_size) {
655 				int len = rem > gro_size ? gro_size : rem;
656 				skb_fill_page_desc(skb, frag_id++,
657 						   old_rx_pg.page,
658 						   old_rx_pg.offset + offset,
659 						   len);
660 				if (offset)
661 					get_page(old_rx_pg.page);
662 				offset += len;
663 			}
664 		}
665 
666 		skb->data_len += frag_len;
667 		skb->truesize += SGE_PAGES;
668 		skb->len += frag_len;
669 
670 		frag_size -= frag_len;
671 	}
672 
673 	return 0;
674 }
675 
676 static void bnx2x_frag_free(const struct bnx2x_fastpath *fp, void *data)
677 {
678 	if (fp->rx_frag_size)
679 		skb_free_frag(data);
680 	else
681 		kfree(data);
682 }
683 
684 static void *bnx2x_frag_alloc(const struct bnx2x_fastpath *fp, gfp_t gfp_mask)
685 {
686 	if (fp->rx_frag_size) {
687 		/* GFP_KERNEL allocations are used only during initialization */
688 		if (unlikely(gfpflags_allow_blocking(gfp_mask)))
689 			return (void *)__get_free_page(gfp_mask);
690 
691 		return napi_alloc_frag(fp->rx_frag_size);
692 	}
693 
694 	return kmalloc(fp->rx_buf_size + NET_SKB_PAD, gfp_mask);
695 }
696 
697 #ifdef CONFIG_INET
698 static void bnx2x_gro_ip_csum(struct bnx2x *bp, struct sk_buff *skb)
699 {
700 	const struct iphdr *iph = ip_hdr(skb);
701 	struct tcphdr *th;
702 
703 	skb_set_transport_header(skb, sizeof(struct iphdr));
704 	th = tcp_hdr(skb);
705 
706 	th->check = ~tcp_v4_check(skb->len - skb_transport_offset(skb),
707 				  iph->saddr, iph->daddr, 0);
708 }
709 
710 static void bnx2x_gro_ipv6_csum(struct bnx2x *bp, struct sk_buff *skb)
711 {
712 	struct ipv6hdr *iph = ipv6_hdr(skb);
713 	struct tcphdr *th;
714 
715 	skb_set_transport_header(skb, sizeof(struct ipv6hdr));
716 	th = tcp_hdr(skb);
717 
718 	th->check = ~tcp_v6_check(skb->len - skb_transport_offset(skb),
719 				  &iph->saddr, &iph->daddr, 0);
720 }
721 
722 static void bnx2x_gro_csum(struct bnx2x *bp, struct sk_buff *skb,
723 			    void (*gro_func)(struct bnx2x*, struct sk_buff*))
724 {
725 	skb_reset_network_header(skb);
726 	gro_func(bp, skb);
727 	tcp_gro_complete(skb);
728 }
729 #endif
730 
731 static void bnx2x_gro_receive(struct bnx2x *bp, struct bnx2x_fastpath *fp,
732 			       struct sk_buff *skb)
733 {
734 #ifdef CONFIG_INET
735 	if (skb_shinfo(skb)->gso_size) {
736 		switch (be16_to_cpu(skb->protocol)) {
737 		case ETH_P_IP:
738 			bnx2x_gro_csum(bp, skb, bnx2x_gro_ip_csum);
739 			break;
740 		case ETH_P_IPV6:
741 			bnx2x_gro_csum(bp, skb, bnx2x_gro_ipv6_csum);
742 			break;
743 		default:
744 			netdev_WARN_ONCE(bp->dev,
745 					 "Error: FW GRO supports only IPv4/IPv6, not 0x%04x\n",
746 					 be16_to_cpu(skb->protocol));
747 		}
748 	}
749 #endif
750 	skb_record_rx_queue(skb, fp->rx_queue);
751 	napi_gro_receive(&fp->napi, skb);
752 }
753 
754 static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
755 			   struct bnx2x_agg_info *tpa_info,
756 			   u16 pages,
757 			   struct eth_end_agg_rx_cqe *cqe,
758 			   u16 cqe_idx)
759 {
760 	struct sw_rx_bd *rx_buf = &tpa_info->first_buf;
761 	u8 pad = tpa_info->placement_offset;
762 	u16 len = tpa_info->len_on_bd;
763 	struct sk_buff *skb = NULL;
764 	u8 *new_data, *data = rx_buf->data;
765 	u8 old_tpa_state = tpa_info->tpa_state;
766 
767 	tpa_info->tpa_state = BNX2X_TPA_STOP;
768 
769 	/* If we there was an error during the handling of the TPA_START -
770 	 * drop this aggregation.
771 	 */
772 	if (old_tpa_state == BNX2X_TPA_ERROR)
773 		goto drop;
774 
775 	/* Try to allocate the new data */
776 	new_data = bnx2x_frag_alloc(fp, GFP_ATOMIC);
777 	/* Unmap skb in the pool anyway, as we are going to change
778 	   pool entry status to BNX2X_TPA_STOP even if new skb allocation
779 	   fails. */
780 	dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
781 			 fp->rx_buf_size, DMA_FROM_DEVICE);
782 	if (likely(new_data))
783 		skb = build_skb(data, fp->rx_frag_size);
784 
785 	if (likely(skb)) {
786 #ifdef BNX2X_STOP_ON_ERROR
787 		if (pad + len > fp->rx_buf_size) {
788 			BNX2X_ERR("skb_put is about to fail...  pad %d  len %d  rx_buf_size %d\n",
789 				  pad, len, fp->rx_buf_size);
790 			bnx2x_panic();
791 			return;
792 		}
793 #endif
794 
795 		skb_reserve(skb, pad + NET_SKB_PAD);
796 		skb_put(skb, len);
797 		skb_set_hash(skb, tpa_info->rxhash, tpa_info->rxhash_type);
798 
799 		skb->protocol = eth_type_trans(skb, bp->dev);
800 		skb->ip_summed = CHECKSUM_UNNECESSARY;
801 
802 		if (!bnx2x_fill_frag_skb(bp, fp, tpa_info, pages,
803 					 skb, cqe, cqe_idx)) {
804 			if (tpa_info->parsing_flags & PARSING_FLAGS_VLAN)
805 				__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), tpa_info->vlan_tag);
806 			bnx2x_gro_receive(bp, fp, skb);
807 		} else {
808 			DP(NETIF_MSG_RX_STATUS,
809 			   "Failed to allocate new pages - dropping packet!\n");
810 			dev_kfree_skb_any(skb);
811 		}
812 
813 		/* put new data in bin */
814 		rx_buf->data = new_data;
815 
816 		return;
817 	}
818 	if (new_data)
819 		bnx2x_frag_free(fp, new_data);
820 drop:
821 	/* drop the packet and keep the buffer in the bin */
822 	DP(NETIF_MSG_RX_STATUS,
823 	   "Failed to allocate or map a new skb - dropping packet!\n");
824 	bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed++;
825 }
826 
827 static int bnx2x_alloc_rx_data(struct bnx2x *bp, struct bnx2x_fastpath *fp,
828 			       u16 index, gfp_t gfp_mask)
829 {
830 	u8 *data;
831 	struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[index];
832 	struct eth_rx_bd *rx_bd = &fp->rx_desc_ring[index];
833 	dma_addr_t mapping;
834 
835 	data = bnx2x_frag_alloc(fp, gfp_mask);
836 	if (unlikely(data == NULL))
837 		return -ENOMEM;
838 
839 	mapping = dma_map_single(&bp->pdev->dev, data + NET_SKB_PAD,
840 				 fp->rx_buf_size,
841 				 DMA_FROM_DEVICE);
842 	if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
843 		bnx2x_frag_free(fp, data);
844 		BNX2X_ERR("Can't map rx data\n");
845 		return -ENOMEM;
846 	}
847 
848 	rx_buf->data = data;
849 	dma_unmap_addr_set(rx_buf, mapping, mapping);
850 
851 	rx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
852 	rx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
853 
854 	return 0;
855 }
856 
857 static
858 void bnx2x_csum_validate(struct sk_buff *skb, union eth_rx_cqe *cqe,
859 				 struct bnx2x_fastpath *fp,
860 				 struct bnx2x_eth_q_stats *qstats)
861 {
862 	/* Do nothing if no L4 csum validation was done.
863 	 * We do not check whether IP csum was validated. For IPv4 we assume
864 	 * that if the card got as far as validating the L4 csum, it also
865 	 * validated the IP csum. IPv6 has no IP csum.
866 	 */
867 	if (cqe->fast_path_cqe.status_flags &
868 	    ETH_FAST_PATH_RX_CQE_L4_XSUM_NO_VALIDATION_FLG)
869 		return;
870 
871 	/* If L4 validation was done, check if an error was found. */
872 
873 	if (cqe->fast_path_cqe.type_error_flags &
874 	    (ETH_FAST_PATH_RX_CQE_IP_BAD_XSUM_FLG |
875 	     ETH_FAST_PATH_RX_CQE_L4_BAD_XSUM_FLG))
876 		qstats->hw_csum_err++;
877 	else
878 		skb->ip_summed = CHECKSUM_UNNECESSARY;
879 }
880 
881 static int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
882 {
883 	struct bnx2x *bp = fp->bp;
884 	u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
885 	u16 sw_comp_cons, sw_comp_prod;
886 	int rx_pkt = 0;
887 	union eth_rx_cqe *cqe;
888 	struct eth_fast_path_rx_cqe *cqe_fp;
889 
890 #ifdef BNX2X_STOP_ON_ERROR
891 	if (unlikely(bp->panic))
892 		return 0;
893 #endif
894 	if (budget <= 0)
895 		return rx_pkt;
896 
897 	bd_cons = fp->rx_bd_cons;
898 	bd_prod = fp->rx_bd_prod;
899 	bd_prod_fw = bd_prod;
900 	sw_comp_cons = fp->rx_comp_cons;
901 	sw_comp_prod = fp->rx_comp_prod;
902 
903 	comp_ring_cons = RCQ_BD(sw_comp_cons);
904 	cqe = &fp->rx_comp_ring[comp_ring_cons];
905 	cqe_fp = &cqe->fast_path_cqe;
906 
907 	DP(NETIF_MSG_RX_STATUS,
908 	   "queue[%d]: sw_comp_cons %u\n", fp->index, sw_comp_cons);
909 
910 	while (BNX2X_IS_CQE_COMPLETED(cqe_fp)) {
911 		struct sw_rx_bd *rx_buf = NULL;
912 		struct sk_buff *skb;
913 		u8 cqe_fp_flags;
914 		enum eth_rx_cqe_type cqe_fp_type;
915 		u16 len, pad, queue;
916 		u8 *data;
917 		u32 rxhash;
918 		enum pkt_hash_types rxhash_type;
919 
920 #ifdef BNX2X_STOP_ON_ERROR
921 		if (unlikely(bp->panic))
922 			return 0;
923 #endif
924 
925 		bd_prod = RX_BD(bd_prod);
926 		bd_cons = RX_BD(bd_cons);
927 
928 		/* A rmb() is required to ensure that the CQE is not read
929 		 * before it is written by the adapter DMA.  PCI ordering
930 		 * rules will make sure the other fields are written before
931 		 * the marker at the end of struct eth_fast_path_rx_cqe
932 		 * but without rmb() a weakly ordered processor can process
933 		 * stale data.  Without the barrier TPA state-machine might
934 		 * enter inconsistent state and kernel stack might be
935 		 * provided with incorrect packet description - these lead
936 		 * to various kernel crashed.
937 		 */
938 		rmb();
939 
940 		cqe_fp_flags = cqe_fp->type_error_flags;
941 		cqe_fp_type = cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE;
942 
943 		DP(NETIF_MSG_RX_STATUS,
944 		   "CQE type %x  err %x  status %x  queue %x  vlan %x  len %u\n",
945 		   CQE_TYPE(cqe_fp_flags),
946 		   cqe_fp_flags, cqe_fp->status_flags,
947 		   le32_to_cpu(cqe_fp->rss_hash_result),
948 		   le16_to_cpu(cqe_fp->vlan_tag),
949 		   le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len));
950 
951 		/* is this a slowpath msg? */
952 		if (unlikely(CQE_TYPE_SLOW(cqe_fp_type))) {
953 			bnx2x_sp_event(fp, cqe);
954 			goto next_cqe;
955 		}
956 
957 		rx_buf = &fp->rx_buf_ring[bd_cons];
958 		data = rx_buf->data;
959 
960 		if (!CQE_TYPE_FAST(cqe_fp_type)) {
961 			struct bnx2x_agg_info *tpa_info;
962 			u16 frag_size, pages;
963 #ifdef BNX2X_STOP_ON_ERROR
964 			/* sanity check */
965 			if (fp->mode == TPA_MODE_DISABLED &&
966 			    (CQE_TYPE_START(cqe_fp_type) ||
967 			     CQE_TYPE_STOP(cqe_fp_type)))
968 				BNX2X_ERR("START/STOP packet while TPA disabled, type %x\n",
969 					  CQE_TYPE(cqe_fp_type));
970 #endif
971 
972 			if (CQE_TYPE_START(cqe_fp_type)) {
973 				u16 queue = cqe_fp->queue_index;
974 				DP(NETIF_MSG_RX_STATUS,
975 				   "calling tpa_start on queue %d\n",
976 				   queue);
977 
978 				bnx2x_tpa_start(fp, queue,
979 						bd_cons, bd_prod,
980 						cqe_fp);
981 
982 				goto next_rx;
983 			}
984 			queue = cqe->end_agg_cqe.queue_index;
985 			tpa_info = &fp->tpa_info[queue];
986 			DP(NETIF_MSG_RX_STATUS,
987 			   "calling tpa_stop on queue %d\n",
988 			   queue);
989 
990 			frag_size = le16_to_cpu(cqe->end_agg_cqe.pkt_len) -
991 				    tpa_info->len_on_bd;
992 
993 			if (fp->mode == TPA_MODE_GRO)
994 				pages = (frag_size + tpa_info->full_page - 1) /
995 					 tpa_info->full_page;
996 			else
997 				pages = SGE_PAGE_ALIGN(frag_size) >>
998 					SGE_PAGE_SHIFT;
999 
1000 			bnx2x_tpa_stop(bp, fp, tpa_info, pages,
1001 				       &cqe->end_agg_cqe, comp_ring_cons);
1002 #ifdef BNX2X_STOP_ON_ERROR
1003 			if (bp->panic)
1004 				return 0;
1005 #endif
1006 
1007 			bnx2x_update_sge_prod(fp, pages, &cqe->end_agg_cqe);
1008 			goto next_cqe;
1009 		}
1010 		/* non TPA */
1011 		len = le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len);
1012 		pad = cqe_fp->placement_offset;
1013 		dma_sync_single_for_cpu(&bp->pdev->dev,
1014 					dma_unmap_addr(rx_buf, mapping),
1015 					pad + RX_COPY_THRESH,
1016 					DMA_FROM_DEVICE);
1017 		pad += NET_SKB_PAD;
1018 		prefetch(data + pad); /* speedup eth_type_trans() */
1019 		/* is this an error packet? */
1020 		if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
1021 			DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
1022 			   "ERROR  flags %x  rx packet %u\n",
1023 			   cqe_fp_flags, sw_comp_cons);
1024 			bnx2x_fp_qstats(bp, fp)->rx_err_discard_pkt++;
1025 			goto reuse_rx;
1026 		}
1027 
1028 		/* Since we don't have a jumbo ring
1029 		 * copy small packets if mtu > 1500
1030 		 */
1031 		if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
1032 		    (len <= RX_COPY_THRESH)) {
1033 			skb = napi_alloc_skb(&fp->napi, len);
1034 			if (skb == NULL) {
1035 				DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
1036 				   "ERROR  packet dropped because of alloc failure\n");
1037 				bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
1038 				goto reuse_rx;
1039 			}
1040 			memcpy(skb->data, data + pad, len);
1041 			bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
1042 		} else {
1043 			if (likely(bnx2x_alloc_rx_data(bp, fp, bd_prod,
1044 						       GFP_ATOMIC) == 0)) {
1045 				dma_unmap_single(&bp->pdev->dev,
1046 						 dma_unmap_addr(rx_buf, mapping),
1047 						 fp->rx_buf_size,
1048 						 DMA_FROM_DEVICE);
1049 				skb = build_skb(data, fp->rx_frag_size);
1050 				if (unlikely(!skb)) {
1051 					bnx2x_frag_free(fp, data);
1052 					bnx2x_fp_qstats(bp, fp)->
1053 							rx_skb_alloc_failed++;
1054 					goto next_rx;
1055 				}
1056 				skb_reserve(skb, pad);
1057 			} else {
1058 				DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
1059 				   "ERROR  packet dropped because of alloc failure\n");
1060 				bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
1061 reuse_rx:
1062 				bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
1063 				goto next_rx;
1064 			}
1065 		}
1066 
1067 		skb_put(skb, len);
1068 		skb->protocol = eth_type_trans(skb, bp->dev);
1069 
1070 		/* Set Toeplitz hash for a none-LRO skb */
1071 		rxhash = bnx2x_get_rxhash(bp, cqe_fp, &rxhash_type);
1072 		skb_set_hash(skb, rxhash, rxhash_type);
1073 
1074 		skb_checksum_none_assert(skb);
1075 
1076 		if (bp->dev->features & NETIF_F_RXCSUM)
1077 			bnx2x_csum_validate(skb, cqe, fp,
1078 					    bnx2x_fp_qstats(bp, fp));
1079 
1080 		skb_record_rx_queue(skb, fp->rx_queue);
1081 
1082 		/* Check if this packet was timestamped */
1083 		if (unlikely(cqe->fast_path_cqe.type_error_flags &
1084 			     (1 << ETH_FAST_PATH_RX_CQE_PTP_PKT_SHIFT)))
1085 			bnx2x_set_rx_ts(bp, skb);
1086 
1087 		if (le16_to_cpu(cqe_fp->pars_flags.flags) &
1088 		    PARSING_FLAGS_VLAN)
1089 			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
1090 					       le16_to_cpu(cqe_fp->vlan_tag));
1091 
1092 		napi_gro_receive(&fp->napi, skb);
1093 next_rx:
1094 		rx_buf->data = NULL;
1095 
1096 		bd_cons = NEXT_RX_IDX(bd_cons);
1097 		bd_prod = NEXT_RX_IDX(bd_prod);
1098 		bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
1099 		rx_pkt++;
1100 next_cqe:
1101 		sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
1102 		sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
1103 
1104 		/* mark CQE as free */
1105 		BNX2X_SEED_CQE(cqe_fp);
1106 
1107 		if (rx_pkt == budget)
1108 			break;
1109 
1110 		comp_ring_cons = RCQ_BD(sw_comp_cons);
1111 		cqe = &fp->rx_comp_ring[comp_ring_cons];
1112 		cqe_fp = &cqe->fast_path_cqe;
1113 	} /* while */
1114 
1115 	fp->rx_bd_cons = bd_cons;
1116 	fp->rx_bd_prod = bd_prod_fw;
1117 	fp->rx_comp_cons = sw_comp_cons;
1118 	fp->rx_comp_prod = sw_comp_prod;
1119 
1120 	/* Update producers */
1121 	bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
1122 			     fp->rx_sge_prod);
1123 
1124 	return rx_pkt;
1125 }
1126 
1127 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
1128 {
1129 	struct bnx2x_fastpath *fp = fp_cookie;
1130 	struct bnx2x *bp = fp->bp;
1131 	u8 cos;
1132 
1133 	DP(NETIF_MSG_INTR,
1134 	   "got an MSI-X interrupt on IDX:SB [fp %d fw_sd %d igusb %d]\n",
1135 	   fp->index, fp->fw_sb_id, fp->igu_sb_id);
1136 
1137 	bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
1138 
1139 #ifdef BNX2X_STOP_ON_ERROR
1140 	if (unlikely(bp->panic))
1141 		return IRQ_HANDLED;
1142 #endif
1143 
1144 	/* Handle Rx and Tx according to MSI-X vector */
1145 	for_each_cos_in_tx_queue(fp, cos)
1146 		prefetch(fp->txdata_ptr[cos]->tx_cons_sb);
1147 
1148 	prefetch(&fp->sb_running_index[SM_RX_ID]);
1149 	napi_schedule_irqoff(&bnx2x_fp(bp, fp->index, napi));
1150 
1151 	return IRQ_HANDLED;
1152 }
1153 
1154 /* HW Lock for shared dual port PHYs */
1155 void bnx2x_acquire_phy_lock(struct bnx2x *bp)
1156 {
1157 	mutex_lock(&bp->port.phy_mutex);
1158 
1159 	bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1160 }
1161 
1162 void bnx2x_release_phy_lock(struct bnx2x *bp)
1163 {
1164 	bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1165 
1166 	mutex_unlock(&bp->port.phy_mutex);
1167 }
1168 
1169 /* calculates MF speed according to current linespeed and MF configuration */
1170 u16 bnx2x_get_mf_speed(struct bnx2x *bp)
1171 {
1172 	u16 line_speed = bp->link_vars.line_speed;
1173 	if (IS_MF(bp)) {
1174 		u16 maxCfg = bnx2x_extract_max_cfg(bp,
1175 						   bp->mf_config[BP_VN(bp)]);
1176 
1177 		/* Calculate the current MAX line speed limit for the MF
1178 		 * devices
1179 		 */
1180 		if (IS_MF_PERCENT_BW(bp))
1181 			line_speed = (line_speed * maxCfg) / 100;
1182 		else { /* SD mode */
1183 			u16 vn_max_rate = maxCfg * 100;
1184 
1185 			if (vn_max_rate < line_speed)
1186 				line_speed = vn_max_rate;
1187 		}
1188 	}
1189 
1190 	return line_speed;
1191 }
1192 
1193 /**
1194  * bnx2x_fill_report_data - fill link report data to report
1195  *
1196  * @bp:		driver handle
1197  * @data:	link state to update
1198  *
1199  * It uses a none-atomic bit operations because is called under the mutex.
1200  */
1201 static void bnx2x_fill_report_data(struct bnx2x *bp,
1202 				   struct bnx2x_link_report_data *data)
1203 {
1204 	memset(data, 0, sizeof(*data));
1205 
1206 	if (IS_PF(bp)) {
1207 		/* Fill the report data: effective line speed */
1208 		data->line_speed = bnx2x_get_mf_speed(bp);
1209 
1210 		/* Link is down */
1211 		if (!bp->link_vars.link_up || (bp->flags & MF_FUNC_DIS))
1212 			__set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1213 				  &data->link_report_flags);
1214 
1215 		if (!BNX2X_NUM_ETH_QUEUES(bp))
1216 			__set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1217 				  &data->link_report_flags);
1218 
1219 		/* Full DUPLEX */
1220 		if (bp->link_vars.duplex == DUPLEX_FULL)
1221 			__set_bit(BNX2X_LINK_REPORT_FD,
1222 				  &data->link_report_flags);
1223 
1224 		/* Rx Flow Control is ON */
1225 		if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX)
1226 			__set_bit(BNX2X_LINK_REPORT_RX_FC_ON,
1227 				  &data->link_report_flags);
1228 
1229 		/* Tx Flow Control is ON */
1230 		if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX)
1231 			__set_bit(BNX2X_LINK_REPORT_TX_FC_ON,
1232 				  &data->link_report_flags);
1233 	} else { /* VF */
1234 		*data = bp->vf_link_vars;
1235 	}
1236 }
1237 
1238 /**
1239  * bnx2x_link_report - report link status to OS.
1240  *
1241  * @bp:		driver handle
1242  *
1243  * Calls the __bnx2x_link_report() under the same locking scheme
1244  * as a link/PHY state managing code to ensure a consistent link
1245  * reporting.
1246  */
1247 
1248 void bnx2x_link_report(struct bnx2x *bp)
1249 {
1250 	bnx2x_acquire_phy_lock(bp);
1251 	__bnx2x_link_report(bp);
1252 	bnx2x_release_phy_lock(bp);
1253 }
1254 
1255 /**
1256  * __bnx2x_link_report - report link status to OS.
1257  *
1258  * @bp:		driver handle
1259  *
1260  * None atomic implementation.
1261  * Should be called under the phy_lock.
1262  */
1263 void __bnx2x_link_report(struct bnx2x *bp)
1264 {
1265 	struct bnx2x_link_report_data cur_data;
1266 
1267 	if (bp->force_link_down) {
1268 		bp->link_vars.link_up = 0;
1269 		return;
1270 	}
1271 
1272 	/* reread mf_cfg */
1273 	if (IS_PF(bp) && !CHIP_IS_E1(bp))
1274 		bnx2x_read_mf_cfg(bp);
1275 
1276 	/* Read the current link report info */
1277 	bnx2x_fill_report_data(bp, &cur_data);
1278 
1279 	/* Don't report link down or exactly the same link status twice */
1280 	if (!memcmp(&cur_data, &bp->last_reported_link, sizeof(cur_data)) ||
1281 	    (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1282 		      &bp->last_reported_link.link_report_flags) &&
1283 	     test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1284 		      &cur_data.link_report_flags)))
1285 		return;
1286 
1287 	bp->link_cnt++;
1288 
1289 	/* We are going to report a new link parameters now -
1290 	 * remember the current data for the next time.
1291 	 */
1292 	memcpy(&bp->last_reported_link, &cur_data, sizeof(cur_data));
1293 
1294 	/* propagate status to VFs */
1295 	if (IS_PF(bp))
1296 		bnx2x_iov_link_update(bp);
1297 
1298 	if (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1299 		     &cur_data.link_report_flags)) {
1300 		netif_carrier_off(bp->dev);
1301 		netdev_err(bp->dev, "NIC Link is Down\n");
1302 		return;
1303 	} else {
1304 		const char *duplex;
1305 		const char *flow;
1306 
1307 		netif_carrier_on(bp->dev);
1308 
1309 		if (test_and_clear_bit(BNX2X_LINK_REPORT_FD,
1310 				       &cur_data.link_report_flags))
1311 			duplex = "full";
1312 		else
1313 			duplex = "half";
1314 
1315 		/* Handle the FC at the end so that only these flags would be
1316 		 * possibly set. This way we may easily check if there is no FC
1317 		 * enabled.
1318 		 */
1319 		if (cur_data.link_report_flags) {
1320 			if (test_bit(BNX2X_LINK_REPORT_RX_FC_ON,
1321 				     &cur_data.link_report_flags)) {
1322 				if (test_bit(BNX2X_LINK_REPORT_TX_FC_ON,
1323 				     &cur_data.link_report_flags))
1324 					flow = "ON - receive & transmit";
1325 				else
1326 					flow = "ON - receive";
1327 			} else {
1328 				flow = "ON - transmit";
1329 			}
1330 		} else {
1331 			flow = "none";
1332 		}
1333 		netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n",
1334 			    cur_data.line_speed, duplex, flow);
1335 	}
1336 }
1337 
1338 static void bnx2x_set_next_page_sgl(struct bnx2x_fastpath *fp)
1339 {
1340 	int i;
1341 
1342 	for (i = 1; i <= NUM_RX_SGE_PAGES; i++) {
1343 		struct eth_rx_sge *sge;
1344 
1345 		sge = &fp->rx_sge_ring[RX_SGE_CNT * i - 2];
1346 		sge->addr_hi =
1347 			cpu_to_le32(U64_HI(fp->rx_sge_mapping +
1348 			BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1349 
1350 		sge->addr_lo =
1351 			cpu_to_le32(U64_LO(fp->rx_sge_mapping +
1352 			BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1353 	}
1354 }
1355 
1356 static void bnx2x_free_tpa_pool(struct bnx2x *bp,
1357 				struct bnx2x_fastpath *fp, int last)
1358 {
1359 	int i;
1360 
1361 	for (i = 0; i < last; i++) {
1362 		struct bnx2x_agg_info *tpa_info = &fp->tpa_info[i];
1363 		struct sw_rx_bd *first_buf = &tpa_info->first_buf;
1364 		u8 *data = first_buf->data;
1365 
1366 		if (data == NULL) {
1367 			DP(NETIF_MSG_IFDOWN, "tpa bin %d empty on free\n", i);
1368 			continue;
1369 		}
1370 		if (tpa_info->tpa_state == BNX2X_TPA_START)
1371 			dma_unmap_single(&bp->pdev->dev,
1372 					 dma_unmap_addr(first_buf, mapping),
1373 					 fp->rx_buf_size, DMA_FROM_DEVICE);
1374 		bnx2x_frag_free(fp, data);
1375 		first_buf->data = NULL;
1376 	}
1377 }
1378 
1379 void bnx2x_init_rx_rings_cnic(struct bnx2x *bp)
1380 {
1381 	int j;
1382 
1383 	for_each_rx_queue_cnic(bp, j) {
1384 		struct bnx2x_fastpath *fp = &bp->fp[j];
1385 
1386 		fp->rx_bd_cons = 0;
1387 
1388 		/* Activate BD ring */
1389 		/* Warning!
1390 		 * this will generate an interrupt (to the TSTORM)
1391 		 * must only be done after chip is initialized
1392 		 */
1393 		bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1394 				     fp->rx_sge_prod);
1395 	}
1396 }
1397 
1398 void bnx2x_init_rx_rings(struct bnx2x *bp)
1399 {
1400 	int func = BP_FUNC(bp);
1401 	u16 ring_prod;
1402 	int i, j;
1403 
1404 	/* Allocate TPA resources */
1405 	for_each_eth_queue(bp, j) {
1406 		struct bnx2x_fastpath *fp = &bp->fp[j];
1407 
1408 		DP(NETIF_MSG_IFUP,
1409 		   "mtu %d  rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size);
1410 
1411 		if (fp->mode != TPA_MODE_DISABLED) {
1412 			/* Fill the per-aggregation pool */
1413 			for (i = 0; i < MAX_AGG_QS(bp); i++) {
1414 				struct bnx2x_agg_info *tpa_info =
1415 					&fp->tpa_info[i];
1416 				struct sw_rx_bd *first_buf =
1417 					&tpa_info->first_buf;
1418 
1419 				first_buf->data =
1420 					bnx2x_frag_alloc(fp, GFP_KERNEL);
1421 				if (!first_buf->data) {
1422 					BNX2X_ERR("Failed to allocate TPA skb pool for queue[%d] - disabling TPA on this queue!\n",
1423 						  j);
1424 					bnx2x_free_tpa_pool(bp, fp, i);
1425 					fp->mode = TPA_MODE_DISABLED;
1426 					break;
1427 				}
1428 				dma_unmap_addr_set(first_buf, mapping, 0);
1429 				tpa_info->tpa_state = BNX2X_TPA_STOP;
1430 			}
1431 
1432 			/* "next page" elements initialization */
1433 			bnx2x_set_next_page_sgl(fp);
1434 
1435 			/* set SGEs bit mask */
1436 			bnx2x_init_sge_ring_bit_mask(fp);
1437 
1438 			/* Allocate SGEs and initialize the ring elements */
1439 			for (i = 0, ring_prod = 0;
1440 			     i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
1441 
1442 				if (bnx2x_alloc_rx_sge(bp, fp, ring_prod,
1443 						       GFP_KERNEL) < 0) {
1444 					BNX2X_ERR("was only able to allocate %d rx sges\n",
1445 						  i);
1446 					BNX2X_ERR("disabling TPA for queue[%d]\n",
1447 						  j);
1448 					/* Cleanup already allocated elements */
1449 					bnx2x_free_rx_sge_range(bp, fp,
1450 								ring_prod);
1451 					bnx2x_free_tpa_pool(bp, fp,
1452 							    MAX_AGG_QS(bp));
1453 					fp->mode = TPA_MODE_DISABLED;
1454 					ring_prod = 0;
1455 					break;
1456 				}
1457 				ring_prod = NEXT_SGE_IDX(ring_prod);
1458 			}
1459 
1460 			fp->rx_sge_prod = ring_prod;
1461 		}
1462 	}
1463 
1464 	for_each_eth_queue(bp, j) {
1465 		struct bnx2x_fastpath *fp = &bp->fp[j];
1466 
1467 		fp->rx_bd_cons = 0;
1468 
1469 		/* Activate BD ring */
1470 		/* Warning!
1471 		 * this will generate an interrupt (to the TSTORM)
1472 		 * must only be done after chip is initialized
1473 		 */
1474 		bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1475 				     fp->rx_sge_prod);
1476 
1477 		if (j != 0)
1478 			continue;
1479 
1480 		if (CHIP_IS_E1(bp)) {
1481 			REG_WR(bp, BAR_USTRORM_INTMEM +
1482 			       USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
1483 			       U64_LO(fp->rx_comp_mapping));
1484 			REG_WR(bp, BAR_USTRORM_INTMEM +
1485 			       USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
1486 			       U64_HI(fp->rx_comp_mapping));
1487 		}
1488 	}
1489 }
1490 
1491 static void bnx2x_free_tx_skbs_queue(struct bnx2x_fastpath *fp)
1492 {
1493 	u8 cos;
1494 	struct bnx2x *bp = fp->bp;
1495 
1496 	for_each_cos_in_tx_queue(fp, cos) {
1497 		struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
1498 		unsigned pkts_compl = 0, bytes_compl = 0;
1499 
1500 		u16 sw_prod = txdata->tx_pkt_prod;
1501 		u16 sw_cons = txdata->tx_pkt_cons;
1502 
1503 		while (sw_cons != sw_prod) {
1504 			bnx2x_free_tx_pkt(bp, txdata, TX_BD(sw_cons),
1505 					  &pkts_compl, &bytes_compl);
1506 			sw_cons++;
1507 		}
1508 
1509 		netdev_tx_reset_queue(
1510 			netdev_get_tx_queue(bp->dev,
1511 					    txdata->txq_index));
1512 	}
1513 }
1514 
1515 static void bnx2x_free_tx_skbs_cnic(struct bnx2x *bp)
1516 {
1517 	int i;
1518 
1519 	for_each_tx_queue_cnic(bp, i) {
1520 		bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1521 	}
1522 }
1523 
1524 static void bnx2x_free_tx_skbs(struct bnx2x *bp)
1525 {
1526 	int i;
1527 
1528 	for_each_eth_queue(bp, i) {
1529 		bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1530 	}
1531 }
1532 
1533 static void bnx2x_free_rx_bds(struct bnx2x_fastpath *fp)
1534 {
1535 	struct bnx2x *bp = fp->bp;
1536 	int i;
1537 
1538 	/* ring wasn't allocated */
1539 	if (fp->rx_buf_ring == NULL)
1540 		return;
1541 
1542 	for (i = 0; i < NUM_RX_BD; i++) {
1543 		struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
1544 		u8 *data = rx_buf->data;
1545 
1546 		if (data == NULL)
1547 			continue;
1548 		dma_unmap_single(&bp->pdev->dev,
1549 				 dma_unmap_addr(rx_buf, mapping),
1550 				 fp->rx_buf_size, DMA_FROM_DEVICE);
1551 
1552 		rx_buf->data = NULL;
1553 		bnx2x_frag_free(fp, data);
1554 	}
1555 }
1556 
1557 static void bnx2x_free_rx_skbs_cnic(struct bnx2x *bp)
1558 {
1559 	int j;
1560 
1561 	for_each_rx_queue_cnic(bp, j) {
1562 		bnx2x_free_rx_bds(&bp->fp[j]);
1563 	}
1564 }
1565 
1566 static void bnx2x_free_rx_skbs(struct bnx2x *bp)
1567 {
1568 	int j;
1569 
1570 	for_each_eth_queue(bp, j) {
1571 		struct bnx2x_fastpath *fp = &bp->fp[j];
1572 
1573 		bnx2x_free_rx_bds(fp);
1574 
1575 		if (fp->mode != TPA_MODE_DISABLED)
1576 			bnx2x_free_tpa_pool(bp, fp, MAX_AGG_QS(bp));
1577 	}
1578 }
1579 
1580 static void bnx2x_free_skbs_cnic(struct bnx2x *bp)
1581 {
1582 	bnx2x_free_tx_skbs_cnic(bp);
1583 	bnx2x_free_rx_skbs_cnic(bp);
1584 }
1585 
1586 void bnx2x_free_skbs(struct bnx2x *bp)
1587 {
1588 	bnx2x_free_tx_skbs(bp);
1589 	bnx2x_free_rx_skbs(bp);
1590 }
1591 
1592 void bnx2x_update_max_mf_config(struct bnx2x *bp, u32 value)
1593 {
1594 	/* load old values */
1595 	u32 mf_cfg = bp->mf_config[BP_VN(bp)];
1596 
1597 	if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) {
1598 		/* leave all but MAX value */
1599 		mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK;
1600 
1601 		/* set new MAX value */
1602 		mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT)
1603 				& FUNC_MF_CFG_MAX_BW_MASK;
1604 
1605 		bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg);
1606 	}
1607 }
1608 
1609 /**
1610  * bnx2x_free_msix_irqs - free previously requested MSI-X IRQ vectors
1611  *
1612  * @bp:		driver handle
1613  * @nvecs:	number of vectors to be released
1614  */
1615 static void bnx2x_free_msix_irqs(struct bnx2x *bp, int nvecs)
1616 {
1617 	int i, offset = 0;
1618 
1619 	if (nvecs == offset)
1620 		return;
1621 
1622 	/* VFs don't have a default SB */
1623 	if (IS_PF(bp)) {
1624 		free_irq(bp->msix_table[offset].vector, bp->dev);
1625 		DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
1626 		   bp->msix_table[offset].vector);
1627 		offset++;
1628 	}
1629 
1630 	if (CNIC_SUPPORT(bp)) {
1631 		if (nvecs == offset)
1632 			return;
1633 		offset++;
1634 	}
1635 
1636 	for_each_eth_queue(bp, i) {
1637 		if (nvecs == offset)
1638 			return;
1639 		DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq\n",
1640 		   i, bp->msix_table[offset].vector);
1641 
1642 		free_irq(bp->msix_table[offset++].vector, &bp->fp[i]);
1643 	}
1644 }
1645 
1646 void bnx2x_free_irq(struct bnx2x *bp)
1647 {
1648 	if (bp->flags & USING_MSIX_FLAG &&
1649 	    !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1650 		int nvecs = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_SUPPORT(bp);
1651 
1652 		/* vfs don't have a default status block */
1653 		if (IS_PF(bp))
1654 			nvecs++;
1655 
1656 		bnx2x_free_msix_irqs(bp, nvecs);
1657 	} else {
1658 		free_irq(bp->dev->irq, bp->dev);
1659 	}
1660 }
1661 
1662 int bnx2x_enable_msix(struct bnx2x *bp)
1663 {
1664 	int msix_vec = 0, i, rc;
1665 
1666 	/* VFs don't have a default status block */
1667 	if (IS_PF(bp)) {
1668 		bp->msix_table[msix_vec].entry = msix_vec;
1669 		BNX2X_DEV_INFO("msix_table[0].entry = %d (slowpath)\n",
1670 			       bp->msix_table[0].entry);
1671 		msix_vec++;
1672 	}
1673 
1674 	/* Cnic requires an msix vector for itself */
1675 	if (CNIC_SUPPORT(bp)) {
1676 		bp->msix_table[msix_vec].entry = msix_vec;
1677 		BNX2X_DEV_INFO("msix_table[%d].entry = %d (CNIC)\n",
1678 			       msix_vec, bp->msix_table[msix_vec].entry);
1679 		msix_vec++;
1680 	}
1681 
1682 	/* We need separate vectors for ETH queues only (not FCoE) */
1683 	for_each_eth_queue(bp, i) {
1684 		bp->msix_table[msix_vec].entry = msix_vec;
1685 		BNX2X_DEV_INFO("msix_table[%d].entry = %d (fastpath #%u)\n",
1686 			       msix_vec, msix_vec, i);
1687 		msix_vec++;
1688 	}
1689 
1690 	DP(BNX2X_MSG_SP, "about to request enable msix with %d vectors\n",
1691 	   msix_vec);
1692 
1693 	rc = pci_enable_msix_range(bp->pdev, &bp->msix_table[0],
1694 				   BNX2X_MIN_MSIX_VEC_CNT(bp), msix_vec);
1695 	/*
1696 	 * reconfigure number of tx/rx queues according to available
1697 	 * MSI-X vectors
1698 	 */
1699 	if (rc == -ENOSPC) {
1700 		/* Get by with single vector */
1701 		rc = pci_enable_msix_range(bp->pdev, &bp->msix_table[0], 1, 1);
1702 		if (rc < 0) {
1703 			BNX2X_DEV_INFO("Single MSI-X is not attainable rc %d\n",
1704 				       rc);
1705 			goto no_msix;
1706 		}
1707 
1708 		BNX2X_DEV_INFO("Using single MSI-X vector\n");
1709 		bp->flags |= USING_SINGLE_MSIX_FLAG;
1710 
1711 		BNX2X_DEV_INFO("set number of queues to 1\n");
1712 		bp->num_ethernet_queues = 1;
1713 		bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1714 	} else if (rc < 0) {
1715 		BNX2X_DEV_INFO("MSI-X is not attainable rc %d\n", rc);
1716 		goto no_msix;
1717 	} else if (rc < msix_vec) {
1718 		/* how less vectors we will have? */
1719 		int diff = msix_vec - rc;
1720 
1721 		BNX2X_DEV_INFO("Trying to use less MSI-X vectors: %d\n", rc);
1722 
1723 		/*
1724 		 * decrease number of queues by number of unallocated entries
1725 		 */
1726 		bp->num_ethernet_queues -= diff;
1727 		bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1728 
1729 		BNX2X_DEV_INFO("New queue configuration set: %d\n",
1730 			       bp->num_queues);
1731 	}
1732 
1733 	bp->flags |= USING_MSIX_FLAG;
1734 
1735 	return 0;
1736 
1737 no_msix:
1738 	/* fall to INTx if not enough memory */
1739 	if (rc == -ENOMEM)
1740 		bp->flags |= DISABLE_MSI_FLAG;
1741 
1742 	return rc;
1743 }
1744 
1745 static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1746 {
1747 	int i, rc, offset = 0;
1748 
1749 	/* no default status block for vf */
1750 	if (IS_PF(bp)) {
1751 		rc = request_irq(bp->msix_table[offset++].vector,
1752 				 bnx2x_msix_sp_int, 0,
1753 				 bp->dev->name, bp->dev);
1754 		if (rc) {
1755 			BNX2X_ERR("request sp irq failed\n");
1756 			return -EBUSY;
1757 		}
1758 	}
1759 
1760 	if (CNIC_SUPPORT(bp))
1761 		offset++;
1762 
1763 	for_each_eth_queue(bp, i) {
1764 		struct bnx2x_fastpath *fp = &bp->fp[i];
1765 		snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1766 			 bp->dev->name, i);
1767 
1768 		rc = request_irq(bp->msix_table[offset].vector,
1769 				 bnx2x_msix_fp_int, 0, fp->name, fp);
1770 		if (rc) {
1771 			BNX2X_ERR("request fp #%d irq (%d) failed  rc %d\n", i,
1772 			      bp->msix_table[offset].vector, rc);
1773 			bnx2x_free_msix_irqs(bp, offset);
1774 			return -EBUSY;
1775 		}
1776 
1777 		offset++;
1778 	}
1779 
1780 	i = BNX2X_NUM_ETH_QUEUES(bp);
1781 	if (IS_PF(bp)) {
1782 		offset = 1 + CNIC_SUPPORT(bp);
1783 		netdev_info(bp->dev,
1784 			    "using MSI-X  IRQs: sp %d  fp[%d] %d ... fp[%d] %d\n",
1785 			    bp->msix_table[0].vector,
1786 			    0, bp->msix_table[offset].vector,
1787 			    i - 1, bp->msix_table[offset + i - 1].vector);
1788 	} else {
1789 		offset = CNIC_SUPPORT(bp);
1790 		netdev_info(bp->dev,
1791 			    "using MSI-X  IRQs: fp[%d] %d ... fp[%d] %d\n",
1792 			    0, bp->msix_table[offset].vector,
1793 			    i - 1, bp->msix_table[offset + i - 1].vector);
1794 	}
1795 	return 0;
1796 }
1797 
1798 int bnx2x_enable_msi(struct bnx2x *bp)
1799 {
1800 	int rc;
1801 
1802 	rc = pci_enable_msi(bp->pdev);
1803 	if (rc) {
1804 		BNX2X_DEV_INFO("MSI is not attainable\n");
1805 		return -1;
1806 	}
1807 	bp->flags |= USING_MSI_FLAG;
1808 
1809 	return 0;
1810 }
1811 
1812 static int bnx2x_req_irq(struct bnx2x *bp)
1813 {
1814 	unsigned long flags;
1815 	unsigned int irq;
1816 
1817 	if (bp->flags & (USING_MSI_FLAG | USING_MSIX_FLAG))
1818 		flags = 0;
1819 	else
1820 		flags = IRQF_SHARED;
1821 
1822 	if (bp->flags & USING_MSIX_FLAG)
1823 		irq = bp->msix_table[0].vector;
1824 	else
1825 		irq = bp->pdev->irq;
1826 
1827 	return request_irq(irq, bnx2x_interrupt, flags, bp->dev->name, bp->dev);
1828 }
1829 
1830 static int bnx2x_setup_irqs(struct bnx2x *bp)
1831 {
1832 	int rc = 0;
1833 	if (bp->flags & USING_MSIX_FLAG &&
1834 	    !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1835 		rc = bnx2x_req_msix_irqs(bp);
1836 		if (rc)
1837 			return rc;
1838 	} else {
1839 		rc = bnx2x_req_irq(bp);
1840 		if (rc) {
1841 			BNX2X_ERR("IRQ request failed  rc %d, aborting\n", rc);
1842 			return rc;
1843 		}
1844 		if (bp->flags & USING_MSI_FLAG) {
1845 			bp->dev->irq = bp->pdev->irq;
1846 			netdev_info(bp->dev, "using MSI IRQ %d\n",
1847 				    bp->dev->irq);
1848 		}
1849 		if (bp->flags & USING_MSIX_FLAG) {
1850 			bp->dev->irq = bp->msix_table[0].vector;
1851 			netdev_info(bp->dev, "using MSIX IRQ %d\n",
1852 				    bp->dev->irq);
1853 		}
1854 	}
1855 
1856 	return 0;
1857 }
1858 
1859 static void bnx2x_napi_enable_cnic(struct bnx2x *bp)
1860 {
1861 	int i;
1862 
1863 	for_each_rx_queue_cnic(bp, i) {
1864 		napi_enable(&bnx2x_fp(bp, i, napi));
1865 	}
1866 }
1867 
1868 static void bnx2x_napi_enable(struct bnx2x *bp)
1869 {
1870 	int i;
1871 
1872 	for_each_eth_queue(bp, i) {
1873 		napi_enable(&bnx2x_fp(bp, i, napi));
1874 	}
1875 }
1876 
1877 static void bnx2x_napi_disable_cnic(struct bnx2x *bp)
1878 {
1879 	int i;
1880 
1881 	for_each_rx_queue_cnic(bp, i) {
1882 		napi_disable(&bnx2x_fp(bp, i, napi));
1883 	}
1884 }
1885 
1886 static void bnx2x_napi_disable(struct bnx2x *bp)
1887 {
1888 	int i;
1889 
1890 	for_each_eth_queue(bp, i) {
1891 		napi_disable(&bnx2x_fp(bp, i, napi));
1892 	}
1893 }
1894 
1895 void bnx2x_netif_start(struct bnx2x *bp)
1896 {
1897 	if (netif_running(bp->dev)) {
1898 		bnx2x_napi_enable(bp);
1899 		if (CNIC_LOADED(bp))
1900 			bnx2x_napi_enable_cnic(bp);
1901 		bnx2x_int_enable(bp);
1902 		if (bp->state == BNX2X_STATE_OPEN)
1903 			netif_tx_wake_all_queues(bp->dev);
1904 	}
1905 }
1906 
1907 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1908 {
1909 	bnx2x_int_disable_sync(bp, disable_hw);
1910 	bnx2x_napi_disable(bp);
1911 	if (CNIC_LOADED(bp))
1912 		bnx2x_napi_disable_cnic(bp);
1913 }
1914 
1915 u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb,
1916 		       struct net_device *sb_dev)
1917 {
1918 	struct bnx2x *bp = netdev_priv(dev);
1919 
1920 	if (CNIC_LOADED(bp) && !NO_FCOE(bp)) {
1921 		struct ethhdr *hdr = (struct ethhdr *)skb->data;
1922 		u16 ether_type = ntohs(hdr->h_proto);
1923 
1924 		/* Skip VLAN tag if present */
1925 		if (ether_type == ETH_P_8021Q) {
1926 			struct vlan_ethhdr *vhdr =
1927 				(struct vlan_ethhdr *)skb->data;
1928 
1929 			ether_type = ntohs(vhdr->h_vlan_encapsulated_proto);
1930 		}
1931 
1932 		/* If ethertype is FCoE or FIP - use FCoE ring */
1933 		if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP))
1934 			return bnx2x_fcoe_tx(bp, txq_index);
1935 	}
1936 
1937 	/* select a non-FCoE queue */
1938 	return netdev_pick_tx(dev, skb, NULL) %
1939 			(BNX2X_NUM_ETH_QUEUES(bp) * bp->max_cos);
1940 }
1941 
1942 void bnx2x_set_num_queues(struct bnx2x *bp)
1943 {
1944 	/* RSS queues */
1945 	bp->num_ethernet_queues = bnx2x_calc_num_queues(bp);
1946 
1947 	/* override in STORAGE SD modes */
1948 	if (IS_MF_STORAGE_ONLY(bp))
1949 		bp->num_ethernet_queues = 1;
1950 
1951 	/* Add special queues */
1952 	bp->num_cnic_queues = CNIC_SUPPORT(bp); /* For FCOE */
1953 	bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1954 
1955 	BNX2X_DEV_INFO("set number of queues to %d\n", bp->num_queues);
1956 }
1957 
1958 /**
1959  * bnx2x_set_real_num_queues - configure netdev->real_num_[tx,rx]_queues
1960  *
1961  * @bp:		Driver handle
1962  * @include_cnic: handle cnic case
1963  *
1964  * We currently support for at most 16 Tx queues for each CoS thus we will
1965  * allocate a multiple of 16 for ETH L2 rings according to the value of the
1966  * bp->max_cos.
1967  *
1968  * If there is an FCoE L2 queue the appropriate Tx queue will have the next
1969  * index after all ETH L2 indices.
1970  *
1971  * If the actual number of Tx queues (for each CoS) is less than 16 then there
1972  * will be the holes at the end of each group of 16 ETh L2 indices (0..15,
1973  * 16..31,...) with indices that are not coupled with any real Tx queue.
1974  *
1975  * The proper configuration of skb->queue_mapping is handled by
1976  * bnx2x_select_queue() and __skb_tx_hash().
1977  *
1978  * bnx2x_setup_tc() takes care of the proper TC mappings so that __skb_tx_hash()
1979  * will return a proper Tx index if TC is enabled (netdev->num_tc > 0).
1980  */
1981 static int bnx2x_set_real_num_queues(struct bnx2x *bp, int include_cnic)
1982 {
1983 	int rc, tx, rx;
1984 
1985 	tx = BNX2X_NUM_ETH_QUEUES(bp) * bp->max_cos;
1986 	rx = BNX2X_NUM_ETH_QUEUES(bp);
1987 
1988 /* account for fcoe queue */
1989 	if (include_cnic && !NO_FCOE(bp)) {
1990 		rx++;
1991 		tx++;
1992 	}
1993 
1994 	rc = netif_set_real_num_tx_queues(bp->dev, tx);
1995 	if (rc) {
1996 		BNX2X_ERR("Failed to set real number of Tx queues: %d\n", rc);
1997 		return rc;
1998 	}
1999 	rc = netif_set_real_num_rx_queues(bp->dev, rx);
2000 	if (rc) {
2001 		BNX2X_ERR("Failed to set real number of Rx queues: %d\n", rc);
2002 		return rc;
2003 	}
2004 
2005 	DP(NETIF_MSG_IFUP, "Setting real num queues to (tx, rx) (%d, %d)\n",
2006 			  tx, rx);
2007 
2008 	return rc;
2009 }
2010 
2011 static void bnx2x_set_rx_buf_size(struct bnx2x *bp)
2012 {
2013 	int i;
2014 
2015 	for_each_queue(bp, i) {
2016 		struct bnx2x_fastpath *fp = &bp->fp[i];
2017 		u32 mtu;
2018 
2019 		/* Always use a mini-jumbo MTU for the FCoE L2 ring */
2020 		if (IS_FCOE_IDX(i))
2021 			/*
2022 			 * Although there are no IP frames expected to arrive to
2023 			 * this ring we still want to add an
2024 			 * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer
2025 			 * overrun attack.
2026 			 */
2027 			mtu = BNX2X_FCOE_MINI_JUMBO_MTU;
2028 		else
2029 			mtu = bp->dev->mtu;
2030 		fp->rx_buf_size = BNX2X_FW_RX_ALIGN_START +
2031 				  IP_HEADER_ALIGNMENT_PADDING +
2032 				  ETH_OVERHEAD +
2033 				  mtu +
2034 				  BNX2X_FW_RX_ALIGN_END;
2035 		fp->rx_buf_size = SKB_DATA_ALIGN(fp->rx_buf_size);
2036 		/* Note : rx_buf_size doesn't take into account NET_SKB_PAD */
2037 		if (fp->rx_buf_size + NET_SKB_PAD <= PAGE_SIZE)
2038 			fp->rx_frag_size = fp->rx_buf_size + NET_SKB_PAD;
2039 		else
2040 			fp->rx_frag_size = 0;
2041 	}
2042 }
2043 
2044 static int bnx2x_init_rss(struct bnx2x *bp)
2045 {
2046 	int i;
2047 	u8 num_eth_queues = BNX2X_NUM_ETH_QUEUES(bp);
2048 
2049 	/* Prepare the initial contents for the indirection table if RSS is
2050 	 * enabled
2051 	 */
2052 	for (i = 0; i < sizeof(bp->rss_conf_obj.ind_table); i++)
2053 		bp->rss_conf_obj.ind_table[i] =
2054 			bp->fp->cl_id +
2055 			ethtool_rxfh_indir_default(i, num_eth_queues);
2056 
2057 	/*
2058 	 * For 57710 and 57711 SEARCHER configuration (rss_keys) is
2059 	 * per-port, so if explicit configuration is needed , do it only
2060 	 * for a PMF.
2061 	 *
2062 	 * For 57712 and newer on the other hand it's a per-function
2063 	 * configuration.
2064 	 */
2065 	return bnx2x_config_rss_eth(bp, bp->port.pmf || !CHIP_IS_E1x(bp));
2066 }
2067 
2068 int bnx2x_rss(struct bnx2x *bp, struct bnx2x_rss_config_obj *rss_obj,
2069 	      bool config_hash, bool enable)
2070 {
2071 	struct bnx2x_config_rss_params params = {NULL};
2072 
2073 	/* Although RSS is meaningless when there is a single HW queue we
2074 	 * still need it enabled in order to have HW Rx hash generated.
2075 	 *
2076 	 * if (!is_eth_multi(bp))
2077 	 *      bp->multi_mode = ETH_RSS_MODE_DISABLED;
2078 	 */
2079 
2080 	params.rss_obj = rss_obj;
2081 
2082 	__set_bit(RAMROD_COMP_WAIT, &params.ramrod_flags);
2083 
2084 	if (enable) {
2085 		__set_bit(BNX2X_RSS_MODE_REGULAR, &params.rss_flags);
2086 
2087 		/* RSS configuration */
2088 		__set_bit(BNX2X_RSS_IPV4, &params.rss_flags);
2089 		__set_bit(BNX2X_RSS_IPV4_TCP, &params.rss_flags);
2090 		__set_bit(BNX2X_RSS_IPV6, &params.rss_flags);
2091 		__set_bit(BNX2X_RSS_IPV6_TCP, &params.rss_flags);
2092 		if (rss_obj->udp_rss_v4)
2093 			__set_bit(BNX2X_RSS_IPV4_UDP, &params.rss_flags);
2094 		if (rss_obj->udp_rss_v6)
2095 			__set_bit(BNX2X_RSS_IPV6_UDP, &params.rss_flags);
2096 
2097 		if (!CHIP_IS_E1x(bp)) {
2098 			/* valid only for TUNN_MODE_VXLAN tunnel mode */
2099 			__set_bit(BNX2X_RSS_IPV4_VXLAN, &params.rss_flags);
2100 			__set_bit(BNX2X_RSS_IPV6_VXLAN, &params.rss_flags);
2101 
2102 			/* valid only for TUNN_MODE_GRE tunnel mode */
2103 			__set_bit(BNX2X_RSS_TUNN_INNER_HDRS, &params.rss_flags);
2104 		}
2105 	} else {
2106 		__set_bit(BNX2X_RSS_MODE_DISABLED, &params.rss_flags);
2107 	}
2108 
2109 	/* Hash bits */
2110 	params.rss_result_mask = MULTI_MASK;
2111 
2112 	memcpy(params.ind_table, rss_obj->ind_table, sizeof(params.ind_table));
2113 
2114 	if (config_hash) {
2115 		/* RSS keys */
2116 		netdev_rss_key_fill(params.rss_key, T_ETH_RSS_KEY * 4);
2117 		__set_bit(BNX2X_RSS_SET_SRCH, &params.rss_flags);
2118 	}
2119 
2120 	if (IS_PF(bp))
2121 		return bnx2x_config_rss(bp, &params);
2122 	else
2123 		return bnx2x_vfpf_config_rss(bp, &params);
2124 }
2125 
2126 static int bnx2x_init_hw(struct bnx2x *bp, u32 load_code)
2127 {
2128 	struct bnx2x_func_state_params func_params = {NULL};
2129 
2130 	/* Prepare parameters for function state transitions */
2131 	__set_bit(RAMROD_COMP_WAIT, &func_params.ramrod_flags);
2132 
2133 	func_params.f_obj = &bp->func_obj;
2134 	func_params.cmd = BNX2X_F_CMD_HW_INIT;
2135 
2136 	func_params.params.hw_init.load_phase = load_code;
2137 
2138 	return bnx2x_func_state_change(bp, &func_params);
2139 }
2140 
2141 /*
2142  * Cleans the object that have internal lists without sending
2143  * ramrods. Should be run when interrupts are disabled.
2144  */
2145 void bnx2x_squeeze_objects(struct bnx2x *bp)
2146 {
2147 	int rc;
2148 	unsigned long ramrod_flags = 0, vlan_mac_flags = 0;
2149 	struct bnx2x_mcast_ramrod_params rparam = {NULL};
2150 	struct bnx2x_vlan_mac_obj *mac_obj = &bp->sp_objs->mac_obj;
2151 
2152 	/***************** Cleanup MACs' object first *************************/
2153 
2154 	/* Wait for completion of requested */
2155 	__set_bit(RAMROD_COMP_WAIT, &ramrod_flags);
2156 	/* Perform a dry cleanup */
2157 	__set_bit(RAMROD_DRV_CLR_ONLY, &ramrod_flags);
2158 
2159 	/* Clean ETH primary MAC */
2160 	__set_bit(BNX2X_ETH_MAC, &vlan_mac_flags);
2161 	rc = mac_obj->delete_all(bp, &bp->sp_objs->mac_obj, &vlan_mac_flags,
2162 				 &ramrod_flags);
2163 	if (rc != 0)
2164 		BNX2X_ERR("Failed to clean ETH MACs: %d\n", rc);
2165 
2166 	/* Cleanup UC list */
2167 	vlan_mac_flags = 0;
2168 	__set_bit(BNX2X_UC_LIST_MAC, &vlan_mac_flags);
2169 	rc = mac_obj->delete_all(bp, mac_obj, &vlan_mac_flags,
2170 				 &ramrod_flags);
2171 	if (rc != 0)
2172 		BNX2X_ERR("Failed to clean UC list MACs: %d\n", rc);
2173 
2174 	/***************** Now clean mcast object *****************************/
2175 	rparam.mcast_obj = &bp->mcast_obj;
2176 	__set_bit(RAMROD_DRV_CLR_ONLY, &rparam.ramrod_flags);
2177 
2178 	/* Add a DEL command... - Since we're doing a driver cleanup only,
2179 	 * we take a lock surrounding both the initial send and the CONTs,
2180 	 * as we don't want a true completion to disrupt us in the middle.
2181 	 */
2182 	netif_addr_lock_bh(bp->dev);
2183 	rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_DEL);
2184 	if (rc < 0)
2185 		BNX2X_ERR("Failed to add a new DEL command to a multi-cast object: %d\n",
2186 			  rc);
2187 
2188 	/* ...and wait until all pending commands are cleared */
2189 	rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2190 	while (rc != 0) {
2191 		if (rc < 0) {
2192 			BNX2X_ERR("Failed to clean multi-cast object: %d\n",
2193 				  rc);
2194 			netif_addr_unlock_bh(bp->dev);
2195 			return;
2196 		}
2197 
2198 		rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2199 	}
2200 	netif_addr_unlock_bh(bp->dev);
2201 }
2202 
2203 #ifndef BNX2X_STOP_ON_ERROR
2204 #define LOAD_ERROR_EXIT(bp, label) \
2205 	do { \
2206 		(bp)->state = BNX2X_STATE_ERROR; \
2207 		goto label; \
2208 	} while (0)
2209 
2210 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
2211 	do { \
2212 		bp->cnic_loaded = false; \
2213 		goto label; \
2214 	} while (0)
2215 #else /*BNX2X_STOP_ON_ERROR*/
2216 #define LOAD_ERROR_EXIT(bp, label) \
2217 	do { \
2218 		(bp)->state = BNX2X_STATE_ERROR; \
2219 		(bp)->panic = 1; \
2220 		return -EBUSY; \
2221 	} while (0)
2222 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
2223 	do { \
2224 		bp->cnic_loaded = false; \
2225 		(bp)->panic = 1; \
2226 		return -EBUSY; \
2227 	} while (0)
2228 #endif /*BNX2X_STOP_ON_ERROR*/
2229 
2230 static void bnx2x_free_fw_stats_mem(struct bnx2x *bp)
2231 {
2232 	BNX2X_PCI_FREE(bp->fw_stats, bp->fw_stats_mapping,
2233 		       bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2234 	return;
2235 }
2236 
2237 static int bnx2x_alloc_fw_stats_mem(struct bnx2x *bp)
2238 {
2239 	int num_groups, vf_headroom = 0;
2240 	int is_fcoe_stats = NO_FCOE(bp) ? 0 : 1;
2241 
2242 	/* number of queues for statistics is number of eth queues + FCoE */
2243 	u8 num_queue_stats = BNX2X_NUM_ETH_QUEUES(bp) + is_fcoe_stats;
2244 
2245 	/* Total number of FW statistics requests =
2246 	 * 1 for port stats + 1 for PF stats + potential 2 for FCoE (fcoe proper
2247 	 * and fcoe l2 queue) stats + num of queues (which includes another 1
2248 	 * for fcoe l2 queue if applicable)
2249 	 */
2250 	bp->fw_stats_num = 2 + is_fcoe_stats + num_queue_stats;
2251 
2252 	/* vf stats appear in the request list, but their data is allocated by
2253 	 * the VFs themselves. We don't include them in the bp->fw_stats_num as
2254 	 * it is used to determine where to place the vf stats queries in the
2255 	 * request struct
2256 	 */
2257 	if (IS_SRIOV(bp))
2258 		vf_headroom = bnx2x_vf_headroom(bp);
2259 
2260 	/* Request is built from stats_query_header and an array of
2261 	 * stats_query_cmd_group each of which contains
2262 	 * STATS_QUERY_CMD_COUNT rules. The real number or requests is
2263 	 * configured in the stats_query_header.
2264 	 */
2265 	num_groups =
2266 		(((bp->fw_stats_num + vf_headroom) / STATS_QUERY_CMD_COUNT) +
2267 		 (((bp->fw_stats_num + vf_headroom) % STATS_QUERY_CMD_COUNT) ?
2268 		 1 : 0));
2269 
2270 	DP(BNX2X_MSG_SP, "stats fw_stats_num %d, vf headroom %d, num_groups %d\n",
2271 	   bp->fw_stats_num, vf_headroom, num_groups);
2272 	bp->fw_stats_req_sz = sizeof(struct stats_query_header) +
2273 		num_groups * sizeof(struct stats_query_cmd_group);
2274 
2275 	/* Data for statistics requests + stats_counter
2276 	 * stats_counter holds per-STORM counters that are incremented
2277 	 * when STORM has finished with the current request.
2278 	 * memory for FCoE offloaded statistics are counted anyway,
2279 	 * even if they will not be sent.
2280 	 * VF stats are not accounted for here as the data of VF stats is stored
2281 	 * in memory allocated by the VF, not here.
2282 	 */
2283 	bp->fw_stats_data_sz = sizeof(struct per_port_stats) +
2284 		sizeof(struct per_pf_stats) +
2285 		sizeof(struct fcoe_statistics_params) +
2286 		sizeof(struct per_queue_stats) * num_queue_stats +
2287 		sizeof(struct stats_counter);
2288 
2289 	bp->fw_stats = BNX2X_PCI_ALLOC(&bp->fw_stats_mapping,
2290 				       bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2291 	if (!bp->fw_stats)
2292 		goto alloc_mem_err;
2293 
2294 	/* Set shortcuts */
2295 	bp->fw_stats_req = (struct bnx2x_fw_stats_req *)bp->fw_stats;
2296 	bp->fw_stats_req_mapping = bp->fw_stats_mapping;
2297 	bp->fw_stats_data = (struct bnx2x_fw_stats_data *)
2298 		((u8 *)bp->fw_stats + bp->fw_stats_req_sz);
2299 	bp->fw_stats_data_mapping = bp->fw_stats_mapping +
2300 		bp->fw_stats_req_sz;
2301 
2302 	DP(BNX2X_MSG_SP, "statistics request base address set to %x %x\n",
2303 	   U64_HI(bp->fw_stats_req_mapping),
2304 	   U64_LO(bp->fw_stats_req_mapping));
2305 	DP(BNX2X_MSG_SP, "statistics data base address set to %x %x\n",
2306 	   U64_HI(bp->fw_stats_data_mapping),
2307 	   U64_LO(bp->fw_stats_data_mapping));
2308 	return 0;
2309 
2310 alloc_mem_err:
2311 	bnx2x_free_fw_stats_mem(bp);
2312 	BNX2X_ERR("Can't allocate FW stats memory\n");
2313 	return -ENOMEM;
2314 }
2315 
2316 /* send load request to mcp and analyze response */
2317 static int bnx2x_nic_load_request(struct bnx2x *bp, u32 *load_code)
2318 {
2319 	u32 param;
2320 
2321 	/* init fw_seq */
2322 	bp->fw_seq =
2323 		(SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_mb_header) &
2324 		 DRV_MSG_SEQ_NUMBER_MASK);
2325 	BNX2X_DEV_INFO("fw_seq 0x%08x\n", bp->fw_seq);
2326 
2327 	/* Get current FW pulse sequence */
2328 	bp->fw_drv_pulse_wr_seq =
2329 		(SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_pulse_mb) &
2330 		 DRV_PULSE_SEQ_MASK);
2331 	BNX2X_DEV_INFO("drv_pulse 0x%x\n", bp->fw_drv_pulse_wr_seq);
2332 
2333 	param = DRV_MSG_CODE_LOAD_REQ_WITH_LFA;
2334 
2335 	if (IS_MF_SD(bp) && bnx2x_port_after_undi(bp))
2336 		param |= DRV_MSG_CODE_LOAD_REQ_FORCE_LFA;
2337 
2338 	/* load request */
2339 	(*load_code) = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ, param);
2340 
2341 	/* if mcp fails to respond we must abort */
2342 	if (!(*load_code)) {
2343 		BNX2X_ERR("MCP response failure, aborting\n");
2344 		return -EBUSY;
2345 	}
2346 
2347 	/* If mcp refused (e.g. other port is in diagnostic mode) we
2348 	 * must abort
2349 	 */
2350 	if ((*load_code) == FW_MSG_CODE_DRV_LOAD_REFUSED) {
2351 		BNX2X_ERR("MCP refused load request, aborting\n");
2352 		return -EBUSY;
2353 	}
2354 	return 0;
2355 }
2356 
2357 /* check whether another PF has already loaded FW to chip. In
2358  * virtualized environments a pf from another VM may have already
2359  * initialized the device including loading FW
2360  */
2361 int bnx2x_compare_fw_ver(struct bnx2x *bp, u32 load_code, bool print_err)
2362 {
2363 	/* is another pf loaded on this engine? */
2364 	if (load_code != FW_MSG_CODE_DRV_LOAD_COMMON_CHIP &&
2365 	    load_code != FW_MSG_CODE_DRV_LOAD_COMMON) {
2366 		/* build my FW version dword */
2367 		u32 my_fw = (BCM_5710_FW_MAJOR_VERSION) +
2368 			(BCM_5710_FW_MINOR_VERSION << 8) +
2369 			(BCM_5710_FW_REVISION_VERSION << 16) +
2370 			(BCM_5710_FW_ENGINEERING_VERSION << 24);
2371 
2372 		/* read loaded FW from chip */
2373 		u32 loaded_fw = REG_RD(bp, XSEM_REG_PRAM);
2374 
2375 		DP(BNX2X_MSG_SP, "loaded fw %x, my fw %x\n",
2376 		   loaded_fw, my_fw);
2377 
2378 		/* abort nic load if version mismatch */
2379 		if (my_fw != loaded_fw) {
2380 			if (print_err)
2381 				BNX2X_ERR("bnx2x with FW %x was already loaded which mismatches my %x FW. Aborting\n",
2382 					  loaded_fw, my_fw);
2383 			else
2384 				BNX2X_DEV_INFO("bnx2x with FW %x was already loaded which mismatches my %x FW, possibly due to MF UNDI\n",
2385 					       loaded_fw, my_fw);
2386 			return -EBUSY;
2387 		}
2388 	}
2389 	return 0;
2390 }
2391 
2392 /* returns the "mcp load_code" according to global load_count array */
2393 static int bnx2x_nic_load_no_mcp(struct bnx2x *bp, int port)
2394 {
2395 	int path = BP_PATH(bp);
2396 
2397 	DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d]      %d, %d, %d\n",
2398 	   path, bnx2x_load_count[path][0], bnx2x_load_count[path][1],
2399 	   bnx2x_load_count[path][2]);
2400 	bnx2x_load_count[path][0]++;
2401 	bnx2x_load_count[path][1 + port]++;
2402 	DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d]  %d, %d, %d\n",
2403 	   path, bnx2x_load_count[path][0], bnx2x_load_count[path][1],
2404 	   bnx2x_load_count[path][2]);
2405 	if (bnx2x_load_count[path][0] == 1)
2406 		return FW_MSG_CODE_DRV_LOAD_COMMON;
2407 	else if (bnx2x_load_count[path][1 + port] == 1)
2408 		return FW_MSG_CODE_DRV_LOAD_PORT;
2409 	else
2410 		return FW_MSG_CODE_DRV_LOAD_FUNCTION;
2411 }
2412 
2413 /* mark PMF if applicable */
2414 static void bnx2x_nic_load_pmf(struct bnx2x *bp, u32 load_code)
2415 {
2416 	if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2417 	    (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
2418 	    (load_code == FW_MSG_CODE_DRV_LOAD_PORT)) {
2419 		bp->port.pmf = 1;
2420 		/* We need the barrier to ensure the ordering between the
2421 		 * writing to bp->port.pmf here and reading it from the
2422 		 * bnx2x_periodic_task().
2423 		 */
2424 		smp_mb();
2425 	} else {
2426 		bp->port.pmf = 0;
2427 	}
2428 
2429 	DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf);
2430 }
2431 
2432 static void bnx2x_nic_load_afex_dcc(struct bnx2x *bp, int load_code)
2433 {
2434 	if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2435 	     (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
2436 	    (bp->common.shmem2_base)) {
2437 		if (SHMEM2_HAS(bp, dcc_support))
2438 			SHMEM2_WR(bp, dcc_support,
2439 				  (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
2440 				   SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
2441 		if (SHMEM2_HAS(bp, afex_driver_support))
2442 			SHMEM2_WR(bp, afex_driver_support,
2443 				  SHMEM_AFEX_SUPPORTED_VERSION_ONE);
2444 	}
2445 
2446 	/* Set AFEX default VLAN tag to an invalid value */
2447 	bp->afex_def_vlan_tag = -1;
2448 }
2449 
2450 /**
2451  * bnx2x_bz_fp - zero content of the fastpath structure.
2452  *
2453  * @bp:		driver handle
2454  * @index:	fastpath index to be zeroed
2455  *
2456  * Makes sure the contents of the bp->fp[index].napi is kept
2457  * intact.
2458  */
2459 static void bnx2x_bz_fp(struct bnx2x *bp, int index)
2460 {
2461 	struct bnx2x_fastpath *fp = &bp->fp[index];
2462 	int cos;
2463 	struct napi_struct orig_napi = fp->napi;
2464 	struct bnx2x_agg_info *orig_tpa_info = fp->tpa_info;
2465 
2466 	/* bzero bnx2x_fastpath contents */
2467 	if (fp->tpa_info)
2468 		memset(fp->tpa_info, 0, ETH_MAX_AGGREGATION_QUEUES_E1H_E2 *
2469 		       sizeof(struct bnx2x_agg_info));
2470 	memset(fp, 0, sizeof(*fp));
2471 
2472 	/* Restore the NAPI object as it has been already initialized */
2473 	fp->napi = orig_napi;
2474 	fp->tpa_info = orig_tpa_info;
2475 	fp->bp = bp;
2476 	fp->index = index;
2477 	if (IS_ETH_FP(fp))
2478 		fp->max_cos = bp->max_cos;
2479 	else
2480 		/* Special queues support only one CoS */
2481 		fp->max_cos = 1;
2482 
2483 	/* Init txdata pointers */
2484 	if (IS_FCOE_FP(fp))
2485 		fp->txdata_ptr[0] = &bp->bnx2x_txq[FCOE_TXQ_IDX(bp)];
2486 	if (IS_ETH_FP(fp))
2487 		for_each_cos_in_tx_queue(fp, cos)
2488 			fp->txdata_ptr[cos] = &bp->bnx2x_txq[cos *
2489 				BNX2X_NUM_ETH_QUEUES(bp) + index];
2490 
2491 	/* set the tpa flag for each queue. The tpa flag determines the queue
2492 	 * minimal size so it must be set prior to queue memory allocation
2493 	 */
2494 	if (bp->dev->features & NETIF_F_LRO)
2495 		fp->mode = TPA_MODE_LRO;
2496 	else if (bp->dev->features & NETIF_F_GRO_HW)
2497 		fp->mode = TPA_MODE_GRO;
2498 	else
2499 		fp->mode = TPA_MODE_DISABLED;
2500 
2501 	/* We don't want TPA if it's disabled in bp
2502 	 * or if this is an FCoE L2 ring.
2503 	 */
2504 	if (bp->disable_tpa || IS_FCOE_FP(fp))
2505 		fp->mode = TPA_MODE_DISABLED;
2506 }
2507 
2508 void bnx2x_set_os_driver_state(struct bnx2x *bp, u32 state)
2509 {
2510 	u32 cur;
2511 
2512 	if (!IS_MF_BD(bp) || !SHMEM2_HAS(bp, os_driver_state) || IS_VF(bp))
2513 		return;
2514 
2515 	cur = SHMEM2_RD(bp, os_driver_state[BP_FW_MB_IDX(bp)]);
2516 	DP(NETIF_MSG_IFUP, "Driver state %08x-->%08x\n",
2517 	   cur, state);
2518 
2519 	SHMEM2_WR(bp, os_driver_state[BP_FW_MB_IDX(bp)], state);
2520 }
2521 
2522 int bnx2x_load_cnic(struct bnx2x *bp)
2523 {
2524 	int i, rc, port = BP_PORT(bp);
2525 
2526 	DP(NETIF_MSG_IFUP, "Starting CNIC-related load\n");
2527 
2528 	mutex_init(&bp->cnic_mutex);
2529 
2530 	if (IS_PF(bp)) {
2531 		rc = bnx2x_alloc_mem_cnic(bp);
2532 		if (rc) {
2533 			BNX2X_ERR("Unable to allocate bp memory for cnic\n");
2534 			LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2535 		}
2536 	}
2537 
2538 	rc = bnx2x_alloc_fp_mem_cnic(bp);
2539 	if (rc) {
2540 		BNX2X_ERR("Unable to allocate memory for cnic fps\n");
2541 		LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2542 	}
2543 
2544 	/* Update the number of queues with the cnic queues */
2545 	rc = bnx2x_set_real_num_queues(bp, 1);
2546 	if (rc) {
2547 		BNX2X_ERR("Unable to set real_num_queues including cnic\n");
2548 		LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2549 	}
2550 
2551 	/* Add all CNIC NAPI objects */
2552 	bnx2x_add_all_napi_cnic(bp);
2553 	DP(NETIF_MSG_IFUP, "cnic napi added\n");
2554 	bnx2x_napi_enable_cnic(bp);
2555 
2556 	rc = bnx2x_init_hw_func_cnic(bp);
2557 	if (rc)
2558 		LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic1);
2559 
2560 	bnx2x_nic_init_cnic(bp);
2561 
2562 	if (IS_PF(bp)) {
2563 		/* Enable Timer scan */
2564 		REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 1);
2565 
2566 		/* setup cnic queues */
2567 		for_each_cnic_queue(bp, i) {
2568 			rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
2569 			if (rc) {
2570 				BNX2X_ERR("Queue setup failed\n");
2571 				LOAD_ERROR_EXIT(bp, load_error_cnic2);
2572 			}
2573 		}
2574 	}
2575 
2576 	/* Initialize Rx filter. */
2577 	bnx2x_set_rx_mode_inner(bp);
2578 
2579 	/* re-read iscsi info */
2580 	bnx2x_get_iscsi_info(bp);
2581 	bnx2x_setup_cnic_irq_info(bp);
2582 	bnx2x_setup_cnic_info(bp);
2583 	bp->cnic_loaded = true;
2584 	if (bp->state == BNX2X_STATE_OPEN)
2585 		bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
2586 
2587 	DP(NETIF_MSG_IFUP, "Ending successfully CNIC-related load\n");
2588 
2589 	return 0;
2590 
2591 #ifndef BNX2X_STOP_ON_ERROR
2592 load_error_cnic2:
2593 	/* Disable Timer scan */
2594 	REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 0);
2595 
2596 load_error_cnic1:
2597 	bnx2x_napi_disable_cnic(bp);
2598 	/* Update the number of queues without the cnic queues */
2599 	if (bnx2x_set_real_num_queues(bp, 0))
2600 		BNX2X_ERR("Unable to set real_num_queues not including cnic\n");
2601 load_error_cnic0:
2602 	BNX2X_ERR("CNIC-related load failed\n");
2603 	bnx2x_free_fp_mem_cnic(bp);
2604 	bnx2x_free_mem_cnic(bp);
2605 	return rc;
2606 #endif /* ! BNX2X_STOP_ON_ERROR */
2607 }
2608 
2609 /* must be called with rtnl_lock */
2610 int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
2611 {
2612 	int port = BP_PORT(bp);
2613 	int i, rc = 0, load_code = 0;
2614 
2615 	DP(NETIF_MSG_IFUP, "Starting NIC load\n");
2616 	DP(NETIF_MSG_IFUP,
2617 	   "CNIC is %s\n", CNIC_ENABLED(bp) ? "enabled" : "disabled");
2618 
2619 #ifdef BNX2X_STOP_ON_ERROR
2620 	if (unlikely(bp->panic)) {
2621 		BNX2X_ERR("Can't load NIC when there is panic\n");
2622 		return -EPERM;
2623 	}
2624 #endif
2625 
2626 	bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
2627 
2628 	/* zero the structure w/o any lock, before SP handler is initialized */
2629 	memset(&bp->last_reported_link, 0, sizeof(bp->last_reported_link));
2630 	__set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
2631 		&bp->last_reported_link.link_report_flags);
2632 
2633 	if (IS_PF(bp))
2634 		/* must be called before memory allocation and HW init */
2635 		bnx2x_ilt_set_info(bp);
2636 
2637 	/*
2638 	 * Zero fastpath structures preserving invariants like napi, which are
2639 	 * allocated only once, fp index, max_cos, bp pointer.
2640 	 * Also set fp->mode and txdata_ptr.
2641 	 */
2642 	DP(NETIF_MSG_IFUP, "num queues: %d", bp->num_queues);
2643 	for_each_queue(bp, i)
2644 		bnx2x_bz_fp(bp, i);
2645 	memset(bp->bnx2x_txq, 0, (BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS +
2646 				  bp->num_cnic_queues) *
2647 				  sizeof(struct bnx2x_fp_txdata));
2648 
2649 	bp->fcoe_init = false;
2650 
2651 	/* Set the receive queues buffer size */
2652 	bnx2x_set_rx_buf_size(bp);
2653 
2654 	if (IS_PF(bp)) {
2655 		rc = bnx2x_alloc_mem(bp);
2656 		if (rc) {
2657 			BNX2X_ERR("Unable to allocate bp memory\n");
2658 			return rc;
2659 		}
2660 	}
2661 
2662 	/* need to be done after alloc mem, since it's self adjusting to amount
2663 	 * of memory available for RSS queues
2664 	 */
2665 	rc = bnx2x_alloc_fp_mem(bp);
2666 	if (rc) {
2667 		BNX2X_ERR("Unable to allocate memory for fps\n");
2668 		LOAD_ERROR_EXIT(bp, load_error0);
2669 	}
2670 
2671 	/* Allocated memory for FW statistics  */
2672 	if (bnx2x_alloc_fw_stats_mem(bp))
2673 		LOAD_ERROR_EXIT(bp, load_error0);
2674 
2675 	/* request pf to initialize status blocks */
2676 	if (IS_VF(bp)) {
2677 		rc = bnx2x_vfpf_init(bp);
2678 		if (rc)
2679 			LOAD_ERROR_EXIT(bp, load_error0);
2680 	}
2681 
2682 	/* As long as bnx2x_alloc_mem() may possibly update
2683 	 * bp->num_queues, bnx2x_set_real_num_queues() should always
2684 	 * come after it. At this stage cnic queues are not counted.
2685 	 */
2686 	rc = bnx2x_set_real_num_queues(bp, 0);
2687 	if (rc) {
2688 		BNX2X_ERR("Unable to set real_num_queues\n");
2689 		LOAD_ERROR_EXIT(bp, load_error0);
2690 	}
2691 
2692 	/* configure multi cos mappings in kernel.
2693 	 * this configuration may be overridden by a multi class queue
2694 	 * discipline or by a dcbx negotiation result.
2695 	 */
2696 	bnx2x_setup_tc(bp->dev, bp->max_cos);
2697 
2698 	/* Add all NAPI objects */
2699 	bnx2x_add_all_napi(bp);
2700 	DP(NETIF_MSG_IFUP, "napi added\n");
2701 	bnx2x_napi_enable(bp);
2702 
2703 	if (IS_PF(bp)) {
2704 		/* set pf load just before approaching the MCP */
2705 		bnx2x_set_pf_load(bp);
2706 
2707 		/* if mcp exists send load request and analyze response */
2708 		if (!BP_NOMCP(bp)) {
2709 			/* attempt to load pf */
2710 			rc = bnx2x_nic_load_request(bp, &load_code);
2711 			if (rc)
2712 				LOAD_ERROR_EXIT(bp, load_error1);
2713 
2714 			/* what did mcp say? */
2715 			rc = bnx2x_compare_fw_ver(bp, load_code, true);
2716 			if (rc) {
2717 				bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2718 				LOAD_ERROR_EXIT(bp, load_error2);
2719 			}
2720 		} else {
2721 			load_code = bnx2x_nic_load_no_mcp(bp, port);
2722 		}
2723 
2724 		/* mark pmf if applicable */
2725 		bnx2x_nic_load_pmf(bp, load_code);
2726 
2727 		/* Init Function state controlling object */
2728 		bnx2x__init_func_obj(bp);
2729 
2730 		/* Initialize HW */
2731 		rc = bnx2x_init_hw(bp, load_code);
2732 		if (rc) {
2733 			BNX2X_ERR("HW init failed, aborting\n");
2734 			bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2735 			LOAD_ERROR_EXIT(bp, load_error2);
2736 		}
2737 	}
2738 
2739 	bnx2x_pre_irq_nic_init(bp);
2740 
2741 	/* Connect to IRQs */
2742 	rc = bnx2x_setup_irqs(bp);
2743 	if (rc) {
2744 		BNX2X_ERR("setup irqs failed\n");
2745 		if (IS_PF(bp))
2746 			bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2747 		LOAD_ERROR_EXIT(bp, load_error2);
2748 	}
2749 
2750 	/* Init per-function objects */
2751 	if (IS_PF(bp)) {
2752 		/* Setup NIC internals and enable interrupts */
2753 		bnx2x_post_irq_nic_init(bp, load_code);
2754 
2755 		bnx2x_init_bp_objs(bp);
2756 		bnx2x_iov_nic_init(bp);
2757 
2758 		/* Set AFEX default VLAN tag to an invalid value */
2759 		bp->afex_def_vlan_tag = -1;
2760 		bnx2x_nic_load_afex_dcc(bp, load_code);
2761 		bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
2762 		rc = bnx2x_func_start(bp);
2763 		if (rc) {
2764 			BNX2X_ERR("Function start failed!\n");
2765 			bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2766 
2767 			LOAD_ERROR_EXIT(bp, load_error3);
2768 		}
2769 
2770 		/* Send LOAD_DONE command to MCP */
2771 		if (!BP_NOMCP(bp)) {
2772 			load_code = bnx2x_fw_command(bp,
2773 						     DRV_MSG_CODE_LOAD_DONE, 0);
2774 			if (!load_code) {
2775 				BNX2X_ERR("MCP response failure, aborting\n");
2776 				rc = -EBUSY;
2777 				LOAD_ERROR_EXIT(bp, load_error3);
2778 			}
2779 		}
2780 
2781 		/* initialize FW coalescing state machines in RAM */
2782 		bnx2x_update_coalesce(bp);
2783 	}
2784 
2785 	/* setup the leading queue */
2786 	rc = bnx2x_setup_leading(bp);
2787 	if (rc) {
2788 		BNX2X_ERR("Setup leading failed!\n");
2789 		LOAD_ERROR_EXIT(bp, load_error3);
2790 	}
2791 
2792 	/* set up the rest of the queues */
2793 	for_each_nondefault_eth_queue(bp, i) {
2794 		if (IS_PF(bp))
2795 			rc = bnx2x_setup_queue(bp, &bp->fp[i], false);
2796 		else /* VF */
2797 			rc = bnx2x_vfpf_setup_q(bp, &bp->fp[i], false);
2798 		if (rc) {
2799 			BNX2X_ERR("Queue %d setup failed\n", i);
2800 			LOAD_ERROR_EXIT(bp, load_error3);
2801 		}
2802 	}
2803 
2804 	/* setup rss */
2805 	rc = bnx2x_init_rss(bp);
2806 	if (rc) {
2807 		BNX2X_ERR("PF RSS init failed\n");
2808 		LOAD_ERROR_EXIT(bp, load_error3);
2809 	}
2810 
2811 	/* Now when Clients are configured we are ready to work */
2812 	bp->state = BNX2X_STATE_OPEN;
2813 
2814 	/* Configure a ucast MAC */
2815 	if (IS_PF(bp))
2816 		rc = bnx2x_set_eth_mac(bp, true);
2817 	else /* vf */
2818 		rc = bnx2x_vfpf_config_mac(bp, bp->dev->dev_addr, bp->fp->index,
2819 					   true);
2820 	if (rc) {
2821 		BNX2X_ERR("Setting Ethernet MAC failed\n");
2822 		LOAD_ERROR_EXIT(bp, load_error3);
2823 	}
2824 
2825 	if (IS_PF(bp) && bp->pending_max) {
2826 		bnx2x_update_max_mf_config(bp, bp->pending_max);
2827 		bp->pending_max = 0;
2828 	}
2829 
2830 	bp->force_link_down = false;
2831 	if (bp->port.pmf) {
2832 		rc = bnx2x_initial_phy_init(bp, load_mode);
2833 		if (rc)
2834 			LOAD_ERROR_EXIT(bp, load_error3);
2835 	}
2836 	bp->link_params.feature_config_flags &= ~FEATURE_CONFIG_BOOT_FROM_SAN;
2837 
2838 	/* Start fast path */
2839 
2840 	/* Re-configure vlan filters */
2841 	rc = bnx2x_vlan_reconfigure_vid(bp);
2842 	if (rc)
2843 		LOAD_ERROR_EXIT(bp, load_error3);
2844 
2845 	/* Initialize Rx filter. */
2846 	bnx2x_set_rx_mode_inner(bp);
2847 
2848 	if (bp->flags & PTP_SUPPORTED) {
2849 		bnx2x_register_phc(bp);
2850 		bnx2x_init_ptp(bp);
2851 		bnx2x_configure_ptp_filters(bp);
2852 	}
2853 	/* Start Tx */
2854 	switch (load_mode) {
2855 	case LOAD_NORMAL:
2856 		/* Tx queue should be only re-enabled */
2857 		netif_tx_wake_all_queues(bp->dev);
2858 		break;
2859 
2860 	case LOAD_OPEN:
2861 		netif_tx_start_all_queues(bp->dev);
2862 		smp_mb__after_atomic();
2863 		break;
2864 
2865 	case LOAD_DIAG:
2866 	case LOAD_LOOPBACK_EXT:
2867 		bp->state = BNX2X_STATE_DIAG;
2868 		break;
2869 
2870 	default:
2871 		break;
2872 	}
2873 
2874 	if (bp->port.pmf)
2875 		bnx2x_update_drv_flags(bp, 1 << DRV_FLAGS_PORT_MASK, 0);
2876 	else
2877 		bnx2x__link_status_update(bp);
2878 
2879 	/* start the timer */
2880 	mod_timer(&bp->timer, jiffies + bp->current_interval);
2881 
2882 	if (CNIC_ENABLED(bp))
2883 		bnx2x_load_cnic(bp);
2884 
2885 	if (IS_PF(bp))
2886 		bnx2x_schedule_sp_rtnl(bp, BNX2X_SP_RTNL_GET_DRV_VERSION, 0);
2887 
2888 	if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2889 		/* mark driver is loaded in shmem2 */
2890 		u32 val;
2891 		val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2892 		val &= ~DRV_FLAGS_MTU_MASK;
2893 		val |= (bp->dev->mtu << DRV_FLAGS_MTU_SHIFT);
2894 		SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2895 			  val | DRV_FLAGS_CAPABILITIES_LOADED_SUPPORTED |
2896 			  DRV_FLAGS_CAPABILITIES_LOADED_L2);
2897 	}
2898 
2899 	/* Wait for all pending SP commands to complete */
2900 	if (IS_PF(bp) && !bnx2x_wait_sp_comp(bp, ~0x0UL)) {
2901 		BNX2X_ERR("Timeout waiting for SP elements to complete\n");
2902 		bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
2903 		return -EBUSY;
2904 	}
2905 
2906 	/* Update driver data for On-Chip MFW dump. */
2907 	if (IS_PF(bp))
2908 		bnx2x_update_mfw_dump(bp);
2909 
2910 	/* If PMF - send ADMIN DCBX msg to MFW to initiate DCBX FSM */
2911 	if (bp->port.pmf && (bp->state != BNX2X_STATE_DIAG))
2912 		bnx2x_dcbx_init(bp, false);
2913 
2914 	if (!IS_MF_SD_STORAGE_PERSONALITY_ONLY(bp))
2915 		bnx2x_set_os_driver_state(bp, OS_DRIVER_STATE_ACTIVE);
2916 
2917 	DP(NETIF_MSG_IFUP, "Ending successfully NIC load\n");
2918 
2919 	return 0;
2920 
2921 #ifndef BNX2X_STOP_ON_ERROR
2922 load_error3:
2923 	if (IS_PF(bp)) {
2924 		bnx2x_int_disable_sync(bp, 1);
2925 
2926 		/* Clean queueable objects */
2927 		bnx2x_squeeze_objects(bp);
2928 	}
2929 
2930 	/* Free SKBs, SGEs, TPA pool and driver internals */
2931 	bnx2x_free_skbs(bp);
2932 	for_each_rx_queue(bp, i)
2933 		bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2934 
2935 	/* Release IRQs */
2936 	bnx2x_free_irq(bp);
2937 load_error2:
2938 	if (IS_PF(bp) && !BP_NOMCP(bp)) {
2939 		bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
2940 		bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
2941 	}
2942 
2943 	bp->port.pmf = 0;
2944 load_error1:
2945 	bnx2x_napi_disable(bp);
2946 	bnx2x_del_all_napi(bp);
2947 
2948 	/* clear pf_load status, as it was already set */
2949 	if (IS_PF(bp))
2950 		bnx2x_clear_pf_load(bp);
2951 load_error0:
2952 	bnx2x_free_fw_stats_mem(bp);
2953 	bnx2x_free_fp_mem(bp);
2954 	bnx2x_free_mem(bp);
2955 
2956 	return rc;
2957 #endif /* ! BNX2X_STOP_ON_ERROR */
2958 }
2959 
2960 int bnx2x_drain_tx_queues(struct bnx2x *bp)
2961 {
2962 	u8 rc = 0, cos, i;
2963 
2964 	/* Wait until tx fastpath tasks complete */
2965 	for_each_tx_queue(bp, i) {
2966 		struct bnx2x_fastpath *fp = &bp->fp[i];
2967 
2968 		for_each_cos_in_tx_queue(fp, cos)
2969 			rc = bnx2x_clean_tx_queue(bp, fp->txdata_ptr[cos]);
2970 		if (rc)
2971 			return rc;
2972 	}
2973 	return 0;
2974 }
2975 
2976 /* must be called with rtnl_lock */
2977 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode, bool keep_link)
2978 {
2979 	int i;
2980 	bool global = false;
2981 
2982 	DP(NETIF_MSG_IFUP, "Starting NIC unload\n");
2983 
2984 	if (!IS_MF_SD_STORAGE_PERSONALITY_ONLY(bp))
2985 		bnx2x_set_os_driver_state(bp, OS_DRIVER_STATE_DISABLED);
2986 
2987 	/* mark driver is unloaded in shmem2 */
2988 	if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2989 		u32 val;
2990 		val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2991 		SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2992 			  val & ~DRV_FLAGS_CAPABILITIES_LOADED_L2);
2993 	}
2994 
2995 	if (IS_PF(bp) && bp->recovery_state != BNX2X_RECOVERY_DONE &&
2996 	    (bp->state == BNX2X_STATE_CLOSED ||
2997 	     bp->state == BNX2X_STATE_ERROR)) {
2998 		/* We can get here if the driver has been unloaded
2999 		 * during parity error recovery and is either waiting for a
3000 		 * leader to complete or for other functions to unload and
3001 		 * then ifdown has been issued. In this case we want to
3002 		 * unload and let other functions to complete a recovery
3003 		 * process.
3004 		 */
3005 		bp->recovery_state = BNX2X_RECOVERY_DONE;
3006 		bp->is_leader = 0;
3007 		bnx2x_release_leader_lock(bp);
3008 		smp_mb();
3009 
3010 		DP(NETIF_MSG_IFDOWN, "Releasing a leadership...\n");
3011 		BNX2X_ERR("Can't unload in closed or error state\n");
3012 		return -EINVAL;
3013 	}
3014 
3015 	/* Nothing to do during unload if previous bnx2x_nic_load()
3016 	 * have not completed successfully - all resources are released.
3017 	 *
3018 	 * we can get here only after unsuccessful ndo_* callback, during which
3019 	 * dev->IFF_UP flag is still on.
3020 	 */
3021 	if (bp->state == BNX2X_STATE_CLOSED || bp->state == BNX2X_STATE_ERROR)
3022 		return 0;
3023 
3024 	/* It's important to set the bp->state to the value different from
3025 	 * BNX2X_STATE_OPEN and only then stop the Tx. Otherwise bnx2x_tx_int()
3026 	 * may restart the Tx from the NAPI context (see bnx2x_tx_int()).
3027 	 */
3028 	bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
3029 	smp_mb();
3030 
3031 	/* indicate to VFs that the PF is going down */
3032 	bnx2x_iov_channel_down(bp);
3033 
3034 	if (CNIC_LOADED(bp))
3035 		bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
3036 
3037 	/* Stop Tx */
3038 	bnx2x_tx_disable(bp);
3039 	netdev_reset_tc(bp->dev);
3040 
3041 	bp->rx_mode = BNX2X_RX_MODE_NONE;
3042 
3043 	del_timer_sync(&bp->timer);
3044 
3045 	if (IS_PF(bp) && !BP_NOMCP(bp)) {
3046 		/* Set ALWAYS_ALIVE bit in shmem */
3047 		bp->fw_drv_pulse_wr_seq |= DRV_PULSE_ALWAYS_ALIVE;
3048 		bnx2x_drv_pulse(bp);
3049 		bnx2x_stats_handle(bp, STATS_EVENT_STOP);
3050 		bnx2x_save_statistics(bp);
3051 	}
3052 
3053 	/* wait till consumers catch up with producers in all queues.
3054 	 * If we're recovering, FW can't write to host so no reason
3055 	 * to wait for the queues to complete all Tx.
3056 	 */
3057 	if (unload_mode != UNLOAD_RECOVERY)
3058 		bnx2x_drain_tx_queues(bp);
3059 
3060 	/* if VF indicate to PF this function is going down (PF will delete sp
3061 	 * elements and clear initializations
3062 	 */
3063 	if (IS_VF(bp)) {
3064 		bnx2x_clear_vlan_info(bp);
3065 		bnx2x_vfpf_close_vf(bp);
3066 	} else if (unload_mode != UNLOAD_RECOVERY) {
3067 		/* if this is a normal/close unload need to clean up chip*/
3068 		bnx2x_chip_cleanup(bp, unload_mode, keep_link);
3069 	} else {
3070 		/* Send the UNLOAD_REQUEST to the MCP */
3071 		bnx2x_send_unload_req(bp, unload_mode);
3072 
3073 		/* Prevent transactions to host from the functions on the
3074 		 * engine that doesn't reset global blocks in case of global
3075 		 * attention once global blocks are reset and gates are opened
3076 		 * (the engine which leader will perform the recovery
3077 		 * last).
3078 		 */
3079 		if (!CHIP_IS_E1x(bp))
3080 			bnx2x_pf_disable(bp);
3081 
3082 		/* Disable HW interrupts, NAPI */
3083 		bnx2x_netif_stop(bp, 1);
3084 		/* Delete all NAPI objects */
3085 		bnx2x_del_all_napi(bp);
3086 		if (CNIC_LOADED(bp))
3087 			bnx2x_del_all_napi_cnic(bp);
3088 		/* Release IRQs */
3089 		bnx2x_free_irq(bp);
3090 
3091 		/* Report UNLOAD_DONE to MCP */
3092 		bnx2x_send_unload_done(bp, false);
3093 	}
3094 
3095 	/*
3096 	 * At this stage no more interrupts will arrive so we may safely clean
3097 	 * the queueable objects here in case they failed to get cleaned so far.
3098 	 */
3099 	if (IS_PF(bp))
3100 		bnx2x_squeeze_objects(bp);
3101 
3102 	/* There should be no more pending SP commands at this stage */
3103 	bp->sp_state = 0;
3104 
3105 	bp->port.pmf = 0;
3106 
3107 	/* clear pending work in rtnl task */
3108 	bp->sp_rtnl_state = 0;
3109 	smp_mb();
3110 
3111 	/* Free SKBs, SGEs, TPA pool and driver internals */
3112 	bnx2x_free_skbs(bp);
3113 	if (CNIC_LOADED(bp))
3114 		bnx2x_free_skbs_cnic(bp);
3115 	for_each_rx_queue(bp, i)
3116 		bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
3117 
3118 	bnx2x_free_fp_mem(bp);
3119 	if (CNIC_LOADED(bp))
3120 		bnx2x_free_fp_mem_cnic(bp);
3121 
3122 	if (IS_PF(bp)) {
3123 		if (CNIC_LOADED(bp))
3124 			bnx2x_free_mem_cnic(bp);
3125 	}
3126 	bnx2x_free_mem(bp);
3127 
3128 	bp->state = BNX2X_STATE_CLOSED;
3129 	bp->cnic_loaded = false;
3130 
3131 	/* Clear driver version indication in shmem */
3132 	if (IS_PF(bp) && !BP_NOMCP(bp))
3133 		bnx2x_update_mng_version(bp);
3134 
3135 	/* Check if there are pending parity attentions. If there are - set
3136 	 * RECOVERY_IN_PROGRESS.
3137 	 */
3138 	if (IS_PF(bp) && bnx2x_chk_parity_attn(bp, &global, false)) {
3139 		bnx2x_set_reset_in_progress(bp);
3140 
3141 		/* Set RESET_IS_GLOBAL if needed */
3142 		if (global)
3143 			bnx2x_set_reset_global(bp);
3144 	}
3145 
3146 	/* The last driver must disable a "close the gate" if there is no
3147 	 * parity attention or "process kill" pending.
3148 	 */
3149 	if (IS_PF(bp) &&
3150 	    !bnx2x_clear_pf_load(bp) &&
3151 	    bnx2x_reset_is_done(bp, BP_PATH(bp)))
3152 		bnx2x_disable_close_the_gate(bp);
3153 
3154 	DP(NETIF_MSG_IFUP, "Ending NIC unload\n");
3155 
3156 	return 0;
3157 }
3158 
3159 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
3160 {
3161 	u16 pmcsr;
3162 
3163 	/* If there is no power capability, silently succeed */
3164 	if (!bp->pdev->pm_cap) {
3165 		BNX2X_DEV_INFO("No power capability. Breaking.\n");
3166 		return 0;
3167 	}
3168 
3169 	pci_read_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL, &pmcsr);
3170 
3171 	switch (state) {
3172 	case PCI_D0:
3173 		pci_write_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL,
3174 				      ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
3175 				       PCI_PM_CTRL_PME_STATUS));
3176 
3177 		if (pmcsr & PCI_PM_CTRL_STATE_MASK)
3178 			/* delay required during transition out of D3hot */
3179 			msleep(20);
3180 		break;
3181 
3182 	case PCI_D3hot:
3183 		/* If there are other clients above don't
3184 		   shut down the power */
3185 		if (atomic_read(&bp->pdev->enable_cnt) != 1)
3186 			return 0;
3187 		/* Don't shut down the power for emulation and FPGA */
3188 		if (CHIP_REV_IS_SLOW(bp))
3189 			return 0;
3190 
3191 		pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
3192 		pmcsr |= 3;
3193 
3194 		if (bp->wol)
3195 			pmcsr |= PCI_PM_CTRL_PME_ENABLE;
3196 
3197 		pci_write_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL,
3198 				      pmcsr);
3199 
3200 		/* No more memory access after this point until
3201 		* device is brought back to D0.
3202 		*/
3203 		break;
3204 
3205 	default:
3206 		dev_err(&bp->pdev->dev, "Can't support state = %d\n", state);
3207 		return -EINVAL;
3208 	}
3209 	return 0;
3210 }
3211 
3212 /*
3213  * net_device service functions
3214  */
3215 static int bnx2x_poll(struct napi_struct *napi, int budget)
3216 {
3217 	struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
3218 						 napi);
3219 	struct bnx2x *bp = fp->bp;
3220 	int rx_work_done;
3221 	u8 cos;
3222 
3223 #ifdef BNX2X_STOP_ON_ERROR
3224 	if (unlikely(bp->panic)) {
3225 		napi_complete(napi);
3226 		return 0;
3227 	}
3228 #endif
3229 	for_each_cos_in_tx_queue(fp, cos)
3230 		if (bnx2x_tx_queue_has_work(fp->txdata_ptr[cos]))
3231 			bnx2x_tx_int(bp, fp->txdata_ptr[cos]);
3232 
3233 	rx_work_done = (bnx2x_has_rx_work(fp)) ? bnx2x_rx_int(fp, budget) : 0;
3234 
3235 	if (rx_work_done < budget) {
3236 		/* No need to update SB for FCoE L2 ring as long as
3237 		 * it's connected to the default SB and the SB
3238 		 * has been updated when NAPI was scheduled.
3239 		 */
3240 		if (IS_FCOE_FP(fp)) {
3241 			napi_complete_done(napi, rx_work_done);
3242 		} else {
3243 			bnx2x_update_fpsb_idx(fp);
3244 			/* bnx2x_has_rx_work() reads the status block,
3245 			 * thus we need to ensure that status block indices
3246 			 * have been actually read (bnx2x_update_fpsb_idx)
3247 			 * prior to this check (bnx2x_has_rx_work) so that
3248 			 * we won't write the "newer" value of the status block
3249 			 * to IGU (if there was a DMA right after
3250 			 * bnx2x_has_rx_work and if there is no rmb, the memory
3251 			 * reading (bnx2x_update_fpsb_idx) may be postponed
3252 			 * to right before bnx2x_ack_sb). In this case there
3253 			 * will never be another interrupt until there is
3254 			 * another update of the status block, while there
3255 			 * is still unhandled work.
3256 			 */
3257 			rmb();
3258 
3259 			if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
3260 				if (napi_complete_done(napi, rx_work_done)) {
3261 					/* Re-enable interrupts */
3262 					DP(NETIF_MSG_RX_STATUS,
3263 					   "Update index to %d\n", fp->fp_hc_idx);
3264 					bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
3265 						     le16_to_cpu(fp->fp_hc_idx),
3266 						     IGU_INT_ENABLE, 1);
3267 				}
3268 			} else {
3269 				rx_work_done = budget;
3270 			}
3271 		}
3272 	}
3273 
3274 	return rx_work_done;
3275 }
3276 
3277 /* we split the first BD into headers and data BDs
3278  * to ease the pain of our fellow microcode engineers
3279  * we use one mapping for both BDs
3280  */
3281 static u16 bnx2x_tx_split(struct bnx2x *bp,
3282 			  struct bnx2x_fp_txdata *txdata,
3283 			  struct sw_tx_bd *tx_buf,
3284 			  struct eth_tx_start_bd **tx_bd, u16 hlen,
3285 			  u16 bd_prod)
3286 {
3287 	struct eth_tx_start_bd *h_tx_bd = *tx_bd;
3288 	struct eth_tx_bd *d_tx_bd;
3289 	dma_addr_t mapping;
3290 	int old_len = le16_to_cpu(h_tx_bd->nbytes);
3291 
3292 	/* first fix first BD */
3293 	h_tx_bd->nbytes = cpu_to_le16(hlen);
3294 
3295 	DP(NETIF_MSG_TX_QUEUED,	"TSO split header size is %d (%x:%x)\n",
3296 	   h_tx_bd->nbytes, h_tx_bd->addr_hi, h_tx_bd->addr_lo);
3297 
3298 	/* now get a new data BD
3299 	 * (after the pbd) and fill it */
3300 	bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3301 	d_tx_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3302 
3303 	mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
3304 			   le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
3305 
3306 	d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3307 	d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3308 	d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
3309 
3310 	/* this marks the BD as one that has no individual mapping */
3311 	tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
3312 
3313 	DP(NETIF_MSG_TX_QUEUED,
3314 	   "TSO split data size is %d (%x:%x)\n",
3315 	   d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
3316 
3317 	/* update tx_bd */
3318 	*tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
3319 
3320 	return bd_prod;
3321 }
3322 
3323 #define bswab32(b32) ((__force __le32) swab32((__force __u32) (b32)))
3324 #define bswab16(b16) ((__force __le16) swab16((__force __u16) (b16)))
3325 static __le16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
3326 {
3327 	__sum16 tsum = (__force __sum16) csum;
3328 
3329 	if (fix > 0)
3330 		tsum = ~csum_fold(csum_sub((__force __wsum) csum,
3331 				  csum_partial(t_header - fix, fix, 0)));
3332 
3333 	else if (fix < 0)
3334 		tsum = ~csum_fold(csum_add((__force __wsum) csum,
3335 				  csum_partial(t_header, -fix, 0)));
3336 
3337 	return bswab16(tsum);
3338 }
3339 
3340 static u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
3341 {
3342 	u32 rc;
3343 	__u8 prot = 0;
3344 	__be16 protocol;
3345 
3346 	if (skb->ip_summed != CHECKSUM_PARTIAL)
3347 		return XMIT_PLAIN;
3348 
3349 	protocol = vlan_get_protocol(skb);
3350 	if (protocol == htons(ETH_P_IPV6)) {
3351 		rc = XMIT_CSUM_V6;
3352 		prot = ipv6_hdr(skb)->nexthdr;
3353 	} else {
3354 		rc = XMIT_CSUM_V4;
3355 		prot = ip_hdr(skb)->protocol;
3356 	}
3357 
3358 	if (!CHIP_IS_E1x(bp) && skb->encapsulation) {
3359 		if (inner_ip_hdr(skb)->version == 6) {
3360 			rc |= XMIT_CSUM_ENC_V6;
3361 			if (inner_ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3362 				rc |= XMIT_CSUM_TCP;
3363 		} else {
3364 			rc |= XMIT_CSUM_ENC_V4;
3365 			if (inner_ip_hdr(skb)->protocol == IPPROTO_TCP)
3366 				rc |= XMIT_CSUM_TCP;
3367 		}
3368 	}
3369 	if (prot == IPPROTO_TCP)
3370 		rc |= XMIT_CSUM_TCP;
3371 
3372 	if (skb_is_gso(skb)) {
3373 		if (skb_is_gso_v6(skb)) {
3374 			rc |= (XMIT_GSO_V6 | XMIT_CSUM_TCP);
3375 			if (rc & XMIT_CSUM_ENC)
3376 				rc |= XMIT_GSO_ENC_V6;
3377 		} else {
3378 			rc |= (XMIT_GSO_V4 | XMIT_CSUM_TCP);
3379 			if (rc & XMIT_CSUM_ENC)
3380 				rc |= XMIT_GSO_ENC_V4;
3381 		}
3382 	}
3383 
3384 	return rc;
3385 }
3386 
3387 /* VXLAN: 4 = 1 (for linear data BD) + 3 (2 for PBD and last BD) */
3388 #define BNX2X_NUM_VXLAN_TSO_WIN_SUB_BDS         4
3389 
3390 /* Regular: 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
3391 #define BNX2X_NUM_TSO_WIN_SUB_BDS               3
3392 
3393 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - BDS_PER_TX_PKT)
3394 /* check if packet requires linearization (packet is too fragmented)
3395    no need to check fragmentation if page size > 8K (there will be no
3396    violation to FW restrictions) */
3397 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
3398 			     u32 xmit_type)
3399 {
3400 	int first_bd_sz = 0, num_tso_win_sub = BNX2X_NUM_TSO_WIN_SUB_BDS;
3401 	int to_copy = 0, hlen = 0;
3402 
3403 	if (xmit_type & XMIT_GSO_ENC)
3404 		num_tso_win_sub = BNX2X_NUM_VXLAN_TSO_WIN_SUB_BDS;
3405 
3406 	if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - num_tso_win_sub)) {
3407 		if (xmit_type & XMIT_GSO) {
3408 			unsigned short lso_mss = skb_shinfo(skb)->gso_size;
3409 			int wnd_size = MAX_FETCH_BD - num_tso_win_sub;
3410 			/* Number of windows to check */
3411 			int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
3412 			int wnd_idx = 0;
3413 			int frag_idx = 0;
3414 			u32 wnd_sum = 0;
3415 
3416 			/* Headers length */
3417 			if (xmit_type & XMIT_GSO_ENC)
3418 				hlen = (int)(skb_inner_transport_header(skb) -
3419 					     skb->data) +
3420 					     inner_tcp_hdrlen(skb);
3421 			else
3422 				hlen = (int)(skb_transport_header(skb) -
3423 					     skb->data) + tcp_hdrlen(skb);
3424 
3425 			/* Amount of data (w/o headers) on linear part of SKB*/
3426 			first_bd_sz = skb_headlen(skb) - hlen;
3427 
3428 			wnd_sum  = first_bd_sz;
3429 
3430 			/* Calculate the first sum - it's special */
3431 			for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
3432 				wnd_sum +=
3433 					skb_frag_size(&skb_shinfo(skb)->frags[frag_idx]);
3434 
3435 			/* If there was data on linear skb data - check it */
3436 			if (first_bd_sz > 0) {
3437 				if (unlikely(wnd_sum < lso_mss)) {
3438 					to_copy = 1;
3439 					goto exit_lbl;
3440 				}
3441 
3442 				wnd_sum -= first_bd_sz;
3443 			}
3444 
3445 			/* Others are easier: run through the frag list and
3446 			   check all windows */
3447 			for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
3448 				wnd_sum +=
3449 			  skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1]);
3450 
3451 				if (unlikely(wnd_sum < lso_mss)) {
3452 					to_copy = 1;
3453 					break;
3454 				}
3455 				wnd_sum -=
3456 					skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx]);
3457 			}
3458 		} else {
3459 			/* in non-LSO too fragmented packet should always
3460 			   be linearized */
3461 			to_copy = 1;
3462 		}
3463 	}
3464 
3465 exit_lbl:
3466 	if (unlikely(to_copy))
3467 		DP(NETIF_MSG_TX_QUEUED,
3468 		   "Linearization IS REQUIRED for %s packet. num_frags %d  hlen %d  first_bd_sz %d\n",
3469 		   (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
3470 		   skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
3471 
3472 	return to_copy;
3473 }
3474 #endif
3475 
3476 /**
3477  * bnx2x_set_pbd_gso - update PBD in GSO case.
3478  *
3479  * @skb:	packet skb
3480  * @pbd:	parse BD
3481  * @xmit_type:	xmit flags
3482  */
3483 static void bnx2x_set_pbd_gso(struct sk_buff *skb,
3484 			      struct eth_tx_parse_bd_e1x *pbd,
3485 			      u32 xmit_type)
3486 {
3487 	pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
3488 	pbd->tcp_send_seq = bswab32(tcp_hdr(skb)->seq);
3489 	pbd->tcp_flags = pbd_tcp_flags(tcp_hdr(skb));
3490 
3491 	if (xmit_type & XMIT_GSO_V4) {
3492 		pbd->ip_id = bswab16(ip_hdr(skb)->id);
3493 		pbd->tcp_pseudo_csum =
3494 			bswab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
3495 						   ip_hdr(skb)->daddr,
3496 						   0, IPPROTO_TCP, 0));
3497 	} else {
3498 		pbd->tcp_pseudo_csum =
3499 			bswab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3500 						 &ipv6_hdr(skb)->daddr,
3501 						 0, IPPROTO_TCP, 0));
3502 	}
3503 
3504 	pbd->global_data |=
3505 		cpu_to_le16(ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN);
3506 }
3507 
3508 /**
3509  * bnx2x_set_pbd_csum_enc - update PBD with checksum and return header length
3510  *
3511  * @bp:			driver handle
3512  * @skb:		packet skb
3513  * @parsing_data:	data to be updated
3514  * @xmit_type:		xmit flags
3515  *
3516  * 57712/578xx related, when skb has encapsulation
3517  */
3518 static u8 bnx2x_set_pbd_csum_enc(struct bnx2x *bp, struct sk_buff *skb,
3519 				 u32 *parsing_data, u32 xmit_type)
3520 {
3521 	*parsing_data |=
3522 		((((u8 *)skb_inner_transport_header(skb) - skb->data) >> 1) <<
3523 		ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3524 		ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3525 
3526 	if (xmit_type & XMIT_CSUM_TCP) {
3527 		*parsing_data |= ((inner_tcp_hdrlen(skb) / 4) <<
3528 			ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3529 			ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3530 
3531 		return skb_inner_transport_header(skb) +
3532 			inner_tcp_hdrlen(skb) - skb->data;
3533 	}
3534 
3535 	/* We support checksum offload for TCP and UDP only.
3536 	 * No need to pass the UDP header length - it's a constant.
3537 	 */
3538 	return skb_inner_transport_header(skb) +
3539 		sizeof(struct udphdr) - skb->data;
3540 }
3541 
3542 /**
3543  * bnx2x_set_pbd_csum_e2 - update PBD with checksum and return header length
3544  *
3545  * @bp:			driver handle
3546  * @skb:		packet skb
3547  * @parsing_data:	data to be updated
3548  * @xmit_type:		xmit flags
3549  *
3550  * 57712/578xx related
3551  */
3552 static u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
3553 				u32 *parsing_data, u32 xmit_type)
3554 {
3555 	*parsing_data |=
3556 		((((u8 *)skb_transport_header(skb) - skb->data) >> 1) <<
3557 		ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3558 		ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3559 
3560 	if (xmit_type & XMIT_CSUM_TCP) {
3561 		*parsing_data |= ((tcp_hdrlen(skb) / 4) <<
3562 			ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3563 			ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3564 
3565 		return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data;
3566 	}
3567 	/* We support checksum offload for TCP and UDP only.
3568 	 * No need to pass the UDP header length - it's a constant.
3569 	 */
3570 	return skb_transport_header(skb) + sizeof(struct udphdr) - skb->data;
3571 }
3572 
3573 /* set FW indication according to inner or outer protocols if tunneled */
3574 static void bnx2x_set_sbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3575 			       struct eth_tx_start_bd *tx_start_bd,
3576 			       u32 xmit_type)
3577 {
3578 	tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
3579 
3580 	if (xmit_type & (XMIT_CSUM_ENC_V6 | XMIT_CSUM_V6))
3581 		tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IPV6;
3582 
3583 	if (!(xmit_type & XMIT_CSUM_TCP))
3584 		tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IS_UDP;
3585 }
3586 
3587 /**
3588  * bnx2x_set_pbd_csum - update PBD with checksum and return header length
3589  *
3590  * @bp:		driver handle
3591  * @skb:	packet skb
3592  * @pbd:	parse BD to be updated
3593  * @xmit_type:	xmit flags
3594  */
3595 static u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3596 			     struct eth_tx_parse_bd_e1x *pbd,
3597 			     u32 xmit_type)
3598 {
3599 	u8 hlen = (skb_network_header(skb) - skb->data) >> 1;
3600 
3601 	/* for now NS flag is not used in Linux */
3602 	pbd->global_data =
3603 		cpu_to_le16(hlen |
3604 			    ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3605 			     ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT));
3606 
3607 	pbd->ip_hlen_w = (skb_transport_header(skb) -
3608 			skb_network_header(skb)) >> 1;
3609 
3610 	hlen += pbd->ip_hlen_w;
3611 
3612 	/* We support checksum offload for TCP and UDP only */
3613 	if (xmit_type & XMIT_CSUM_TCP)
3614 		hlen += tcp_hdrlen(skb) / 2;
3615 	else
3616 		hlen += sizeof(struct udphdr) / 2;
3617 
3618 	pbd->total_hlen_w = cpu_to_le16(hlen);
3619 	hlen = hlen*2;
3620 
3621 	if (xmit_type & XMIT_CSUM_TCP) {
3622 		pbd->tcp_pseudo_csum = bswab16(tcp_hdr(skb)->check);
3623 
3624 	} else {
3625 		s8 fix = SKB_CS_OFF(skb); /* signed! */
3626 
3627 		DP(NETIF_MSG_TX_QUEUED,
3628 		   "hlen %d  fix %d  csum before fix %x\n",
3629 		   le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb));
3630 
3631 		/* HW bug: fixup the CSUM */
3632 		pbd->tcp_pseudo_csum =
3633 			bnx2x_csum_fix(skb_transport_header(skb),
3634 				       SKB_CS(skb), fix);
3635 
3636 		DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
3637 		   pbd->tcp_pseudo_csum);
3638 	}
3639 
3640 	return hlen;
3641 }
3642 
3643 static void bnx2x_update_pbds_gso_enc(struct sk_buff *skb,
3644 				      struct eth_tx_parse_bd_e2 *pbd_e2,
3645 				      struct eth_tx_parse_2nd_bd *pbd2,
3646 				      u16 *global_data,
3647 				      u32 xmit_type)
3648 {
3649 	u16 hlen_w = 0;
3650 	u8 outerip_off, outerip_len = 0;
3651 
3652 	/* from outer IP to transport */
3653 	hlen_w = (skb_inner_transport_header(skb) -
3654 		  skb_network_header(skb)) >> 1;
3655 
3656 	/* transport len */
3657 	hlen_w += inner_tcp_hdrlen(skb) >> 1;
3658 
3659 	pbd2->fw_ip_hdr_to_payload_w = hlen_w;
3660 
3661 	/* outer IP header info */
3662 	if (xmit_type & XMIT_CSUM_V4) {
3663 		struct iphdr *iph = ip_hdr(skb);
3664 		u32 csum = (__force u32)(~iph->check) -
3665 			   (__force u32)iph->tot_len -
3666 			   (__force u32)iph->frag_off;
3667 
3668 		outerip_len = iph->ihl << 1;
3669 
3670 		pbd2->fw_ip_csum_wo_len_flags_frag =
3671 			bswab16(csum_fold((__force __wsum)csum));
3672 	} else {
3673 		pbd2->fw_ip_hdr_to_payload_w =
3674 			hlen_w - ((sizeof(struct ipv6hdr)) >> 1);
3675 		pbd_e2->data.tunnel_data.flags |=
3676 			ETH_TUNNEL_DATA_IPV6_OUTER;
3677 	}
3678 
3679 	pbd2->tcp_send_seq = bswab32(inner_tcp_hdr(skb)->seq);
3680 
3681 	pbd2->tcp_flags = pbd_tcp_flags(inner_tcp_hdr(skb));
3682 
3683 	/* inner IP header info */
3684 	if (xmit_type & XMIT_CSUM_ENC_V4) {
3685 		pbd2->hw_ip_id = bswab16(inner_ip_hdr(skb)->id);
3686 
3687 		pbd_e2->data.tunnel_data.pseudo_csum =
3688 			bswab16(~csum_tcpudp_magic(
3689 					inner_ip_hdr(skb)->saddr,
3690 					inner_ip_hdr(skb)->daddr,
3691 					0, IPPROTO_TCP, 0));
3692 	} else {
3693 		pbd_e2->data.tunnel_data.pseudo_csum =
3694 			bswab16(~csum_ipv6_magic(
3695 					&inner_ipv6_hdr(skb)->saddr,
3696 					&inner_ipv6_hdr(skb)->daddr,
3697 					0, IPPROTO_TCP, 0));
3698 	}
3699 
3700 	outerip_off = (skb_network_header(skb) - skb->data) >> 1;
3701 
3702 	*global_data |=
3703 		outerip_off |
3704 		(outerip_len <<
3705 			ETH_TX_PARSE_2ND_BD_IP_HDR_LEN_OUTER_W_SHIFT) |
3706 		((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3707 			ETH_TX_PARSE_2ND_BD_LLC_SNAP_EN_SHIFT);
3708 
3709 	if (ip_hdr(skb)->protocol == IPPROTO_UDP) {
3710 		SET_FLAG(*global_data, ETH_TX_PARSE_2ND_BD_TUNNEL_UDP_EXIST, 1);
3711 		pbd2->tunnel_udp_hdr_start_w = skb_transport_offset(skb) >> 1;
3712 	}
3713 }
3714 
3715 static inline void bnx2x_set_ipv6_ext_e2(struct sk_buff *skb, u32 *parsing_data,
3716 					 u32 xmit_type)
3717 {
3718 	struct ipv6hdr *ipv6;
3719 
3720 	if (!(xmit_type & (XMIT_GSO_ENC_V6 | XMIT_GSO_V6)))
3721 		return;
3722 
3723 	if (xmit_type & XMIT_GSO_ENC_V6)
3724 		ipv6 = inner_ipv6_hdr(skb);
3725 	else /* XMIT_GSO_V6 */
3726 		ipv6 = ipv6_hdr(skb);
3727 
3728 	if (ipv6->nexthdr == NEXTHDR_IPV6)
3729 		*parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR;
3730 }
3731 
3732 /* called with netif_tx_lock
3733  * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
3734  * netif_wake_queue()
3735  */
3736 netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
3737 {
3738 	struct bnx2x *bp = netdev_priv(dev);
3739 
3740 	struct netdev_queue *txq;
3741 	struct bnx2x_fp_txdata *txdata;
3742 	struct sw_tx_bd *tx_buf;
3743 	struct eth_tx_start_bd *tx_start_bd, *first_bd;
3744 	struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
3745 	struct eth_tx_parse_bd_e1x *pbd_e1x = NULL;
3746 	struct eth_tx_parse_bd_e2 *pbd_e2 = NULL;
3747 	struct eth_tx_parse_2nd_bd *pbd2 = NULL;
3748 	u32 pbd_e2_parsing_data = 0;
3749 	u16 pkt_prod, bd_prod;
3750 	int nbd, txq_index;
3751 	dma_addr_t mapping;
3752 	u32 xmit_type = bnx2x_xmit_type(bp, skb);
3753 	int i;
3754 	u8 hlen = 0;
3755 	__le16 pkt_size = 0;
3756 	struct ethhdr *eth;
3757 	u8 mac_type = UNICAST_ADDRESS;
3758 
3759 #ifdef BNX2X_STOP_ON_ERROR
3760 	if (unlikely(bp->panic))
3761 		return NETDEV_TX_BUSY;
3762 #endif
3763 
3764 	txq_index = skb_get_queue_mapping(skb);
3765 	txq = netdev_get_tx_queue(dev, txq_index);
3766 
3767 	BUG_ON(txq_index >= MAX_ETH_TXQ_IDX(bp) + (CNIC_LOADED(bp) ? 1 : 0));
3768 
3769 	txdata = &bp->bnx2x_txq[txq_index];
3770 
3771 	/* enable this debug print to view the transmission queue being used
3772 	DP(NETIF_MSG_TX_QUEUED, "indices: txq %d, fp %d, txdata %d\n",
3773 	   txq_index, fp_index, txdata_index); */
3774 
3775 	/* enable this debug print to view the transmission details
3776 	DP(NETIF_MSG_TX_QUEUED,
3777 	   "transmitting packet cid %d fp index %d txdata_index %d tx_data ptr %p fp pointer %p\n",
3778 	   txdata->cid, fp_index, txdata_index, txdata, fp); */
3779 
3780 	if (unlikely(bnx2x_tx_avail(bp, txdata) <
3781 			skb_shinfo(skb)->nr_frags +
3782 			BDS_PER_TX_PKT +
3783 			NEXT_CNT_PER_TX_PKT(MAX_BDS_PER_TX_PKT))) {
3784 		/* Handle special storage cases separately */
3785 		if (txdata->tx_ring_size == 0) {
3786 			struct bnx2x_eth_q_stats *q_stats =
3787 				bnx2x_fp_qstats(bp, txdata->parent_fp);
3788 			q_stats->driver_filtered_tx_pkt++;
3789 			dev_kfree_skb(skb);
3790 			return NETDEV_TX_OK;
3791 		}
3792 		bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
3793 		netif_tx_stop_queue(txq);
3794 		BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
3795 
3796 		return NETDEV_TX_BUSY;
3797 	}
3798 
3799 	DP(NETIF_MSG_TX_QUEUED,
3800 	   "queue[%d]: SKB: summed %x  protocol %x protocol(%x,%x) gso type %x  xmit_type %x len %d\n",
3801 	   txq_index, skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
3802 	   ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type,
3803 	   skb->len);
3804 
3805 	eth = (struct ethhdr *)skb->data;
3806 
3807 	/* set flag according to packet type (UNICAST_ADDRESS is default)*/
3808 	if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
3809 		if (is_broadcast_ether_addr(eth->h_dest))
3810 			mac_type = BROADCAST_ADDRESS;
3811 		else
3812 			mac_type = MULTICAST_ADDRESS;
3813 	}
3814 
3815 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - BDS_PER_TX_PKT)
3816 	/* First, check if we need to linearize the skb (due to FW
3817 	   restrictions). No need to check fragmentation if page size > 8K
3818 	   (there will be no violation to FW restrictions) */
3819 	if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
3820 		/* Statistics of linearization */
3821 		bp->lin_cnt++;
3822 		if (skb_linearize(skb) != 0) {
3823 			DP(NETIF_MSG_TX_QUEUED,
3824 			   "SKB linearization failed - silently dropping this SKB\n");
3825 			dev_kfree_skb_any(skb);
3826 			return NETDEV_TX_OK;
3827 		}
3828 	}
3829 #endif
3830 	/* Map skb linear data for DMA */
3831 	mapping = dma_map_single(&bp->pdev->dev, skb->data,
3832 				 skb_headlen(skb), DMA_TO_DEVICE);
3833 	if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
3834 		DP(NETIF_MSG_TX_QUEUED,
3835 		   "SKB mapping failed - silently dropping this SKB\n");
3836 		dev_kfree_skb_any(skb);
3837 		return NETDEV_TX_OK;
3838 	}
3839 	/*
3840 	Please read carefully. First we use one BD which we mark as start,
3841 	then we have a parsing info BD (used for TSO or xsum),
3842 	and only then we have the rest of the TSO BDs.
3843 	(don't forget to mark the last one as last,
3844 	and to unmap only AFTER you write to the BD ...)
3845 	And above all, all pdb sizes are in words - NOT DWORDS!
3846 	*/
3847 
3848 	/* get current pkt produced now - advance it just before sending packet
3849 	 * since mapping of pages may fail and cause packet to be dropped
3850 	 */
3851 	pkt_prod = txdata->tx_pkt_prod;
3852 	bd_prod = TX_BD(txdata->tx_bd_prod);
3853 
3854 	/* get a tx_buf and first BD
3855 	 * tx_start_bd may be changed during SPLIT,
3856 	 * but first_bd will always stay first
3857 	 */
3858 	tx_buf = &txdata->tx_buf_ring[TX_BD(pkt_prod)];
3859 	tx_start_bd = &txdata->tx_desc_ring[bd_prod].start_bd;
3860 	first_bd = tx_start_bd;
3861 
3862 	tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
3863 
3864 	if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
3865 		if (!(bp->flags & TX_TIMESTAMPING_EN)) {
3866 			bp->eth_stats.ptp_skip_tx_ts++;
3867 			BNX2X_ERR("Tx timestamping was not enabled, this packet will not be timestamped\n");
3868 		} else if (bp->ptp_tx_skb) {
3869 			bp->eth_stats.ptp_skip_tx_ts++;
3870 			netdev_err_once(bp->dev,
3871 					"Device supports only a single outstanding packet to timestamp, this packet won't be timestamped\n");
3872 		} else {
3873 			skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
3874 			/* schedule check for Tx timestamp */
3875 			bp->ptp_tx_skb = skb_get(skb);
3876 			bp->ptp_tx_start = jiffies;
3877 			schedule_work(&bp->ptp_task);
3878 		}
3879 	}
3880 
3881 	/* header nbd: indirectly zero other flags! */
3882 	tx_start_bd->general_data = 1 << ETH_TX_START_BD_HDR_NBDS_SHIFT;
3883 
3884 	/* remember the first BD of the packet */
3885 	tx_buf->first_bd = txdata->tx_bd_prod;
3886 	tx_buf->skb = skb;
3887 	tx_buf->flags = 0;
3888 
3889 	DP(NETIF_MSG_TX_QUEUED,
3890 	   "sending pkt %u @%p  next_idx %u  bd %u @%p\n",
3891 	   pkt_prod, tx_buf, txdata->tx_pkt_prod, bd_prod, tx_start_bd);
3892 
3893 	if (skb_vlan_tag_present(skb)) {
3894 		tx_start_bd->vlan_or_ethertype =
3895 		    cpu_to_le16(skb_vlan_tag_get(skb));
3896 		tx_start_bd->bd_flags.as_bitfield |=
3897 		    (X_ETH_OUTBAND_VLAN << ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
3898 	} else {
3899 		/* when transmitting in a vf, start bd must hold the ethertype
3900 		 * for fw to enforce it
3901 		 */
3902 		u16 vlan_tci = 0;
3903 #ifndef BNX2X_STOP_ON_ERROR
3904 		if (IS_VF(bp)) {
3905 #endif
3906 			/* Still need to consider inband vlan for enforced */
3907 			if (__vlan_get_tag(skb, &vlan_tci)) {
3908 				tx_start_bd->vlan_or_ethertype =
3909 					cpu_to_le16(ntohs(eth->h_proto));
3910 			} else {
3911 				tx_start_bd->bd_flags.as_bitfield |=
3912 					(X_ETH_INBAND_VLAN <<
3913 					 ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
3914 				tx_start_bd->vlan_or_ethertype =
3915 					cpu_to_le16(vlan_tci);
3916 			}
3917 #ifndef BNX2X_STOP_ON_ERROR
3918 		} else {
3919 			/* used by FW for packet accounting */
3920 			tx_start_bd->vlan_or_ethertype = cpu_to_le16(pkt_prod);
3921 		}
3922 #endif
3923 	}
3924 
3925 	nbd = 2; /* start_bd + pbd + frags (updated when pages are mapped) */
3926 
3927 	/* turn on parsing and get a BD */
3928 	bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3929 
3930 	if (xmit_type & XMIT_CSUM)
3931 		bnx2x_set_sbd_csum(bp, skb, tx_start_bd, xmit_type);
3932 
3933 	if (!CHIP_IS_E1x(bp)) {
3934 		pbd_e2 = &txdata->tx_desc_ring[bd_prod].parse_bd_e2;
3935 		memset(pbd_e2, 0, sizeof(struct eth_tx_parse_bd_e2));
3936 
3937 		if (xmit_type & XMIT_CSUM_ENC) {
3938 			u16 global_data = 0;
3939 
3940 			/* Set PBD in enc checksum offload case */
3941 			hlen = bnx2x_set_pbd_csum_enc(bp, skb,
3942 						      &pbd_e2_parsing_data,
3943 						      xmit_type);
3944 
3945 			/* turn on 2nd parsing and get a BD */
3946 			bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3947 
3948 			pbd2 = &txdata->tx_desc_ring[bd_prod].parse_2nd_bd;
3949 
3950 			memset(pbd2, 0, sizeof(*pbd2));
3951 
3952 			pbd_e2->data.tunnel_data.ip_hdr_start_inner_w =
3953 				(skb_inner_network_header(skb) -
3954 				 skb->data) >> 1;
3955 
3956 			if (xmit_type & XMIT_GSO_ENC)
3957 				bnx2x_update_pbds_gso_enc(skb, pbd_e2, pbd2,
3958 							  &global_data,
3959 							  xmit_type);
3960 
3961 			pbd2->global_data = cpu_to_le16(global_data);
3962 
3963 			/* add addition parse BD indication to start BD */
3964 			SET_FLAG(tx_start_bd->general_data,
3965 				 ETH_TX_START_BD_PARSE_NBDS, 1);
3966 			/* set encapsulation flag in start BD */
3967 			SET_FLAG(tx_start_bd->general_data,
3968 				 ETH_TX_START_BD_TUNNEL_EXIST, 1);
3969 
3970 			tx_buf->flags |= BNX2X_HAS_SECOND_PBD;
3971 
3972 			nbd++;
3973 		} else if (xmit_type & XMIT_CSUM) {
3974 			/* Set PBD in checksum offload case w/o encapsulation */
3975 			hlen = bnx2x_set_pbd_csum_e2(bp, skb,
3976 						     &pbd_e2_parsing_data,
3977 						     xmit_type);
3978 		}
3979 
3980 		bnx2x_set_ipv6_ext_e2(skb, &pbd_e2_parsing_data, xmit_type);
3981 		/* Add the macs to the parsing BD if this is a vf or if
3982 		 * Tx Switching is enabled.
3983 		 */
3984 		if (IS_VF(bp)) {
3985 			/* override GRE parameters in BD */
3986 			bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.src_hi,
3987 					      &pbd_e2->data.mac_addr.src_mid,
3988 					      &pbd_e2->data.mac_addr.src_lo,
3989 					      eth->h_source);
3990 
3991 			bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.dst_hi,
3992 					      &pbd_e2->data.mac_addr.dst_mid,
3993 					      &pbd_e2->data.mac_addr.dst_lo,
3994 					      eth->h_dest);
3995 		} else {
3996 			if (bp->flags & TX_SWITCHING)
3997 				bnx2x_set_fw_mac_addr(
3998 						&pbd_e2->data.mac_addr.dst_hi,
3999 						&pbd_e2->data.mac_addr.dst_mid,
4000 						&pbd_e2->data.mac_addr.dst_lo,
4001 						eth->h_dest);
4002 #ifdef BNX2X_STOP_ON_ERROR
4003 			/* Enforce security is always set in Stop on Error -
4004 			 * source mac should be present in the parsing BD
4005 			 */
4006 			bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.src_hi,
4007 					      &pbd_e2->data.mac_addr.src_mid,
4008 					      &pbd_e2->data.mac_addr.src_lo,
4009 					      eth->h_source);
4010 #endif
4011 		}
4012 
4013 		SET_FLAG(pbd_e2_parsing_data,
4014 			 ETH_TX_PARSE_BD_E2_ETH_ADDR_TYPE, mac_type);
4015 	} else {
4016 		u16 global_data = 0;
4017 		pbd_e1x = &txdata->tx_desc_ring[bd_prod].parse_bd_e1x;
4018 		memset(pbd_e1x, 0, sizeof(struct eth_tx_parse_bd_e1x));
4019 		/* Set PBD in checksum offload case */
4020 		if (xmit_type & XMIT_CSUM)
4021 			hlen = bnx2x_set_pbd_csum(bp, skb, pbd_e1x, xmit_type);
4022 
4023 		SET_FLAG(global_data,
4024 			 ETH_TX_PARSE_BD_E1X_ETH_ADDR_TYPE, mac_type);
4025 		pbd_e1x->global_data |= cpu_to_le16(global_data);
4026 	}
4027 
4028 	/* Setup the data pointer of the first BD of the packet */
4029 	tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
4030 	tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
4031 	tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
4032 	pkt_size = tx_start_bd->nbytes;
4033 
4034 	DP(NETIF_MSG_TX_QUEUED,
4035 	   "first bd @%p  addr (%x:%x)  nbytes %d  flags %x  vlan %x\n",
4036 	   tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
4037 	   le16_to_cpu(tx_start_bd->nbytes),
4038 	   tx_start_bd->bd_flags.as_bitfield,
4039 	   le16_to_cpu(tx_start_bd->vlan_or_ethertype));
4040 
4041 	if (xmit_type & XMIT_GSO) {
4042 
4043 		DP(NETIF_MSG_TX_QUEUED,
4044 		   "TSO packet len %d  hlen %d  total len %d  tso size %d\n",
4045 		   skb->len, hlen, skb_headlen(skb),
4046 		   skb_shinfo(skb)->gso_size);
4047 
4048 		tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
4049 
4050 		if (unlikely(skb_headlen(skb) > hlen)) {
4051 			nbd++;
4052 			bd_prod = bnx2x_tx_split(bp, txdata, tx_buf,
4053 						 &tx_start_bd, hlen,
4054 						 bd_prod);
4055 		}
4056 		if (!CHIP_IS_E1x(bp))
4057 			pbd_e2_parsing_data |=
4058 				(skb_shinfo(skb)->gso_size <<
4059 				 ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT) &
4060 				 ETH_TX_PARSE_BD_E2_LSO_MSS;
4061 		else
4062 			bnx2x_set_pbd_gso(skb, pbd_e1x, xmit_type);
4063 	}
4064 
4065 	/* Set the PBD's parsing_data field if not zero
4066 	 * (for the chips newer than 57711).
4067 	 */
4068 	if (pbd_e2_parsing_data)
4069 		pbd_e2->parsing_data = cpu_to_le32(pbd_e2_parsing_data);
4070 
4071 	tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
4072 
4073 	/* Handle fragmented skb */
4074 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
4075 		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
4076 
4077 		mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 0,
4078 					   skb_frag_size(frag), DMA_TO_DEVICE);
4079 		if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
4080 			unsigned int pkts_compl = 0, bytes_compl = 0;
4081 
4082 			DP(NETIF_MSG_TX_QUEUED,
4083 			   "Unable to map page - dropping packet...\n");
4084 
4085 			/* we need unmap all buffers already mapped
4086 			 * for this SKB;
4087 			 * first_bd->nbd need to be properly updated
4088 			 * before call to bnx2x_free_tx_pkt
4089 			 */
4090 			first_bd->nbd = cpu_to_le16(nbd);
4091 			bnx2x_free_tx_pkt(bp, txdata,
4092 					  TX_BD(txdata->tx_pkt_prod),
4093 					  &pkts_compl, &bytes_compl);
4094 			return NETDEV_TX_OK;
4095 		}
4096 
4097 		bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
4098 		tx_data_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
4099 		if (total_pkt_bd == NULL)
4100 			total_pkt_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
4101 
4102 		tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
4103 		tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
4104 		tx_data_bd->nbytes = cpu_to_le16(skb_frag_size(frag));
4105 		le16_add_cpu(&pkt_size, skb_frag_size(frag));
4106 		nbd++;
4107 
4108 		DP(NETIF_MSG_TX_QUEUED,
4109 		   "frag %d  bd @%p  addr (%x:%x)  nbytes %d\n",
4110 		   i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
4111 		   le16_to_cpu(tx_data_bd->nbytes));
4112 	}
4113 
4114 	DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
4115 
4116 	/* update with actual num BDs */
4117 	first_bd->nbd = cpu_to_le16(nbd);
4118 
4119 	bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
4120 
4121 	/* now send a tx doorbell, counting the next BD
4122 	 * if the packet contains or ends with it
4123 	 */
4124 	if (TX_BD_POFF(bd_prod) < nbd)
4125 		nbd++;
4126 
4127 	/* total_pkt_bytes should be set on the first data BD if
4128 	 * it's not an LSO packet and there is more than one
4129 	 * data BD. In this case pkt_size is limited by an MTU value.
4130 	 * However we prefer to set it for an LSO packet (while we don't
4131 	 * have to) in order to save some CPU cycles in a none-LSO
4132 	 * case, when we much more care about them.
4133 	 */
4134 	if (total_pkt_bd != NULL)
4135 		total_pkt_bd->total_pkt_bytes = pkt_size;
4136 
4137 	if (pbd_e1x)
4138 		DP(NETIF_MSG_TX_QUEUED,
4139 		   "PBD (E1X) @%p  ip_data %x  ip_hlen %u  ip_id %u  lso_mss %u  tcp_flags %x  xsum %x  seq %u  hlen %u\n",
4140 		   pbd_e1x, pbd_e1x->global_data, pbd_e1x->ip_hlen_w,
4141 		   pbd_e1x->ip_id, pbd_e1x->lso_mss, pbd_e1x->tcp_flags,
4142 		   pbd_e1x->tcp_pseudo_csum, pbd_e1x->tcp_send_seq,
4143 		    le16_to_cpu(pbd_e1x->total_hlen_w));
4144 	if (pbd_e2)
4145 		DP(NETIF_MSG_TX_QUEUED,
4146 		   "PBD (E2) @%p  dst %x %x %x src %x %x %x parsing_data %x\n",
4147 		   pbd_e2,
4148 		   pbd_e2->data.mac_addr.dst_hi,
4149 		   pbd_e2->data.mac_addr.dst_mid,
4150 		   pbd_e2->data.mac_addr.dst_lo,
4151 		   pbd_e2->data.mac_addr.src_hi,
4152 		   pbd_e2->data.mac_addr.src_mid,
4153 		   pbd_e2->data.mac_addr.src_lo,
4154 		   pbd_e2->parsing_data);
4155 	DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d  bd %u\n", nbd, bd_prod);
4156 
4157 	netdev_tx_sent_queue(txq, skb->len);
4158 
4159 	skb_tx_timestamp(skb);
4160 
4161 	txdata->tx_pkt_prod++;
4162 	/*
4163 	 * Make sure that the BD data is updated before updating the producer
4164 	 * since FW might read the BD right after the producer is updated.
4165 	 * This is only applicable for weak-ordered memory model archs such
4166 	 * as IA-64. The following barrier is also mandatory since FW will
4167 	 * assumes packets must have BDs.
4168 	 */
4169 	wmb();
4170 
4171 	txdata->tx_db.data.prod += nbd;
4172 	/* make sure descriptor update is observed by HW */
4173 	wmb();
4174 
4175 	DOORBELL_RELAXED(bp, txdata->cid, txdata->tx_db.raw);
4176 
4177 	txdata->tx_bd_prod += nbd;
4178 
4179 	if (unlikely(bnx2x_tx_avail(bp, txdata) < MAX_DESC_PER_TX_PKT)) {
4180 		netif_tx_stop_queue(txq);
4181 
4182 		/* paired memory barrier is in bnx2x_tx_int(), we have to keep
4183 		 * ordering of set_bit() in netif_tx_stop_queue() and read of
4184 		 * fp->bd_tx_cons */
4185 		smp_mb();
4186 
4187 		bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
4188 		if (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT)
4189 			netif_tx_wake_queue(txq);
4190 	}
4191 	txdata->tx_pkt++;
4192 
4193 	return NETDEV_TX_OK;
4194 }
4195 
4196 void bnx2x_get_c2s_mapping(struct bnx2x *bp, u8 *c2s_map, u8 *c2s_default)
4197 {
4198 	int mfw_vn = BP_FW_MB_IDX(bp);
4199 	u32 tmp;
4200 
4201 	/* If the shmem shouldn't affect configuration, reflect */
4202 	if (!IS_MF_BD(bp)) {
4203 		int i;
4204 
4205 		for (i = 0; i < BNX2X_MAX_PRIORITY; i++)
4206 			c2s_map[i] = i;
4207 		*c2s_default = 0;
4208 
4209 		return;
4210 	}
4211 
4212 	tmp = SHMEM2_RD(bp, c2s_pcp_map_lower[mfw_vn]);
4213 	tmp = (__force u32)be32_to_cpu((__force __be32)tmp);
4214 	c2s_map[0] = tmp & 0xff;
4215 	c2s_map[1] = (tmp >> 8) & 0xff;
4216 	c2s_map[2] = (tmp >> 16) & 0xff;
4217 	c2s_map[3] = (tmp >> 24) & 0xff;
4218 
4219 	tmp = SHMEM2_RD(bp, c2s_pcp_map_upper[mfw_vn]);
4220 	tmp = (__force u32)be32_to_cpu((__force __be32)tmp);
4221 	c2s_map[4] = tmp & 0xff;
4222 	c2s_map[5] = (tmp >> 8) & 0xff;
4223 	c2s_map[6] = (tmp >> 16) & 0xff;
4224 	c2s_map[7] = (tmp >> 24) & 0xff;
4225 
4226 	tmp = SHMEM2_RD(bp, c2s_pcp_map_default[mfw_vn]);
4227 	tmp = (__force u32)be32_to_cpu((__force __be32)tmp);
4228 	*c2s_default = (tmp >> (8 * mfw_vn)) & 0xff;
4229 }
4230 
4231 /**
4232  * bnx2x_setup_tc - routine to configure net_device for multi tc
4233  *
4234  * @dev: net device to configure
4235  * @num_tc: number of traffic classes to enable
4236  *
4237  * callback connected to the ndo_setup_tc function pointer
4238  */
4239 int bnx2x_setup_tc(struct net_device *dev, u8 num_tc)
4240 {
4241 	struct bnx2x *bp = netdev_priv(dev);
4242 	u8 c2s_map[BNX2X_MAX_PRIORITY], c2s_def;
4243 	int cos, prio, count, offset;
4244 
4245 	/* setup tc must be called under rtnl lock */
4246 	ASSERT_RTNL();
4247 
4248 	/* no traffic classes requested. Aborting */
4249 	if (!num_tc) {
4250 		netdev_reset_tc(dev);
4251 		return 0;
4252 	}
4253 
4254 	/* requested to support too many traffic classes */
4255 	if (num_tc > bp->max_cos) {
4256 		BNX2X_ERR("support for too many traffic classes requested: %d. Max supported is %d\n",
4257 			  num_tc, bp->max_cos);
4258 		return -EINVAL;
4259 	}
4260 
4261 	/* declare amount of supported traffic classes */
4262 	if (netdev_set_num_tc(dev, num_tc)) {
4263 		BNX2X_ERR("failed to declare %d traffic classes\n", num_tc);
4264 		return -EINVAL;
4265 	}
4266 
4267 	bnx2x_get_c2s_mapping(bp, c2s_map, &c2s_def);
4268 
4269 	/* configure priority to traffic class mapping */
4270 	for (prio = 0; prio < BNX2X_MAX_PRIORITY; prio++) {
4271 		int outer_prio = c2s_map[prio];
4272 
4273 		netdev_set_prio_tc_map(dev, prio, bp->prio_to_cos[outer_prio]);
4274 		DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
4275 		   "mapping priority %d to tc %d\n",
4276 		   outer_prio, bp->prio_to_cos[outer_prio]);
4277 	}
4278 
4279 	/* Use this configuration to differentiate tc0 from other COSes
4280 	   This can be used for ets or pfc, and save the effort of setting
4281 	   up a multio class queue disc or negotiating DCBX with a switch
4282 	netdev_set_prio_tc_map(dev, 0, 0);
4283 	DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", 0, 0);
4284 	for (prio = 1; prio < 16; prio++) {
4285 		netdev_set_prio_tc_map(dev, prio, 1);
4286 		DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", prio, 1);
4287 	} */
4288 
4289 	/* configure traffic class to transmission queue mapping */
4290 	for (cos = 0; cos < bp->max_cos; cos++) {
4291 		count = BNX2X_NUM_ETH_QUEUES(bp);
4292 		offset = cos * BNX2X_NUM_NON_CNIC_QUEUES(bp);
4293 		netdev_set_tc_queue(dev, cos, count, offset);
4294 		DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
4295 		   "mapping tc %d to offset %d count %d\n",
4296 		   cos, offset, count);
4297 	}
4298 
4299 	return 0;
4300 }
4301 
4302 int __bnx2x_setup_tc(struct net_device *dev, enum tc_setup_type type,
4303 		     void *type_data)
4304 {
4305 	struct tc_mqprio_qopt *mqprio = type_data;
4306 
4307 	if (type != TC_SETUP_QDISC_MQPRIO)
4308 		return -EOPNOTSUPP;
4309 
4310 	mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
4311 
4312 	return bnx2x_setup_tc(dev, mqprio->num_tc);
4313 }
4314 
4315 /* called with rtnl_lock */
4316 int bnx2x_change_mac_addr(struct net_device *dev, void *p)
4317 {
4318 	struct sockaddr *addr = p;
4319 	struct bnx2x *bp = netdev_priv(dev);
4320 	int rc = 0;
4321 
4322 	if (!is_valid_ether_addr(addr->sa_data)) {
4323 		BNX2X_ERR("Requested MAC address is not valid\n");
4324 		return -EINVAL;
4325 	}
4326 
4327 	if (IS_MF_STORAGE_ONLY(bp)) {
4328 		BNX2X_ERR("Can't change address on STORAGE ONLY function\n");
4329 		return -EINVAL;
4330 	}
4331 
4332 	if (netif_running(dev))  {
4333 		rc = bnx2x_set_eth_mac(bp, false);
4334 		if (rc)
4335 			return rc;
4336 	}
4337 
4338 	memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
4339 
4340 	if (netif_running(dev))
4341 		rc = bnx2x_set_eth_mac(bp, true);
4342 
4343 	if (IS_PF(bp) && SHMEM2_HAS(bp, curr_cfg))
4344 		SHMEM2_WR(bp, curr_cfg, CURR_CFG_MET_OS);
4345 
4346 	return rc;
4347 }
4348 
4349 static void bnx2x_free_fp_mem_at(struct bnx2x *bp, int fp_index)
4350 {
4351 	union host_hc_status_block *sb = &bnx2x_fp(bp, fp_index, status_blk);
4352 	struct bnx2x_fastpath *fp = &bp->fp[fp_index];
4353 	u8 cos;
4354 
4355 	/* Common */
4356 
4357 	if (IS_FCOE_IDX(fp_index)) {
4358 		memset(sb, 0, sizeof(union host_hc_status_block));
4359 		fp->status_blk_mapping = 0;
4360 	} else {
4361 		/* status blocks */
4362 		if (!CHIP_IS_E1x(bp))
4363 			BNX2X_PCI_FREE(sb->e2_sb,
4364 				       bnx2x_fp(bp, fp_index,
4365 						status_blk_mapping),
4366 				       sizeof(struct host_hc_status_block_e2));
4367 		else
4368 			BNX2X_PCI_FREE(sb->e1x_sb,
4369 				       bnx2x_fp(bp, fp_index,
4370 						status_blk_mapping),
4371 				       sizeof(struct host_hc_status_block_e1x));
4372 	}
4373 
4374 	/* Rx */
4375 	if (!skip_rx_queue(bp, fp_index)) {
4376 		bnx2x_free_rx_bds(fp);
4377 
4378 		/* fastpath rx rings: rx_buf rx_desc rx_comp */
4379 		BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_buf_ring));
4380 		BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_desc_ring),
4381 			       bnx2x_fp(bp, fp_index, rx_desc_mapping),
4382 			       sizeof(struct eth_rx_bd) * NUM_RX_BD);
4383 
4384 		BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_comp_ring),
4385 			       bnx2x_fp(bp, fp_index, rx_comp_mapping),
4386 			       sizeof(struct eth_fast_path_rx_cqe) *
4387 			       NUM_RCQ_BD);
4388 
4389 		/* SGE ring */
4390 		BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_page_ring));
4391 		BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_sge_ring),
4392 			       bnx2x_fp(bp, fp_index, rx_sge_mapping),
4393 			       BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4394 	}
4395 
4396 	/* Tx */
4397 	if (!skip_tx_queue(bp, fp_index)) {
4398 		/* fastpath tx rings: tx_buf tx_desc */
4399 		for_each_cos_in_tx_queue(fp, cos) {
4400 			struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
4401 
4402 			DP(NETIF_MSG_IFDOWN,
4403 			   "freeing tx memory of fp %d cos %d cid %d\n",
4404 			   fp_index, cos, txdata->cid);
4405 
4406 			BNX2X_FREE(txdata->tx_buf_ring);
4407 			BNX2X_PCI_FREE(txdata->tx_desc_ring,
4408 				txdata->tx_desc_mapping,
4409 				sizeof(union eth_tx_bd_types) * NUM_TX_BD);
4410 		}
4411 	}
4412 	/* end of fastpath */
4413 }
4414 
4415 static void bnx2x_free_fp_mem_cnic(struct bnx2x *bp)
4416 {
4417 	int i;
4418 	for_each_cnic_queue(bp, i)
4419 		bnx2x_free_fp_mem_at(bp, i);
4420 }
4421 
4422 void bnx2x_free_fp_mem(struct bnx2x *bp)
4423 {
4424 	int i;
4425 	for_each_eth_queue(bp, i)
4426 		bnx2x_free_fp_mem_at(bp, i);
4427 }
4428 
4429 static void set_sb_shortcuts(struct bnx2x *bp, int index)
4430 {
4431 	union host_hc_status_block status_blk = bnx2x_fp(bp, index, status_blk);
4432 	if (!CHIP_IS_E1x(bp)) {
4433 		bnx2x_fp(bp, index, sb_index_values) =
4434 			(__le16 *)status_blk.e2_sb->sb.index_values;
4435 		bnx2x_fp(bp, index, sb_running_index) =
4436 			(__le16 *)status_blk.e2_sb->sb.running_index;
4437 	} else {
4438 		bnx2x_fp(bp, index, sb_index_values) =
4439 			(__le16 *)status_blk.e1x_sb->sb.index_values;
4440 		bnx2x_fp(bp, index, sb_running_index) =
4441 			(__le16 *)status_blk.e1x_sb->sb.running_index;
4442 	}
4443 }
4444 
4445 /* Returns the number of actually allocated BDs */
4446 static int bnx2x_alloc_rx_bds(struct bnx2x_fastpath *fp,
4447 			      int rx_ring_size)
4448 {
4449 	struct bnx2x *bp = fp->bp;
4450 	u16 ring_prod, cqe_ring_prod;
4451 	int i, failure_cnt = 0;
4452 
4453 	fp->rx_comp_cons = 0;
4454 	cqe_ring_prod = ring_prod = 0;
4455 
4456 	/* This routine is called only during fo init so
4457 	 * fp->eth_q_stats.rx_skb_alloc_failed = 0
4458 	 */
4459 	for (i = 0; i < rx_ring_size; i++) {
4460 		if (bnx2x_alloc_rx_data(bp, fp, ring_prod, GFP_KERNEL) < 0) {
4461 			failure_cnt++;
4462 			continue;
4463 		}
4464 		ring_prod = NEXT_RX_IDX(ring_prod);
4465 		cqe_ring_prod = NEXT_RCQ_IDX(cqe_ring_prod);
4466 		WARN_ON(ring_prod <= (i - failure_cnt));
4467 	}
4468 
4469 	if (failure_cnt)
4470 		BNX2X_ERR("was only able to allocate %d rx skbs on queue[%d]\n",
4471 			  i - failure_cnt, fp->index);
4472 
4473 	fp->rx_bd_prod = ring_prod;
4474 	/* Limit the CQE producer by the CQE ring size */
4475 	fp->rx_comp_prod = min_t(u16, NUM_RCQ_RINGS*RCQ_DESC_CNT,
4476 			       cqe_ring_prod);
4477 
4478 	bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed += failure_cnt;
4479 
4480 	return i - failure_cnt;
4481 }
4482 
4483 static void bnx2x_set_next_page_rx_cq(struct bnx2x_fastpath *fp)
4484 {
4485 	int i;
4486 
4487 	for (i = 1; i <= NUM_RCQ_RINGS; i++) {
4488 		struct eth_rx_cqe_next_page *nextpg;
4489 
4490 		nextpg = (struct eth_rx_cqe_next_page *)
4491 			&fp->rx_comp_ring[RCQ_DESC_CNT * i - 1];
4492 		nextpg->addr_hi =
4493 			cpu_to_le32(U64_HI(fp->rx_comp_mapping +
4494 				   BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4495 		nextpg->addr_lo =
4496 			cpu_to_le32(U64_LO(fp->rx_comp_mapping +
4497 				   BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4498 	}
4499 }
4500 
4501 static int bnx2x_alloc_fp_mem_at(struct bnx2x *bp, int index)
4502 {
4503 	union host_hc_status_block *sb;
4504 	struct bnx2x_fastpath *fp = &bp->fp[index];
4505 	int ring_size = 0;
4506 	u8 cos;
4507 	int rx_ring_size = 0;
4508 
4509 	if (!bp->rx_ring_size && IS_MF_STORAGE_ONLY(bp)) {
4510 		rx_ring_size = MIN_RX_SIZE_NONTPA;
4511 		bp->rx_ring_size = rx_ring_size;
4512 	} else if (!bp->rx_ring_size) {
4513 		rx_ring_size = MAX_RX_AVAIL/BNX2X_NUM_RX_QUEUES(bp);
4514 
4515 		if (CHIP_IS_E3(bp)) {
4516 			u32 cfg = SHMEM_RD(bp,
4517 					   dev_info.port_hw_config[BP_PORT(bp)].
4518 					   default_cfg);
4519 
4520 			/* Decrease ring size for 1G functions */
4521 			if ((cfg & PORT_HW_CFG_NET_SERDES_IF_MASK) ==
4522 			    PORT_HW_CFG_NET_SERDES_IF_SGMII)
4523 				rx_ring_size /= 10;
4524 		}
4525 
4526 		/* allocate at least number of buffers required by FW */
4527 		rx_ring_size = max_t(int, bp->disable_tpa ? MIN_RX_SIZE_NONTPA :
4528 				     MIN_RX_SIZE_TPA, rx_ring_size);
4529 
4530 		bp->rx_ring_size = rx_ring_size;
4531 	} else /* if rx_ring_size specified - use it */
4532 		rx_ring_size = bp->rx_ring_size;
4533 
4534 	DP(BNX2X_MSG_SP, "calculated rx_ring_size %d\n", rx_ring_size);
4535 
4536 	/* Common */
4537 	sb = &bnx2x_fp(bp, index, status_blk);
4538 
4539 	if (!IS_FCOE_IDX(index)) {
4540 		/* status blocks */
4541 		if (!CHIP_IS_E1x(bp)) {
4542 			sb->e2_sb = BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, status_blk_mapping),
4543 						    sizeof(struct host_hc_status_block_e2));
4544 			if (!sb->e2_sb)
4545 				goto alloc_mem_err;
4546 		} else {
4547 			sb->e1x_sb = BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, status_blk_mapping),
4548 						     sizeof(struct host_hc_status_block_e1x));
4549 			if (!sb->e1x_sb)
4550 				goto alloc_mem_err;
4551 		}
4552 	}
4553 
4554 	/* FCoE Queue uses Default SB and doesn't ACK the SB, thus no need to
4555 	 * set shortcuts for it.
4556 	 */
4557 	if (!IS_FCOE_IDX(index))
4558 		set_sb_shortcuts(bp, index);
4559 
4560 	/* Tx */
4561 	if (!skip_tx_queue(bp, index)) {
4562 		/* fastpath tx rings: tx_buf tx_desc */
4563 		for_each_cos_in_tx_queue(fp, cos) {
4564 			struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
4565 
4566 			DP(NETIF_MSG_IFUP,
4567 			   "allocating tx memory of fp %d cos %d\n",
4568 			   index, cos);
4569 
4570 			txdata->tx_buf_ring = kcalloc(NUM_TX_BD,
4571 						      sizeof(struct sw_tx_bd),
4572 						      GFP_KERNEL);
4573 			if (!txdata->tx_buf_ring)
4574 				goto alloc_mem_err;
4575 			txdata->tx_desc_ring = BNX2X_PCI_ALLOC(&txdata->tx_desc_mapping,
4576 							       sizeof(union eth_tx_bd_types) * NUM_TX_BD);
4577 			if (!txdata->tx_desc_ring)
4578 				goto alloc_mem_err;
4579 		}
4580 	}
4581 
4582 	/* Rx */
4583 	if (!skip_rx_queue(bp, index)) {
4584 		/* fastpath rx rings: rx_buf rx_desc rx_comp */
4585 		bnx2x_fp(bp, index, rx_buf_ring) =
4586 			kcalloc(NUM_RX_BD, sizeof(struct sw_rx_bd), GFP_KERNEL);
4587 		if (!bnx2x_fp(bp, index, rx_buf_ring))
4588 			goto alloc_mem_err;
4589 		bnx2x_fp(bp, index, rx_desc_ring) =
4590 			BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, rx_desc_mapping),
4591 					sizeof(struct eth_rx_bd) * NUM_RX_BD);
4592 		if (!bnx2x_fp(bp, index, rx_desc_ring))
4593 			goto alloc_mem_err;
4594 
4595 		/* Seed all CQEs by 1s */
4596 		bnx2x_fp(bp, index, rx_comp_ring) =
4597 			BNX2X_PCI_FALLOC(&bnx2x_fp(bp, index, rx_comp_mapping),
4598 					 sizeof(struct eth_fast_path_rx_cqe) * NUM_RCQ_BD);
4599 		if (!bnx2x_fp(bp, index, rx_comp_ring))
4600 			goto alloc_mem_err;
4601 
4602 		/* SGE ring */
4603 		bnx2x_fp(bp, index, rx_page_ring) =
4604 			kcalloc(NUM_RX_SGE, sizeof(struct sw_rx_page),
4605 				GFP_KERNEL);
4606 		if (!bnx2x_fp(bp, index, rx_page_ring))
4607 			goto alloc_mem_err;
4608 		bnx2x_fp(bp, index, rx_sge_ring) =
4609 			BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, rx_sge_mapping),
4610 					BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4611 		if (!bnx2x_fp(bp, index, rx_sge_ring))
4612 			goto alloc_mem_err;
4613 		/* RX BD ring */
4614 		bnx2x_set_next_page_rx_bd(fp);
4615 
4616 		/* CQ ring */
4617 		bnx2x_set_next_page_rx_cq(fp);
4618 
4619 		/* BDs */
4620 		ring_size = bnx2x_alloc_rx_bds(fp, rx_ring_size);
4621 		if (ring_size < rx_ring_size)
4622 			goto alloc_mem_err;
4623 	}
4624 
4625 	return 0;
4626 
4627 /* handles low memory cases */
4628 alloc_mem_err:
4629 	BNX2X_ERR("Unable to allocate full memory for queue %d (size %d)\n",
4630 						index, ring_size);
4631 	/* FW will drop all packets if queue is not big enough,
4632 	 * In these cases we disable the queue
4633 	 * Min size is different for OOO, TPA and non-TPA queues
4634 	 */
4635 	if (ring_size < (fp->mode == TPA_MODE_DISABLED ?
4636 				MIN_RX_SIZE_NONTPA : MIN_RX_SIZE_TPA)) {
4637 			/* release memory allocated for this queue */
4638 			bnx2x_free_fp_mem_at(bp, index);
4639 			return -ENOMEM;
4640 	}
4641 	return 0;
4642 }
4643 
4644 static int bnx2x_alloc_fp_mem_cnic(struct bnx2x *bp)
4645 {
4646 	if (!NO_FCOE(bp))
4647 		/* FCoE */
4648 		if (bnx2x_alloc_fp_mem_at(bp, FCOE_IDX(bp)))
4649 			/* we will fail load process instead of mark
4650 			 * NO_FCOE_FLAG
4651 			 */
4652 			return -ENOMEM;
4653 
4654 	return 0;
4655 }
4656 
4657 static int bnx2x_alloc_fp_mem(struct bnx2x *bp)
4658 {
4659 	int i;
4660 
4661 	/* 1. Allocate FP for leading - fatal if error
4662 	 * 2. Allocate RSS - fix number of queues if error
4663 	 */
4664 
4665 	/* leading */
4666 	if (bnx2x_alloc_fp_mem_at(bp, 0))
4667 		return -ENOMEM;
4668 
4669 	/* RSS */
4670 	for_each_nondefault_eth_queue(bp, i)
4671 		if (bnx2x_alloc_fp_mem_at(bp, i))
4672 			break;
4673 
4674 	/* handle memory failures */
4675 	if (i != BNX2X_NUM_ETH_QUEUES(bp)) {
4676 		int delta = BNX2X_NUM_ETH_QUEUES(bp) - i;
4677 
4678 		WARN_ON(delta < 0);
4679 		bnx2x_shrink_eth_fp(bp, delta);
4680 		if (CNIC_SUPPORT(bp))
4681 			/* move non eth FPs next to last eth FP
4682 			 * must be done in that order
4683 			 * FCOE_IDX < FWD_IDX < OOO_IDX
4684 			 */
4685 
4686 			/* move FCoE fp even NO_FCOE_FLAG is on */
4687 			bnx2x_move_fp(bp, FCOE_IDX(bp), FCOE_IDX(bp) - delta);
4688 		bp->num_ethernet_queues -= delta;
4689 		bp->num_queues = bp->num_ethernet_queues +
4690 				 bp->num_cnic_queues;
4691 		BNX2X_ERR("Adjusted num of queues from %d to %d\n",
4692 			  bp->num_queues + delta, bp->num_queues);
4693 	}
4694 
4695 	return 0;
4696 }
4697 
4698 void bnx2x_free_mem_bp(struct bnx2x *bp)
4699 {
4700 	int i;
4701 
4702 	for (i = 0; i < bp->fp_array_size; i++)
4703 		kfree(bp->fp[i].tpa_info);
4704 	kfree(bp->fp);
4705 	kfree(bp->sp_objs);
4706 	kfree(bp->fp_stats);
4707 	kfree(bp->bnx2x_txq);
4708 	kfree(bp->msix_table);
4709 	kfree(bp->ilt);
4710 }
4711 
4712 int bnx2x_alloc_mem_bp(struct bnx2x *bp)
4713 {
4714 	struct bnx2x_fastpath *fp;
4715 	struct msix_entry *tbl;
4716 	struct bnx2x_ilt *ilt;
4717 	int msix_table_size = 0;
4718 	int fp_array_size, txq_array_size;
4719 	int i;
4720 
4721 	/*
4722 	 * The biggest MSI-X table we might need is as a maximum number of fast
4723 	 * path IGU SBs plus default SB (for PF only).
4724 	 */
4725 	msix_table_size = bp->igu_sb_cnt;
4726 	if (IS_PF(bp))
4727 		msix_table_size++;
4728 	BNX2X_DEV_INFO("msix_table_size %d\n", msix_table_size);
4729 
4730 	/* fp array: RSS plus CNIC related L2 queues */
4731 	fp_array_size = BNX2X_MAX_RSS_COUNT(bp) + CNIC_SUPPORT(bp);
4732 	bp->fp_array_size = fp_array_size;
4733 	BNX2X_DEV_INFO("fp_array_size %d\n", bp->fp_array_size);
4734 
4735 	fp = kcalloc(bp->fp_array_size, sizeof(*fp), GFP_KERNEL);
4736 	if (!fp)
4737 		goto alloc_err;
4738 	for (i = 0; i < bp->fp_array_size; i++) {
4739 		fp[i].tpa_info =
4740 			kcalloc(ETH_MAX_AGGREGATION_QUEUES_E1H_E2,
4741 				sizeof(struct bnx2x_agg_info), GFP_KERNEL);
4742 		if (!(fp[i].tpa_info))
4743 			goto alloc_err;
4744 	}
4745 
4746 	bp->fp = fp;
4747 
4748 	/* allocate sp objs */
4749 	bp->sp_objs = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_sp_objs),
4750 			      GFP_KERNEL);
4751 	if (!bp->sp_objs)
4752 		goto alloc_err;
4753 
4754 	/* allocate fp_stats */
4755 	bp->fp_stats = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_fp_stats),
4756 			       GFP_KERNEL);
4757 	if (!bp->fp_stats)
4758 		goto alloc_err;
4759 
4760 	/* Allocate memory for the transmission queues array */
4761 	txq_array_size =
4762 		BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS + CNIC_SUPPORT(bp);
4763 	BNX2X_DEV_INFO("txq_array_size %d", txq_array_size);
4764 
4765 	bp->bnx2x_txq = kcalloc(txq_array_size, sizeof(struct bnx2x_fp_txdata),
4766 				GFP_KERNEL);
4767 	if (!bp->bnx2x_txq)
4768 		goto alloc_err;
4769 
4770 	/* msix table */
4771 	tbl = kcalloc(msix_table_size, sizeof(*tbl), GFP_KERNEL);
4772 	if (!tbl)
4773 		goto alloc_err;
4774 	bp->msix_table = tbl;
4775 
4776 	/* ilt */
4777 	ilt = kzalloc(sizeof(*ilt), GFP_KERNEL);
4778 	if (!ilt)
4779 		goto alloc_err;
4780 	bp->ilt = ilt;
4781 
4782 	return 0;
4783 alloc_err:
4784 	bnx2x_free_mem_bp(bp);
4785 	return -ENOMEM;
4786 }
4787 
4788 int bnx2x_reload_if_running(struct net_device *dev)
4789 {
4790 	struct bnx2x *bp = netdev_priv(dev);
4791 
4792 	if (unlikely(!netif_running(dev)))
4793 		return 0;
4794 
4795 	bnx2x_nic_unload(bp, UNLOAD_NORMAL, true);
4796 	return bnx2x_nic_load(bp, LOAD_NORMAL);
4797 }
4798 
4799 int bnx2x_get_cur_phy_idx(struct bnx2x *bp)
4800 {
4801 	u32 sel_phy_idx = 0;
4802 	if (bp->link_params.num_phys <= 1)
4803 		return INT_PHY;
4804 
4805 	if (bp->link_vars.link_up) {
4806 		sel_phy_idx = EXT_PHY1;
4807 		/* In case link is SERDES, check if the EXT_PHY2 is the one */
4808 		if ((bp->link_vars.link_status & LINK_STATUS_SERDES_LINK) &&
4809 		    (bp->link_params.phy[EXT_PHY2].supported & SUPPORTED_FIBRE))
4810 			sel_phy_idx = EXT_PHY2;
4811 	} else {
4812 
4813 		switch (bnx2x_phy_selection(&bp->link_params)) {
4814 		case PORT_HW_CFG_PHY_SELECTION_HARDWARE_DEFAULT:
4815 		case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY:
4816 		case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY_PRIORITY:
4817 		       sel_phy_idx = EXT_PHY1;
4818 		       break;
4819 		case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY:
4820 		case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY_PRIORITY:
4821 		       sel_phy_idx = EXT_PHY2;
4822 		       break;
4823 		}
4824 	}
4825 
4826 	return sel_phy_idx;
4827 }
4828 int bnx2x_get_link_cfg_idx(struct bnx2x *bp)
4829 {
4830 	u32 sel_phy_idx = bnx2x_get_cur_phy_idx(bp);
4831 	/*
4832 	 * The selected activated PHY is always after swapping (in case PHY
4833 	 * swapping is enabled). So when swapping is enabled, we need to reverse
4834 	 * the configuration
4835 	 */
4836 
4837 	if (bp->link_params.multi_phy_config &
4838 	    PORT_HW_CFG_PHY_SWAPPED_ENABLED) {
4839 		if (sel_phy_idx == EXT_PHY1)
4840 			sel_phy_idx = EXT_PHY2;
4841 		else if (sel_phy_idx == EXT_PHY2)
4842 			sel_phy_idx = EXT_PHY1;
4843 	}
4844 	return LINK_CONFIG_IDX(sel_phy_idx);
4845 }
4846 
4847 #ifdef NETDEV_FCOE_WWNN
4848 int bnx2x_fcoe_get_wwn(struct net_device *dev, u64 *wwn, int type)
4849 {
4850 	struct bnx2x *bp = netdev_priv(dev);
4851 	struct cnic_eth_dev *cp = &bp->cnic_eth_dev;
4852 
4853 	switch (type) {
4854 	case NETDEV_FCOE_WWNN:
4855 		*wwn = HILO_U64(cp->fcoe_wwn_node_name_hi,
4856 				cp->fcoe_wwn_node_name_lo);
4857 		break;
4858 	case NETDEV_FCOE_WWPN:
4859 		*wwn = HILO_U64(cp->fcoe_wwn_port_name_hi,
4860 				cp->fcoe_wwn_port_name_lo);
4861 		break;
4862 	default:
4863 		BNX2X_ERR("Wrong WWN type requested - %d\n", type);
4864 		return -EINVAL;
4865 	}
4866 
4867 	return 0;
4868 }
4869 #endif
4870 
4871 /* called with rtnl_lock */
4872 int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
4873 {
4874 	struct bnx2x *bp = netdev_priv(dev);
4875 
4876 	if (pci_num_vf(bp->pdev)) {
4877 		DP(BNX2X_MSG_IOV, "VFs are enabled, can not change MTU\n");
4878 		return -EPERM;
4879 	}
4880 
4881 	if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
4882 		BNX2X_ERR("Can't perform change MTU during parity recovery\n");
4883 		return -EAGAIN;
4884 	}
4885 
4886 	/* This does not race with packet allocation
4887 	 * because the actual alloc size is
4888 	 * only updated as part of load
4889 	 */
4890 	dev->mtu = new_mtu;
4891 
4892 	if (!bnx2x_mtu_allows_gro(new_mtu))
4893 		dev->features &= ~NETIF_F_GRO_HW;
4894 
4895 	if (IS_PF(bp) && SHMEM2_HAS(bp, curr_cfg))
4896 		SHMEM2_WR(bp, curr_cfg, CURR_CFG_MET_OS);
4897 
4898 	return bnx2x_reload_if_running(dev);
4899 }
4900 
4901 netdev_features_t bnx2x_fix_features(struct net_device *dev,
4902 				     netdev_features_t features)
4903 {
4904 	struct bnx2x *bp = netdev_priv(dev);
4905 
4906 	if (pci_num_vf(bp->pdev)) {
4907 		netdev_features_t changed = dev->features ^ features;
4908 
4909 		/* Revert the requested changes in features if they
4910 		 * would require internal reload of PF in bnx2x_set_features().
4911 		 */
4912 		if (!(features & NETIF_F_RXCSUM) && !bp->disable_tpa) {
4913 			features &= ~NETIF_F_RXCSUM;
4914 			features |= dev->features & NETIF_F_RXCSUM;
4915 		}
4916 
4917 		if (changed & NETIF_F_LOOPBACK) {
4918 			features &= ~NETIF_F_LOOPBACK;
4919 			features |= dev->features & NETIF_F_LOOPBACK;
4920 		}
4921 	}
4922 
4923 	/* TPA requires Rx CSUM offloading */
4924 	if (!(features & NETIF_F_RXCSUM))
4925 		features &= ~NETIF_F_LRO;
4926 
4927 	if (!(features & NETIF_F_GRO) || !bnx2x_mtu_allows_gro(dev->mtu))
4928 		features &= ~NETIF_F_GRO_HW;
4929 	if (features & NETIF_F_GRO_HW)
4930 		features &= ~NETIF_F_LRO;
4931 
4932 	return features;
4933 }
4934 
4935 int bnx2x_set_features(struct net_device *dev, netdev_features_t features)
4936 {
4937 	struct bnx2x *bp = netdev_priv(dev);
4938 	netdev_features_t changes = features ^ dev->features;
4939 	bool bnx2x_reload = false;
4940 	int rc;
4941 
4942 	/* VFs or non SRIOV PFs should be able to change loopback feature */
4943 	if (!pci_num_vf(bp->pdev)) {
4944 		if (features & NETIF_F_LOOPBACK) {
4945 			if (bp->link_params.loopback_mode != LOOPBACK_BMAC) {
4946 				bp->link_params.loopback_mode = LOOPBACK_BMAC;
4947 				bnx2x_reload = true;
4948 			}
4949 		} else {
4950 			if (bp->link_params.loopback_mode != LOOPBACK_NONE) {
4951 				bp->link_params.loopback_mode = LOOPBACK_NONE;
4952 				bnx2x_reload = true;
4953 			}
4954 		}
4955 	}
4956 
4957 	/* Don't care about GRO changes */
4958 	changes &= ~NETIF_F_GRO;
4959 
4960 	if (changes)
4961 		bnx2x_reload = true;
4962 
4963 	if (bnx2x_reload) {
4964 		if (bp->recovery_state == BNX2X_RECOVERY_DONE) {
4965 			dev->features = features;
4966 			rc = bnx2x_reload_if_running(dev);
4967 			return rc ? rc : 1;
4968 		}
4969 		/* else: bnx2x_nic_load() will be called at end of recovery */
4970 	}
4971 
4972 	return 0;
4973 }
4974 
4975 void bnx2x_tx_timeout(struct net_device *dev, unsigned int txqueue)
4976 {
4977 	struct bnx2x *bp = netdev_priv(dev);
4978 
4979 	/* We want the information of the dump logged,
4980 	 * but calling bnx2x_panic() would kill all chances of recovery.
4981 	 */
4982 	if (!bp->panic)
4983 #ifndef BNX2X_STOP_ON_ERROR
4984 		bnx2x_panic_dump(bp, false);
4985 #else
4986 		bnx2x_panic();
4987 #endif
4988 
4989 	/* This allows the netif to be shutdown gracefully before resetting */
4990 	bnx2x_schedule_sp_rtnl(bp, BNX2X_SP_RTNL_TX_TIMEOUT, 0);
4991 }
4992 
4993 static int __maybe_unused bnx2x_suspend(struct device *dev_d)
4994 {
4995 	struct pci_dev *pdev = to_pci_dev(dev_d);
4996 	struct net_device *dev = pci_get_drvdata(pdev);
4997 	struct bnx2x *bp;
4998 
4999 	if (!dev) {
5000 		dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
5001 		return -ENODEV;
5002 	}
5003 	bp = netdev_priv(dev);
5004 
5005 	rtnl_lock();
5006 
5007 	if (!netif_running(dev)) {
5008 		rtnl_unlock();
5009 		return 0;
5010 	}
5011 
5012 	netif_device_detach(dev);
5013 
5014 	bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
5015 
5016 	rtnl_unlock();
5017 
5018 	return 0;
5019 }
5020 
5021 static int __maybe_unused bnx2x_resume(struct device *dev_d)
5022 {
5023 	struct pci_dev *pdev = to_pci_dev(dev_d);
5024 	struct net_device *dev = pci_get_drvdata(pdev);
5025 	struct bnx2x *bp;
5026 	int rc;
5027 
5028 	if (!dev) {
5029 		dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
5030 		return -ENODEV;
5031 	}
5032 	bp = netdev_priv(dev);
5033 
5034 	if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
5035 		BNX2X_ERR("Handling parity error recovery. Try again later\n");
5036 		return -EAGAIN;
5037 	}
5038 
5039 	rtnl_lock();
5040 
5041 	if (!netif_running(dev)) {
5042 		rtnl_unlock();
5043 		return 0;
5044 	}
5045 
5046 	netif_device_attach(dev);
5047 
5048 	rc = bnx2x_nic_load(bp, LOAD_OPEN);
5049 
5050 	rtnl_unlock();
5051 
5052 	return rc;
5053 }
5054 
5055 SIMPLE_DEV_PM_OPS(bnx2x_pm_ops, bnx2x_suspend, bnx2x_resume);
5056 
5057 void bnx2x_set_ctx_validation(struct bnx2x *bp, struct eth_context *cxt,
5058 			      u32 cid)
5059 {
5060 	if (!cxt) {
5061 		BNX2X_ERR("bad context pointer %p\n", cxt);
5062 		return;
5063 	}
5064 
5065 	/* ustorm cxt validation */
5066 	cxt->ustorm_ag_context.cdu_usage =
5067 		CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
5068 			CDU_REGION_NUMBER_UCM_AG, ETH_CONNECTION_TYPE);
5069 	/* xcontext validation */
5070 	cxt->xstorm_ag_context.cdu_reserved =
5071 		CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
5072 			CDU_REGION_NUMBER_XCM_AG, ETH_CONNECTION_TYPE);
5073 }
5074 
5075 static void storm_memset_hc_timeout(struct bnx2x *bp, u8 port,
5076 				    u8 fw_sb_id, u8 sb_index,
5077 				    u8 ticks)
5078 {
5079 	u32 addr = BAR_CSTRORM_INTMEM +
5080 		   CSTORM_STATUS_BLOCK_DATA_TIMEOUT_OFFSET(fw_sb_id, sb_index);
5081 	REG_WR8(bp, addr, ticks);
5082 	DP(NETIF_MSG_IFUP,
5083 	   "port %x fw_sb_id %d sb_index %d ticks %d\n",
5084 	   port, fw_sb_id, sb_index, ticks);
5085 }
5086 
5087 static void storm_memset_hc_disable(struct bnx2x *bp, u8 port,
5088 				    u16 fw_sb_id, u8 sb_index,
5089 				    u8 disable)
5090 {
5091 	u32 enable_flag = disable ? 0 : (1 << HC_INDEX_DATA_HC_ENABLED_SHIFT);
5092 	u32 addr = BAR_CSTRORM_INTMEM +
5093 		   CSTORM_STATUS_BLOCK_DATA_FLAGS_OFFSET(fw_sb_id, sb_index);
5094 	u8 flags = REG_RD8(bp, addr);
5095 	/* clear and set */
5096 	flags &= ~HC_INDEX_DATA_HC_ENABLED;
5097 	flags |= enable_flag;
5098 	REG_WR8(bp, addr, flags);
5099 	DP(NETIF_MSG_IFUP,
5100 	   "port %x fw_sb_id %d sb_index %d disable %d\n",
5101 	   port, fw_sb_id, sb_index, disable);
5102 }
5103 
5104 void bnx2x_update_coalesce_sb_index(struct bnx2x *bp, u8 fw_sb_id,
5105 				    u8 sb_index, u8 disable, u16 usec)
5106 {
5107 	int port = BP_PORT(bp);
5108 	u8 ticks = usec / BNX2X_BTR;
5109 
5110 	storm_memset_hc_timeout(bp, port, fw_sb_id, sb_index, ticks);
5111 
5112 	disable = disable ? 1 : (usec ? 0 : 1);
5113 	storm_memset_hc_disable(bp, port, fw_sb_id, sb_index, disable);
5114 }
5115 
5116 void bnx2x_schedule_sp_rtnl(struct bnx2x *bp, enum sp_rtnl_flag flag,
5117 			    u32 verbose)
5118 {
5119 	smp_mb__before_atomic();
5120 	set_bit(flag, &bp->sp_rtnl_state);
5121 	smp_mb__after_atomic();
5122 	DP((BNX2X_MSG_SP | verbose), "Scheduling sp_rtnl task [Flag: %d]\n",
5123 	   flag);
5124 	schedule_delayed_work(&bp->sp_rtnl_task, 0);
5125 }
5126