1 /* bnx2x_cmn.c: QLogic Everest network driver.
2  *
3  * Copyright (c) 2007-2013 Broadcom Corporation
4  * Copyright (c) 2014 QLogic Corporation
5  * All rights reserved
6  *
7  * This program is free software; you can redistribute it and/or modify
8  * it under the terms of the GNU General Public License as published by
9  * the Free Software Foundation.
10  *
11  * Maintained by: Ariel Elior <ariel.elior@qlogic.com>
12  * Written by: Eliezer Tamir
13  * Based on code from Michael Chan's bnx2 driver
14  * UDP CSUM errata workaround by Arik Gendelman
15  * Slowpath and fastpath rework by Vladislav Zolotarov
16  * Statistics and Link management by Yitchak Gertner
17  *
18  */
19 
20 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
21 
22 #include <linux/etherdevice.h>
23 #include <linux/if_vlan.h>
24 #include <linux/interrupt.h>
25 #include <linux/ip.h>
26 #include <linux/crash_dump.h>
27 #include <net/tcp.h>
28 #include <net/ipv6.h>
29 #include <net/ip6_checksum.h>
30 #include <net/busy_poll.h>
31 #include <linux/prefetch.h>
32 #include "bnx2x_cmn.h"
33 #include "bnx2x_init.h"
34 #include "bnx2x_sp.h"
35 
36 static void bnx2x_free_fp_mem_cnic(struct bnx2x *bp);
37 static int bnx2x_alloc_fp_mem_cnic(struct bnx2x *bp);
38 static int bnx2x_alloc_fp_mem(struct bnx2x *bp);
39 static int bnx2x_poll(struct napi_struct *napi, int budget);
40 
41 static void bnx2x_add_all_napi_cnic(struct bnx2x *bp)
42 {
43 	int i;
44 
45 	/* Add NAPI objects */
46 	for_each_rx_queue_cnic(bp, i) {
47 		netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi),
48 			       bnx2x_poll, NAPI_POLL_WEIGHT);
49 	}
50 }
51 
52 static void bnx2x_add_all_napi(struct bnx2x *bp)
53 {
54 	int i;
55 
56 	/* Add NAPI objects */
57 	for_each_eth_queue(bp, i) {
58 		netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi),
59 			       bnx2x_poll, NAPI_POLL_WEIGHT);
60 	}
61 }
62 
63 static int bnx2x_calc_num_queues(struct bnx2x *bp)
64 {
65 	int nq = bnx2x_num_queues ? : netif_get_num_default_rss_queues();
66 
67 	/* Reduce memory usage in kdump environment by using only one queue */
68 	if (is_kdump_kernel())
69 		nq = 1;
70 
71 	nq = clamp(nq, 1, BNX2X_MAX_QUEUES(bp));
72 	return nq;
73 }
74 
75 /**
76  * bnx2x_move_fp - move content of the fastpath structure.
77  *
78  * @bp:		driver handle
79  * @from:	source FP index
80  * @to:		destination FP index
81  *
82  * Makes sure the contents of the bp->fp[to].napi is kept
83  * intact. This is done by first copying the napi struct from
84  * the target to the source, and then mem copying the entire
85  * source onto the target. Update txdata pointers and related
86  * content.
87  */
88 static inline void bnx2x_move_fp(struct bnx2x *bp, int from, int to)
89 {
90 	struct bnx2x_fastpath *from_fp = &bp->fp[from];
91 	struct bnx2x_fastpath *to_fp = &bp->fp[to];
92 	struct bnx2x_sp_objs *from_sp_objs = &bp->sp_objs[from];
93 	struct bnx2x_sp_objs *to_sp_objs = &bp->sp_objs[to];
94 	struct bnx2x_fp_stats *from_fp_stats = &bp->fp_stats[from];
95 	struct bnx2x_fp_stats *to_fp_stats = &bp->fp_stats[to];
96 	int old_max_eth_txqs, new_max_eth_txqs;
97 	int old_txdata_index = 0, new_txdata_index = 0;
98 	struct bnx2x_agg_info *old_tpa_info = to_fp->tpa_info;
99 
100 	/* Copy the NAPI object as it has been already initialized */
101 	from_fp->napi = to_fp->napi;
102 
103 	/* Move bnx2x_fastpath contents */
104 	memcpy(to_fp, from_fp, sizeof(*to_fp));
105 	to_fp->index = to;
106 
107 	/* Retain the tpa_info of the original `to' version as we don't want
108 	 * 2 FPs to contain the same tpa_info pointer.
109 	 */
110 	to_fp->tpa_info = old_tpa_info;
111 
112 	/* move sp_objs contents as well, as their indices match fp ones */
113 	memcpy(to_sp_objs, from_sp_objs, sizeof(*to_sp_objs));
114 
115 	/* move fp_stats contents as well, as their indices match fp ones */
116 	memcpy(to_fp_stats, from_fp_stats, sizeof(*to_fp_stats));
117 
118 	/* Update txdata pointers in fp and move txdata content accordingly:
119 	 * Each fp consumes 'max_cos' txdata structures, so the index should be
120 	 * decremented by max_cos x delta.
121 	 */
122 
123 	old_max_eth_txqs = BNX2X_NUM_ETH_QUEUES(bp) * (bp)->max_cos;
124 	new_max_eth_txqs = (BNX2X_NUM_ETH_QUEUES(bp) - from + to) *
125 				(bp)->max_cos;
126 	if (from == FCOE_IDX(bp)) {
127 		old_txdata_index = old_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
128 		new_txdata_index = new_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
129 	}
130 
131 	memcpy(&bp->bnx2x_txq[new_txdata_index],
132 	       &bp->bnx2x_txq[old_txdata_index],
133 	       sizeof(struct bnx2x_fp_txdata));
134 	to_fp->txdata_ptr[0] = &bp->bnx2x_txq[new_txdata_index];
135 }
136 
137 /**
138  * bnx2x_fill_fw_str - Fill buffer with FW version string.
139  *
140  * @bp:        driver handle
141  * @buf:       character buffer to fill with the fw name
142  * @buf_len:   length of the above buffer
143  *
144  */
145 void bnx2x_fill_fw_str(struct bnx2x *bp, char *buf, size_t buf_len)
146 {
147 	if (IS_PF(bp)) {
148 		u8 phy_fw_ver[PHY_FW_VER_LEN];
149 
150 		phy_fw_ver[0] = '\0';
151 		bnx2x_get_ext_phy_fw_version(&bp->link_params,
152 					     phy_fw_ver, PHY_FW_VER_LEN);
153 		strlcpy(buf, bp->fw_ver, buf_len);
154 		snprintf(buf + strlen(bp->fw_ver), 32 - strlen(bp->fw_ver),
155 			 "bc %d.%d.%d%s%s",
156 			 (bp->common.bc_ver & 0xff0000) >> 16,
157 			 (bp->common.bc_ver & 0xff00) >> 8,
158 			 (bp->common.bc_ver & 0xff),
159 			 ((phy_fw_ver[0] != '\0') ? " phy " : ""), phy_fw_ver);
160 	} else {
161 		bnx2x_vf_fill_fw_str(bp, buf, buf_len);
162 	}
163 }
164 
165 /**
166  * bnx2x_shrink_eth_fp - guarantees fastpath structures stay intact
167  *
168  * @bp:	driver handle
169  * @delta:	number of eth queues which were not allocated
170  */
171 static void bnx2x_shrink_eth_fp(struct bnx2x *bp, int delta)
172 {
173 	int i, cos, old_eth_num = BNX2X_NUM_ETH_QUEUES(bp);
174 
175 	/* Queue pointer cannot be re-set on an fp-basis, as moving pointer
176 	 * backward along the array could cause memory to be overridden
177 	 */
178 	for (cos = 1; cos < bp->max_cos; cos++) {
179 		for (i = 0; i < old_eth_num - delta; i++) {
180 			struct bnx2x_fastpath *fp = &bp->fp[i];
181 			int new_idx = cos * (old_eth_num - delta) + i;
182 
183 			memcpy(&bp->bnx2x_txq[new_idx], fp->txdata_ptr[cos],
184 			       sizeof(struct bnx2x_fp_txdata));
185 			fp->txdata_ptr[cos] = &bp->bnx2x_txq[new_idx];
186 		}
187 	}
188 }
189 
190 int bnx2x_load_count[2][3] = { {0} }; /* per-path: 0-common, 1-port0, 2-port1 */
191 
192 /* free skb in the packet ring at pos idx
193  * return idx of last bd freed
194  */
195 static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata,
196 			     u16 idx, unsigned int *pkts_compl,
197 			     unsigned int *bytes_compl)
198 {
199 	struct sw_tx_bd *tx_buf = &txdata->tx_buf_ring[idx];
200 	struct eth_tx_start_bd *tx_start_bd;
201 	struct eth_tx_bd *tx_data_bd;
202 	struct sk_buff *skb = tx_buf->skb;
203 	u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
204 	int nbd;
205 	u16 split_bd_len = 0;
206 
207 	/* prefetch skb end pointer to speedup dev_kfree_skb() */
208 	prefetch(&skb->end);
209 
210 	DP(NETIF_MSG_TX_DONE, "fp[%d]: pkt_idx %d  buff @(%p)->skb %p\n",
211 	   txdata->txq_index, idx, tx_buf, skb);
212 
213 	tx_start_bd = &txdata->tx_desc_ring[bd_idx].start_bd;
214 
215 	nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
216 #ifdef BNX2X_STOP_ON_ERROR
217 	if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
218 		BNX2X_ERR("BAD nbd!\n");
219 		bnx2x_panic();
220 	}
221 #endif
222 	new_cons = nbd + tx_buf->first_bd;
223 
224 	/* Get the next bd */
225 	bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
226 
227 	/* Skip a parse bd... */
228 	--nbd;
229 	bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
230 
231 	if (tx_buf->flags & BNX2X_HAS_SECOND_PBD) {
232 		/* Skip second parse bd... */
233 		--nbd;
234 		bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
235 	}
236 
237 	/* TSO headers+data bds share a common mapping. See bnx2x_tx_split() */
238 	if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
239 		tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
240 		split_bd_len = BD_UNMAP_LEN(tx_data_bd);
241 		--nbd;
242 		bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
243 	}
244 
245 	/* unmap first bd */
246 	dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
247 			 BD_UNMAP_LEN(tx_start_bd) + split_bd_len,
248 			 DMA_TO_DEVICE);
249 
250 	/* now free frags */
251 	while (nbd > 0) {
252 
253 		tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
254 		dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
255 			       BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
256 		if (--nbd)
257 			bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
258 	}
259 
260 	/* release skb */
261 	WARN_ON(!skb);
262 	if (likely(skb)) {
263 		(*pkts_compl)++;
264 		(*bytes_compl) += skb->len;
265 		dev_kfree_skb_any(skb);
266 	}
267 
268 	tx_buf->first_bd = 0;
269 	tx_buf->skb = NULL;
270 
271 	return new_cons;
272 }
273 
274 int bnx2x_tx_int(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata)
275 {
276 	struct netdev_queue *txq;
277 	u16 hw_cons, sw_cons, bd_cons = txdata->tx_bd_cons;
278 	unsigned int pkts_compl = 0, bytes_compl = 0;
279 
280 #ifdef BNX2X_STOP_ON_ERROR
281 	if (unlikely(bp->panic))
282 		return -1;
283 #endif
284 
285 	txq = netdev_get_tx_queue(bp->dev, txdata->txq_index);
286 	hw_cons = le16_to_cpu(*txdata->tx_cons_sb);
287 	sw_cons = txdata->tx_pkt_cons;
288 
289 	while (sw_cons != hw_cons) {
290 		u16 pkt_cons;
291 
292 		pkt_cons = TX_BD(sw_cons);
293 
294 		DP(NETIF_MSG_TX_DONE,
295 		   "queue[%d]: hw_cons %u  sw_cons %u  pkt_cons %u\n",
296 		   txdata->txq_index, hw_cons, sw_cons, pkt_cons);
297 
298 		bd_cons = bnx2x_free_tx_pkt(bp, txdata, pkt_cons,
299 					    &pkts_compl, &bytes_compl);
300 
301 		sw_cons++;
302 	}
303 
304 	netdev_tx_completed_queue(txq, pkts_compl, bytes_compl);
305 
306 	txdata->tx_pkt_cons = sw_cons;
307 	txdata->tx_bd_cons = bd_cons;
308 
309 	/* Need to make the tx_bd_cons update visible to start_xmit()
310 	 * before checking for netif_tx_queue_stopped().  Without the
311 	 * memory barrier, there is a small possibility that
312 	 * start_xmit() will miss it and cause the queue to be stopped
313 	 * forever.
314 	 * On the other hand we need an rmb() here to ensure the proper
315 	 * ordering of bit testing in the following
316 	 * netif_tx_queue_stopped(txq) call.
317 	 */
318 	smp_mb();
319 
320 	if (unlikely(netif_tx_queue_stopped(txq))) {
321 		/* Taking tx_lock() is needed to prevent re-enabling the queue
322 		 * while it's empty. This could have happen if rx_action() gets
323 		 * suspended in bnx2x_tx_int() after the condition before
324 		 * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
325 		 *
326 		 * stops the queue->sees fresh tx_bd_cons->releases the queue->
327 		 * sends some packets consuming the whole queue again->
328 		 * stops the queue
329 		 */
330 
331 		__netif_tx_lock(txq, smp_processor_id());
332 
333 		if ((netif_tx_queue_stopped(txq)) &&
334 		    (bp->state == BNX2X_STATE_OPEN) &&
335 		    (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT))
336 			netif_tx_wake_queue(txq);
337 
338 		__netif_tx_unlock(txq);
339 	}
340 	return 0;
341 }
342 
343 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
344 					     u16 idx)
345 {
346 	u16 last_max = fp->last_max_sge;
347 
348 	if (SUB_S16(idx, last_max) > 0)
349 		fp->last_max_sge = idx;
350 }
351 
352 static inline void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
353 					 u16 sge_len,
354 					 struct eth_end_agg_rx_cqe *cqe)
355 {
356 	struct bnx2x *bp = fp->bp;
357 	u16 last_max, last_elem, first_elem;
358 	u16 delta = 0;
359 	u16 i;
360 
361 	if (!sge_len)
362 		return;
363 
364 	/* First mark all used pages */
365 	for (i = 0; i < sge_len; i++)
366 		BIT_VEC64_CLEAR_BIT(fp->sge_mask,
367 			RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[i])));
368 
369 	DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
370 	   sge_len - 1, le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
371 
372 	/* Here we assume that the last SGE index is the biggest */
373 	prefetch((void *)(fp->sge_mask));
374 	bnx2x_update_last_max_sge(fp,
375 		le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
376 
377 	last_max = RX_SGE(fp->last_max_sge);
378 	last_elem = last_max >> BIT_VEC64_ELEM_SHIFT;
379 	first_elem = RX_SGE(fp->rx_sge_prod) >> BIT_VEC64_ELEM_SHIFT;
380 
381 	/* If ring is not full */
382 	if (last_elem + 1 != first_elem)
383 		last_elem++;
384 
385 	/* Now update the prod */
386 	for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
387 		if (likely(fp->sge_mask[i]))
388 			break;
389 
390 		fp->sge_mask[i] = BIT_VEC64_ELEM_ONE_MASK;
391 		delta += BIT_VEC64_ELEM_SZ;
392 	}
393 
394 	if (delta > 0) {
395 		fp->rx_sge_prod += delta;
396 		/* clear page-end entries */
397 		bnx2x_clear_sge_mask_next_elems(fp);
398 	}
399 
400 	DP(NETIF_MSG_RX_STATUS,
401 	   "fp->last_max_sge = %d  fp->rx_sge_prod = %d\n",
402 	   fp->last_max_sge, fp->rx_sge_prod);
403 }
404 
405 /* Get Toeplitz hash value in the skb using the value from the
406  * CQE (calculated by HW).
407  */
408 static u32 bnx2x_get_rxhash(const struct bnx2x *bp,
409 			    const struct eth_fast_path_rx_cqe *cqe,
410 			    enum pkt_hash_types *rxhash_type)
411 {
412 	/* Get Toeplitz hash from CQE */
413 	if ((bp->dev->features & NETIF_F_RXHASH) &&
414 	    (cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG)) {
415 		enum eth_rss_hash_type htype;
416 
417 		htype = cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_TYPE;
418 		*rxhash_type = ((htype == TCP_IPV4_HASH_TYPE) ||
419 				(htype == TCP_IPV6_HASH_TYPE)) ?
420 			       PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3;
421 
422 		return le32_to_cpu(cqe->rss_hash_result);
423 	}
424 	*rxhash_type = PKT_HASH_TYPE_NONE;
425 	return 0;
426 }
427 
428 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
429 			    u16 cons, u16 prod,
430 			    struct eth_fast_path_rx_cqe *cqe)
431 {
432 	struct bnx2x *bp = fp->bp;
433 	struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
434 	struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
435 	struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
436 	dma_addr_t mapping;
437 	struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
438 	struct sw_rx_bd *first_buf = &tpa_info->first_buf;
439 
440 	/* print error if current state != stop */
441 	if (tpa_info->tpa_state != BNX2X_TPA_STOP)
442 		BNX2X_ERR("start of bin not in stop [%d]\n", queue);
443 
444 	/* Try to map an empty data buffer from the aggregation info  */
445 	mapping = dma_map_single(&bp->pdev->dev,
446 				 first_buf->data + NET_SKB_PAD,
447 				 fp->rx_buf_size, DMA_FROM_DEVICE);
448 	/*
449 	 *  ...if it fails - move the skb from the consumer to the producer
450 	 *  and set the current aggregation state as ERROR to drop it
451 	 *  when TPA_STOP arrives.
452 	 */
453 
454 	if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
455 		/* Move the BD from the consumer to the producer */
456 		bnx2x_reuse_rx_data(fp, cons, prod);
457 		tpa_info->tpa_state = BNX2X_TPA_ERROR;
458 		return;
459 	}
460 
461 	/* move empty data from pool to prod */
462 	prod_rx_buf->data = first_buf->data;
463 	dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
464 	/* point prod_bd to new data */
465 	prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
466 	prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
467 
468 	/* move partial skb from cons to pool (don't unmap yet) */
469 	*first_buf = *cons_rx_buf;
470 
471 	/* mark bin state as START */
472 	tpa_info->parsing_flags =
473 		le16_to_cpu(cqe->pars_flags.flags);
474 	tpa_info->vlan_tag = le16_to_cpu(cqe->vlan_tag);
475 	tpa_info->tpa_state = BNX2X_TPA_START;
476 	tpa_info->len_on_bd = le16_to_cpu(cqe->len_on_bd);
477 	tpa_info->placement_offset = cqe->placement_offset;
478 	tpa_info->rxhash = bnx2x_get_rxhash(bp, cqe, &tpa_info->rxhash_type);
479 	if (fp->mode == TPA_MODE_GRO) {
480 		u16 gro_size = le16_to_cpu(cqe->pkt_len_or_gro_seg_len);
481 		tpa_info->full_page = SGE_PAGES / gro_size * gro_size;
482 		tpa_info->gro_size = gro_size;
483 	}
484 
485 #ifdef BNX2X_STOP_ON_ERROR
486 	fp->tpa_queue_used |= (1 << queue);
487 	DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
488 	   fp->tpa_queue_used);
489 #endif
490 }
491 
492 /* Timestamp option length allowed for TPA aggregation:
493  *
494  *		nop nop kind length echo val
495  */
496 #define TPA_TSTAMP_OPT_LEN	12
497 /**
498  * bnx2x_set_gro_params - compute GRO values
499  *
500  * @skb:		packet skb
501  * @parsing_flags:	parsing flags from the START CQE
502  * @len_on_bd:		total length of the first packet for the
503  *			aggregation.
504  * @pkt_len:		length of all segments
505  *
506  * Approximate value of the MSS for this aggregation calculated using
507  * the first packet of it.
508  * Compute number of aggregated segments, and gso_type.
509  */
510 static void bnx2x_set_gro_params(struct sk_buff *skb, u16 parsing_flags,
511 				 u16 len_on_bd, unsigned int pkt_len,
512 				 u16 num_of_coalesced_segs)
513 {
514 	/* TPA aggregation won't have either IP options or TCP options
515 	 * other than timestamp or IPv6 extension headers.
516 	 */
517 	u16 hdrs_len = ETH_HLEN + sizeof(struct tcphdr);
518 
519 	if (GET_FLAG(parsing_flags, PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
520 	    PRS_FLAG_OVERETH_IPV6) {
521 		hdrs_len += sizeof(struct ipv6hdr);
522 		skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
523 	} else {
524 		hdrs_len += sizeof(struct iphdr);
525 		skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
526 	}
527 
528 	/* Check if there was a TCP timestamp, if there is it's will
529 	 * always be 12 bytes length: nop nop kind length echo val.
530 	 *
531 	 * Otherwise FW would close the aggregation.
532 	 */
533 	if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG)
534 		hdrs_len += TPA_TSTAMP_OPT_LEN;
535 
536 	skb_shinfo(skb)->gso_size = len_on_bd - hdrs_len;
537 
538 	/* tcp_gro_complete() will copy NAPI_GRO_CB(skb)->count
539 	 * to skb_shinfo(skb)->gso_segs
540 	 */
541 	NAPI_GRO_CB(skb)->count = num_of_coalesced_segs;
542 }
543 
544 static int bnx2x_alloc_rx_sge(struct bnx2x *bp, struct bnx2x_fastpath *fp,
545 			      u16 index, gfp_t gfp_mask)
546 {
547 	struct sw_rx_page *sw_buf = &fp->rx_page_ring[index];
548 	struct eth_rx_sge *sge = &fp->rx_sge_ring[index];
549 	struct bnx2x_alloc_pool *pool = &fp->page_pool;
550 	dma_addr_t mapping;
551 
552 	if (!pool->page) {
553 		pool->page = alloc_pages(gfp_mask, PAGES_PER_SGE_SHIFT);
554 		if (unlikely(!pool->page))
555 			return -ENOMEM;
556 
557 		pool->offset = 0;
558 	}
559 
560 	mapping = dma_map_page(&bp->pdev->dev, pool->page,
561 			       pool->offset, SGE_PAGE_SIZE, DMA_FROM_DEVICE);
562 	if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
563 		BNX2X_ERR("Can't map sge\n");
564 		return -ENOMEM;
565 	}
566 
567 	sw_buf->page = pool->page;
568 	sw_buf->offset = pool->offset;
569 
570 	dma_unmap_addr_set(sw_buf, mapping, mapping);
571 
572 	sge->addr_hi = cpu_to_le32(U64_HI(mapping));
573 	sge->addr_lo = cpu_to_le32(U64_LO(mapping));
574 
575 	pool->offset += SGE_PAGE_SIZE;
576 	if (PAGE_SIZE - pool->offset >= SGE_PAGE_SIZE)
577 		get_page(pool->page);
578 	else
579 		pool->page = NULL;
580 	return 0;
581 }
582 
583 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
584 			       struct bnx2x_agg_info *tpa_info,
585 			       u16 pages,
586 			       struct sk_buff *skb,
587 			       struct eth_end_agg_rx_cqe *cqe,
588 			       u16 cqe_idx)
589 {
590 	struct sw_rx_page *rx_pg, old_rx_pg;
591 	u32 i, frag_len, frag_size;
592 	int err, j, frag_id = 0;
593 	u16 len_on_bd = tpa_info->len_on_bd;
594 	u16 full_page = 0, gro_size = 0;
595 
596 	frag_size = le16_to_cpu(cqe->pkt_len) - len_on_bd;
597 
598 	if (fp->mode == TPA_MODE_GRO) {
599 		gro_size = tpa_info->gro_size;
600 		full_page = tpa_info->full_page;
601 	}
602 
603 	/* This is needed in order to enable forwarding support */
604 	if (frag_size)
605 		bnx2x_set_gro_params(skb, tpa_info->parsing_flags, len_on_bd,
606 				     le16_to_cpu(cqe->pkt_len),
607 				     le16_to_cpu(cqe->num_of_coalesced_segs));
608 
609 #ifdef BNX2X_STOP_ON_ERROR
610 	if (pages > min_t(u32, 8, MAX_SKB_FRAGS) * SGE_PAGES) {
611 		BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
612 			  pages, cqe_idx);
613 		BNX2X_ERR("cqe->pkt_len = %d\n", cqe->pkt_len);
614 		bnx2x_panic();
615 		return -EINVAL;
616 	}
617 #endif
618 
619 	/* Run through the SGL and compose the fragmented skb */
620 	for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
621 		u16 sge_idx = RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[j]));
622 
623 		/* FW gives the indices of the SGE as if the ring is an array
624 		   (meaning that "next" element will consume 2 indices) */
625 		if (fp->mode == TPA_MODE_GRO)
626 			frag_len = min_t(u32, frag_size, (u32)full_page);
627 		else /* LRO */
628 			frag_len = min_t(u32, frag_size, (u32)SGE_PAGES);
629 
630 		rx_pg = &fp->rx_page_ring[sge_idx];
631 		old_rx_pg = *rx_pg;
632 
633 		/* If we fail to allocate a substitute page, we simply stop
634 		   where we are and drop the whole packet */
635 		err = bnx2x_alloc_rx_sge(bp, fp, sge_idx, GFP_ATOMIC);
636 		if (unlikely(err)) {
637 			bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
638 			return err;
639 		}
640 
641 		dma_unmap_page(&bp->pdev->dev,
642 			       dma_unmap_addr(&old_rx_pg, mapping),
643 			       SGE_PAGE_SIZE, DMA_FROM_DEVICE);
644 		/* Add one frag and update the appropriate fields in the skb */
645 		if (fp->mode == TPA_MODE_LRO)
646 			skb_fill_page_desc(skb, j, old_rx_pg.page,
647 					   old_rx_pg.offset, frag_len);
648 		else { /* GRO */
649 			int rem;
650 			int offset = 0;
651 			for (rem = frag_len; rem > 0; rem -= gro_size) {
652 				int len = rem > gro_size ? gro_size : rem;
653 				skb_fill_page_desc(skb, frag_id++,
654 						   old_rx_pg.page,
655 						   old_rx_pg.offset + offset,
656 						   len);
657 				if (offset)
658 					get_page(old_rx_pg.page);
659 				offset += len;
660 			}
661 		}
662 
663 		skb->data_len += frag_len;
664 		skb->truesize += SGE_PAGES;
665 		skb->len += frag_len;
666 
667 		frag_size -= frag_len;
668 	}
669 
670 	return 0;
671 }
672 
673 static void bnx2x_frag_free(const struct bnx2x_fastpath *fp, void *data)
674 {
675 	if (fp->rx_frag_size)
676 		skb_free_frag(data);
677 	else
678 		kfree(data);
679 }
680 
681 static void *bnx2x_frag_alloc(const struct bnx2x_fastpath *fp, gfp_t gfp_mask)
682 {
683 	if (fp->rx_frag_size) {
684 		/* GFP_KERNEL allocations are used only during initialization */
685 		if (unlikely(gfpflags_allow_blocking(gfp_mask)))
686 			return (void *)__get_free_page(gfp_mask);
687 
688 		return netdev_alloc_frag(fp->rx_frag_size);
689 	}
690 
691 	return kmalloc(fp->rx_buf_size + NET_SKB_PAD, gfp_mask);
692 }
693 
694 #ifdef CONFIG_INET
695 static void bnx2x_gro_ip_csum(struct bnx2x *bp, struct sk_buff *skb)
696 {
697 	const struct iphdr *iph = ip_hdr(skb);
698 	struct tcphdr *th;
699 
700 	skb_set_transport_header(skb, sizeof(struct iphdr));
701 	th = tcp_hdr(skb);
702 
703 	th->check = ~tcp_v4_check(skb->len - skb_transport_offset(skb),
704 				  iph->saddr, iph->daddr, 0);
705 }
706 
707 static void bnx2x_gro_ipv6_csum(struct bnx2x *bp, struct sk_buff *skb)
708 {
709 	struct ipv6hdr *iph = ipv6_hdr(skb);
710 	struct tcphdr *th;
711 
712 	skb_set_transport_header(skb, sizeof(struct ipv6hdr));
713 	th = tcp_hdr(skb);
714 
715 	th->check = ~tcp_v6_check(skb->len - skb_transport_offset(skb),
716 				  &iph->saddr, &iph->daddr, 0);
717 }
718 
719 static void bnx2x_gro_csum(struct bnx2x *bp, struct sk_buff *skb,
720 			    void (*gro_func)(struct bnx2x*, struct sk_buff*))
721 {
722 	skb_reset_network_header(skb);
723 	gro_func(bp, skb);
724 	tcp_gro_complete(skb);
725 }
726 #endif
727 
728 static void bnx2x_gro_receive(struct bnx2x *bp, struct bnx2x_fastpath *fp,
729 			       struct sk_buff *skb)
730 {
731 #ifdef CONFIG_INET
732 	if (skb_shinfo(skb)->gso_size) {
733 		switch (be16_to_cpu(skb->protocol)) {
734 		case ETH_P_IP:
735 			bnx2x_gro_csum(bp, skb, bnx2x_gro_ip_csum);
736 			break;
737 		case ETH_P_IPV6:
738 			bnx2x_gro_csum(bp, skb, bnx2x_gro_ipv6_csum);
739 			break;
740 		default:
741 			netdev_WARN_ONCE(bp->dev,
742 					 "Error: FW GRO supports only IPv4/IPv6, not 0x%04x\n",
743 					 be16_to_cpu(skb->protocol));
744 		}
745 	}
746 #endif
747 	skb_record_rx_queue(skb, fp->rx_queue);
748 	napi_gro_receive(&fp->napi, skb);
749 }
750 
751 static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
752 			   struct bnx2x_agg_info *tpa_info,
753 			   u16 pages,
754 			   struct eth_end_agg_rx_cqe *cqe,
755 			   u16 cqe_idx)
756 {
757 	struct sw_rx_bd *rx_buf = &tpa_info->first_buf;
758 	u8 pad = tpa_info->placement_offset;
759 	u16 len = tpa_info->len_on_bd;
760 	struct sk_buff *skb = NULL;
761 	u8 *new_data, *data = rx_buf->data;
762 	u8 old_tpa_state = tpa_info->tpa_state;
763 
764 	tpa_info->tpa_state = BNX2X_TPA_STOP;
765 
766 	/* If we there was an error during the handling of the TPA_START -
767 	 * drop this aggregation.
768 	 */
769 	if (old_tpa_state == BNX2X_TPA_ERROR)
770 		goto drop;
771 
772 	/* Try to allocate the new data */
773 	new_data = bnx2x_frag_alloc(fp, GFP_ATOMIC);
774 	/* Unmap skb in the pool anyway, as we are going to change
775 	   pool entry status to BNX2X_TPA_STOP even if new skb allocation
776 	   fails. */
777 	dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
778 			 fp->rx_buf_size, DMA_FROM_DEVICE);
779 	if (likely(new_data))
780 		skb = build_skb(data, fp->rx_frag_size);
781 
782 	if (likely(skb)) {
783 #ifdef BNX2X_STOP_ON_ERROR
784 		if (pad + len > fp->rx_buf_size) {
785 			BNX2X_ERR("skb_put is about to fail...  pad %d  len %d  rx_buf_size %d\n",
786 				  pad, len, fp->rx_buf_size);
787 			bnx2x_panic();
788 			return;
789 		}
790 #endif
791 
792 		skb_reserve(skb, pad + NET_SKB_PAD);
793 		skb_put(skb, len);
794 		skb_set_hash(skb, tpa_info->rxhash, tpa_info->rxhash_type);
795 
796 		skb->protocol = eth_type_trans(skb, bp->dev);
797 		skb->ip_summed = CHECKSUM_UNNECESSARY;
798 
799 		if (!bnx2x_fill_frag_skb(bp, fp, tpa_info, pages,
800 					 skb, cqe, cqe_idx)) {
801 			if (tpa_info->parsing_flags & PARSING_FLAGS_VLAN)
802 				__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), tpa_info->vlan_tag);
803 			bnx2x_gro_receive(bp, fp, skb);
804 		} else {
805 			DP(NETIF_MSG_RX_STATUS,
806 			   "Failed to allocate new pages - dropping packet!\n");
807 			dev_kfree_skb_any(skb);
808 		}
809 
810 		/* put new data in bin */
811 		rx_buf->data = new_data;
812 
813 		return;
814 	}
815 	if (new_data)
816 		bnx2x_frag_free(fp, new_data);
817 drop:
818 	/* drop the packet and keep the buffer in the bin */
819 	DP(NETIF_MSG_RX_STATUS,
820 	   "Failed to allocate or map a new skb - dropping packet!\n");
821 	bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed++;
822 }
823 
824 static int bnx2x_alloc_rx_data(struct bnx2x *bp, struct bnx2x_fastpath *fp,
825 			       u16 index, gfp_t gfp_mask)
826 {
827 	u8 *data;
828 	struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[index];
829 	struct eth_rx_bd *rx_bd = &fp->rx_desc_ring[index];
830 	dma_addr_t mapping;
831 
832 	data = bnx2x_frag_alloc(fp, gfp_mask);
833 	if (unlikely(data == NULL))
834 		return -ENOMEM;
835 
836 	mapping = dma_map_single(&bp->pdev->dev, data + NET_SKB_PAD,
837 				 fp->rx_buf_size,
838 				 DMA_FROM_DEVICE);
839 	if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
840 		bnx2x_frag_free(fp, data);
841 		BNX2X_ERR("Can't map rx data\n");
842 		return -ENOMEM;
843 	}
844 
845 	rx_buf->data = data;
846 	dma_unmap_addr_set(rx_buf, mapping, mapping);
847 
848 	rx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
849 	rx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
850 
851 	return 0;
852 }
853 
854 static
855 void bnx2x_csum_validate(struct sk_buff *skb, union eth_rx_cqe *cqe,
856 				 struct bnx2x_fastpath *fp,
857 				 struct bnx2x_eth_q_stats *qstats)
858 {
859 	/* Do nothing if no L4 csum validation was done.
860 	 * We do not check whether IP csum was validated. For IPv4 we assume
861 	 * that if the card got as far as validating the L4 csum, it also
862 	 * validated the IP csum. IPv6 has no IP csum.
863 	 */
864 	if (cqe->fast_path_cqe.status_flags &
865 	    ETH_FAST_PATH_RX_CQE_L4_XSUM_NO_VALIDATION_FLG)
866 		return;
867 
868 	/* If L4 validation was done, check if an error was found. */
869 
870 	if (cqe->fast_path_cqe.type_error_flags &
871 	    (ETH_FAST_PATH_RX_CQE_IP_BAD_XSUM_FLG |
872 	     ETH_FAST_PATH_RX_CQE_L4_BAD_XSUM_FLG))
873 		qstats->hw_csum_err++;
874 	else
875 		skb->ip_summed = CHECKSUM_UNNECESSARY;
876 }
877 
878 static int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
879 {
880 	struct bnx2x *bp = fp->bp;
881 	u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
882 	u16 sw_comp_cons, sw_comp_prod;
883 	int rx_pkt = 0;
884 	union eth_rx_cqe *cqe;
885 	struct eth_fast_path_rx_cqe *cqe_fp;
886 
887 #ifdef BNX2X_STOP_ON_ERROR
888 	if (unlikely(bp->panic))
889 		return 0;
890 #endif
891 	if (budget <= 0)
892 		return rx_pkt;
893 
894 	bd_cons = fp->rx_bd_cons;
895 	bd_prod = fp->rx_bd_prod;
896 	bd_prod_fw = bd_prod;
897 	sw_comp_cons = fp->rx_comp_cons;
898 	sw_comp_prod = fp->rx_comp_prod;
899 
900 	comp_ring_cons = RCQ_BD(sw_comp_cons);
901 	cqe = &fp->rx_comp_ring[comp_ring_cons];
902 	cqe_fp = &cqe->fast_path_cqe;
903 
904 	DP(NETIF_MSG_RX_STATUS,
905 	   "queue[%d]: sw_comp_cons %u\n", fp->index, sw_comp_cons);
906 
907 	while (BNX2X_IS_CQE_COMPLETED(cqe_fp)) {
908 		struct sw_rx_bd *rx_buf = NULL;
909 		struct sk_buff *skb;
910 		u8 cqe_fp_flags;
911 		enum eth_rx_cqe_type cqe_fp_type;
912 		u16 len, pad, queue;
913 		u8 *data;
914 		u32 rxhash;
915 		enum pkt_hash_types rxhash_type;
916 
917 #ifdef BNX2X_STOP_ON_ERROR
918 		if (unlikely(bp->panic))
919 			return 0;
920 #endif
921 
922 		bd_prod = RX_BD(bd_prod);
923 		bd_cons = RX_BD(bd_cons);
924 
925 		/* A rmb() is required to ensure that the CQE is not read
926 		 * before it is written by the adapter DMA.  PCI ordering
927 		 * rules will make sure the other fields are written before
928 		 * the marker at the end of struct eth_fast_path_rx_cqe
929 		 * but without rmb() a weakly ordered processor can process
930 		 * stale data.  Without the barrier TPA state-machine might
931 		 * enter inconsistent state and kernel stack might be
932 		 * provided with incorrect packet description - these lead
933 		 * to various kernel crashed.
934 		 */
935 		rmb();
936 
937 		cqe_fp_flags = cqe_fp->type_error_flags;
938 		cqe_fp_type = cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE;
939 
940 		DP(NETIF_MSG_RX_STATUS,
941 		   "CQE type %x  err %x  status %x  queue %x  vlan %x  len %u\n",
942 		   CQE_TYPE(cqe_fp_flags),
943 		   cqe_fp_flags, cqe_fp->status_flags,
944 		   le32_to_cpu(cqe_fp->rss_hash_result),
945 		   le16_to_cpu(cqe_fp->vlan_tag),
946 		   le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len));
947 
948 		/* is this a slowpath msg? */
949 		if (unlikely(CQE_TYPE_SLOW(cqe_fp_type))) {
950 			bnx2x_sp_event(fp, cqe);
951 			goto next_cqe;
952 		}
953 
954 		rx_buf = &fp->rx_buf_ring[bd_cons];
955 		data = rx_buf->data;
956 
957 		if (!CQE_TYPE_FAST(cqe_fp_type)) {
958 			struct bnx2x_agg_info *tpa_info;
959 			u16 frag_size, pages;
960 #ifdef BNX2X_STOP_ON_ERROR
961 			/* sanity check */
962 			if (fp->mode == TPA_MODE_DISABLED &&
963 			    (CQE_TYPE_START(cqe_fp_type) ||
964 			     CQE_TYPE_STOP(cqe_fp_type)))
965 				BNX2X_ERR("START/STOP packet while TPA disabled, type %x\n",
966 					  CQE_TYPE(cqe_fp_type));
967 #endif
968 
969 			if (CQE_TYPE_START(cqe_fp_type)) {
970 				u16 queue = cqe_fp->queue_index;
971 				DP(NETIF_MSG_RX_STATUS,
972 				   "calling tpa_start on queue %d\n",
973 				   queue);
974 
975 				bnx2x_tpa_start(fp, queue,
976 						bd_cons, bd_prod,
977 						cqe_fp);
978 
979 				goto next_rx;
980 			}
981 			queue = cqe->end_agg_cqe.queue_index;
982 			tpa_info = &fp->tpa_info[queue];
983 			DP(NETIF_MSG_RX_STATUS,
984 			   "calling tpa_stop on queue %d\n",
985 			   queue);
986 
987 			frag_size = le16_to_cpu(cqe->end_agg_cqe.pkt_len) -
988 				    tpa_info->len_on_bd;
989 
990 			if (fp->mode == TPA_MODE_GRO)
991 				pages = (frag_size + tpa_info->full_page - 1) /
992 					 tpa_info->full_page;
993 			else
994 				pages = SGE_PAGE_ALIGN(frag_size) >>
995 					SGE_PAGE_SHIFT;
996 
997 			bnx2x_tpa_stop(bp, fp, tpa_info, pages,
998 				       &cqe->end_agg_cqe, comp_ring_cons);
999 #ifdef BNX2X_STOP_ON_ERROR
1000 			if (bp->panic)
1001 				return 0;
1002 #endif
1003 
1004 			bnx2x_update_sge_prod(fp, pages, &cqe->end_agg_cqe);
1005 			goto next_cqe;
1006 		}
1007 		/* non TPA */
1008 		len = le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len);
1009 		pad = cqe_fp->placement_offset;
1010 		dma_sync_single_for_cpu(&bp->pdev->dev,
1011 					dma_unmap_addr(rx_buf, mapping),
1012 					pad + RX_COPY_THRESH,
1013 					DMA_FROM_DEVICE);
1014 		pad += NET_SKB_PAD;
1015 		prefetch(data + pad); /* speedup eth_type_trans() */
1016 		/* is this an error packet? */
1017 		if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
1018 			DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
1019 			   "ERROR  flags %x  rx packet %u\n",
1020 			   cqe_fp_flags, sw_comp_cons);
1021 			bnx2x_fp_qstats(bp, fp)->rx_err_discard_pkt++;
1022 			goto reuse_rx;
1023 		}
1024 
1025 		/* Since we don't have a jumbo ring
1026 		 * copy small packets if mtu > 1500
1027 		 */
1028 		if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
1029 		    (len <= RX_COPY_THRESH)) {
1030 			skb = napi_alloc_skb(&fp->napi, len);
1031 			if (skb == NULL) {
1032 				DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
1033 				   "ERROR  packet dropped because of alloc failure\n");
1034 				bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
1035 				goto reuse_rx;
1036 			}
1037 			memcpy(skb->data, data + pad, len);
1038 			bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
1039 		} else {
1040 			if (likely(bnx2x_alloc_rx_data(bp, fp, bd_prod,
1041 						       GFP_ATOMIC) == 0)) {
1042 				dma_unmap_single(&bp->pdev->dev,
1043 						 dma_unmap_addr(rx_buf, mapping),
1044 						 fp->rx_buf_size,
1045 						 DMA_FROM_DEVICE);
1046 				skb = build_skb(data, fp->rx_frag_size);
1047 				if (unlikely(!skb)) {
1048 					bnx2x_frag_free(fp, data);
1049 					bnx2x_fp_qstats(bp, fp)->
1050 							rx_skb_alloc_failed++;
1051 					goto next_rx;
1052 				}
1053 				skb_reserve(skb, pad);
1054 			} else {
1055 				DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
1056 				   "ERROR  packet dropped because of alloc failure\n");
1057 				bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
1058 reuse_rx:
1059 				bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
1060 				goto next_rx;
1061 			}
1062 		}
1063 
1064 		skb_put(skb, len);
1065 		skb->protocol = eth_type_trans(skb, bp->dev);
1066 
1067 		/* Set Toeplitz hash for a none-LRO skb */
1068 		rxhash = bnx2x_get_rxhash(bp, cqe_fp, &rxhash_type);
1069 		skb_set_hash(skb, rxhash, rxhash_type);
1070 
1071 		skb_checksum_none_assert(skb);
1072 
1073 		if (bp->dev->features & NETIF_F_RXCSUM)
1074 			bnx2x_csum_validate(skb, cqe, fp,
1075 					    bnx2x_fp_qstats(bp, fp));
1076 
1077 		skb_record_rx_queue(skb, fp->rx_queue);
1078 
1079 		/* Check if this packet was timestamped */
1080 		if (unlikely(cqe->fast_path_cqe.type_error_flags &
1081 			     (1 << ETH_FAST_PATH_RX_CQE_PTP_PKT_SHIFT)))
1082 			bnx2x_set_rx_ts(bp, skb);
1083 
1084 		if (le16_to_cpu(cqe_fp->pars_flags.flags) &
1085 		    PARSING_FLAGS_VLAN)
1086 			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
1087 					       le16_to_cpu(cqe_fp->vlan_tag));
1088 
1089 		napi_gro_receive(&fp->napi, skb);
1090 next_rx:
1091 		rx_buf->data = NULL;
1092 
1093 		bd_cons = NEXT_RX_IDX(bd_cons);
1094 		bd_prod = NEXT_RX_IDX(bd_prod);
1095 		bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
1096 		rx_pkt++;
1097 next_cqe:
1098 		sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
1099 		sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
1100 
1101 		/* mark CQE as free */
1102 		BNX2X_SEED_CQE(cqe_fp);
1103 
1104 		if (rx_pkt == budget)
1105 			break;
1106 
1107 		comp_ring_cons = RCQ_BD(sw_comp_cons);
1108 		cqe = &fp->rx_comp_ring[comp_ring_cons];
1109 		cqe_fp = &cqe->fast_path_cqe;
1110 	} /* while */
1111 
1112 	fp->rx_bd_cons = bd_cons;
1113 	fp->rx_bd_prod = bd_prod_fw;
1114 	fp->rx_comp_cons = sw_comp_cons;
1115 	fp->rx_comp_prod = sw_comp_prod;
1116 
1117 	/* Update producers */
1118 	bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
1119 			     fp->rx_sge_prod);
1120 
1121 	return rx_pkt;
1122 }
1123 
1124 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
1125 {
1126 	struct bnx2x_fastpath *fp = fp_cookie;
1127 	struct bnx2x *bp = fp->bp;
1128 	u8 cos;
1129 
1130 	DP(NETIF_MSG_INTR,
1131 	   "got an MSI-X interrupt on IDX:SB [fp %d fw_sd %d igusb %d]\n",
1132 	   fp->index, fp->fw_sb_id, fp->igu_sb_id);
1133 
1134 	bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
1135 
1136 #ifdef BNX2X_STOP_ON_ERROR
1137 	if (unlikely(bp->panic))
1138 		return IRQ_HANDLED;
1139 #endif
1140 
1141 	/* Handle Rx and Tx according to MSI-X vector */
1142 	for_each_cos_in_tx_queue(fp, cos)
1143 		prefetch(fp->txdata_ptr[cos]->tx_cons_sb);
1144 
1145 	prefetch(&fp->sb_running_index[SM_RX_ID]);
1146 	napi_schedule_irqoff(&bnx2x_fp(bp, fp->index, napi));
1147 
1148 	return IRQ_HANDLED;
1149 }
1150 
1151 /* HW Lock for shared dual port PHYs */
1152 void bnx2x_acquire_phy_lock(struct bnx2x *bp)
1153 {
1154 	mutex_lock(&bp->port.phy_mutex);
1155 
1156 	bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1157 }
1158 
1159 void bnx2x_release_phy_lock(struct bnx2x *bp)
1160 {
1161 	bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1162 
1163 	mutex_unlock(&bp->port.phy_mutex);
1164 }
1165 
1166 /* calculates MF speed according to current linespeed and MF configuration */
1167 u16 bnx2x_get_mf_speed(struct bnx2x *bp)
1168 {
1169 	u16 line_speed = bp->link_vars.line_speed;
1170 	if (IS_MF(bp)) {
1171 		u16 maxCfg = bnx2x_extract_max_cfg(bp,
1172 						   bp->mf_config[BP_VN(bp)]);
1173 
1174 		/* Calculate the current MAX line speed limit for the MF
1175 		 * devices
1176 		 */
1177 		if (IS_MF_PERCENT_BW(bp))
1178 			line_speed = (line_speed * maxCfg) / 100;
1179 		else { /* SD mode */
1180 			u16 vn_max_rate = maxCfg * 100;
1181 
1182 			if (vn_max_rate < line_speed)
1183 				line_speed = vn_max_rate;
1184 		}
1185 	}
1186 
1187 	return line_speed;
1188 }
1189 
1190 /**
1191  * bnx2x_fill_report_data - fill link report data to report
1192  *
1193  * @bp:		driver handle
1194  * @data:	link state to update
1195  *
1196  * It uses a none-atomic bit operations because is called under the mutex.
1197  */
1198 static void bnx2x_fill_report_data(struct bnx2x *bp,
1199 				   struct bnx2x_link_report_data *data)
1200 {
1201 	memset(data, 0, sizeof(*data));
1202 
1203 	if (IS_PF(bp)) {
1204 		/* Fill the report data: effective line speed */
1205 		data->line_speed = bnx2x_get_mf_speed(bp);
1206 
1207 		/* Link is down */
1208 		if (!bp->link_vars.link_up || (bp->flags & MF_FUNC_DIS))
1209 			__set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1210 				  &data->link_report_flags);
1211 
1212 		if (!BNX2X_NUM_ETH_QUEUES(bp))
1213 			__set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1214 				  &data->link_report_flags);
1215 
1216 		/* Full DUPLEX */
1217 		if (bp->link_vars.duplex == DUPLEX_FULL)
1218 			__set_bit(BNX2X_LINK_REPORT_FD,
1219 				  &data->link_report_flags);
1220 
1221 		/* Rx Flow Control is ON */
1222 		if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX)
1223 			__set_bit(BNX2X_LINK_REPORT_RX_FC_ON,
1224 				  &data->link_report_flags);
1225 
1226 		/* Tx Flow Control is ON */
1227 		if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX)
1228 			__set_bit(BNX2X_LINK_REPORT_TX_FC_ON,
1229 				  &data->link_report_flags);
1230 	} else { /* VF */
1231 		*data = bp->vf_link_vars;
1232 	}
1233 }
1234 
1235 /**
1236  * bnx2x_link_report - report link status to OS.
1237  *
1238  * @bp:		driver handle
1239  *
1240  * Calls the __bnx2x_link_report() under the same locking scheme
1241  * as a link/PHY state managing code to ensure a consistent link
1242  * reporting.
1243  */
1244 
1245 void bnx2x_link_report(struct bnx2x *bp)
1246 {
1247 	bnx2x_acquire_phy_lock(bp);
1248 	__bnx2x_link_report(bp);
1249 	bnx2x_release_phy_lock(bp);
1250 }
1251 
1252 /**
1253  * __bnx2x_link_report - report link status to OS.
1254  *
1255  * @bp:		driver handle
1256  *
1257  * None atomic implementation.
1258  * Should be called under the phy_lock.
1259  */
1260 void __bnx2x_link_report(struct bnx2x *bp)
1261 {
1262 	struct bnx2x_link_report_data cur_data;
1263 
1264 	/* reread mf_cfg */
1265 	if (IS_PF(bp) && !CHIP_IS_E1(bp))
1266 		bnx2x_read_mf_cfg(bp);
1267 
1268 	/* Read the current link report info */
1269 	bnx2x_fill_report_data(bp, &cur_data);
1270 
1271 	/* Don't report link down or exactly the same link status twice */
1272 	if (!memcmp(&cur_data, &bp->last_reported_link, sizeof(cur_data)) ||
1273 	    (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1274 		      &bp->last_reported_link.link_report_flags) &&
1275 	     test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1276 		      &cur_data.link_report_flags)))
1277 		return;
1278 
1279 	bp->link_cnt++;
1280 
1281 	/* We are going to report a new link parameters now -
1282 	 * remember the current data for the next time.
1283 	 */
1284 	memcpy(&bp->last_reported_link, &cur_data, sizeof(cur_data));
1285 
1286 	/* propagate status to VFs */
1287 	if (IS_PF(bp))
1288 		bnx2x_iov_link_update(bp);
1289 
1290 	if (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1291 		     &cur_data.link_report_flags)) {
1292 		netif_carrier_off(bp->dev);
1293 		netdev_err(bp->dev, "NIC Link is Down\n");
1294 		return;
1295 	} else {
1296 		const char *duplex;
1297 		const char *flow;
1298 
1299 		netif_carrier_on(bp->dev);
1300 
1301 		if (test_and_clear_bit(BNX2X_LINK_REPORT_FD,
1302 				       &cur_data.link_report_flags))
1303 			duplex = "full";
1304 		else
1305 			duplex = "half";
1306 
1307 		/* Handle the FC at the end so that only these flags would be
1308 		 * possibly set. This way we may easily check if there is no FC
1309 		 * enabled.
1310 		 */
1311 		if (cur_data.link_report_flags) {
1312 			if (test_bit(BNX2X_LINK_REPORT_RX_FC_ON,
1313 				     &cur_data.link_report_flags)) {
1314 				if (test_bit(BNX2X_LINK_REPORT_TX_FC_ON,
1315 				     &cur_data.link_report_flags))
1316 					flow = "ON - receive & transmit";
1317 				else
1318 					flow = "ON - receive";
1319 			} else {
1320 				flow = "ON - transmit";
1321 			}
1322 		} else {
1323 			flow = "none";
1324 		}
1325 		netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n",
1326 			    cur_data.line_speed, duplex, flow);
1327 	}
1328 }
1329 
1330 static void bnx2x_set_next_page_sgl(struct bnx2x_fastpath *fp)
1331 {
1332 	int i;
1333 
1334 	for (i = 1; i <= NUM_RX_SGE_PAGES; i++) {
1335 		struct eth_rx_sge *sge;
1336 
1337 		sge = &fp->rx_sge_ring[RX_SGE_CNT * i - 2];
1338 		sge->addr_hi =
1339 			cpu_to_le32(U64_HI(fp->rx_sge_mapping +
1340 			BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1341 
1342 		sge->addr_lo =
1343 			cpu_to_le32(U64_LO(fp->rx_sge_mapping +
1344 			BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1345 	}
1346 }
1347 
1348 static void bnx2x_free_tpa_pool(struct bnx2x *bp,
1349 				struct bnx2x_fastpath *fp, int last)
1350 {
1351 	int i;
1352 
1353 	for (i = 0; i < last; i++) {
1354 		struct bnx2x_agg_info *tpa_info = &fp->tpa_info[i];
1355 		struct sw_rx_bd *first_buf = &tpa_info->first_buf;
1356 		u8 *data = first_buf->data;
1357 
1358 		if (data == NULL) {
1359 			DP(NETIF_MSG_IFDOWN, "tpa bin %d empty on free\n", i);
1360 			continue;
1361 		}
1362 		if (tpa_info->tpa_state == BNX2X_TPA_START)
1363 			dma_unmap_single(&bp->pdev->dev,
1364 					 dma_unmap_addr(first_buf, mapping),
1365 					 fp->rx_buf_size, DMA_FROM_DEVICE);
1366 		bnx2x_frag_free(fp, data);
1367 		first_buf->data = NULL;
1368 	}
1369 }
1370 
1371 void bnx2x_init_rx_rings_cnic(struct bnx2x *bp)
1372 {
1373 	int j;
1374 
1375 	for_each_rx_queue_cnic(bp, j) {
1376 		struct bnx2x_fastpath *fp = &bp->fp[j];
1377 
1378 		fp->rx_bd_cons = 0;
1379 
1380 		/* Activate BD ring */
1381 		/* Warning!
1382 		 * this will generate an interrupt (to the TSTORM)
1383 		 * must only be done after chip is initialized
1384 		 */
1385 		bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1386 				     fp->rx_sge_prod);
1387 	}
1388 }
1389 
1390 void bnx2x_init_rx_rings(struct bnx2x *bp)
1391 {
1392 	int func = BP_FUNC(bp);
1393 	u16 ring_prod;
1394 	int i, j;
1395 
1396 	/* Allocate TPA resources */
1397 	for_each_eth_queue(bp, j) {
1398 		struct bnx2x_fastpath *fp = &bp->fp[j];
1399 
1400 		DP(NETIF_MSG_IFUP,
1401 		   "mtu %d  rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size);
1402 
1403 		if (fp->mode != TPA_MODE_DISABLED) {
1404 			/* Fill the per-aggregation pool */
1405 			for (i = 0; i < MAX_AGG_QS(bp); i++) {
1406 				struct bnx2x_agg_info *tpa_info =
1407 					&fp->tpa_info[i];
1408 				struct sw_rx_bd *first_buf =
1409 					&tpa_info->first_buf;
1410 
1411 				first_buf->data =
1412 					bnx2x_frag_alloc(fp, GFP_KERNEL);
1413 				if (!first_buf->data) {
1414 					BNX2X_ERR("Failed to allocate TPA skb pool for queue[%d] - disabling TPA on this queue!\n",
1415 						  j);
1416 					bnx2x_free_tpa_pool(bp, fp, i);
1417 					fp->mode = TPA_MODE_DISABLED;
1418 					break;
1419 				}
1420 				dma_unmap_addr_set(first_buf, mapping, 0);
1421 				tpa_info->tpa_state = BNX2X_TPA_STOP;
1422 			}
1423 
1424 			/* "next page" elements initialization */
1425 			bnx2x_set_next_page_sgl(fp);
1426 
1427 			/* set SGEs bit mask */
1428 			bnx2x_init_sge_ring_bit_mask(fp);
1429 
1430 			/* Allocate SGEs and initialize the ring elements */
1431 			for (i = 0, ring_prod = 0;
1432 			     i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
1433 
1434 				if (bnx2x_alloc_rx_sge(bp, fp, ring_prod,
1435 						       GFP_KERNEL) < 0) {
1436 					BNX2X_ERR("was only able to allocate %d rx sges\n",
1437 						  i);
1438 					BNX2X_ERR("disabling TPA for queue[%d]\n",
1439 						  j);
1440 					/* Cleanup already allocated elements */
1441 					bnx2x_free_rx_sge_range(bp, fp,
1442 								ring_prod);
1443 					bnx2x_free_tpa_pool(bp, fp,
1444 							    MAX_AGG_QS(bp));
1445 					fp->mode = TPA_MODE_DISABLED;
1446 					ring_prod = 0;
1447 					break;
1448 				}
1449 				ring_prod = NEXT_SGE_IDX(ring_prod);
1450 			}
1451 
1452 			fp->rx_sge_prod = ring_prod;
1453 		}
1454 	}
1455 
1456 	for_each_eth_queue(bp, j) {
1457 		struct bnx2x_fastpath *fp = &bp->fp[j];
1458 
1459 		fp->rx_bd_cons = 0;
1460 
1461 		/* Activate BD ring */
1462 		/* Warning!
1463 		 * this will generate an interrupt (to the TSTORM)
1464 		 * must only be done after chip is initialized
1465 		 */
1466 		bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1467 				     fp->rx_sge_prod);
1468 
1469 		if (j != 0)
1470 			continue;
1471 
1472 		if (CHIP_IS_E1(bp)) {
1473 			REG_WR(bp, BAR_USTRORM_INTMEM +
1474 			       USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
1475 			       U64_LO(fp->rx_comp_mapping));
1476 			REG_WR(bp, BAR_USTRORM_INTMEM +
1477 			       USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
1478 			       U64_HI(fp->rx_comp_mapping));
1479 		}
1480 	}
1481 }
1482 
1483 static void bnx2x_free_tx_skbs_queue(struct bnx2x_fastpath *fp)
1484 {
1485 	u8 cos;
1486 	struct bnx2x *bp = fp->bp;
1487 
1488 	for_each_cos_in_tx_queue(fp, cos) {
1489 		struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
1490 		unsigned pkts_compl = 0, bytes_compl = 0;
1491 
1492 		u16 sw_prod = txdata->tx_pkt_prod;
1493 		u16 sw_cons = txdata->tx_pkt_cons;
1494 
1495 		while (sw_cons != sw_prod) {
1496 			bnx2x_free_tx_pkt(bp, txdata, TX_BD(sw_cons),
1497 					  &pkts_compl, &bytes_compl);
1498 			sw_cons++;
1499 		}
1500 
1501 		netdev_tx_reset_queue(
1502 			netdev_get_tx_queue(bp->dev,
1503 					    txdata->txq_index));
1504 	}
1505 }
1506 
1507 static void bnx2x_free_tx_skbs_cnic(struct bnx2x *bp)
1508 {
1509 	int i;
1510 
1511 	for_each_tx_queue_cnic(bp, i) {
1512 		bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1513 	}
1514 }
1515 
1516 static void bnx2x_free_tx_skbs(struct bnx2x *bp)
1517 {
1518 	int i;
1519 
1520 	for_each_eth_queue(bp, i) {
1521 		bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1522 	}
1523 }
1524 
1525 static void bnx2x_free_rx_bds(struct bnx2x_fastpath *fp)
1526 {
1527 	struct bnx2x *bp = fp->bp;
1528 	int i;
1529 
1530 	/* ring wasn't allocated */
1531 	if (fp->rx_buf_ring == NULL)
1532 		return;
1533 
1534 	for (i = 0; i < NUM_RX_BD; i++) {
1535 		struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
1536 		u8 *data = rx_buf->data;
1537 
1538 		if (data == NULL)
1539 			continue;
1540 		dma_unmap_single(&bp->pdev->dev,
1541 				 dma_unmap_addr(rx_buf, mapping),
1542 				 fp->rx_buf_size, DMA_FROM_DEVICE);
1543 
1544 		rx_buf->data = NULL;
1545 		bnx2x_frag_free(fp, data);
1546 	}
1547 }
1548 
1549 static void bnx2x_free_rx_skbs_cnic(struct bnx2x *bp)
1550 {
1551 	int j;
1552 
1553 	for_each_rx_queue_cnic(bp, j) {
1554 		bnx2x_free_rx_bds(&bp->fp[j]);
1555 	}
1556 }
1557 
1558 static void bnx2x_free_rx_skbs(struct bnx2x *bp)
1559 {
1560 	int j;
1561 
1562 	for_each_eth_queue(bp, j) {
1563 		struct bnx2x_fastpath *fp = &bp->fp[j];
1564 
1565 		bnx2x_free_rx_bds(fp);
1566 
1567 		if (fp->mode != TPA_MODE_DISABLED)
1568 			bnx2x_free_tpa_pool(bp, fp, MAX_AGG_QS(bp));
1569 	}
1570 }
1571 
1572 static void bnx2x_free_skbs_cnic(struct bnx2x *bp)
1573 {
1574 	bnx2x_free_tx_skbs_cnic(bp);
1575 	bnx2x_free_rx_skbs_cnic(bp);
1576 }
1577 
1578 void bnx2x_free_skbs(struct bnx2x *bp)
1579 {
1580 	bnx2x_free_tx_skbs(bp);
1581 	bnx2x_free_rx_skbs(bp);
1582 }
1583 
1584 void bnx2x_update_max_mf_config(struct bnx2x *bp, u32 value)
1585 {
1586 	/* load old values */
1587 	u32 mf_cfg = bp->mf_config[BP_VN(bp)];
1588 
1589 	if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) {
1590 		/* leave all but MAX value */
1591 		mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK;
1592 
1593 		/* set new MAX value */
1594 		mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT)
1595 				& FUNC_MF_CFG_MAX_BW_MASK;
1596 
1597 		bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg);
1598 	}
1599 }
1600 
1601 /**
1602  * bnx2x_free_msix_irqs - free previously requested MSI-X IRQ vectors
1603  *
1604  * @bp:		driver handle
1605  * @nvecs:	number of vectors to be released
1606  */
1607 static void bnx2x_free_msix_irqs(struct bnx2x *bp, int nvecs)
1608 {
1609 	int i, offset = 0;
1610 
1611 	if (nvecs == offset)
1612 		return;
1613 
1614 	/* VFs don't have a default SB */
1615 	if (IS_PF(bp)) {
1616 		free_irq(bp->msix_table[offset].vector, bp->dev);
1617 		DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
1618 		   bp->msix_table[offset].vector);
1619 		offset++;
1620 	}
1621 
1622 	if (CNIC_SUPPORT(bp)) {
1623 		if (nvecs == offset)
1624 			return;
1625 		offset++;
1626 	}
1627 
1628 	for_each_eth_queue(bp, i) {
1629 		if (nvecs == offset)
1630 			return;
1631 		DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq\n",
1632 		   i, bp->msix_table[offset].vector);
1633 
1634 		free_irq(bp->msix_table[offset++].vector, &bp->fp[i]);
1635 	}
1636 }
1637 
1638 void bnx2x_free_irq(struct bnx2x *bp)
1639 {
1640 	if (bp->flags & USING_MSIX_FLAG &&
1641 	    !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1642 		int nvecs = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_SUPPORT(bp);
1643 
1644 		/* vfs don't have a default status block */
1645 		if (IS_PF(bp))
1646 			nvecs++;
1647 
1648 		bnx2x_free_msix_irqs(bp, nvecs);
1649 	} else {
1650 		free_irq(bp->dev->irq, bp->dev);
1651 	}
1652 }
1653 
1654 int bnx2x_enable_msix(struct bnx2x *bp)
1655 {
1656 	int msix_vec = 0, i, rc;
1657 
1658 	/* VFs don't have a default status block */
1659 	if (IS_PF(bp)) {
1660 		bp->msix_table[msix_vec].entry = msix_vec;
1661 		BNX2X_DEV_INFO("msix_table[0].entry = %d (slowpath)\n",
1662 			       bp->msix_table[0].entry);
1663 		msix_vec++;
1664 	}
1665 
1666 	/* Cnic requires an msix vector for itself */
1667 	if (CNIC_SUPPORT(bp)) {
1668 		bp->msix_table[msix_vec].entry = msix_vec;
1669 		BNX2X_DEV_INFO("msix_table[%d].entry = %d (CNIC)\n",
1670 			       msix_vec, bp->msix_table[msix_vec].entry);
1671 		msix_vec++;
1672 	}
1673 
1674 	/* We need separate vectors for ETH queues only (not FCoE) */
1675 	for_each_eth_queue(bp, i) {
1676 		bp->msix_table[msix_vec].entry = msix_vec;
1677 		BNX2X_DEV_INFO("msix_table[%d].entry = %d (fastpath #%u)\n",
1678 			       msix_vec, msix_vec, i);
1679 		msix_vec++;
1680 	}
1681 
1682 	DP(BNX2X_MSG_SP, "about to request enable msix with %d vectors\n",
1683 	   msix_vec);
1684 
1685 	rc = pci_enable_msix_range(bp->pdev, &bp->msix_table[0],
1686 				   BNX2X_MIN_MSIX_VEC_CNT(bp), msix_vec);
1687 	/*
1688 	 * reconfigure number of tx/rx queues according to available
1689 	 * MSI-X vectors
1690 	 */
1691 	if (rc == -ENOSPC) {
1692 		/* Get by with single vector */
1693 		rc = pci_enable_msix_range(bp->pdev, &bp->msix_table[0], 1, 1);
1694 		if (rc < 0) {
1695 			BNX2X_DEV_INFO("Single MSI-X is not attainable rc %d\n",
1696 				       rc);
1697 			goto no_msix;
1698 		}
1699 
1700 		BNX2X_DEV_INFO("Using single MSI-X vector\n");
1701 		bp->flags |= USING_SINGLE_MSIX_FLAG;
1702 
1703 		BNX2X_DEV_INFO("set number of queues to 1\n");
1704 		bp->num_ethernet_queues = 1;
1705 		bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1706 	} else if (rc < 0) {
1707 		BNX2X_DEV_INFO("MSI-X is not attainable rc %d\n", rc);
1708 		goto no_msix;
1709 	} else if (rc < msix_vec) {
1710 		/* how less vectors we will have? */
1711 		int diff = msix_vec - rc;
1712 
1713 		BNX2X_DEV_INFO("Trying to use less MSI-X vectors: %d\n", rc);
1714 
1715 		/*
1716 		 * decrease number of queues by number of unallocated entries
1717 		 */
1718 		bp->num_ethernet_queues -= diff;
1719 		bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1720 
1721 		BNX2X_DEV_INFO("New queue configuration set: %d\n",
1722 			       bp->num_queues);
1723 	}
1724 
1725 	bp->flags |= USING_MSIX_FLAG;
1726 
1727 	return 0;
1728 
1729 no_msix:
1730 	/* fall to INTx if not enough memory */
1731 	if (rc == -ENOMEM)
1732 		bp->flags |= DISABLE_MSI_FLAG;
1733 
1734 	return rc;
1735 }
1736 
1737 static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1738 {
1739 	int i, rc, offset = 0;
1740 
1741 	/* no default status block for vf */
1742 	if (IS_PF(bp)) {
1743 		rc = request_irq(bp->msix_table[offset++].vector,
1744 				 bnx2x_msix_sp_int, 0,
1745 				 bp->dev->name, bp->dev);
1746 		if (rc) {
1747 			BNX2X_ERR("request sp irq failed\n");
1748 			return -EBUSY;
1749 		}
1750 	}
1751 
1752 	if (CNIC_SUPPORT(bp))
1753 		offset++;
1754 
1755 	for_each_eth_queue(bp, i) {
1756 		struct bnx2x_fastpath *fp = &bp->fp[i];
1757 		snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1758 			 bp->dev->name, i);
1759 
1760 		rc = request_irq(bp->msix_table[offset].vector,
1761 				 bnx2x_msix_fp_int, 0, fp->name, fp);
1762 		if (rc) {
1763 			BNX2X_ERR("request fp #%d irq (%d) failed  rc %d\n", i,
1764 			      bp->msix_table[offset].vector, rc);
1765 			bnx2x_free_msix_irqs(bp, offset);
1766 			return -EBUSY;
1767 		}
1768 
1769 		offset++;
1770 	}
1771 
1772 	i = BNX2X_NUM_ETH_QUEUES(bp);
1773 	if (IS_PF(bp)) {
1774 		offset = 1 + CNIC_SUPPORT(bp);
1775 		netdev_info(bp->dev,
1776 			    "using MSI-X  IRQs: sp %d  fp[%d] %d ... fp[%d] %d\n",
1777 			    bp->msix_table[0].vector,
1778 			    0, bp->msix_table[offset].vector,
1779 			    i - 1, bp->msix_table[offset + i - 1].vector);
1780 	} else {
1781 		offset = CNIC_SUPPORT(bp);
1782 		netdev_info(bp->dev,
1783 			    "using MSI-X  IRQs: fp[%d] %d ... fp[%d] %d\n",
1784 			    0, bp->msix_table[offset].vector,
1785 			    i - 1, bp->msix_table[offset + i - 1].vector);
1786 	}
1787 	return 0;
1788 }
1789 
1790 int bnx2x_enable_msi(struct bnx2x *bp)
1791 {
1792 	int rc;
1793 
1794 	rc = pci_enable_msi(bp->pdev);
1795 	if (rc) {
1796 		BNX2X_DEV_INFO("MSI is not attainable\n");
1797 		return -1;
1798 	}
1799 	bp->flags |= USING_MSI_FLAG;
1800 
1801 	return 0;
1802 }
1803 
1804 static int bnx2x_req_irq(struct bnx2x *bp)
1805 {
1806 	unsigned long flags;
1807 	unsigned int irq;
1808 
1809 	if (bp->flags & (USING_MSI_FLAG | USING_MSIX_FLAG))
1810 		flags = 0;
1811 	else
1812 		flags = IRQF_SHARED;
1813 
1814 	if (bp->flags & USING_MSIX_FLAG)
1815 		irq = bp->msix_table[0].vector;
1816 	else
1817 		irq = bp->pdev->irq;
1818 
1819 	return request_irq(irq, bnx2x_interrupt, flags, bp->dev->name, bp->dev);
1820 }
1821 
1822 static int bnx2x_setup_irqs(struct bnx2x *bp)
1823 {
1824 	int rc = 0;
1825 	if (bp->flags & USING_MSIX_FLAG &&
1826 	    !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1827 		rc = bnx2x_req_msix_irqs(bp);
1828 		if (rc)
1829 			return rc;
1830 	} else {
1831 		rc = bnx2x_req_irq(bp);
1832 		if (rc) {
1833 			BNX2X_ERR("IRQ request failed  rc %d, aborting\n", rc);
1834 			return rc;
1835 		}
1836 		if (bp->flags & USING_MSI_FLAG) {
1837 			bp->dev->irq = bp->pdev->irq;
1838 			netdev_info(bp->dev, "using MSI IRQ %d\n",
1839 				    bp->dev->irq);
1840 		}
1841 		if (bp->flags & USING_MSIX_FLAG) {
1842 			bp->dev->irq = bp->msix_table[0].vector;
1843 			netdev_info(bp->dev, "using MSIX IRQ %d\n",
1844 				    bp->dev->irq);
1845 		}
1846 	}
1847 
1848 	return 0;
1849 }
1850 
1851 static void bnx2x_napi_enable_cnic(struct bnx2x *bp)
1852 {
1853 	int i;
1854 
1855 	for_each_rx_queue_cnic(bp, i) {
1856 		napi_enable(&bnx2x_fp(bp, i, napi));
1857 	}
1858 }
1859 
1860 static void bnx2x_napi_enable(struct bnx2x *bp)
1861 {
1862 	int i;
1863 
1864 	for_each_eth_queue(bp, i) {
1865 		napi_enable(&bnx2x_fp(bp, i, napi));
1866 	}
1867 }
1868 
1869 static void bnx2x_napi_disable_cnic(struct bnx2x *bp)
1870 {
1871 	int i;
1872 
1873 	for_each_rx_queue_cnic(bp, i) {
1874 		napi_disable(&bnx2x_fp(bp, i, napi));
1875 	}
1876 }
1877 
1878 static void bnx2x_napi_disable(struct bnx2x *bp)
1879 {
1880 	int i;
1881 
1882 	for_each_eth_queue(bp, i) {
1883 		napi_disable(&bnx2x_fp(bp, i, napi));
1884 	}
1885 }
1886 
1887 void bnx2x_netif_start(struct bnx2x *bp)
1888 {
1889 	if (netif_running(bp->dev)) {
1890 		bnx2x_napi_enable(bp);
1891 		if (CNIC_LOADED(bp))
1892 			bnx2x_napi_enable_cnic(bp);
1893 		bnx2x_int_enable(bp);
1894 		if (bp->state == BNX2X_STATE_OPEN)
1895 			netif_tx_wake_all_queues(bp->dev);
1896 	}
1897 }
1898 
1899 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1900 {
1901 	bnx2x_int_disable_sync(bp, disable_hw);
1902 	bnx2x_napi_disable(bp);
1903 	if (CNIC_LOADED(bp))
1904 		bnx2x_napi_disable_cnic(bp);
1905 }
1906 
1907 u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb,
1908 		       void *accel_priv, select_queue_fallback_t fallback)
1909 {
1910 	struct bnx2x *bp = netdev_priv(dev);
1911 
1912 	if (CNIC_LOADED(bp) && !NO_FCOE(bp)) {
1913 		struct ethhdr *hdr = (struct ethhdr *)skb->data;
1914 		u16 ether_type = ntohs(hdr->h_proto);
1915 
1916 		/* Skip VLAN tag if present */
1917 		if (ether_type == ETH_P_8021Q) {
1918 			struct vlan_ethhdr *vhdr =
1919 				(struct vlan_ethhdr *)skb->data;
1920 
1921 			ether_type = ntohs(vhdr->h_vlan_encapsulated_proto);
1922 		}
1923 
1924 		/* If ethertype is FCoE or FIP - use FCoE ring */
1925 		if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP))
1926 			return bnx2x_fcoe_tx(bp, txq_index);
1927 	}
1928 
1929 	/* select a non-FCoE queue */
1930 	return fallback(dev, skb) % (BNX2X_NUM_ETH_QUEUES(bp) * bp->max_cos);
1931 }
1932 
1933 void bnx2x_set_num_queues(struct bnx2x *bp)
1934 {
1935 	/* RSS queues */
1936 	bp->num_ethernet_queues = bnx2x_calc_num_queues(bp);
1937 
1938 	/* override in STORAGE SD modes */
1939 	if (IS_MF_STORAGE_ONLY(bp))
1940 		bp->num_ethernet_queues = 1;
1941 
1942 	/* Add special queues */
1943 	bp->num_cnic_queues = CNIC_SUPPORT(bp); /* For FCOE */
1944 	bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1945 
1946 	BNX2X_DEV_INFO("set number of queues to %d\n", bp->num_queues);
1947 }
1948 
1949 /**
1950  * bnx2x_set_real_num_queues - configure netdev->real_num_[tx,rx]_queues
1951  *
1952  * @bp:		Driver handle
1953  *
1954  * We currently support for at most 16 Tx queues for each CoS thus we will
1955  * allocate a multiple of 16 for ETH L2 rings according to the value of the
1956  * bp->max_cos.
1957  *
1958  * If there is an FCoE L2 queue the appropriate Tx queue will have the next
1959  * index after all ETH L2 indices.
1960  *
1961  * If the actual number of Tx queues (for each CoS) is less than 16 then there
1962  * will be the holes at the end of each group of 16 ETh L2 indices (0..15,
1963  * 16..31,...) with indices that are not coupled with any real Tx queue.
1964  *
1965  * The proper configuration of skb->queue_mapping is handled by
1966  * bnx2x_select_queue() and __skb_tx_hash().
1967  *
1968  * bnx2x_setup_tc() takes care of the proper TC mappings so that __skb_tx_hash()
1969  * will return a proper Tx index if TC is enabled (netdev->num_tc > 0).
1970  */
1971 static int bnx2x_set_real_num_queues(struct bnx2x *bp, int include_cnic)
1972 {
1973 	int rc, tx, rx;
1974 
1975 	tx = BNX2X_NUM_ETH_QUEUES(bp) * bp->max_cos;
1976 	rx = BNX2X_NUM_ETH_QUEUES(bp);
1977 
1978 /* account for fcoe queue */
1979 	if (include_cnic && !NO_FCOE(bp)) {
1980 		rx++;
1981 		tx++;
1982 	}
1983 
1984 	rc = netif_set_real_num_tx_queues(bp->dev, tx);
1985 	if (rc) {
1986 		BNX2X_ERR("Failed to set real number of Tx queues: %d\n", rc);
1987 		return rc;
1988 	}
1989 	rc = netif_set_real_num_rx_queues(bp->dev, rx);
1990 	if (rc) {
1991 		BNX2X_ERR("Failed to set real number of Rx queues: %d\n", rc);
1992 		return rc;
1993 	}
1994 
1995 	DP(NETIF_MSG_IFUP, "Setting real num queues to (tx, rx) (%d, %d)\n",
1996 			  tx, rx);
1997 
1998 	return rc;
1999 }
2000 
2001 static void bnx2x_set_rx_buf_size(struct bnx2x *bp)
2002 {
2003 	int i;
2004 
2005 	for_each_queue(bp, i) {
2006 		struct bnx2x_fastpath *fp = &bp->fp[i];
2007 		u32 mtu;
2008 
2009 		/* Always use a mini-jumbo MTU for the FCoE L2 ring */
2010 		if (IS_FCOE_IDX(i))
2011 			/*
2012 			 * Although there are no IP frames expected to arrive to
2013 			 * this ring we still want to add an
2014 			 * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer
2015 			 * overrun attack.
2016 			 */
2017 			mtu = BNX2X_FCOE_MINI_JUMBO_MTU;
2018 		else
2019 			mtu = bp->dev->mtu;
2020 		fp->rx_buf_size = BNX2X_FW_RX_ALIGN_START +
2021 				  IP_HEADER_ALIGNMENT_PADDING +
2022 				  ETH_OVERHEAD +
2023 				  mtu +
2024 				  BNX2X_FW_RX_ALIGN_END;
2025 		fp->rx_buf_size = SKB_DATA_ALIGN(fp->rx_buf_size);
2026 		/* Note : rx_buf_size doesn't take into account NET_SKB_PAD */
2027 		if (fp->rx_buf_size + NET_SKB_PAD <= PAGE_SIZE)
2028 			fp->rx_frag_size = fp->rx_buf_size + NET_SKB_PAD;
2029 		else
2030 			fp->rx_frag_size = 0;
2031 	}
2032 }
2033 
2034 static int bnx2x_init_rss(struct bnx2x *bp)
2035 {
2036 	int i;
2037 	u8 num_eth_queues = BNX2X_NUM_ETH_QUEUES(bp);
2038 
2039 	/* Prepare the initial contents for the indirection table if RSS is
2040 	 * enabled
2041 	 */
2042 	for (i = 0; i < sizeof(bp->rss_conf_obj.ind_table); i++)
2043 		bp->rss_conf_obj.ind_table[i] =
2044 			bp->fp->cl_id +
2045 			ethtool_rxfh_indir_default(i, num_eth_queues);
2046 
2047 	/*
2048 	 * For 57710 and 57711 SEARCHER configuration (rss_keys) is
2049 	 * per-port, so if explicit configuration is needed , do it only
2050 	 * for a PMF.
2051 	 *
2052 	 * For 57712 and newer on the other hand it's a per-function
2053 	 * configuration.
2054 	 */
2055 	return bnx2x_config_rss_eth(bp, bp->port.pmf || !CHIP_IS_E1x(bp));
2056 }
2057 
2058 int bnx2x_rss(struct bnx2x *bp, struct bnx2x_rss_config_obj *rss_obj,
2059 	      bool config_hash, bool enable)
2060 {
2061 	struct bnx2x_config_rss_params params = {NULL};
2062 
2063 	/* Although RSS is meaningless when there is a single HW queue we
2064 	 * still need it enabled in order to have HW Rx hash generated.
2065 	 *
2066 	 * if (!is_eth_multi(bp))
2067 	 *      bp->multi_mode = ETH_RSS_MODE_DISABLED;
2068 	 */
2069 
2070 	params.rss_obj = rss_obj;
2071 
2072 	__set_bit(RAMROD_COMP_WAIT, &params.ramrod_flags);
2073 
2074 	if (enable) {
2075 		__set_bit(BNX2X_RSS_MODE_REGULAR, &params.rss_flags);
2076 
2077 		/* RSS configuration */
2078 		__set_bit(BNX2X_RSS_IPV4, &params.rss_flags);
2079 		__set_bit(BNX2X_RSS_IPV4_TCP, &params.rss_flags);
2080 		__set_bit(BNX2X_RSS_IPV6, &params.rss_flags);
2081 		__set_bit(BNX2X_RSS_IPV6_TCP, &params.rss_flags);
2082 		if (rss_obj->udp_rss_v4)
2083 			__set_bit(BNX2X_RSS_IPV4_UDP, &params.rss_flags);
2084 		if (rss_obj->udp_rss_v6)
2085 			__set_bit(BNX2X_RSS_IPV6_UDP, &params.rss_flags);
2086 
2087 		if (!CHIP_IS_E1x(bp)) {
2088 			/* valid only for TUNN_MODE_VXLAN tunnel mode */
2089 			__set_bit(BNX2X_RSS_IPV4_VXLAN, &params.rss_flags);
2090 			__set_bit(BNX2X_RSS_IPV6_VXLAN, &params.rss_flags);
2091 
2092 			/* valid only for TUNN_MODE_GRE tunnel mode */
2093 			__set_bit(BNX2X_RSS_TUNN_INNER_HDRS, &params.rss_flags);
2094 		}
2095 	} else {
2096 		__set_bit(BNX2X_RSS_MODE_DISABLED, &params.rss_flags);
2097 	}
2098 
2099 	/* Hash bits */
2100 	params.rss_result_mask = MULTI_MASK;
2101 
2102 	memcpy(params.ind_table, rss_obj->ind_table, sizeof(params.ind_table));
2103 
2104 	if (config_hash) {
2105 		/* RSS keys */
2106 		netdev_rss_key_fill(params.rss_key, T_ETH_RSS_KEY * 4);
2107 		__set_bit(BNX2X_RSS_SET_SRCH, &params.rss_flags);
2108 	}
2109 
2110 	if (IS_PF(bp))
2111 		return bnx2x_config_rss(bp, &params);
2112 	else
2113 		return bnx2x_vfpf_config_rss(bp, &params);
2114 }
2115 
2116 static int bnx2x_init_hw(struct bnx2x *bp, u32 load_code)
2117 {
2118 	struct bnx2x_func_state_params func_params = {NULL};
2119 
2120 	/* Prepare parameters for function state transitions */
2121 	__set_bit(RAMROD_COMP_WAIT, &func_params.ramrod_flags);
2122 
2123 	func_params.f_obj = &bp->func_obj;
2124 	func_params.cmd = BNX2X_F_CMD_HW_INIT;
2125 
2126 	func_params.params.hw_init.load_phase = load_code;
2127 
2128 	return bnx2x_func_state_change(bp, &func_params);
2129 }
2130 
2131 /*
2132  * Cleans the object that have internal lists without sending
2133  * ramrods. Should be run when interrupts are disabled.
2134  */
2135 void bnx2x_squeeze_objects(struct bnx2x *bp)
2136 {
2137 	int rc;
2138 	unsigned long ramrod_flags = 0, vlan_mac_flags = 0;
2139 	struct bnx2x_mcast_ramrod_params rparam = {NULL};
2140 	struct bnx2x_vlan_mac_obj *mac_obj = &bp->sp_objs->mac_obj;
2141 
2142 	/***************** Cleanup MACs' object first *************************/
2143 
2144 	/* Wait for completion of requested */
2145 	__set_bit(RAMROD_COMP_WAIT, &ramrod_flags);
2146 	/* Perform a dry cleanup */
2147 	__set_bit(RAMROD_DRV_CLR_ONLY, &ramrod_flags);
2148 
2149 	/* Clean ETH primary MAC */
2150 	__set_bit(BNX2X_ETH_MAC, &vlan_mac_flags);
2151 	rc = mac_obj->delete_all(bp, &bp->sp_objs->mac_obj, &vlan_mac_flags,
2152 				 &ramrod_flags);
2153 	if (rc != 0)
2154 		BNX2X_ERR("Failed to clean ETH MACs: %d\n", rc);
2155 
2156 	/* Cleanup UC list */
2157 	vlan_mac_flags = 0;
2158 	__set_bit(BNX2X_UC_LIST_MAC, &vlan_mac_flags);
2159 	rc = mac_obj->delete_all(bp, mac_obj, &vlan_mac_flags,
2160 				 &ramrod_flags);
2161 	if (rc != 0)
2162 		BNX2X_ERR("Failed to clean UC list MACs: %d\n", rc);
2163 
2164 	/***************** Now clean mcast object *****************************/
2165 	rparam.mcast_obj = &bp->mcast_obj;
2166 	__set_bit(RAMROD_DRV_CLR_ONLY, &rparam.ramrod_flags);
2167 
2168 	/* Add a DEL command... - Since we're doing a driver cleanup only,
2169 	 * we take a lock surrounding both the initial send and the CONTs,
2170 	 * as we don't want a true completion to disrupt us in the middle.
2171 	 */
2172 	netif_addr_lock_bh(bp->dev);
2173 	rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_DEL);
2174 	if (rc < 0)
2175 		BNX2X_ERR("Failed to add a new DEL command to a multi-cast object: %d\n",
2176 			  rc);
2177 
2178 	/* ...and wait until all pending commands are cleared */
2179 	rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2180 	while (rc != 0) {
2181 		if (rc < 0) {
2182 			BNX2X_ERR("Failed to clean multi-cast object: %d\n",
2183 				  rc);
2184 			netif_addr_unlock_bh(bp->dev);
2185 			return;
2186 		}
2187 
2188 		rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2189 	}
2190 	netif_addr_unlock_bh(bp->dev);
2191 }
2192 
2193 #ifndef BNX2X_STOP_ON_ERROR
2194 #define LOAD_ERROR_EXIT(bp, label) \
2195 	do { \
2196 		(bp)->state = BNX2X_STATE_ERROR; \
2197 		goto label; \
2198 	} while (0)
2199 
2200 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
2201 	do { \
2202 		bp->cnic_loaded = false; \
2203 		goto label; \
2204 	} while (0)
2205 #else /*BNX2X_STOP_ON_ERROR*/
2206 #define LOAD_ERROR_EXIT(bp, label) \
2207 	do { \
2208 		(bp)->state = BNX2X_STATE_ERROR; \
2209 		(bp)->panic = 1; \
2210 		return -EBUSY; \
2211 	} while (0)
2212 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
2213 	do { \
2214 		bp->cnic_loaded = false; \
2215 		(bp)->panic = 1; \
2216 		return -EBUSY; \
2217 	} while (0)
2218 #endif /*BNX2X_STOP_ON_ERROR*/
2219 
2220 static void bnx2x_free_fw_stats_mem(struct bnx2x *bp)
2221 {
2222 	BNX2X_PCI_FREE(bp->fw_stats, bp->fw_stats_mapping,
2223 		       bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2224 	return;
2225 }
2226 
2227 static int bnx2x_alloc_fw_stats_mem(struct bnx2x *bp)
2228 {
2229 	int num_groups, vf_headroom = 0;
2230 	int is_fcoe_stats = NO_FCOE(bp) ? 0 : 1;
2231 
2232 	/* number of queues for statistics is number of eth queues + FCoE */
2233 	u8 num_queue_stats = BNX2X_NUM_ETH_QUEUES(bp) + is_fcoe_stats;
2234 
2235 	/* Total number of FW statistics requests =
2236 	 * 1 for port stats + 1 for PF stats + potential 2 for FCoE (fcoe proper
2237 	 * and fcoe l2 queue) stats + num of queues (which includes another 1
2238 	 * for fcoe l2 queue if applicable)
2239 	 */
2240 	bp->fw_stats_num = 2 + is_fcoe_stats + num_queue_stats;
2241 
2242 	/* vf stats appear in the request list, but their data is allocated by
2243 	 * the VFs themselves. We don't include them in the bp->fw_stats_num as
2244 	 * it is used to determine where to place the vf stats queries in the
2245 	 * request struct
2246 	 */
2247 	if (IS_SRIOV(bp))
2248 		vf_headroom = bnx2x_vf_headroom(bp);
2249 
2250 	/* Request is built from stats_query_header and an array of
2251 	 * stats_query_cmd_group each of which contains
2252 	 * STATS_QUERY_CMD_COUNT rules. The real number or requests is
2253 	 * configured in the stats_query_header.
2254 	 */
2255 	num_groups =
2256 		(((bp->fw_stats_num + vf_headroom) / STATS_QUERY_CMD_COUNT) +
2257 		 (((bp->fw_stats_num + vf_headroom) % STATS_QUERY_CMD_COUNT) ?
2258 		 1 : 0));
2259 
2260 	DP(BNX2X_MSG_SP, "stats fw_stats_num %d, vf headroom %d, num_groups %d\n",
2261 	   bp->fw_stats_num, vf_headroom, num_groups);
2262 	bp->fw_stats_req_sz = sizeof(struct stats_query_header) +
2263 		num_groups * sizeof(struct stats_query_cmd_group);
2264 
2265 	/* Data for statistics requests + stats_counter
2266 	 * stats_counter holds per-STORM counters that are incremented
2267 	 * when STORM has finished with the current request.
2268 	 * memory for FCoE offloaded statistics are counted anyway,
2269 	 * even if they will not be sent.
2270 	 * VF stats are not accounted for here as the data of VF stats is stored
2271 	 * in memory allocated by the VF, not here.
2272 	 */
2273 	bp->fw_stats_data_sz = sizeof(struct per_port_stats) +
2274 		sizeof(struct per_pf_stats) +
2275 		sizeof(struct fcoe_statistics_params) +
2276 		sizeof(struct per_queue_stats) * num_queue_stats +
2277 		sizeof(struct stats_counter);
2278 
2279 	bp->fw_stats = BNX2X_PCI_ALLOC(&bp->fw_stats_mapping,
2280 				       bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2281 	if (!bp->fw_stats)
2282 		goto alloc_mem_err;
2283 
2284 	/* Set shortcuts */
2285 	bp->fw_stats_req = (struct bnx2x_fw_stats_req *)bp->fw_stats;
2286 	bp->fw_stats_req_mapping = bp->fw_stats_mapping;
2287 	bp->fw_stats_data = (struct bnx2x_fw_stats_data *)
2288 		((u8 *)bp->fw_stats + bp->fw_stats_req_sz);
2289 	bp->fw_stats_data_mapping = bp->fw_stats_mapping +
2290 		bp->fw_stats_req_sz;
2291 
2292 	DP(BNX2X_MSG_SP, "statistics request base address set to %x %x\n",
2293 	   U64_HI(bp->fw_stats_req_mapping),
2294 	   U64_LO(bp->fw_stats_req_mapping));
2295 	DP(BNX2X_MSG_SP, "statistics data base address set to %x %x\n",
2296 	   U64_HI(bp->fw_stats_data_mapping),
2297 	   U64_LO(bp->fw_stats_data_mapping));
2298 	return 0;
2299 
2300 alloc_mem_err:
2301 	bnx2x_free_fw_stats_mem(bp);
2302 	BNX2X_ERR("Can't allocate FW stats memory\n");
2303 	return -ENOMEM;
2304 }
2305 
2306 /* send load request to mcp and analyze response */
2307 static int bnx2x_nic_load_request(struct bnx2x *bp, u32 *load_code)
2308 {
2309 	u32 param;
2310 
2311 	/* init fw_seq */
2312 	bp->fw_seq =
2313 		(SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_mb_header) &
2314 		 DRV_MSG_SEQ_NUMBER_MASK);
2315 	BNX2X_DEV_INFO("fw_seq 0x%08x\n", bp->fw_seq);
2316 
2317 	/* Get current FW pulse sequence */
2318 	bp->fw_drv_pulse_wr_seq =
2319 		(SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_pulse_mb) &
2320 		 DRV_PULSE_SEQ_MASK);
2321 	BNX2X_DEV_INFO("drv_pulse 0x%x\n", bp->fw_drv_pulse_wr_seq);
2322 
2323 	param = DRV_MSG_CODE_LOAD_REQ_WITH_LFA;
2324 
2325 	if (IS_MF_SD(bp) && bnx2x_port_after_undi(bp))
2326 		param |= DRV_MSG_CODE_LOAD_REQ_FORCE_LFA;
2327 
2328 	/* load request */
2329 	(*load_code) = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ, param);
2330 
2331 	/* if mcp fails to respond we must abort */
2332 	if (!(*load_code)) {
2333 		BNX2X_ERR("MCP response failure, aborting\n");
2334 		return -EBUSY;
2335 	}
2336 
2337 	/* If mcp refused (e.g. other port is in diagnostic mode) we
2338 	 * must abort
2339 	 */
2340 	if ((*load_code) == FW_MSG_CODE_DRV_LOAD_REFUSED) {
2341 		BNX2X_ERR("MCP refused load request, aborting\n");
2342 		return -EBUSY;
2343 	}
2344 	return 0;
2345 }
2346 
2347 /* check whether another PF has already loaded FW to chip. In
2348  * virtualized environments a pf from another VM may have already
2349  * initialized the device including loading FW
2350  */
2351 int bnx2x_compare_fw_ver(struct bnx2x *bp, u32 load_code, bool print_err)
2352 {
2353 	/* is another pf loaded on this engine? */
2354 	if (load_code != FW_MSG_CODE_DRV_LOAD_COMMON_CHIP &&
2355 	    load_code != FW_MSG_CODE_DRV_LOAD_COMMON) {
2356 		/* build my FW version dword */
2357 		u32 my_fw = (BCM_5710_FW_MAJOR_VERSION) +
2358 			(BCM_5710_FW_MINOR_VERSION << 8) +
2359 			(BCM_5710_FW_REVISION_VERSION << 16) +
2360 			(BCM_5710_FW_ENGINEERING_VERSION << 24);
2361 
2362 		/* read loaded FW from chip */
2363 		u32 loaded_fw = REG_RD(bp, XSEM_REG_PRAM);
2364 
2365 		DP(BNX2X_MSG_SP, "loaded fw %x, my fw %x\n",
2366 		   loaded_fw, my_fw);
2367 
2368 		/* abort nic load if version mismatch */
2369 		if (my_fw != loaded_fw) {
2370 			if (print_err)
2371 				BNX2X_ERR("bnx2x with FW %x was already loaded which mismatches my %x FW. Aborting\n",
2372 					  loaded_fw, my_fw);
2373 			else
2374 				BNX2X_DEV_INFO("bnx2x with FW %x was already loaded which mismatches my %x FW, possibly due to MF UNDI\n",
2375 					       loaded_fw, my_fw);
2376 			return -EBUSY;
2377 		}
2378 	}
2379 	return 0;
2380 }
2381 
2382 /* returns the "mcp load_code" according to global load_count array */
2383 static int bnx2x_nic_load_no_mcp(struct bnx2x *bp, int port)
2384 {
2385 	int path = BP_PATH(bp);
2386 
2387 	DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d]      %d, %d, %d\n",
2388 	   path, bnx2x_load_count[path][0], bnx2x_load_count[path][1],
2389 	   bnx2x_load_count[path][2]);
2390 	bnx2x_load_count[path][0]++;
2391 	bnx2x_load_count[path][1 + port]++;
2392 	DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d]  %d, %d, %d\n",
2393 	   path, bnx2x_load_count[path][0], bnx2x_load_count[path][1],
2394 	   bnx2x_load_count[path][2]);
2395 	if (bnx2x_load_count[path][0] == 1)
2396 		return FW_MSG_CODE_DRV_LOAD_COMMON;
2397 	else if (bnx2x_load_count[path][1 + port] == 1)
2398 		return FW_MSG_CODE_DRV_LOAD_PORT;
2399 	else
2400 		return FW_MSG_CODE_DRV_LOAD_FUNCTION;
2401 }
2402 
2403 /* mark PMF if applicable */
2404 static void bnx2x_nic_load_pmf(struct bnx2x *bp, u32 load_code)
2405 {
2406 	if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2407 	    (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
2408 	    (load_code == FW_MSG_CODE_DRV_LOAD_PORT)) {
2409 		bp->port.pmf = 1;
2410 		/* We need the barrier to ensure the ordering between the
2411 		 * writing to bp->port.pmf here and reading it from the
2412 		 * bnx2x_periodic_task().
2413 		 */
2414 		smp_mb();
2415 	} else {
2416 		bp->port.pmf = 0;
2417 	}
2418 
2419 	DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf);
2420 }
2421 
2422 static void bnx2x_nic_load_afex_dcc(struct bnx2x *bp, int load_code)
2423 {
2424 	if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2425 	     (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
2426 	    (bp->common.shmem2_base)) {
2427 		if (SHMEM2_HAS(bp, dcc_support))
2428 			SHMEM2_WR(bp, dcc_support,
2429 				  (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
2430 				   SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
2431 		if (SHMEM2_HAS(bp, afex_driver_support))
2432 			SHMEM2_WR(bp, afex_driver_support,
2433 				  SHMEM_AFEX_SUPPORTED_VERSION_ONE);
2434 	}
2435 
2436 	/* Set AFEX default VLAN tag to an invalid value */
2437 	bp->afex_def_vlan_tag = -1;
2438 }
2439 
2440 /**
2441  * bnx2x_bz_fp - zero content of the fastpath structure.
2442  *
2443  * @bp:		driver handle
2444  * @index:	fastpath index to be zeroed
2445  *
2446  * Makes sure the contents of the bp->fp[index].napi is kept
2447  * intact.
2448  */
2449 static void bnx2x_bz_fp(struct bnx2x *bp, int index)
2450 {
2451 	struct bnx2x_fastpath *fp = &bp->fp[index];
2452 	int cos;
2453 	struct napi_struct orig_napi = fp->napi;
2454 	struct bnx2x_agg_info *orig_tpa_info = fp->tpa_info;
2455 
2456 	/* bzero bnx2x_fastpath contents */
2457 	if (fp->tpa_info)
2458 		memset(fp->tpa_info, 0, ETH_MAX_AGGREGATION_QUEUES_E1H_E2 *
2459 		       sizeof(struct bnx2x_agg_info));
2460 	memset(fp, 0, sizeof(*fp));
2461 
2462 	/* Restore the NAPI object as it has been already initialized */
2463 	fp->napi = orig_napi;
2464 	fp->tpa_info = orig_tpa_info;
2465 	fp->bp = bp;
2466 	fp->index = index;
2467 	if (IS_ETH_FP(fp))
2468 		fp->max_cos = bp->max_cos;
2469 	else
2470 		/* Special queues support only one CoS */
2471 		fp->max_cos = 1;
2472 
2473 	/* Init txdata pointers */
2474 	if (IS_FCOE_FP(fp))
2475 		fp->txdata_ptr[0] = &bp->bnx2x_txq[FCOE_TXQ_IDX(bp)];
2476 	if (IS_ETH_FP(fp))
2477 		for_each_cos_in_tx_queue(fp, cos)
2478 			fp->txdata_ptr[cos] = &bp->bnx2x_txq[cos *
2479 				BNX2X_NUM_ETH_QUEUES(bp) + index];
2480 
2481 	/* set the tpa flag for each queue. The tpa flag determines the queue
2482 	 * minimal size so it must be set prior to queue memory allocation
2483 	 */
2484 	if (bp->dev->features & NETIF_F_LRO)
2485 		fp->mode = TPA_MODE_LRO;
2486 	else if (bp->dev->features & NETIF_F_GRO_HW)
2487 		fp->mode = TPA_MODE_GRO;
2488 	else
2489 		fp->mode = TPA_MODE_DISABLED;
2490 
2491 	/* We don't want TPA if it's disabled in bp
2492 	 * or if this is an FCoE L2 ring.
2493 	 */
2494 	if (bp->disable_tpa || IS_FCOE_FP(fp))
2495 		fp->mode = TPA_MODE_DISABLED;
2496 }
2497 
2498 void bnx2x_set_os_driver_state(struct bnx2x *bp, u32 state)
2499 {
2500 	u32 cur;
2501 
2502 	if (!IS_MF_BD(bp) || !SHMEM2_HAS(bp, os_driver_state) || IS_VF(bp))
2503 		return;
2504 
2505 	cur = SHMEM2_RD(bp, os_driver_state[BP_FW_MB_IDX(bp)]);
2506 	DP(NETIF_MSG_IFUP, "Driver state %08x-->%08x\n",
2507 	   cur, state);
2508 
2509 	SHMEM2_WR(bp, os_driver_state[BP_FW_MB_IDX(bp)], state);
2510 }
2511 
2512 int bnx2x_load_cnic(struct bnx2x *bp)
2513 {
2514 	int i, rc, port = BP_PORT(bp);
2515 
2516 	DP(NETIF_MSG_IFUP, "Starting CNIC-related load\n");
2517 
2518 	mutex_init(&bp->cnic_mutex);
2519 
2520 	if (IS_PF(bp)) {
2521 		rc = bnx2x_alloc_mem_cnic(bp);
2522 		if (rc) {
2523 			BNX2X_ERR("Unable to allocate bp memory for cnic\n");
2524 			LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2525 		}
2526 	}
2527 
2528 	rc = bnx2x_alloc_fp_mem_cnic(bp);
2529 	if (rc) {
2530 		BNX2X_ERR("Unable to allocate memory for cnic fps\n");
2531 		LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2532 	}
2533 
2534 	/* Update the number of queues with the cnic queues */
2535 	rc = bnx2x_set_real_num_queues(bp, 1);
2536 	if (rc) {
2537 		BNX2X_ERR("Unable to set real_num_queues including cnic\n");
2538 		LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2539 	}
2540 
2541 	/* Add all CNIC NAPI objects */
2542 	bnx2x_add_all_napi_cnic(bp);
2543 	DP(NETIF_MSG_IFUP, "cnic napi added\n");
2544 	bnx2x_napi_enable_cnic(bp);
2545 
2546 	rc = bnx2x_init_hw_func_cnic(bp);
2547 	if (rc)
2548 		LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic1);
2549 
2550 	bnx2x_nic_init_cnic(bp);
2551 
2552 	if (IS_PF(bp)) {
2553 		/* Enable Timer scan */
2554 		REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 1);
2555 
2556 		/* setup cnic queues */
2557 		for_each_cnic_queue(bp, i) {
2558 			rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
2559 			if (rc) {
2560 				BNX2X_ERR("Queue setup failed\n");
2561 				LOAD_ERROR_EXIT(bp, load_error_cnic2);
2562 			}
2563 		}
2564 	}
2565 
2566 	/* Initialize Rx filter. */
2567 	bnx2x_set_rx_mode_inner(bp);
2568 
2569 	/* re-read iscsi info */
2570 	bnx2x_get_iscsi_info(bp);
2571 	bnx2x_setup_cnic_irq_info(bp);
2572 	bnx2x_setup_cnic_info(bp);
2573 	bp->cnic_loaded = true;
2574 	if (bp->state == BNX2X_STATE_OPEN)
2575 		bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
2576 
2577 	DP(NETIF_MSG_IFUP, "Ending successfully CNIC-related load\n");
2578 
2579 	return 0;
2580 
2581 #ifndef BNX2X_STOP_ON_ERROR
2582 load_error_cnic2:
2583 	/* Disable Timer scan */
2584 	REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 0);
2585 
2586 load_error_cnic1:
2587 	bnx2x_napi_disable_cnic(bp);
2588 	/* Update the number of queues without the cnic queues */
2589 	if (bnx2x_set_real_num_queues(bp, 0))
2590 		BNX2X_ERR("Unable to set real_num_queues not including cnic\n");
2591 load_error_cnic0:
2592 	BNX2X_ERR("CNIC-related load failed\n");
2593 	bnx2x_free_fp_mem_cnic(bp);
2594 	bnx2x_free_mem_cnic(bp);
2595 	return rc;
2596 #endif /* ! BNX2X_STOP_ON_ERROR */
2597 }
2598 
2599 /* must be called with rtnl_lock */
2600 int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
2601 {
2602 	int port = BP_PORT(bp);
2603 	int i, rc = 0, load_code = 0;
2604 
2605 	DP(NETIF_MSG_IFUP, "Starting NIC load\n");
2606 	DP(NETIF_MSG_IFUP,
2607 	   "CNIC is %s\n", CNIC_ENABLED(bp) ? "enabled" : "disabled");
2608 
2609 #ifdef BNX2X_STOP_ON_ERROR
2610 	if (unlikely(bp->panic)) {
2611 		BNX2X_ERR("Can't load NIC when there is panic\n");
2612 		return -EPERM;
2613 	}
2614 #endif
2615 
2616 	bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
2617 
2618 	/* zero the structure w/o any lock, before SP handler is initialized */
2619 	memset(&bp->last_reported_link, 0, sizeof(bp->last_reported_link));
2620 	__set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
2621 		&bp->last_reported_link.link_report_flags);
2622 
2623 	if (IS_PF(bp))
2624 		/* must be called before memory allocation and HW init */
2625 		bnx2x_ilt_set_info(bp);
2626 
2627 	/*
2628 	 * Zero fastpath structures preserving invariants like napi, which are
2629 	 * allocated only once, fp index, max_cos, bp pointer.
2630 	 * Also set fp->mode and txdata_ptr.
2631 	 */
2632 	DP(NETIF_MSG_IFUP, "num queues: %d", bp->num_queues);
2633 	for_each_queue(bp, i)
2634 		bnx2x_bz_fp(bp, i);
2635 	memset(bp->bnx2x_txq, 0, (BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS +
2636 				  bp->num_cnic_queues) *
2637 				  sizeof(struct bnx2x_fp_txdata));
2638 
2639 	bp->fcoe_init = false;
2640 
2641 	/* Set the receive queues buffer size */
2642 	bnx2x_set_rx_buf_size(bp);
2643 
2644 	if (IS_PF(bp)) {
2645 		rc = bnx2x_alloc_mem(bp);
2646 		if (rc) {
2647 			BNX2X_ERR("Unable to allocate bp memory\n");
2648 			return rc;
2649 		}
2650 	}
2651 
2652 	/* need to be done after alloc mem, since it's self adjusting to amount
2653 	 * of memory available for RSS queues
2654 	 */
2655 	rc = bnx2x_alloc_fp_mem(bp);
2656 	if (rc) {
2657 		BNX2X_ERR("Unable to allocate memory for fps\n");
2658 		LOAD_ERROR_EXIT(bp, load_error0);
2659 	}
2660 
2661 	/* Allocated memory for FW statistics  */
2662 	if (bnx2x_alloc_fw_stats_mem(bp))
2663 		LOAD_ERROR_EXIT(bp, load_error0);
2664 
2665 	/* request pf to initialize status blocks */
2666 	if (IS_VF(bp)) {
2667 		rc = bnx2x_vfpf_init(bp);
2668 		if (rc)
2669 			LOAD_ERROR_EXIT(bp, load_error0);
2670 	}
2671 
2672 	/* As long as bnx2x_alloc_mem() may possibly update
2673 	 * bp->num_queues, bnx2x_set_real_num_queues() should always
2674 	 * come after it. At this stage cnic queues are not counted.
2675 	 */
2676 	rc = bnx2x_set_real_num_queues(bp, 0);
2677 	if (rc) {
2678 		BNX2X_ERR("Unable to set real_num_queues\n");
2679 		LOAD_ERROR_EXIT(bp, load_error0);
2680 	}
2681 
2682 	/* configure multi cos mappings in kernel.
2683 	 * this configuration may be overridden by a multi class queue
2684 	 * discipline or by a dcbx negotiation result.
2685 	 */
2686 	bnx2x_setup_tc(bp->dev, bp->max_cos);
2687 
2688 	/* Add all NAPI objects */
2689 	bnx2x_add_all_napi(bp);
2690 	DP(NETIF_MSG_IFUP, "napi added\n");
2691 	bnx2x_napi_enable(bp);
2692 
2693 	if (IS_PF(bp)) {
2694 		/* set pf load just before approaching the MCP */
2695 		bnx2x_set_pf_load(bp);
2696 
2697 		/* if mcp exists send load request and analyze response */
2698 		if (!BP_NOMCP(bp)) {
2699 			/* attempt to load pf */
2700 			rc = bnx2x_nic_load_request(bp, &load_code);
2701 			if (rc)
2702 				LOAD_ERROR_EXIT(bp, load_error1);
2703 
2704 			/* what did mcp say? */
2705 			rc = bnx2x_compare_fw_ver(bp, load_code, true);
2706 			if (rc) {
2707 				bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2708 				LOAD_ERROR_EXIT(bp, load_error2);
2709 			}
2710 		} else {
2711 			load_code = bnx2x_nic_load_no_mcp(bp, port);
2712 		}
2713 
2714 		/* mark pmf if applicable */
2715 		bnx2x_nic_load_pmf(bp, load_code);
2716 
2717 		/* Init Function state controlling object */
2718 		bnx2x__init_func_obj(bp);
2719 
2720 		/* Initialize HW */
2721 		rc = bnx2x_init_hw(bp, load_code);
2722 		if (rc) {
2723 			BNX2X_ERR("HW init failed, aborting\n");
2724 			bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2725 			LOAD_ERROR_EXIT(bp, load_error2);
2726 		}
2727 	}
2728 
2729 	bnx2x_pre_irq_nic_init(bp);
2730 
2731 	/* Connect to IRQs */
2732 	rc = bnx2x_setup_irqs(bp);
2733 	if (rc) {
2734 		BNX2X_ERR("setup irqs failed\n");
2735 		if (IS_PF(bp))
2736 			bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2737 		LOAD_ERROR_EXIT(bp, load_error2);
2738 	}
2739 
2740 	/* Init per-function objects */
2741 	if (IS_PF(bp)) {
2742 		/* Setup NIC internals and enable interrupts */
2743 		bnx2x_post_irq_nic_init(bp, load_code);
2744 
2745 		bnx2x_init_bp_objs(bp);
2746 		bnx2x_iov_nic_init(bp);
2747 
2748 		/* Set AFEX default VLAN tag to an invalid value */
2749 		bp->afex_def_vlan_tag = -1;
2750 		bnx2x_nic_load_afex_dcc(bp, load_code);
2751 		bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
2752 		rc = bnx2x_func_start(bp);
2753 		if (rc) {
2754 			BNX2X_ERR("Function start failed!\n");
2755 			bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2756 
2757 			LOAD_ERROR_EXIT(bp, load_error3);
2758 		}
2759 
2760 		/* Send LOAD_DONE command to MCP */
2761 		if (!BP_NOMCP(bp)) {
2762 			load_code = bnx2x_fw_command(bp,
2763 						     DRV_MSG_CODE_LOAD_DONE, 0);
2764 			if (!load_code) {
2765 				BNX2X_ERR("MCP response failure, aborting\n");
2766 				rc = -EBUSY;
2767 				LOAD_ERROR_EXIT(bp, load_error3);
2768 			}
2769 		}
2770 
2771 		/* initialize FW coalescing state machines in RAM */
2772 		bnx2x_update_coalesce(bp);
2773 	}
2774 
2775 	/* setup the leading queue */
2776 	rc = bnx2x_setup_leading(bp);
2777 	if (rc) {
2778 		BNX2X_ERR("Setup leading failed!\n");
2779 		LOAD_ERROR_EXIT(bp, load_error3);
2780 	}
2781 
2782 	/* set up the rest of the queues */
2783 	for_each_nondefault_eth_queue(bp, i) {
2784 		if (IS_PF(bp))
2785 			rc = bnx2x_setup_queue(bp, &bp->fp[i], false);
2786 		else /* VF */
2787 			rc = bnx2x_vfpf_setup_q(bp, &bp->fp[i], false);
2788 		if (rc) {
2789 			BNX2X_ERR("Queue %d setup failed\n", i);
2790 			LOAD_ERROR_EXIT(bp, load_error3);
2791 		}
2792 	}
2793 
2794 	/* setup rss */
2795 	rc = bnx2x_init_rss(bp);
2796 	if (rc) {
2797 		BNX2X_ERR("PF RSS init failed\n");
2798 		LOAD_ERROR_EXIT(bp, load_error3);
2799 	}
2800 
2801 	/* Now when Clients are configured we are ready to work */
2802 	bp->state = BNX2X_STATE_OPEN;
2803 
2804 	/* Configure a ucast MAC */
2805 	if (IS_PF(bp))
2806 		rc = bnx2x_set_eth_mac(bp, true);
2807 	else /* vf */
2808 		rc = bnx2x_vfpf_config_mac(bp, bp->dev->dev_addr, bp->fp->index,
2809 					   true);
2810 	if (rc) {
2811 		BNX2X_ERR("Setting Ethernet MAC failed\n");
2812 		LOAD_ERROR_EXIT(bp, load_error3);
2813 	}
2814 
2815 	if (IS_PF(bp) && bp->pending_max) {
2816 		bnx2x_update_max_mf_config(bp, bp->pending_max);
2817 		bp->pending_max = 0;
2818 	}
2819 
2820 	if (bp->port.pmf) {
2821 		rc = bnx2x_initial_phy_init(bp, load_mode);
2822 		if (rc)
2823 			LOAD_ERROR_EXIT(bp, load_error3);
2824 	}
2825 	bp->link_params.feature_config_flags &= ~FEATURE_CONFIG_BOOT_FROM_SAN;
2826 
2827 	/* Start fast path */
2828 
2829 	/* Re-configure vlan filters */
2830 	rc = bnx2x_vlan_reconfigure_vid(bp);
2831 	if (rc)
2832 		LOAD_ERROR_EXIT(bp, load_error3);
2833 
2834 	/* Initialize Rx filter. */
2835 	bnx2x_set_rx_mode_inner(bp);
2836 
2837 	if (bp->flags & PTP_SUPPORTED) {
2838 		bnx2x_init_ptp(bp);
2839 		bnx2x_configure_ptp_filters(bp);
2840 	}
2841 	/* Start Tx */
2842 	switch (load_mode) {
2843 	case LOAD_NORMAL:
2844 		/* Tx queue should be only re-enabled */
2845 		netif_tx_wake_all_queues(bp->dev);
2846 		break;
2847 
2848 	case LOAD_OPEN:
2849 		netif_tx_start_all_queues(bp->dev);
2850 		smp_mb__after_atomic();
2851 		break;
2852 
2853 	case LOAD_DIAG:
2854 	case LOAD_LOOPBACK_EXT:
2855 		bp->state = BNX2X_STATE_DIAG;
2856 		break;
2857 
2858 	default:
2859 		break;
2860 	}
2861 
2862 	if (bp->port.pmf)
2863 		bnx2x_update_drv_flags(bp, 1 << DRV_FLAGS_PORT_MASK, 0);
2864 	else
2865 		bnx2x__link_status_update(bp);
2866 
2867 	/* start the timer */
2868 	mod_timer(&bp->timer, jiffies + bp->current_interval);
2869 
2870 	if (CNIC_ENABLED(bp))
2871 		bnx2x_load_cnic(bp);
2872 
2873 	if (IS_PF(bp))
2874 		bnx2x_schedule_sp_rtnl(bp, BNX2X_SP_RTNL_GET_DRV_VERSION, 0);
2875 
2876 	if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2877 		/* mark driver is loaded in shmem2 */
2878 		u32 val;
2879 		val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2880 		val &= ~DRV_FLAGS_MTU_MASK;
2881 		val |= (bp->dev->mtu << DRV_FLAGS_MTU_SHIFT);
2882 		SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2883 			  val | DRV_FLAGS_CAPABILITIES_LOADED_SUPPORTED |
2884 			  DRV_FLAGS_CAPABILITIES_LOADED_L2);
2885 	}
2886 
2887 	/* Wait for all pending SP commands to complete */
2888 	if (IS_PF(bp) && !bnx2x_wait_sp_comp(bp, ~0x0UL)) {
2889 		BNX2X_ERR("Timeout waiting for SP elements to complete\n");
2890 		bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
2891 		return -EBUSY;
2892 	}
2893 
2894 	/* Update driver data for On-Chip MFW dump. */
2895 	if (IS_PF(bp))
2896 		bnx2x_update_mfw_dump(bp);
2897 
2898 	/* If PMF - send ADMIN DCBX msg to MFW to initiate DCBX FSM */
2899 	if (bp->port.pmf && (bp->state != BNX2X_STATE_DIAG))
2900 		bnx2x_dcbx_init(bp, false);
2901 
2902 	if (!IS_MF_SD_STORAGE_PERSONALITY_ONLY(bp))
2903 		bnx2x_set_os_driver_state(bp, OS_DRIVER_STATE_ACTIVE);
2904 
2905 	DP(NETIF_MSG_IFUP, "Ending successfully NIC load\n");
2906 
2907 	return 0;
2908 
2909 #ifndef BNX2X_STOP_ON_ERROR
2910 load_error3:
2911 	if (IS_PF(bp)) {
2912 		bnx2x_int_disable_sync(bp, 1);
2913 
2914 		/* Clean queueable objects */
2915 		bnx2x_squeeze_objects(bp);
2916 	}
2917 
2918 	/* Free SKBs, SGEs, TPA pool and driver internals */
2919 	bnx2x_free_skbs(bp);
2920 	for_each_rx_queue(bp, i)
2921 		bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2922 
2923 	/* Release IRQs */
2924 	bnx2x_free_irq(bp);
2925 load_error2:
2926 	if (IS_PF(bp) && !BP_NOMCP(bp)) {
2927 		bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
2928 		bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
2929 	}
2930 
2931 	bp->port.pmf = 0;
2932 load_error1:
2933 	bnx2x_napi_disable(bp);
2934 	bnx2x_del_all_napi(bp);
2935 
2936 	/* clear pf_load status, as it was already set */
2937 	if (IS_PF(bp))
2938 		bnx2x_clear_pf_load(bp);
2939 load_error0:
2940 	bnx2x_free_fw_stats_mem(bp);
2941 	bnx2x_free_fp_mem(bp);
2942 	bnx2x_free_mem(bp);
2943 
2944 	return rc;
2945 #endif /* ! BNX2X_STOP_ON_ERROR */
2946 }
2947 
2948 int bnx2x_drain_tx_queues(struct bnx2x *bp)
2949 {
2950 	u8 rc = 0, cos, i;
2951 
2952 	/* Wait until tx fastpath tasks complete */
2953 	for_each_tx_queue(bp, i) {
2954 		struct bnx2x_fastpath *fp = &bp->fp[i];
2955 
2956 		for_each_cos_in_tx_queue(fp, cos)
2957 			rc = bnx2x_clean_tx_queue(bp, fp->txdata_ptr[cos]);
2958 		if (rc)
2959 			return rc;
2960 	}
2961 	return 0;
2962 }
2963 
2964 /* must be called with rtnl_lock */
2965 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode, bool keep_link)
2966 {
2967 	int i;
2968 	bool global = false;
2969 
2970 	DP(NETIF_MSG_IFUP, "Starting NIC unload\n");
2971 
2972 	if (!IS_MF_SD_STORAGE_PERSONALITY_ONLY(bp))
2973 		bnx2x_set_os_driver_state(bp, OS_DRIVER_STATE_DISABLED);
2974 
2975 	/* mark driver is unloaded in shmem2 */
2976 	if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2977 		u32 val;
2978 		val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2979 		SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2980 			  val & ~DRV_FLAGS_CAPABILITIES_LOADED_L2);
2981 	}
2982 
2983 	if (IS_PF(bp) && bp->recovery_state != BNX2X_RECOVERY_DONE &&
2984 	    (bp->state == BNX2X_STATE_CLOSED ||
2985 	     bp->state == BNX2X_STATE_ERROR)) {
2986 		/* We can get here if the driver has been unloaded
2987 		 * during parity error recovery and is either waiting for a
2988 		 * leader to complete or for other functions to unload and
2989 		 * then ifdown has been issued. In this case we want to
2990 		 * unload and let other functions to complete a recovery
2991 		 * process.
2992 		 */
2993 		bp->recovery_state = BNX2X_RECOVERY_DONE;
2994 		bp->is_leader = 0;
2995 		bnx2x_release_leader_lock(bp);
2996 		smp_mb();
2997 
2998 		DP(NETIF_MSG_IFDOWN, "Releasing a leadership...\n");
2999 		BNX2X_ERR("Can't unload in closed or error state\n");
3000 		return -EINVAL;
3001 	}
3002 
3003 	/* Nothing to do during unload if previous bnx2x_nic_load()
3004 	 * have not completed successfully - all resources are released.
3005 	 *
3006 	 * we can get here only after unsuccessful ndo_* callback, during which
3007 	 * dev->IFF_UP flag is still on.
3008 	 */
3009 	if (bp->state == BNX2X_STATE_CLOSED || bp->state == BNX2X_STATE_ERROR)
3010 		return 0;
3011 
3012 	/* It's important to set the bp->state to the value different from
3013 	 * BNX2X_STATE_OPEN and only then stop the Tx. Otherwise bnx2x_tx_int()
3014 	 * may restart the Tx from the NAPI context (see bnx2x_tx_int()).
3015 	 */
3016 	bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
3017 	smp_mb();
3018 
3019 	/* indicate to VFs that the PF is going down */
3020 	bnx2x_iov_channel_down(bp);
3021 
3022 	if (CNIC_LOADED(bp))
3023 		bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
3024 
3025 	/* Stop Tx */
3026 	bnx2x_tx_disable(bp);
3027 	netdev_reset_tc(bp->dev);
3028 
3029 	bp->rx_mode = BNX2X_RX_MODE_NONE;
3030 
3031 	del_timer_sync(&bp->timer);
3032 
3033 	if (IS_PF(bp) && !BP_NOMCP(bp)) {
3034 		/* Set ALWAYS_ALIVE bit in shmem */
3035 		bp->fw_drv_pulse_wr_seq |= DRV_PULSE_ALWAYS_ALIVE;
3036 		bnx2x_drv_pulse(bp);
3037 		bnx2x_stats_handle(bp, STATS_EVENT_STOP);
3038 		bnx2x_save_statistics(bp);
3039 	}
3040 
3041 	/* wait till consumers catch up with producers in all queues.
3042 	 * If we're recovering, FW can't write to host so no reason
3043 	 * to wait for the queues to complete all Tx.
3044 	 */
3045 	if (unload_mode != UNLOAD_RECOVERY)
3046 		bnx2x_drain_tx_queues(bp);
3047 
3048 	/* if VF indicate to PF this function is going down (PF will delete sp
3049 	 * elements and clear initializations
3050 	 */
3051 	if (IS_VF(bp))
3052 		bnx2x_vfpf_close_vf(bp);
3053 	else if (unload_mode != UNLOAD_RECOVERY)
3054 		/* if this is a normal/close unload need to clean up chip*/
3055 		bnx2x_chip_cleanup(bp, unload_mode, keep_link);
3056 	else {
3057 		/* Send the UNLOAD_REQUEST to the MCP */
3058 		bnx2x_send_unload_req(bp, unload_mode);
3059 
3060 		/* Prevent transactions to host from the functions on the
3061 		 * engine that doesn't reset global blocks in case of global
3062 		 * attention once global blocks are reset and gates are opened
3063 		 * (the engine which leader will perform the recovery
3064 		 * last).
3065 		 */
3066 		if (!CHIP_IS_E1x(bp))
3067 			bnx2x_pf_disable(bp);
3068 
3069 		/* Disable HW interrupts, NAPI */
3070 		bnx2x_netif_stop(bp, 1);
3071 		/* Delete all NAPI objects */
3072 		bnx2x_del_all_napi(bp);
3073 		if (CNIC_LOADED(bp))
3074 			bnx2x_del_all_napi_cnic(bp);
3075 		/* Release IRQs */
3076 		bnx2x_free_irq(bp);
3077 
3078 		/* Report UNLOAD_DONE to MCP */
3079 		bnx2x_send_unload_done(bp, false);
3080 	}
3081 
3082 	/*
3083 	 * At this stage no more interrupts will arrive so we may safely clean
3084 	 * the queueable objects here in case they failed to get cleaned so far.
3085 	 */
3086 	if (IS_PF(bp))
3087 		bnx2x_squeeze_objects(bp);
3088 
3089 	/* There should be no more pending SP commands at this stage */
3090 	bp->sp_state = 0;
3091 
3092 	bp->port.pmf = 0;
3093 
3094 	/* clear pending work in rtnl task */
3095 	bp->sp_rtnl_state = 0;
3096 	smp_mb();
3097 
3098 	/* Free SKBs, SGEs, TPA pool and driver internals */
3099 	bnx2x_free_skbs(bp);
3100 	if (CNIC_LOADED(bp))
3101 		bnx2x_free_skbs_cnic(bp);
3102 	for_each_rx_queue(bp, i)
3103 		bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
3104 
3105 	bnx2x_free_fp_mem(bp);
3106 	if (CNIC_LOADED(bp))
3107 		bnx2x_free_fp_mem_cnic(bp);
3108 
3109 	if (IS_PF(bp)) {
3110 		if (CNIC_LOADED(bp))
3111 			bnx2x_free_mem_cnic(bp);
3112 	}
3113 	bnx2x_free_mem(bp);
3114 
3115 	bp->state = BNX2X_STATE_CLOSED;
3116 	bp->cnic_loaded = false;
3117 
3118 	/* Clear driver version indication in shmem */
3119 	if (IS_PF(bp) && !BP_NOMCP(bp))
3120 		bnx2x_update_mng_version(bp);
3121 
3122 	/* Check if there are pending parity attentions. If there are - set
3123 	 * RECOVERY_IN_PROGRESS.
3124 	 */
3125 	if (IS_PF(bp) && bnx2x_chk_parity_attn(bp, &global, false)) {
3126 		bnx2x_set_reset_in_progress(bp);
3127 
3128 		/* Set RESET_IS_GLOBAL if needed */
3129 		if (global)
3130 			bnx2x_set_reset_global(bp);
3131 	}
3132 
3133 	/* The last driver must disable a "close the gate" if there is no
3134 	 * parity attention or "process kill" pending.
3135 	 */
3136 	if (IS_PF(bp) &&
3137 	    !bnx2x_clear_pf_load(bp) &&
3138 	    bnx2x_reset_is_done(bp, BP_PATH(bp)))
3139 		bnx2x_disable_close_the_gate(bp);
3140 
3141 	DP(NETIF_MSG_IFUP, "Ending NIC unload\n");
3142 
3143 	return 0;
3144 }
3145 
3146 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
3147 {
3148 	u16 pmcsr;
3149 
3150 	/* If there is no power capability, silently succeed */
3151 	if (!bp->pdev->pm_cap) {
3152 		BNX2X_DEV_INFO("No power capability. Breaking.\n");
3153 		return 0;
3154 	}
3155 
3156 	pci_read_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL, &pmcsr);
3157 
3158 	switch (state) {
3159 	case PCI_D0:
3160 		pci_write_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL,
3161 				      ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
3162 				       PCI_PM_CTRL_PME_STATUS));
3163 
3164 		if (pmcsr & PCI_PM_CTRL_STATE_MASK)
3165 			/* delay required during transition out of D3hot */
3166 			msleep(20);
3167 		break;
3168 
3169 	case PCI_D3hot:
3170 		/* If there are other clients above don't
3171 		   shut down the power */
3172 		if (atomic_read(&bp->pdev->enable_cnt) != 1)
3173 			return 0;
3174 		/* Don't shut down the power for emulation and FPGA */
3175 		if (CHIP_REV_IS_SLOW(bp))
3176 			return 0;
3177 
3178 		pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
3179 		pmcsr |= 3;
3180 
3181 		if (bp->wol)
3182 			pmcsr |= PCI_PM_CTRL_PME_ENABLE;
3183 
3184 		pci_write_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL,
3185 				      pmcsr);
3186 
3187 		/* No more memory access after this point until
3188 		* device is brought back to D0.
3189 		*/
3190 		break;
3191 
3192 	default:
3193 		dev_err(&bp->pdev->dev, "Can't support state = %d\n", state);
3194 		return -EINVAL;
3195 	}
3196 	return 0;
3197 }
3198 
3199 /*
3200  * net_device service functions
3201  */
3202 static int bnx2x_poll(struct napi_struct *napi, int budget)
3203 {
3204 	struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
3205 						 napi);
3206 	struct bnx2x *bp = fp->bp;
3207 	int rx_work_done;
3208 	u8 cos;
3209 
3210 #ifdef BNX2X_STOP_ON_ERROR
3211 	if (unlikely(bp->panic)) {
3212 		napi_complete(napi);
3213 		return 0;
3214 	}
3215 #endif
3216 	for_each_cos_in_tx_queue(fp, cos)
3217 		if (bnx2x_tx_queue_has_work(fp->txdata_ptr[cos]))
3218 			bnx2x_tx_int(bp, fp->txdata_ptr[cos]);
3219 
3220 	rx_work_done = (bnx2x_has_rx_work(fp)) ? bnx2x_rx_int(fp, budget) : 0;
3221 
3222 	if (rx_work_done < budget) {
3223 		/* No need to update SB for FCoE L2 ring as long as
3224 		 * it's connected to the default SB and the SB
3225 		 * has been updated when NAPI was scheduled.
3226 		 */
3227 		if (IS_FCOE_FP(fp)) {
3228 			napi_complete_done(napi, rx_work_done);
3229 		} else {
3230 			bnx2x_update_fpsb_idx(fp);
3231 			/* bnx2x_has_rx_work() reads the status block,
3232 			 * thus we need to ensure that status block indices
3233 			 * have been actually read (bnx2x_update_fpsb_idx)
3234 			 * prior to this check (bnx2x_has_rx_work) so that
3235 			 * we won't write the "newer" value of the status block
3236 			 * to IGU (if there was a DMA right after
3237 			 * bnx2x_has_rx_work and if there is no rmb, the memory
3238 			 * reading (bnx2x_update_fpsb_idx) may be postponed
3239 			 * to right before bnx2x_ack_sb). In this case there
3240 			 * will never be another interrupt until there is
3241 			 * another update of the status block, while there
3242 			 * is still unhandled work.
3243 			 */
3244 			rmb();
3245 
3246 			if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
3247 				if (napi_complete_done(napi, rx_work_done)) {
3248 					/* Re-enable interrupts */
3249 					DP(NETIF_MSG_RX_STATUS,
3250 					   "Update index to %d\n", fp->fp_hc_idx);
3251 					bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
3252 						     le16_to_cpu(fp->fp_hc_idx),
3253 						     IGU_INT_ENABLE, 1);
3254 				}
3255 			} else {
3256 				rx_work_done = budget;
3257 			}
3258 		}
3259 	}
3260 
3261 	return rx_work_done;
3262 }
3263 
3264 /* we split the first BD into headers and data BDs
3265  * to ease the pain of our fellow microcode engineers
3266  * we use one mapping for both BDs
3267  */
3268 static u16 bnx2x_tx_split(struct bnx2x *bp,
3269 			  struct bnx2x_fp_txdata *txdata,
3270 			  struct sw_tx_bd *tx_buf,
3271 			  struct eth_tx_start_bd **tx_bd, u16 hlen,
3272 			  u16 bd_prod)
3273 {
3274 	struct eth_tx_start_bd *h_tx_bd = *tx_bd;
3275 	struct eth_tx_bd *d_tx_bd;
3276 	dma_addr_t mapping;
3277 	int old_len = le16_to_cpu(h_tx_bd->nbytes);
3278 
3279 	/* first fix first BD */
3280 	h_tx_bd->nbytes = cpu_to_le16(hlen);
3281 
3282 	DP(NETIF_MSG_TX_QUEUED,	"TSO split header size is %d (%x:%x)\n",
3283 	   h_tx_bd->nbytes, h_tx_bd->addr_hi, h_tx_bd->addr_lo);
3284 
3285 	/* now get a new data BD
3286 	 * (after the pbd) and fill it */
3287 	bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3288 	d_tx_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3289 
3290 	mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
3291 			   le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
3292 
3293 	d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3294 	d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3295 	d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
3296 
3297 	/* this marks the BD as one that has no individual mapping */
3298 	tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
3299 
3300 	DP(NETIF_MSG_TX_QUEUED,
3301 	   "TSO split data size is %d (%x:%x)\n",
3302 	   d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
3303 
3304 	/* update tx_bd */
3305 	*tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
3306 
3307 	return bd_prod;
3308 }
3309 
3310 #define bswab32(b32) ((__force __le32) swab32((__force __u32) (b32)))
3311 #define bswab16(b16) ((__force __le16) swab16((__force __u16) (b16)))
3312 static __le16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
3313 {
3314 	__sum16 tsum = (__force __sum16) csum;
3315 
3316 	if (fix > 0)
3317 		tsum = ~csum_fold(csum_sub((__force __wsum) csum,
3318 				  csum_partial(t_header - fix, fix, 0)));
3319 
3320 	else if (fix < 0)
3321 		tsum = ~csum_fold(csum_add((__force __wsum) csum,
3322 				  csum_partial(t_header, -fix, 0)));
3323 
3324 	return bswab16(tsum);
3325 }
3326 
3327 static u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
3328 {
3329 	u32 rc;
3330 	__u8 prot = 0;
3331 	__be16 protocol;
3332 
3333 	if (skb->ip_summed != CHECKSUM_PARTIAL)
3334 		return XMIT_PLAIN;
3335 
3336 	protocol = vlan_get_protocol(skb);
3337 	if (protocol == htons(ETH_P_IPV6)) {
3338 		rc = XMIT_CSUM_V6;
3339 		prot = ipv6_hdr(skb)->nexthdr;
3340 	} else {
3341 		rc = XMIT_CSUM_V4;
3342 		prot = ip_hdr(skb)->protocol;
3343 	}
3344 
3345 	if (!CHIP_IS_E1x(bp) && skb->encapsulation) {
3346 		if (inner_ip_hdr(skb)->version == 6) {
3347 			rc |= XMIT_CSUM_ENC_V6;
3348 			if (inner_ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3349 				rc |= XMIT_CSUM_TCP;
3350 		} else {
3351 			rc |= XMIT_CSUM_ENC_V4;
3352 			if (inner_ip_hdr(skb)->protocol == IPPROTO_TCP)
3353 				rc |= XMIT_CSUM_TCP;
3354 		}
3355 	}
3356 	if (prot == IPPROTO_TCP)
3357 		rc |= XMIT_CSUM_TCP;
3358 
3359 	if (skb_is_gso(skb)) {
3360 		if (skb_is_gso_v6(skb)) {
3361 			rc |= (XMIT_GSO_V6 | XMIT_CSUM_TCP);
3362 			if (rc & XMIT_CSUM_ENC)
3363 				rc |= XMIT_GSO_ENC_V6;
3364 		} else {
3365 			rc |= (XMIT_GSO_V4 | XMIT_CSUM_TCP);
3366 			if (rc & XMIT_CSUM_ENC)
3367 				rc |= XMIT_GSO_ENC_V4;
3368 		}
3369 	}
3370 
3371 	return rc;
3372 }
3373 
3374 /* VXLAN: 4 = 1 (for linear data BD) + 3 (2 for PBD and last BD) */
3375 #define BNX2X_NUM_VXLAN_TSO_WIN_SUB_BDS         4
3376 
3377 /* Regular: 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
3378 #define BNX2X_NUM_TSO_WIN_SUB_BDS               3
3379 
3380 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - BDS_PER_TX_PKT)
3381 /* check if packet requires linearization (packet is too fragmented)
3382    no need to check fragmentation if page size > 8K (there will be no
3383    violation to FW restrictions) */
3384 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
3385 			     u32 xmit_type)
3386 {
3387 	int first_bd_sz = 0, num_tso_win_sub = BNX2X_NUM_TSO_WIN_SUB_BDS;
3388 	int to_copy = 0, hlen = 0;
3389 
3390 	if (xmit_type & XMIT_GSO_ENC)
3391 		num_tso_win_sub = BNX2X_NUM_VXLAN_TSO_WIN_SUB_BDS;
3392 
3393 	if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - num_tso_win_sub)) {
3394 		if (xmit_type & XMIT_GSO) {
3395 			unsigned short lso_mss = skb_shinfo(skb)->gso_size;
3396 			int wnd_size = MAX_FETCH_BD - num_tso_win_sub;
3397 			/* Number of windows to check */
3398 			int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
3399 			int wnd_idx = 0;
3400 			int frag_idx = 0;
3401 			u32 wnd_sum = 0;
3402 
3403 			/* Headers length */
3404 			if (xmit_type & XMIT_GSO_ENC)
3405 				hlen = (int)(skb_inner_transport_header(skb) -
3406 					     skb->data) +
3407 					     inner_tcp_hdrlen(skb);
3408 			else
3409 				hlen = (int)(skb_transport_header(skb) -
3410 					     skb->data) + tcp_hdrlen(skb);
3411 
3412 			/* Amount of data (w/o headers) on linear part of SKB*/
3413 			first_bd_sz = skb_headlen(skb) - hlen;
3414 
3415 			wnd_sum  = first_bd_sz;
3416 
3417 			/* Calculate the first sum - it's special */
3418 			for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
3419 				wnd_sum +=
3420 					skb_frag_size(&skb_shinfo(skb)->frags[frag_idx]);
3421 
3422 			/* If there was data on linear skb data - check it */
3423 			if (first_bd_sz > 0) {
3424 				if (unlikely(wnd_sum < lso_mss)) {
3425 					to_copy = 1;
3426 					goto exit_lbl;
3427 				}
3428 
3429 				wnd_sum -= first_bd_sz;
3430 			}
3431 
3432 			/* Others are easier: run through the frag list and
3433 			   check all windows */
3434 			for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
3435 				wnd_sum +=
3436 			  skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1]);
3437 
3438 				if (unlikely(wnd_sum < lso_mss)) {
3439 					to_copy = 1;
3440 					break;
3441 				}
3442 				wnd_sum -=
3443 					skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx]);
3444 			}
3445 		} else {
3446 			/* in non-LSO too fragmented packet should always
3447 			   be linearized */
3448 			to_copy = 1;
3449 		}
3450 	}
3451 
3452 exit_lbl:
3453 	if (unlikely(to_copy))
3454 		DP(NETIF_MSG_TX_QUEUED,
3455 		   "Linearization IS REQUIRED for %s packet. num_frags %d  hlen %d  first_bd_sz %d\n",
3456 		   (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
3457 		   skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
3458 
3459 	return to_copy;
3460 }
3461 #endif
3462 
3463 /**
3464  * bnx2x_set_pbd_gso - update PBD in GSO case.
3465  *
3466  * @skb:	packet skb
3467  * @pbd:	parse BD
3468  * @xmit_type:	xmit flags
3469  */
3470 static void bnx2x_set_pbd_gso(struct sk_buff *skb,
3471 			      struct eth_tx_parse_bd_e1x *pbd,
3472 			      u32 xmit_type)
3473 {
3474 	pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
3475 	pbd->tcp_send_seq = bswab32(tcp_hdr(skb)->seq);
3476 	pbd->tcp_flags = pbd_tcp_flags(tcp_hdr(skb));
3477 
3478 	if (xmit_type & XMIT_GSO_V4) {
3479 		pbd->ip_id = bswab16(ip_hdr(skb)->id);
3480 		pbd->tcp_pseudo_csum =
3481 			bswab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
3482 						   ip_hdr(skb)->daddr,
3483 						   0, IPPROTO_TCP, 0));
3484 	} else {
3485 		pbd->tcp_pseudo_csum =
3486 			bswab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3487 						 &ipv6_hdr(skb)->daddr,
3488 						 0, IPPROTO_TCP, 0));
3489 	}
3490 
3491 	pbd->global_data |=
3492 		cpu_to_le16(ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN);
3493 }
3494 
3495 /**
3496  * bnx2x_set_pbd_csum_enc - update PBD with checksum and return header length
3497  *
3498  * @bp:			driver handle
3499  * @skb:		packet skb
3500  * @parsing_data:	data to be updated
3501  * @xmit_type:		xmit flags
3502  *
3503  * 57712/578xx related, when skb has encapsulation
3504  */
3505 static u8 bnx2x_set_pbd_csum_enc(struct bnx2x *bp, struct sk_buff *skb,
3506 				 u32 *parsing_data, u32 xmit_type)
3507 {
3508 	*parsing_data |=
3509 		((((u8 *)skb_inner_transport_header(skb) - skb->data) >> 1) <<
3510 		ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3511 		ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3512 
3513 	if (xmit_type & XMIT_CSUM_TCP) {
3514 		*parsing_data |= ((inner_tcp_hdrlen(skb) / 4) <<
3515 			ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3516 			ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3517 
3518 		return skb_inner_transport_header(skb) +
3519 			inner_tcp_hdrlen(skb) - skb->data;
3520 	}
3521 
3522 	/* We support checksum offload for TCP and UDP only.
3523 	 * No need to pass the UDP header length - it's a constant.
3524 	 */
3525 	return skb_inner_transport_header(skb) +
3526 		sizeof(struct udphdr) - skb->data;
3527 }
3528 
3529 /**
3530  * bnx2x_set_pbd_csum_e2 - update PBD with checksum and return header length
3531  *
3532  * @bp:			driver handle
3533  * @skb:		packet skb
3534  * @parsing_data:	data to be updated
3535  * @xmit_type:		xmit flags
3536  *
3537  * 57712/578xx related
3538  */
3539 static u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
3540 				u32 *parsing_data, u32 xmit_type)
3541 {
3542 	*parsing_data |=
3543 		((((u8 *)skb_transport_header(skb) - skb->data) >> 1) <<
3544 		ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3545 		ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3546 
3547 	if (xmit_type & XMIT_CSUM_TCP) {
3548 		*parsing_data |= ((tcp_hdrlen(skb) / 4) <<
3549 			ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3550 			ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3551 
3552 		return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data;
3553 	}
3554 	/* We support checksum offload for TCP and UDP only.
3555 	 * No need to pass the UDP header length - it's a constant.
3556 	 */
3557 	return skb_transport_header(skb) + sizeof(struct udphdr) - skb->data;
3558 }
3559 
3560 /* set FW indication according to inner or outer protocols if tunneled */
3561 static void bnx2x_set_sbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3562 			       struct eth_tx_start_bd *tx_start_bd,
3563 			       u32 xmit_type)
3564 {
3565 	tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
3566 
3567 	if (xmit_type & (XMIT_CSUM_ENC_V6 | XMIT_CSUM_V6))
3568 		tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IPV6;
3569 
3570 	if (!(xmit_type & XMIT_CSUM_TCP))
3571 		tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IS_UDP;
3572 }
3573 
3574 /**
3575  * bnx2x_set_pbd_csum - update PBD with checksum and return header length
3576  *
3577  * @bp:		driver handle
3578  * @skb:	packet skb
3579  * @pbd:	parse BD to be updated
3580  * @xmit_type:	xmit flags
3581  */
3582 static u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3583 			     struct eth_tx_parse_bd_e1x *pbd,
3584 			     u32 xmit_type)
3585 {
3586 	u8 hlen = (skb_network_header(skb) - skb->data) >> 1;
3587 
3588 	/* for now NS flag is not used in Linux */
3589 	pbd->global_data =
3590 		cpu_to_le16(hlen |
3591 			    ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3592 			     ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT));
3593 
3594 	pbd->ip_hlen_w = (skb_transport_header(skb) -
3595 			skb_network_header(skb)) >> 1;
3596 
3597 	hlen += pbd->ip_hlen_w;
3598 
3599 	/* We support checksum offload for TCP and UDP only */
3600 	if (xmit_type & XMIT_CSUM_TCP)
3601 		hlen += tcp_hdrlen(skb) / 2;
3602 	else
3603 		hlen += sizeof(struct udphdr) / 2;
3604 
3605 	pbd->total_hlen_w = cpu_to_le16(hlen);
3606 	hlen = hlen*2;
3607 
3608 	if (xmit_type & XMIT_CSUM_TCP) {
3609 		pbd->tcp_pseudo_csum = bswab16(tcp_hdr(skb)->check);
3610 
3611 	} else {
3612 		s8 fix = SKB_CS_OFF(skb); /* signed! */
3613 
3614 		DP(NETIF_MSG_TX_QUEUED,
3615 		   "hlen %d  fix %d  csum before fix %x\n",
3616 		   le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb));
3617 
3618 		/* HW bug: fixup the CSUM */
3619 		pbd->tcp_pseudo_csum =
3620 			bnx2x_csum_fix(skb_transport_header(skb),
3621 				       SKB_CS(skb), fix);
3622 
3623 		DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
3624 		   pbd->tcp_pseudo_csum);
3625 	}
3626 
3627 	return hlen;
3628 }
3629 
3630 static void bnx2x_update_pbds_gso_enc(struct sk_buff *skb,
3631 				      struct eth_tx_parse_bd_e2 *pbd_e2,
3632 				      struct eth_tx_parse_2nd_bd *pbd2,
3633 				      u16 *global_data,
3634 				      u32 xmit_type)
3635 {
3636 	u16 hlen_w = 0;
3637 	u8 outerip_off, outerip_len = 0;
3638 
3639 	/* from outer IP to transport */
3640 	hlen_w = (skb_inner_transport_header(skb) -
3641 		  skb_network_header(skb)) >> 1;
3642 
3643 	/* transport len */
3644 	hlen_w += inner_tcp_hdrlen(skb) >> 1;
3645 
3646 	pbd2->fw_ip_hdr_to_payload_w = hlen_w;
3647 
3648 	/* outer IP header info */
3649 	if (xmit_type & XMIT_CSUM_V4) {
3650 		struct iphdr *iph = ip_hdr(skb);
3651 		u32 csum = (__force u32)(~iph->check) -
3652 			   (__force u32)iph->tot_len -
3653 			   (__force u32)iph->frag_off;
3654 
3655 		outerip_len = iph->ihl << 1;
3656 
3657 		pbd2->fw_ip_csum_wo_len_flags_frag =
3658 			bswab16(csum_fold((__force __wsum)csum));
3659 	} else {
3660 		pbd2->fw_ip_hdr_to_payload_w =
3661 			hlen_w - ((sizeof(struct ipv6hdr)) >> 1);
3662 		pbd_e2->data.tunnel_data.flags |=
3663 			ETH_TUNNEL_DATA_IPV6_OUTER;
3664 	}
3665 
3666 	pbd2->tcp_send_seq = bswab32(inner_tcp_hdr(skb)->seq);
3667 
3668 	pbd2->tcp_flags = pbd_tcp_flags(inner_tcp_hdr(skb));
3669 
3670 	/* inner IP header info */
3671 	if (xmit_type & XMIT_CSUM_ENC_V4) {
3672 		pbd2->hw_ip_id = bswab16(inner_ip_hdr(skb)->id);
3673 
3674 		pbd_e2->data.tunnel_data.pseudo_csum =
3675 			bswab16(~csum_tcpudp_magic(
3676 					inner_ip_hdr(skb)->saddr,
3677 					inner_ip_hdr(skb)->daddr,
3678 					0, IPPROTO_TCP, 0));
3679 	} else {
3680 		pbd_e2->data.tunnel_data.pseudo_csum =
3681 			bswab16(~csum_ipv6_magic(
3682 					&inner_ipv6_hdr(skb)->saddr,
3683 					&inner_ipv6_hdr(skb)->daddr,
3684 					0, IPPROTO_TCP, 0));
3685 	}
3686 
3687 	outerip_off = (skb_network_header(skb) - skb->data) >> 1;
3688 
3689 	*global_data |=
3690 		outerip_off |
3691 		(outerip_len <<
3692 			ETH_TX_PARSE_2ND_BD_IP_HDR_LEN_OUTER_W_SHIFT) |
3693 		((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3694 			ETH_TX_PARSE_2ND_BD_LLC_SNAP_EN_SHIFT);
3695 
3696 	if (ip_hdr(skb)->protocol == IPPROTO_UDP) {
3697 		SET_FLAG(*global_data, ETH_TX_PARSE_2ND_BD_TUNNEL_UDP_EXIST, 1);
3698 		pbd2->tunnel_udp_hdr_start_w = skb_transport_offset(skb) >> 1;
3699 	}
3700 }
3701 
3702 static inline void bnx2x_set_ipv6_ext_e2(struct sk_buff *skb, u32 *parsing_data,
3703 					 u32 xmit_type)
3704 {
3705 	struct ipv6hdr *ipv6;
3706 
3707 	if (!(xmit_type & (XMIT_GSO_ENC_V6 | XMIT_GSO_V6)))
3708 		return;
3709 
3710 	if (xmit_type & XMIT_GSO_ENC_V6)
3711 		ipv6 = inner_ipv6_hdr(skb);
3712 	else /* XMIT_GSO_V6 */
3713 		ipv6 = ipv6_hdr(skb);
3714 
3715 	if (ipv6->nexthdr == NEXTHDR_IPV6)
3716 		*parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR;
3717 }
3718 
3719 /* called with netif_tx_lock
3720  * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
3721  * netif_wake_queue()
3722  */
3723 netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
3724 {
3725 	struct bnx2x *bp = netdev_priv(dev);
3726 
3727 	struct netdev_queue *txq;
3728 	struct bnx2x_fp_txdata *txdata;
3729 	struct sw_tx_bd *tx_buf;
3730 	struct eth_tx_start_bd *tx_start_bd, *first_bd;
3731 	struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
3732 	struct eth_tx_parse_bd_e1x *pbd_e1x = NULL;
3733 	struct eth_tx_parse_bd_e2 *pbd_e2 = NULL;
3734 	struct eth_tx_parse_2nd_bd *pbd2 = NULL;
3735 	u32 pbd_e2_parsing_data = 0;
3736 	u16 pkt_prod, bd_prod;
3737 	int nbd, txq_index;
3738 	dma_addr_t mapping;
3739 	u32 xmit_type = bnx2x_xmit_type(bp, skb);
3740 	int i;
3741 	u8 hlen = 0;
3742 	__le16 pkt_size = 0;
3743 	struct ethhdr *eth;
3744 	u8 mac_type = UNICAST_ADDRESS;
3745 
3746 #ifdef BNX2X_STOP_ON_ERROR
3747 	if (unlikely(bp->panic))
3748 		return NETDEV_TX_BUSY;
3749 #endif
3750 
3751 	txq_index = skb_get_queue_mapping(skb);
3752 	txq = netdev_get_tx_queue(dev, txq_index);
3753 
3754 	BUG_ON(txq_index >= MAX_ETH_TXQ_IDX(bp) + (CNIC_LOADED(bp) ? 1 : 0));
3755 
3756 	txdata = &bp->bnx2x_txq[txq_index];
3757 
3758 	/* enable this debug print to view the transmission queue being used
3759 	DP(NETIF_MSG_TX_QUEUED, "indices: txq %d, fp %d, txdata %d\n",
3760 	   txq_index, fp_index, txdata_index); */
3761 
3762 	/* enable this debug print to view the transmission details
3763 	DP(NETIF_MSG_TX_QUEUED,
3764 	   "transmitting packet cid %d fp index %d txdata_index %d tx_data ptr %p fp pointer %p\n",
3765 	   txdata->cid, fp_index, txdata_index, txdata, fp); */
3766 
3767 	if (unlikely(bnx2x_tx_avail(bp, txdata) <
3768 			skb_shinfo(skb)->nr_frags +
3769 			BDS_PER_TX_PKT +
3770 			NEXT_CNT_PER_TX_PKT(MAX_BDS_PER_TX_PKT))) {
3771 		/* Handle special storage cases separately */
3772 		if (txdata->tx_ring_size == 0) {
3773 			struct bnx2x_eth_q_stats *q_stats =
3774 				bnx2x_fp_qstats(bp, txdata->parent_fp);
3775 			q_stats->driver_filtered_tx_pkt++;
3776 			dev_kfree_skb(skb);
3777 			return NETDEV_TX_OK;
3778 		}
3779 		bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
3780 		netif_tx_stop_queue(txq);
3781 		BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
3782 
3783 		return NETDEV_TX_BUSY;
3784 	}
3785 
3786 	DP(NETIF_MSG_TX_QUEUED,
3787 	   "queue[%d]: SKB: summed %x  protocol %x protocol(%x,%x) gso type %x  xmit_type %x len %d\n",
3788 	   txq_index, skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
3789 	   ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type,
3790 	   skb->len);
3791 
3792 	eth = (struct ethhdr *)skb->data;
3793 
3794 	/* set flag according to packet type (UNICAST_ADDRESS is default)*/
3795 	if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
3796 		if (is_broadcast_ether_addr(eth->h_dest))
3797 			mac_type = BROADCAST_ADDRESS;
3798 		else
3799 			mac_type = MULTICAST_ADDRESS;
3800 	}
3801 
3802 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - BDS_PER_TX_PKT)
3803 	/* First, check if we need to linearize the skb (due to FW
3804 	   restrictions). No need to check fragmentation if page size > 8K
3805 	   (there will be no violation to FW restrictions) */
3806 	if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
3807 		/* Statistics of linearization */
3808 		bp->lin_cnt++;
3809 		if (skb_linearize(skb) != 0) {
3810 			DP(NETIF_MSG_TX_QUEUED,
3811 			   "SKB linearization failed - silently dropping this SKB\n");
3812 			dev_kfree_skb_any(skb);
3813 			return NETDEV_TX_OK;
3814 		}
3815 	}
3816 #endif
3817 	/* Map skb linear data for DMA */
3818 	mapping = dma_map_single(&bp->pdev->dev, skb->data,
3819 				 skb_headlen(skb), DMA_TO_DEVICE);
3820 	if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
3821 		DP(NETIF_MSG_TX_QUEUED,
3822 		   "SKB mapping failed - silently dropping this SKB\n");
3823 		dev_kfree_skb_any(skb);
3824 		return NETDEV_TX_OK;
3825 	}
3826 	/*
3827 	Please read carefully. First we use one BD which we mark as start,
3828 	then we have a parsing info BD (used for TSO or xsum),
3829 	and only then we have the rest of the TSO BDs.
3830 	(don't forget to mark the last one as last,
3831 	and to unmap only AFTER you write to the BD ...)
3832 	And above all, all pdb sizes are in words - NOT DWORDS!
3833 	*/
3834 
3835 	/* get current pkt produced now - advance it just before sending packet
3836 	 * since mapping of pages may fail and cause packet to be dropped
3837 	 */
3838 	pkt_prod = txdata->tx_pkt_prod;
3839 	bd_prod = TX_BD(txdata->tx_bd_prod);
3840 
3841 	/* get a tx_buf and first BD
3842 	 * tx_start_bd may be changed during SPLIT,
3843 	 * but first_bd will always stay first
3844 	 */
3845 	tx_buf = &txdata->tx_buf_ring[TX_BD(pkt_prod)];
3846 	tx_start_bd = &txdata->tx_desc_ring[bd_prod].start_bd;
3847 	first_bd = tx_start_bd;
3848 
3849 	tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
3850 
3851 	if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
3852 		if (!(bp->flags & TX_TIMESTAMPING_EN)) {
3853 			BNX2X_ERR("Tx timestamping was not enabled, this packet will not be timestamped\n");
3854 		} else if (bp->ptp_tx_skb) {
3855 			BNX2X_ERR("The device supports only a single outstanding packet to timestamp, this packet will not be timestamped\n");
3856 		} else {
3857 			skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
3858 			/* schedule check for Tx timestamp */
3859 			bp->ptp_tx_skb = skb_get(skb);
3860 			bp->ptp_tx_start = jiffies;
3861 			schedule_work(&bp->ptp_task);
3862 		}
3863 	}
3864 
3865 	/* header nbd: indirectly zero other flags! */
3866 	tx_start_bd->general_data = 1 << ETH_TX_START_BD_HDR_NBDS_SHIFT;
3867 
3868 	/* remember the first BD of the packet */
3869 	tx_buf->first_bd = txdata->tx_bd_prod;
3870 	tx_buf->skb = skb;
3871 	tx_buf->flags = 0;
3872 
3873 	DP(NETIF_MSG_TX_QUEUED,
3874 	   "sending pkt %u @%p  next_idx %u  bd %u @%p\n",
3875 	   pkt_prod, tx_buf, txdata->tx_pkt_prod, bd_prod, tx_start_bd);
3876 
3877 	if (skb_vlan_tag_present(skb)) {
3878 		tx_start_bd->vlan_or_ethertype =
3879 		    cpu_to_le16(skb_vlan_tag_get(skb));
3880 		tx_start_bd->bd_flags.as_bitfield |=
3881 		    (X_ETH_OUTBAND_VLAN << ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
3882 	} else {
3883 		/* when transmitting in a vf, start bd must hold the ethertype
3884 		 * for fw to enforce it
3885 		 */
3886 		u16 vlan_tci = 0;
3887 #ifndef BNX2X_STOP_ON_ERROR
3888 		if (IS_VF(bp)) {
3889 #endif
3890 			/* Still need to consider inband vlan for enforced */
3891 			if (__vlan_get_tag(skb, &vlan_tci)) {
3892 				tx_start_bd->vlan_or_ethertype =
3893 					cpu_to_le16(ntohs(eth->h_proto));
3894 			} else {
3895 				tx_start_bd->bd_flags.as_bitfield |=
3896 					(X_ETH_INBAND_VLAN <<
3897 					 ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
3898 				tx_start_bd->vlan_or_ethertype =
3899 					cpu_to_le16(vlan_tci);
3900 			}
3901 #ifndef BNX2X_STOP_ON_ERROR
3902 		} else {
3903 			/* used by FW for packet accounting */
3904 			tx_start_bd->vlan_or_ethertype = cpu_to_le16(pkt_prod);
3905 		}
3906 #endif
3907 	}
3908 
3909 	nbd = 2; /* start_bd + pbd + frags (updated when pages are mapped) */
3910 
3911 	/* turn on parsing and get a BD */
3912 	bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3913 
3914 	if (xmit_type & XMIT_CSUM)
3915 		bnx2x_set_sbd_csum(bp, skb, tx_start_bd, xmit_type);
3916 
3917 	if (!CHIP_IS_E1x(bp)) {
3918 		pbd_e2 = &txdata->tx_desc_ring[bd_prod].parse_bd_e2;
3919 		memset(pbd_e2, 0, sizeof(struct eth_tx_parse_bd_e2));
3920 
3921 		if (xmit_type & XMIT_CSUM_ENC) {
3922 			u16 global_data = 0;
3923 
3924 			/* Set PBD in enc checksum offload case */
3925 			hlen = bnx2x_set_pbd_csum_enc(bp, skb,
3926 						      &pbd_e2_parsing_data,
3927 						      xmit_type);
3928 
3929 			/* turn on 2nd parsing and get a BD */
3930 			bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3931 
3932 			pbd2 = &txdata->tx_desc_ring[bd_prod].parse_2nd_bd;
3933 
3934 			memset(pbd2, 0, sizeof(*pbd2));
3935 
3936 			pbd_e2->data.tunnel_data.ip_hdr_start_inner_w =
3937 				(skb_inner_network_header(skb) -
3938 				 skb->data) >> 1;
3939 
3940 			if (xmit_type & XMIT_GSO_ENC)
3941 				bnx2x_update_pbds_gso_enc(skb, pbd_e2, pbd2,
3942 							  &global_data,
3943 							  xmit_type);
3944 
3945 			pbd2->global_data = cpu_to_le16(global_data);
3946 
3947 			/* add addition parse BD indication to start BD */
3948 			SET_FLAG(tx_start_bd->general_data,
3949 				 ETH_TX_START_BD_PARSE_NBDS, 1);
3950 			/* set encapsulation flag in start BD */
3951 			SET_FLAG(tx_start_bd->general_data,
3952 				 ETH_TX_START_BD_TUNNEL_EXIST, 1);
3953 
3954 			tx_buf->flags |= BNX2X_HAS_SECOND_PBD;
3955 
3956 			nbd++;
3957 		} else if (xmit_type & XMIT_CSUM) {
3958 			/* Set PBD in checksum offload case w/o encapsulation */
3959 			hlen = bnx2x_set_pbd_csum_e2(bp, skb,
3960 						     &pbd_e2_parsing_data,
3961 						     xmit_type);
3962 		}
3963 
3964 		bnx2x_set_ipv6_ext_e2(skb, &pbd_e2_parsing_data, xmit_type);
3965 		/* Add the macs to the parsing BD if this is a vf or if
3966 		 * Tx Switching is enabled.
3967 		 */
3968 		if (IS_VF(bp)) {
3969 			/* override GRE parameters in BD */
3970 			bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.src_hi,
3971 					      &pbd_e2->data.mac_addr.src_mid,
3972 					      &pbd_e2->data.mac_addr.src_lo,
3973 					      eth->h_source);
3974 
3975 			bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.dst_hi,
3976 					      &pbd_e2->data.mac_addr.dst_mid,
3977 					      &pbd_e2->data.mac_addr.dst_lo,
3978 					      eth->h_dest);
3979 		} else {
3980 			if (bp->flags & TX_SWITCHING)
3981 				bnx2x_set_fw_mac_addr(
3982 						&pbd_e2->data.mac_addr.dst_hi,
3983 						&pbd_e2->data.mac_addr.dst_mid,
3984 						&pbd_e2->data.mac_addr.dst_lo,
3985 						eth->h_dest);
3986 #ifdef BNX2X_STOP_ON_ERROR
3987 			/* Enforce security is always set in Stop on Error -
3988 			 * source mac should be present in the parsing BD
3989 			 */
3990 			bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.src_hi,
3991 					      &pbd_e2->data.mac_addr.src_mid,
3992 					      &pbd_e2->data.mac_addr.src_lo,
3993 					      eth->h_source);
3994 #endif
3995 		}
3996 
3997 		SET_FLAG(pbd_e2_parsing_data,
3998 			 ETH_TX_PARSE_BD_E2_ETH_ADDR_TYPE, mac_type);
3999 	} else {
4000 		u16 global_data = 0;
4001 		pbd_e1x = &txdata->tx_desc_ring[bd_prod].parse_bd_e1x;
4002 		memset(pbd_e1x, 0, sizeof(struct eth_tx_parse_bd_e1x));
4003 		/* Set PBD in checksum offload case */
4004 		if (xmit_type & XMIT_CSUM)
4005 			hlen = bnx2x_set_pbd_csum(bp, skb, pbd_e1x, xmit_type);
4006 
4007 		SET_FLAG(global_data,
4008 			 ETH_TX_PARSE_BD_E1X_ETH_ADDR_TYPE, mac_type);
4009 		pbd_e1x->global_data |= cpu_to_le16(global_data);
4010 	}
4011 
4012 	/* Setup the data pointer of the first BD of the packet */
4013 	tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
4014 	tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
4015 	tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
4016 	pkt_size = tx_start_bd->nbytes;
4017 
4018 	DP(NETIF_MSG_TX_QUEUED,
4019 	   "first bd @%p  addr (%x:%x)  nbytes %d  flags %x  vlan %x\n",
4020 	   tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
4021 	   le16_to_cpu(tx_start_bd->nbytes),
4022 	   tx_start_bd->bd_flags.as_bitfield,
4023 	   le16_to_cpu(tx_start_bd->vlan_or_ethertype));
4024 
4025 	if (xmit_type & XMIT_GSO) {
4026 
4027 		DP(NETIF_MSG_TX_QUEUED,
4028 		   "TSO packet len %d  hlen %d  total len %d  tso size %d\n",
4029 		   skb->len, hlen, skb_headlen(skb),
4030 		   skb_shinfo(skb)->gso_size);
4031 
4032 		tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
4033 
4034 		if (unlikely(skb_headlen(skb) > hlen)) {
4035 			nbd++;
4036 			bd_prod = bnx2x_tx_split(bp, txdata, tx_buf,
4037 						 &tx_start_bd, hlen,
4038 						 bd_prod);
4039 		}
4040 		if (!CHIP_IS_E1x(bp))
4041 			pbd_e2_parsing_data |=
4042 				(skb_shinfo(skb)->gso_size <<
4043 				 ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT) &
4044 				 ETH_TX_PARSE_BD_E2_LSO_MSS;
4045 		else
4046 			bnx2x_set_pbd_gso(skb, pbd_e1x, xmit_type);
4047 	}
4048 
4049 	/* Set the PBD's parsing_data field if not zero
4050 	 * (for the chips newer than 57711).
4051 	 */
4052 	if (pbd_e2_parsing_data)
4053 		pbd_e2->parsing_data = cpu_to_le32(pbd_e2_parsing_data);
4054 
4055 	tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
4056 
4057 	/* Handle fragmented skb */
4058 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
4059 		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
4060 
4061 		mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 0,
4062 					   skb_frag_size(frag), DMA_TO_DEVICE);
4063 		if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
4064 			unsigned int pkts_compl = 0, bytes_compl = 0;
4065 
4066 			DP(NETIF_MSG_TX_QUEUED,
4067 			   "Unable to map page - dropping packet...\n");
4068 
4069 			/* we need unmap all buffers already mapped
4070 			 * for this SKB;
4071 			 * first_bd->nbd need to be properly updated
4072 			 * before call to bnx2x_free_tx_pkt
4073 			 */
4074 			first_bd->nbd = cpu_to_le16(nbd);
4075 			bnx2x_free_tx_pkt(bp, txdata,
4076 					  TX_BD(txdata->tx_pkt_prod),
4077 					  &pkts_compl, &bytes_compl);
4078 			return NETDEV_TX_OK;
4079 		}
4080 
4081 		bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
4082 		tx_data_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
4083 		if (total_pkt_bd == NULL)
4084 			total_pkt_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
4085 
4086 		tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
4087 		tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
4088 		tx_data_bd->nbytes = cpu_to_le16(skb_frag_size(frag));
4089 		le16_add_cpu(&pkt_size, skb_frag_size(frag));
4090 		nbd++;
4091 
4092 		DP(NETIF_MSG_TX_QUEUED,
4093 		   "frag %d  bd @%p  addr (%x:%x)  nbytes %d\n",
4094 		   i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
4095 		   le16_to_cpu(tx_data_bd->nbytes));
4096 	}
4097 
4098 	DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
4099 
4100 	/* update with actual num BDs */
4101 	first_bd->nbd = cpu_to_le16(nbd);
4102 
4103 	bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
4104 
4105 	/* now send a tx doorbell, counting the next BD
4106 	 * if the packet contains or ends with it
4107 	 */
4108 	if (TX_BD_POFF(bd_prod) < nbd)
4109 		nbd++;
4110 
4111 	/* total_pkt_bytes should be set on the first data BD if
4112 	 * it's not an LSO packet and there is more than one
4113 	 * data BD. In this case pkt_size is limited by an MTU value.
4114 	 * However we prefer to set it for an LSO packet (while we don't
4115 	 * have to) in order to save some CPU cycles in a none-LSO
4116 	 * case, when we much more care about them.
4117 	 */
4118 	if (total_pkt_bd != NULL)
4119 		total_pkt_bd->total_pkt_bytes = pkt_size;
4120 
4121 	if (pbd_e1x)
4122 		DP(NETIF_MSG_TX_QUEUED,
4123 		   "PBD (E1X) @%p  ip_data %x  ip_hlen %u  ip_id %u  lso_mss %u  tcp_flags %x  xsum %x  seq %u  hlen %u\n",
4124 		   pbd_e1x, pbd_e1x->global_data, pbd_e1x->ip_hlen_w,
4125 		   pbd_e1x->ip_id, pbd_e1x->lso_mss, pbd_e1x->tcp_flags,
4126 		   pbd_e1x->tcp_pseudo_csum, pbd_e1x->tcp_send_seq,
4127 		    le16_to_cpu(pbd_e1x->total_hlen_w));
4128 	if (pbd_e2)
4129 		DP(NETIF_MSG_TX_QUEUED,
4130 		   "PBD (E2) @%p  dst %x %x %x src %x %x %x parsing_data %x\n",
4131 		   pbd_e2,
4132 		   pbd_e2->data.mac_addr.dst_hi,
4133 		   pbd_e2->data.mac_addr.dst_mid,
4134 		   pbd_e2->data.mac_addr.dst_lo,
4135 		   pbd_e2->data.mac_addr.src_hi,
4136 		   pbd_e2->data.mac_addr.src_mid,
4137 		   pbd_e2->data.mac_addr.src_lo,
4138 		   pbd_e2->parsing_data);
4139 	DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d  bd %u\n", nbd, bd_prod);
4140 
4141 	netdev_tx_sent_queue(txq, skb->len);
4142 
4143 	skb_tx_timestamp(skb);
4144 
4145 	txdata->tx_pkt_prod++;
4146 	/*
4147 	 * Make sure that the BD data is updated before updating the producer
4148 	 * since FW might read the BD right after the producer is updated.
4149 	 * This is only applicable for weak-ordered memory model archs such
4150 	 * as IA-64. The following barrier is also mandatory since FW will
4151 	 * assumes packets must have BDs.
4152 	 */
4153 	wmb();
4154 
4155 	txdata->tx_db.data.prod += nbd;
4156 	barrier();
4157 
4158 	DOORBELL(bp, txdata->cid, txdata->tx_db.raw);
4159 
4160 	mmiowb();
4161 
4162 	txdata->tx_bd_prod += nbd;
4163 
4164 	if (unlikely(bnx2x_tx_avail(bp, txdata) < MAX_DESC_PER_TX_PKT)) {
4165 		netif_tx_stop_queue(txq);
4166 
4167 		/* paired memory barrier is in bnx2x_tx_int(), we have to keep
4168 		 * ordering of set_bit() in netif_tx_stop_queue() and read of
4169 		 * fp->bd_tx_cons */
4170 		smp_mb();
4171 
4172 		bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
4173 		if (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT)
4174 			netif_tx_wake_queue(txq);
4175 	}
4176 	txdata->tx_pkt++;
4177 
4178 	return NETDEV_TX_OK;
4179 }
4180 
4181 void bnx2x_get_c2s_mapping(struct bnx2x *bp, u8 *c2s_map, u8 *c2s_default)
4182 {
4183 	int mfw_vn = BP_FW_MB_IDX(bp);
4184 	u32 tmp;
4185 
4186 	/* If the shmem shouldn't affect configuration, reflect */
4187 	if (!IS_MF_BD(bp)) {
4188 		int i;
4189 
4190 		for (i = 0; i < BNX2X_MAX_PRIORITY; i++)
4191 			c2s_map[i] = i;
4192 		*c2s_default = 0;
4193 
4194 		return;
4195 	}
4196 
4197 	tmp = SHMEM2_RD(bp, c2s_pcp_map_lower[mfw_vn]);
4198 	tmp = (__force u32)be32_to_cpu((__force __be32)tmp);
4199 	c2s_map[0] = tmp & 0xff;
4200 	c2s_map[1] = (tmp >> 8) & 0xff;
4201 	c2s_map[2] = (tmp >> 16) & 0xff;
4202 	c2s_map[3] = (tmp >> 24) & 0xff;
4203 
4204 	tmp = SHMEM2_RD(bp, c2s_pcp_map_upper[mfw_vn]);
4205 	tmp = (__force u32)be32_to_cpu((__force __be32)tmp);
4206 	c2s_map[4] = tmp & 0xff;
4207 	c2s_map[5] = (tmp >> 8) & 0xff;
4208 	c2s_map[6] = (tmp >> 16) & 0xff;
4209 	c2s_map[7] = (tmp >> 24) & 0xff;
4210 
4211 	tmp = SHMEM2_RD(bp, c2s_pcp_map_default[mfw_vn]);
4212 	tmp = (__force u32)be32_to_cpu((__force __be32)tmp);
4213 	*c2s_default = (tmp >> (8 * mfw_vn)) & 0xff;
4214 }
4215 
4216 /**
4217  * bnx2x_setup_tc - routine to configure net_device for multi tc
4218  *
4219  * @netdev: net device to configure
4220  * @tc: number of traffic classes to enable
4221  *
4222  * callback connected to the ndo_setup_tc function pointer
4223  */
4224 int bnx2x_setup_tc(struct net_device *dev, u8 num_tc)
4225 {
4226 	struct bnx2x *bp = netdev_priv(dev);
4227 	u8 c2s_map[BNX2X_MAX_PRIORITY], c2s_def;
4228 	int cos, prio, count, offset;
4229 
4230 	/* setup tc must be called under rtnl lock */
4231 	ASSERT_RTNL();
4232 
4233 	/* no traffic classes requested. Aborting */
4234 	if (!num_tc) {
4235 		netdev_reset_tc(dev);
4236 		return 0;
4237 	}
4238 
4239 	/* requested to support too many traffic classes */
4240 	if (num_tc > bp->max_cos) {
4241 		BNX2X_ERR("support for too many traffic classes requested: %d. Max supported is %d\n",
4242 			  num_tc, bp->max_cos);
4243 		return -EINVAL;
4244 	}
4245 
4246 	/* declare amount of supported traffic classes */
4247 	if (netdev_set_num_tc(dev, num_tc)) {
4248 		BNX2X_ERR("failed to declare %d traffic classes\n", num_tc);
4249 		return -EINVAL;
4250 	}
4251 
4252 	bnx2x_get_c2s_mapping(bp, c2s_map, &c2s_def);
4253 
4254 	/* configure priority to traffic class mapping */
4255 	for (prio = 0; prio < BNX2X_MAX_PRIORITY; prio++) {
4256 		int outer_prio = c2s_map[prio];
4257 
4258 		netdev_set_prio_tc_map(dev, prio, bp->prio_to_cos[outer_prio]);
4259 		DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
4260 		   "mapping priority %d to tc %d\n",
4261 		   outer_prio, bp->prio_to_cos[outer_prio]);
4262 	}
4263 
4264 	/* Use this configuration to differentiate tc0 from other COSes
4265 	   This can be used for ets or pfc, and save the effort of setting
4266 	   up a multio class queue disc or negotiating DCBX with a switch
4267 	netdev_set_prio_tc_map(dev, 0, 0);
4268 	DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", 0, 0);
4269 	for (prio = 1; prio < 16; prio++) {
4270 		netdev_set_prio_tc_map(dev, prio, 1);
4271 		DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", prio, 1);
4272 	} */
4273 
4274 	/* configure traffic class to transmission queue mapping */
4275 	for (cos = 0; cos < bp->max_cos; cos++) {
4276 		count = BNX2X_NUM_ETH_QUEUES(bp);
4277 		offset = cos * BNX2X_NUM_NON_CNIC_QUEUES(bp);
4278 		netdev_set_tc_queue(dev, cos, count, offset);
4279 		DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
4280 		   "mapping tc %d to offset %d count %d\n",
4281 		   cos, offset, count);
4282 	}
4283 
4284 	return 0;
4285 }
4286 
4287 int __bnx2x_setup_tc(struct net_device *dev, enum tc_setup_type type,
4288 		     void *type_data)
4289 {
4290 	struct tc_mqprio_qopt *mqprio = type_data;
4291 
4292 	if (type != TC_SETUP_QDISC_MQPRIO)
4293 		return -EOPNOTSUPP;
4294 
4295 	mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
4296 
4297 	return bnx2x_setup_tc(dev, mqprio->num_tc);
4298 }
4299 
4300 /* called with rtnl_lock */
4301 int bnx2x_change_mac_addr(struct net_device *dev, void *p)
4302 {
4303 	struct sockaddr *addr = p;
4304 	struct bnx2x *bp = netdev_priv(dev);
4305 	int rc = 0;
4306 
4307 	if (!is_valid_ether_addr(addr->sa_data)) {
4308 		BNX2X_ERR("Requested MAC address is not valid\n");
4309 		return -EINVAL;
4310 	}
4311 
4312 	if (IS_MF_STORAGE_ONLY(bp)) {
4313 		BNX2X_ERR("Can't change address on STORAGE ONLY function\n");
4314 		return -EINVAL;
4315 	}
4316 
4317 	if (netif_running(dev))  {
4318 		rc = bnx2x_set_eth_mac(bp, false);
4319 		if (rc)
4320 			return rc;
4321 	}
4322 
4323 	memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
4324 
4325 	if (netif_running(dev))
4326 		rc = bnx2x_set_eth_mac(bp, true);
4327 
4328 	if (IS_PF(bp) && SHMEM2_HAS(bp, curr_cfg))
4329 		SHMEM2_WR(bp, curr_cfg, CURR_CFG_MET_OS);
4330 
4331 	return rc;
4332 }
4333 
4334 static void bnx2x_free_fp_mem_at(struct bnx2x *bp, int fp_index)
4335 {
4336 	union host_hc_status_block *sb = &bnx2x_fp(bp, fp_index, status_blk);
4337 	struct bnx2x_fastpath *fp = &bp->fp[fp_index];
4338 	u8 cos;
4339 
4340 	/* Common */
4341 
4342 	if (IS_FCOE_IDX(fp_index)) {
4343 		memset(sb, 0, sizeof(union host_hc_status_block));
4344 		fp->status_blk_mapping = 0;
4345 	} else {
4346 		/* status blocks */
4347 		if (!CHIP_IS_E1x(bp))
4348 			BNX2X_PCI_FREE(sb->e2_sb,
4349 				       bnx2x_fp(bp, fp_index,
4350 						status_blk_mapping),
4351 				       sizeof(struct host_hc_status_block_e2));
4352 		else
4353 			BNX2X_PCI_FREE(sb->e1x_sb,
4354 				       bnx2x_fp(bp, fp_index,
4355 						status_blk_mapping),
4356 				       sizeof(struct host_hc_status_block_e1x));
4357 	}
4358 
4359 	/* Rx */
4360 	if (!skip_rx_queue(bp, fp_index)) {
4361 		bnx2x_free_rx_bds(fp);
4362 
4363 		/* fastpath rx rings: rx_buf rx_desc rx_comp */
4364 		BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_buf_ring));
4365 		BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_desc_ring),
4366 			       bnx2x_fp(bp, fp_index, rx_desc_mapping),
4367 			       sizeof(struct eth_rx_bd) * NUM_RX_BD);
4368 
4369 		BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_comp_ring),
4370 			       bnx2x_fp(bp, fp_index, rx_comp_mapping),
4371 			       sizeof(struct eth_fast_path_rx_cqe) *
4372 			       NUM_RCQ_BD);
4373 
4374 		/* SGE ring */
4375 		BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_page_ring));
4376 		BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_sge_ring),
4377 			       bnx2x_fp(bp, fp_index, rx_sge_mapping),
4378 			       BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4379 	}
4380 
4381 	/* Tx */
4382 	if (!skip_tx_queue(bp, fp_index)) {
4383 		/* fastpath tx rings: tx_buf tx_desc */
4384 		for_each_cos_in_tx_queue(fp, cos) {
4385 			struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
4386 
4387 			DP(NETIF_MSG_IFDOWN,
4388 			   "freeing tx memory of fp %d cos %d cid %d\n",
4389 			   fp_index, cos, txdata->cid);
4390 
4391 			BNX2X_FREE(txdata->tx_buf_ring);
4392 			BNX2X_PCI_FREE(txdata->tx_desc_ring,
4393 				txdata->tx_desc_mapping,
4394 				sizeof(union eth_tx_bd_types) * NUM_TX_BD);
4395 		}
4396 	}
4397 	/* end of fastpath */
4398 }
4399 
4400 static void bnx2x_free_fp_mem_cnic(struct bnx2x *bp)
4401 {
4402 	int i;
4403 	for_each_cnic_queue(bp, i)
4404 		bnx2x_free_fp_mem_at(bp, i);
4405 }
4406 
4407 void bnx2x_free_fp_mem(struct bnx2x *bp)
4408 {
4409 	int i;
4410 	for_each_eth_queue(bp, i)
4411 		bnx2x_free_fp_mem_at(bp, i);
4412 }
4413 
4414 static void set_sb_shortcuts(struct bnx2x *bp, int index)
4415 {
4416 	union host_hc_status_block status_blk = bnx2x_fp(bp, index, status_blk);
4417 	if (!CHIP_IS_E1x(bp)) {
4418 		bnx2x_fp(bp, index, sb_index_values) =
4419 			(__le16 *)status_blk.e2_sb->sb.index_values;
4420 		bnx2x_fp(bp, index, sb_running_index) =
4421 			(__le16 *)status_blk.e2_sb->sb.running_index;
4422 	} else {
4423 		bnx2x_fp(bp, index, sb_index_values) =
4424 			(__le16 *)status_blk.e1x_sb->sb.index_values;
4425 		bnx2x_fp(bp, index, sb_running_index) =
4426 			(__le16 *)status_blk.e1x_sb->sb.running_index;
4427 	}
4428 }
4429 
4430 /* Returns the number of actually allocated BDs */
4431 static int bnx2x_alloc_rx_bds(struct bnx2x_fastpath *fp,
4432 			      int rx_ring_size)
4433 {
4434 	struct bnx2x *bp = fp->bp;
4435 	u16 ring_prod, cqe_ring_prod;
4436 	int i, failure_cnt = 0;
4437 
4438 	fp->rx_comp_cons = 0;
4439 	cqe_ring_prod = ring_prod = 0;
4440 
4441 	/* This routine is called only during fo init so
4442 	 * fp->eth_q_stats.rx_skb_alloc_failed = 0
4443 	 */
4444 	for (i = 0; i < rx_ring_size; i++) {
4445 		if (bnx2x_alloc_rx_data(bp, fp, ring_prod, GFP_KERNEL) < 0) {
4446 			failure_cnt++;
4447 			continue;
4448 		}
4449 		ring_prod = NEXT_RX_IDX(ring_prod);
4450 		cqe_ring_prod = NEXT_RCQ_IDX(cqe_ring_prod);
4451 		WARN_ON(ring_prod <= (i - failure_cnt));
4452 	}
4453 
4454 	if (failure_cnt)
4455 		BNX2X_ERR("was only able to allocate %d rx skbs on queue[%d]\n",
4456 			  i - failure_cnt, fp->index);
4457 
4458 	fp->rx_bd_prod = ring_prod;
4459 	/* Limit the CQE producer by the CQE ring size */
4460 	fp->rx_comp_prod = min_t(u16, NUM_RCQ_RINGS*RCQ_DESC_CNT,
4461 			       cqe_ring_prod);
4462 
4463 	bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed += failure_cnt;
4464 
4465 	return i - failure_cnt;
4466 }
4467 
4468 static void bnx2x_set_next_page_rx_cq(struct bnx2x_fastpath *fp)
4469 {
4470 	int i;
4471 
4472 	for (i = 1; i <= NUM_RCQ_RINGS; i++) {
4473 		struct eth_rx_cqe_next_page *nextpg;
4474 
4475 		nextpg = (struct eth_rx_cqe_next_page *)
4476 			&fp->rx_comp_ring[RCQ_DESC_CNT * i - 1];
4477 		nextpg->addr_hi =
4478 			cpu_to_le32(U64_HI(fp->rx_comp_mapping +
4479 				   BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4480 		nextpg->addr_lo =
4481 			cpu_to_le32(U64_LO(fp->rx_comp_mapping +
4482 				   BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4483 	}
4484 }
4485 
4486 static int bnx2x_alloc_fp_mem_at(struct bnx2x *bp, int index)
4487 {
4488 	union host_hc_status_block *sb;
4489 	struct bnx2x_fastpath *fp = &bp->fp[index];
4490 	int ring_size = 0;
4491 	u8 cos;
4492 	int rx_ring_size = 0;
4493 
4494 	if (!bp->rx_ring_size && IS_MF_STORAGE_ONLY(bp)) {
4495 		rx_ring_size = MIN_RX_SIZE_NONTPA;
4496 		bp->rx_ring_size = rx_ring_size;
4497 	} else if (!bp->rx_ring_size) {
4498 		rx_ring_size = MAX_RX_AVAIL/BNX2X_NUM_RX_QUEUES(bp);
4499 
4500 		if (CHIP_IS_E3(bp)) {
4501 			u32 cfg = SHMEM_RD(bp,
4502 					   dev_info.port_hw_config[BP_PORT(bp)].
4503 					   default_cfg);
4504 
4505 			/* Decrease ring size for 1G functions */
4506 			if ((cfg & PORT_HW_CFG_NET_SERDES_IF_MASK) ==
4507 			    PORT_HW_CFG_NET_SERDES_IF_SGMII)
4508 				rx_ring_size /= 10;
4509 		}
4510 
4511 		/* allocate at least number of buffers required by FW */
4512 		rx_ring_size = max_t(int, bp->disable_tpa ? MIN_RX_SIZE_NONTPA :
4513 				     MIN_RX_SIZE_TPA, rx_ring_size);
4514 
4515 		bp->rx_ring_size = rx_ring_size;
4516 	} else /* if rx_ring_size specified - use it */
4517 		rx_ring_size = bp->rx_ring_size;
4518 
4519 	DP(BNX2X_MSG_SP, "calculated rx_ring_size %d\n", rx_ring_size);
4520 
4521 	/* Common */
4522 	sb = &bnx2x_fp(bp, index, status_blk);
4523 
4524 	if (!IS_FCOE_IDX(index)) {
4525 		/* status blocks */
4526 		if (!CHIP_IS_E1x(bp)) {
4527 			sb->e2_sb = BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, status_blk_mapping),
4528 						    sizeof(struct host_hc_status_block_e2));
4529 			if (!sb->e2_sb)
4530 				goto alloc_mem_err;
4531 		} else {
4532 			sb->e1x_sb = BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, status_blk_mapping),
4533 						     sizeof(struct host_hc_status_block_e1x));
4534 			if (!sb->e1x_sb)
4535 				goto alloc_mem_err;
4536 		}
4537 	}
4538 
4539 	/* FCoE Queue uses Default SB and doesn't ACK the SB, thus no need to
4540 	 * set shortcuts for it.
4541 	 */
4542 	if (!IS_FCOE_IDX(index))
4543 		set_sb_shortcuts(bp, index);
4544 
4545 	/* Tx */
4546 	if (!skip_tx_queue(bp, index)) {
4547 		/* fastpath tx rings: tx_buf tx_desc */
4548 		for_each_cos_in_tx_queue(fp, cos) {
4549 			struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
4550 
4551 			DP(NETIF_MSG_IFUP,
4552 			   "allocating tx memory of fp %d cos %d\n",
4553 			   index, cos);
4554 
4555 			txdata->tx_buf_ring = kcalloc(NUM_TX_BD,
4556 						      sizeof(struct sw_tx_bd),
4557 						      GFP_KERNEL);
4558 			if (!txdata->tx_buf_ring)
4559 				goto alloc_mem_err;
4560 			txdata->tx_desc_ring = BNX2X_PCI_ALLOC(&txdata->tx_desc_mapping,
4561 							       sizeof(union eth_tx_bd_types) * NUM_TX_BD);
4562 			if (!txdata->tx_desc_ring)
4563 				goto alloc_mem_err;
4564 		}
4565 	}
4566 
4567 	/* Rx */
4568 	if (!skip_rx_queue(bp, index)) {
4569 		/* fastpath rx rings: rx_buf rx_desc rx_comp */
4570 		bnx2x_fp(bp, index, rx_buf_ring) =
4571 			kcalloc(NUM_RX_BD, sizeof(struct sw_rx_bd), GFP_KERNEL);
4572 		if (!bnx2x_fp(bp, index, rx_buf_ring))
4573 			goto alloc_mem_err;
4574 		bnx2x_fp(bp, index, rx_desc_ring) =
4575 			BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, rx_desc_mapping),
4576 					sizeof(struct eth_rx_bd) * NUM_RX_BD);
4577 		if (!bnx2x_fp(bp, index, rx_desc_ring))
4578 			goto alloc_mem_err;
4579 
4580 		/* Seed all CQEs by 1s */
4581 		bnx2x_fp(bp, index, rx_comp_ring) =
4582 			BNX2X_PCI_FALLOC(&bnx2x_fp(bp, index, rx_comp_mapping),
4583 					 sizeof(struct eth_fast_path_rx_cqe) * NUM_RCQ_BD);
4584 		if (!bnx2x_fp(bp, index, rx_comp_ring))
4585 			goto alloc_mem_err;
4586 
4587 		/* SGE ring */
4588 		bnx2x_fp(bp, index, rx_page_ring) =
4589 			kcalloc(NUM_RX_SGE, sizeof(struct sw_rx_page),
4590 				GFP_KERNEL);
4591 		if (!bnx2x_fp(bp, index, rx_page_ring))
4592 			goto alloc_mem_err;
4593 		bnx2x_fp(bp, index, rx_sge_ring) =
4594 			BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, rx_sge_mapping),
4595 					BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4596 		if (!bnx2x_fp(bp, index, rx_sge_ring))
4597 			goto alloc_mem_err;
4598 		/* RX BD ring */
4599 		bnx2x_set_next_page_rx_bd(fp);
4600 
4601 		/* CQ ring */
4602 		bnx2x_set_next_page_rx_cq(fp);
4603 
4604 		/* BDs */
4605 		ring_size = bnx2x_alloc_rx_bds(fp, rx_ring_size);
4606 		if (ring_size < rx_ring_size)
4607 			goto alloc_mem_err;
4608 	}
4609 
4610 	return 0;
4611 
4612 /* handles low memory cases */
4613 alloc_mem_err:
4614 	BNX2X_ERR("Unable to allocate full memory for queue %d (size %d)\n",
4615 						index, ring_size);
4616 	/* FW will drop all packets if queue is not big enough,
4617 	 * In these cases we disable the queue
4618 	 * Min size is different for OOO, TPA and non-TPA queues
4619 	 */
4620 	if (ring_size < (fp->mode == TPA_MODE_DISABLED ?
4621 				MIN_RX_SIZE_NONTPA : MIN_RX_SIZE_TPA)) {
4622 			/* release memory allocated for this queue */
4623 			bnx2x_free_fp_mem_at(bp, index);
4624 			return -ENOMEM;
4625 	}
4626 	return 0;
4627 }
4628 
4629 static int bnx2x_alloc_fp_mem_cnic(struct bnx2x *bp)
4630 {
4631 	if (!NO_FCOE(bp))
4632 		/* FCoE */
4633 		if (bnx2x_alloc_fp_mem_at(bp, FCOE_IDX(bp)))
4634 			/* we will fail load process instead of mark
4635 			 * NO_FCOE_FLAG
4636 			 */
4637 			return -ENOMEM;
4638 
4639 	return 0;
4640 }
4641 
4642 static int bnx2x_alloc_fp_mem(struct bnx2x *bp)
4643 {
4644 	int i;
4645 
4646 	/* 1. Allocate FP for leading - fatal if error
4647 	 * 2. Allocate RSS - fix number of queues if error
4648 	 */
4649 
4650 	/* leading */
4651 	if (bnx2x_alloc_fp_mem_at(bp, 0))
4652 		return -ENOMEM;
4653 
4654 	/* RSS */
4655 	for_each_nondefault_eth_queue(bp, i)
4656 		if (bnx2x_alloc_fp_mem_at(bp, i))
4657 			break;
4658 
4659 	/* handle memory failures */
4660 	if (i != BNX2X_NUM_ETH_QUEUES(bp)) {
4661 		int delta = BNX2X_NUM_ETH_QUEUES(bp) - i;
4662 
4663 		WARN_ON(delta < 0);
4664 		bnx2x_shrink_eth_fp(bp, delta);
4665 		if (CNIC_SUPPORT(bp))
4666 			/* move non eth FPs next to last eth FP
4667 			 * must be done in that order
4668 			 * FCOE_IDX < FWD_IDX < OOO_IDX
4669 			 */
4670 
4671 			/* move FCoE fp even NO_FCOE_FLAG is on */
4672 			bnx2x_move_fp(bp, FCOE_IDX(bp), FCOE_IDX(bp) - delta);
4673 		bp->num_ethernet_queues -= delta;
4674 		bp->num_queues = bp->num_ethernet_queues +
4675 				 bp->num_cnic_queues;
4676 		BNX2X_ERR("Adjusted num of queues from %d to %d\n",
4677 			  bp->num_queues + delta, bp->num_queues);
4678 	}
4679 
4680 	return 0;
4681 }
4682 
4683 void bnx2x_free_mem_bp(struct bnx2x *bp)
4684 {
4685 	int i;
4686 
4687 	for (i = 0; i < bp->fp_array_size; i++)
4688 		kfree(bp->fp[i].tpa_info);
4689 	kfree(bp->fp);
4690 	kfree(bp->sp_objs);
4691 	kfree(bp->fp_stats);
4692 	kfree(bp->bnx2x_txq);
4693 	kfree(bp->msix_table);
4694 	kfree(bp->ilt);
4695 }
4696 
4697 int bnx2x_alloc_mem_bp(struct bnx2x *bp)
4698 {
4699 	struct bnx2x_fastpath *fp;
4700 	struct msix_entry *tbl;
4701 	struct bnx2x_ilt *ilt;
4702 	int msix_table_size = 0;
4703 	int fp_array_size, txq_array_size;
4704 	int i;
4705 
4706 	/*
4707 	 * The biggest MSI-X table we might need is as a maximum number of fast
4708 	 * path IGU SBs plus default SB (for PF only).
4709 	 */
4710 	msix_table_size = bp->igu_sb_cnt;
4711 	if (IS_PF(bp))
4712 		msix_table_size++;
4713 	BNX2X_DEV_INFO("msix_table_size %d\n", msix_table_size);
4714 
4715 	/* fp array: RSS plus CNIC related L2 queues */
4716 	fp_array_size = BNX2X_MAX_RSS_COUNT(bp) + CNIC_SUPPORT(bp);
4717 	bp->fp_array_size = fp_array_size;
4718 	BNX2X_DEV_INFO("fp_array_size %d\n", bp->fp_array_size);
4719 
4720 	fp = kcalloc(bp->fp_array_size, sizeof(*fp), GFP_KERNEL);
4721 	if (!fp)
4722 		goto alloc_err;
4723 	for (i = 0; i < bp->fp_array_size; i++) {
4724 		fp[i].tpa_info =
4725 			kcalloc(ETH_MAX_AGGREGATION_QUEUES_E1H_E2,
4726 				sizeof(struct bnx2x_agg_info), GFP_KERNEL);
4727 		if (!(fp[i].tpa_info))
4728 			goto alloc_err;
4729 	}
4730 
4731 	bp->fp = fp;
4732 
4733 	/* allocate sp objs */
4734 	bp->sp_objs = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_sp_objs),
4735 			      GFP_KERNEL);
4736 	if (!bp->sp_objs)
4737 		goto alloc_err;
4738 
4739 	/* allocate fp_stats */
4740 	bp->fp_stats = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_fp_stats),
4741 			       GFP_KERNEL);
4742 	if (!bp->fp_stats)
4743 		goto alloc_err;
4744 
4745 	/* Allocate memory for the transmission queues array */
4746 	txq_array_size =
4747 		BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS + CNIC_SUPPORT(bp);
4748 	BNX2X_DEV_INFO("txq_array_size %d", txq_array_size);
4749 
4750 	bp->bnx2x_txq = kcalloc(txq_array_size, sizeof(struct bnx2x_fp_txdata),
4751 				GFP_KERNEL);
4752 	if (!bp->bnx2x_txq)
4753 		goto alloc_err;
4754 
4755 	/* msix table */
4756 	tbl = kcalloc(msix_table_size, sizeof(*tbl), GFP_KERNEL);
4757 	if (!tbl)
4758 		goto alloc_err;
4759 	bp->msix_table = tbl;
4760 
4761 	/* ilt */
4762 	ilt = kzalloc(sizeof(*ilt), GFP_KERNEL);
4763 	if (!ilt)
4764 		goto alloc_err;
4765 	bp->ilt = ilt;
4766 
4767 	return 0;
4768 alloc_err:
4769 	bnx2x_free_mem_bp(bp);
4770 	return -ENOMEM;
4771 }
4772 
4773 int bnx2x_reload_if_running(struct net_device *dev)
4774 {
4775 	struct bnx2x *bp = netdev_priv(dev);
4776 
4777 	if (unlikely(!netif_running(dev)))
4778 		return 0;
4779 
4780 	bnx2x_nic_unload(bp, UNLOAD_NORMAL, true);
4781 	return bnx2x_nic_load(bp, LOAD_NORMAL);
4782 }
4783 
4784 int bnx2x_get_cur_phy_idx(struct bnx2x *bp)
4785 {
4786 	u32 sel_phy_idx = 0;
4787 	if (bp->link_params.num_phys <= 1)
4788 		return INT_PHY;
4789 
4790 	if (bp->link_vars.link_up) {
4791 		sel_phy_idx = EXT_PHY1;
4792 		/* In case link is SERDES, check if the EXT_PHY2 is the one */
4793 		if ((bp->link_vars.link_status & LINK_STATUS_SERDES_LINK) &&
4794 		    (bp->link_params.phy[EXT_PHY2].supported & SUPPORTED_FIBRE))
4795 			sel_phy_idx = EXT_PHY2;
4796 	} else {
4797 
4798 		switch (bnx2x_phy_selection(&bp->link_params)) {
4799 		case PORT_HW_CFG_PHY_SELECTION_HARDWARE_DEFAULT:
4800 		case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY:
4801 		case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY_PRIORITY:
4802 		       sel_phy_idx = EXT_PHY1;
4803 		       break;
4804 		case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY:
4805 		case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY_PRIORITY:
4806 		       sel_phy_idx = EXT_PHY2;
4807 		       break;
4808 		}
4809 	}
4810 
4811 	return sel_phy_idx;
4812 }
4813 int bnx2x_get_link_cfg_idx(struct bnx2x *bp)
4814 {
4815 	u32 sel_phy_idx = bnx2x_get_cur_phy_idx(bp);
4816 	/*
4817 	 * The selected activated PHY is always after swapping (in case PHY
4818 	 * swapping is enabled). So when swapping is enabled, we need to reverse
4819 	 * the configuration
4820 	 */
4821 
4822 	if (bp->link_params.multi_phy_config &
4823 	    PORT_HW_CFG_PHY_SWAPPED_ENABLED) {
4824 		if (sel_phy_idx == EXT_PHY1)
4825 			sel_phy_idx = EXT_PHY2;
4826 		else if (sel_phy_idx == EXT_PHY2)
4827 			sel_phy_idx = EXT_PHY1;
4828 	}
4829 	return LINK_CONFIG_IDX(sel_phy_idx);
4830 }
4831 
4832 #ifdef NETDEV_FCOE_WWNN
4833 int bnx2x_fcoe_get_wwn(struct net_device *dev, u64 *wwn, int type)
4834 {
4835 	struct bnx2x *bp = netdev_priv(dev);
4836 	struct cnic_eth_dev *cp = &bp->cnic_eth_dev;
4837 
4838 	switch (type) {
4839 	case NETDEV_FCOE_WWNN:
4840 		*wwn = HILO_U64(cp->fcoe_wwn_node_name_hi,
4841 				cp->fcoe_wwn_node_name_lo);
4842 		break;
4843 	case NETDEV_FCOE_WWPN:
4844 		*wwn = HILO_U64(cp->fcoe_wwn_port_name_hi,
4845 				cp->fcoe_wwn_port_name_lo);
4846 		break;
4847 	default:
4848 		BNX2X_ERR("Wrong WWN type requested - %d\n", type);
4849 		return -EINVAL;
4850 	}
4851 
4852 	return 0;
4853 }
4854 #endif
4855 
4856 /* called with rtnl_lock */
4857 int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
4858 {
4859 	struct bnx2x *bp = netdev_priv(dev);
4860 
4861 	if (pci_num_vf(bp->pdev)) {
4862 		DP(BNX2X_MSG_IOV, "VFs are enabled, can not change MTU\n");
4863 		return -EPERM;
4864 	}
4865 
4866 	if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
4867 		BNX2X_ERR("Can't perform change MTU during parity recovery\n");
4868 		return -EAGAIN;
4869 	}
4870 
4871 	/* This does not race with packet allocation
4872 	 * because the actual alloc size is
4873 	 * only updated as part of load
4874 	 */
4875 	dev->mtu = new_mtu;
4876 
4877 	if (!bnx2x_mtu_allows_gro(new_mtu))
4878 		dev->features &= ~NETIF_F_GRO_HW;
4879 
4880 	if (IS_PF(bp) && SHMEM2_HAS(bp, curr_cfg))
4881 		SHMEM2_WR(bp, curr_cfg, CURR_CFG_MET_OS);
4882 
4883 	return bnx2x_reload_if_running(dev);
4884 }
4885 
4886 netdev_features_t bnx2x_fix_features(struct net_device *dev,
4887 				     netdev_features_t features)
4888 {
4889 	struct bnx2x *bp = netdev_priv(dev);
4890 
4891 	if (pci_num_vf(bp->pdev)) {
4892 		netdev_features_t changed = dev->features ^ features;
4893 
4894 		/* Revert the requested changes in features if they
4895 		 * would require internal reload of PF in bnx2x_set_features().
4896 		 */
4897 		if (!(features & NETIF_F_RXCSUM) && !bp->disable_tpa) {
4898 			features &= ~NETIF_F_RXCSUM;
4899 			features |= dev->features & NETIF_F_RXCSUM;
4900 		}
4901 
4902 		if (changed & NETIF_F_LOOPBACK) {
4903 			features &= ~NETIF_F_LOOPBACK;
4904 			features |= dev->features & NETIF_F_LOOPBACK;
4905 		}
4906 	}
4907 
4908 	/* TPA requires Rx CSUM offloading */
4909 	if (!(features & NETIF_F_RXCSUM))
4910 		features &= ~NETIF_F_LRO;
4911 
4912 	if (!(features & NETIF_F_GRO) || !bnx2x_mtu_allows_gro(dev->mtu))
4913 		features &= ~NETIF_F_GRO_HW;
4914 	if (features & NETIF_F_GRO_HW)
4915 		features &= ~NETIF_F_LRO;
4916 
4917 	return features;
4918 }
4919 
4920 int bnx2x_set_features(struct net_device *dev, netdev_features_t features)
4921 {
4922 	struct bnx2x *bp = netdev_priv(dev);
4923 	netdev_features_t changes = features ^ dev->features;
4924 	bool bnx2x_reload = false;
4925 	int rc;
4926 
4927 	/* VFs or non SRIOV PFs should be able to change loopback feature */
4928 	if (!pci_num_vf(bp->pdev)) {
4929 		if (features & NETIF_F_LOOPBACK) {
4930 			if (bp->link_params.loopback_mode != LOOPBACK_BMAC) {
4931 				bp->link_params.loopback_mode = LOOPBACK_BMAC;
4932 				bnx2x_reload = true;
4933 			}
4934 		} else {
4935 			if (bp->link_params.loopback_mode != LOOPBACK_NONE) {
4936 				bp->link_params.loopback_mode = LOOPBACK_NONE;
4937 				bnx2x_reload = true;
4938 			}
4939 		}
4940 	}
4941 
4942 	/* Don't care about GRO changes */
4943 	changes &= ~NETIF_F_GRO;
4944 
4945 	if (changes)
4946 		bnx2x_reload = true;
4947 
4948 	if (bnx2x_reload) {
4949 		if (bp->recovery_state == BNX2X_RECOVERY_DONE) {
4950 			dev->features = features;
4951 			rc = bnx2x_reload_if_running(dev);
4952 			return rc ? rc : 1;
4953 		}
4954 		/* else: bnx2x_nic_load() will be called at end of recovery */
4955 	}
4956 
4957 	return 0;
4958 }
4959 
4960 void bnx2x_tx_timeout(struct net_device *dev)
4961 {
4962 	struct bnx2x *bp = netdev_priv(dev);
4963 
4964 #ifdef BNX2X_STOP_ON_ERROR
4965 	if (!bp->panic)
4966 		bnx2x_panic();
4967 #endif
4968 
4969 	/* This allows the netif to be shutdown gracefully before resetting */
4970 	bnx2x_schedule_sp_rtnl(bp, BNX2X_SP_RTNL_TX_TIMEOUT, 0);
4971 }
4972 
4973 int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state)
4974 {
4975 	struct net_device *dev = pci_get_drvdata(pdev);
4976 	struct bnx2x *bp;
4977 
4978 	if (!dev) {
4979 		dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
4980 		return -ENODEV;
4981 	}
4982 	bp = netdev_priv(dev);
4983 
4984 	rtnl_lock();
4985 
4986 	pci_save_state(pdev);
4987 
4988 	if (!netif_running(dev)) {
4989 		rtnl_unlock();
4990 		return 0;
4991 	}
4992 
4993 	netif_device_detach(dev);
4994 
4995 	bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
4996 
4997 	bnx2x_set_power_state(bp, pci_choose_state(pdev, state));
4998 
4999 	rtnl_unlock();
5000 
5001 	return 0;
5002 }
5003 
5004 int bnx2x_resume(struct pci_dev *pdev)
5005 {
5006 	struct net_device *dev = pci_get_drvdata(pdev);
5007 	struct bnx2x *bp;
5008 	int rc;
5009 
5010 	if (!dev) {
5011 		dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
5012 		return -ENODEV;
5013 	}
5014 	bp = netdev_priv(dev);
5015 
5016 	if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
5017 		BNX2X_ERR("Handling parity error recovery. Try again later\n");
5018 		return -EAGAIN;
5019 	}
5020 
5021 	rtnl_lock();
5022 
5023 	pci_restore_state(pdev);
5024 
5025 	if (!netif_running(dev)) {
5026 		rtnl_unlock();
5027 		return 0;
5028 	}
5029 
5030 	bnx2x_set_power_state(bp, PCI_D0);
5031 	netif_device_attach(dev);
5032 
5033 	rc = bnx2x_nic_load(bp, LOAD_OPEN);
5034 
5035 	rtnl_unlock();
5036 
5037 	return rc;
5038 }
5039 
5040 void bnx2x_set_ctx_validation(struct bnx2x *bp, struct eth_context *cxt,
5041 			      u32 cid)
5042 {
5043 	if (!cxt) {
5044 		BNX2X_ERR("bad context pointer %p\n", cxt);
5045 		return;
5046 	}
5047 
5048 	/* ustorm cxt validation */
5049 	cxt->ustorm_ag_context.cdu_usage =
5050 		CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
5051 			CDU_REGION_NUMBER_UCM_AG, ETH_CONNECTION_TYPE);
5052 	/* xcontext validation */
5053 	cxt->xstorm_ag_context.cdu_reserved =
5054 		CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
5055 			CDU_REGION_NUMBER_XCM_AG, ETH_CONNECTION_TYPE);
5056 }
5057 
5058 static void storm_memset_hc_timeout(struct bnx2x *bp, u8 port,
5059 				    u8 fw_sb_id, u8 sb_index,
5060 				    u8 ticks)
5061 {
5062 	u32 addr = BAR_CSTRORM_INTMEM +
5063 		   CSTORM_STATUS_BLOCK_DATA_TIMEOUT_OFFSET(fw_sb_id, sb_index);
5064 	REG_WR8(bp, addr, ticks);
5065 	DP(NETIF_MSG_IFUP,
5066 	   "port %x fw_sb_id %d sb_index %d ticks %d\n",
5067 	   port, fw_sb_id, sb_index, ticks);
5068 }
5069 
5070 static void storm_memset_hc_disable(struct bnx2x *bp, u8 port,
5071 				    u16 fw_sb_id, u8 sb_index,
5072 				    u8 disable)
5073 {
5074 	u32 enable_flag = disable ? 0 : (1 << HC_INDEX_DATA_HC_ENABLED_SHIFT);
5075 	u32 addr = BAR_CSTRORM_INTMEM +
5076 		   CSTORM_STATUS_BLOCK_DATA_FLAGS_OFFSET(fw_sb_id, sb_index);
5077 	u8 flags = REG_RD8(bp, addr);
5078 	/* clear and set */
5079 	flags &= ~HC_INDEX_DATA_HC_ENABLED;
5080 	flags |= enable_flag;
5081 	REG_WR8(bp, addr, flags);
5082 	DP(NETIF_MSG_IFUP,
5083 	   "port %x fw_sb_id %d sb_index %d disable %d\n",
5084 	   port, fw_sb_id, sb_index, disable);
5085 }
5086 
5087 void bnx2x_update_coalesce_sb_index(struct bnx2x *bp, u8 fw_sb_id,
5088 				    u8 sb_index, u8 disable, u16 usec)
5089 {
5090 	int port = BP_PORT(bp);
5091 	u8 ticks = usec / BNX2X_BTR;
5092 
5093 	storm_memset_hc_timeout(bp, port, fw_sb_id, sb_index, ticks);
5094 
5095 	disable = disable ? 1 : (usec ? 0 : 1);
5096 	storm_memset_hc_disable(bp, port, fw_sb_id, sb_index, disable);
5097 }
5098 
5099 void bnx2x_schedule_sp_rtnl(struct bnx2x *bp, enum sp_rtnl_flag flag,
5100 			    u32 verbose)
5101 {
5102 	smp_mb__before_atomic();
5103 	set_bit(flag, &bp->sp_rtnl_state);
5104 	smp_mb__after_atomic();
5105 	DP((BNX2X_MSG_SP | verbose), "Scheduling sp_rtnl task [Flag: %d]\n",
5106 	   flag);
5107 	schedule_delayed_work(&bp->sp_rtnl_task, 0);
5108 }
5109