1 /* bnx2x_cmn.c: Broadcom Everest network driver.
2  *
3  * Copyright (c) 2007-2013 Broadcom Corporation
4  *
5  * This program is free software; you can redistribute it and/or modify
6  * it under the terms of the GNU General Public License as published by
7  * the Free Software Foundation.
8  *
9  * Maintained by: Ariel Elior <ariel.elior@qlogic.com>
10  * Written by: Eliezer Tamir
11  * Based on code from Michael Chan's bnx2 driver
12  * UDP CSUM errata workaround by Arik Gendelman
13  * Slowpath and fastpath rework by Vladislav Zolotarov
14  * Statistics and Link management by Yitchak Gertner
15  *
16  */
17 
18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
19 
20 #include <linux/etherdevice.h>
21 #include <linux/if_vlan.h>
22 #include <linux/interrupt.h>
23 #include <linux/ip.h>
24 #include <net/tcp.h>
25 #include <net/ipv6.h>
26 #include <net/ip6_checksum.h>
27 #include <net/busy_poll.h>
28 #include <linux/prefetch.h>
29 #include "bnx2x_cmn.h"
30 #include "bnx2x_init.h"
31 #include "bnx2x_sp.h"
32 
33 static void bnx2x_free_fp_mem_cnic(struct bnx2x *bp);
34 static int bnx2x_alloc_fp_mem_cnic(struct bnx2x *bp);
35 static int bnx2x_alloc_fp_mem(struct bnx2x *bp);
36 static int bnx2x_poll(struct napi_struct *napi, int budget);
37 
38 static void bnx2x_add_all_napi_cnic(struct bnx2x *bp)
39 {
40 	int i;
41 
42 	/* Add NAPI objects */
43 	for_each_rx_queue_cnic(bp, i) {
44 		netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi),
45 			       bnx2x_poll, NAPI_POLL_WEIGHT);
46 		napi_hash_add(&bnx2x_fp(bp, i, napi));
47 	}
48 }
49 
50 static void bnx2x_add_all_napi(struct bnx2x *bp)
51 {
52 	int i;
53 
54 	/* Add NAPI objects */
55 	for_each_eth_queue(bp, i) {
56 		netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi),
57 			       bnx2x_poll, NAPI_POLL_WEIGHT);
58 		napi_hash_add(&bnx2x_fp(bp, i, napi));
59 	}
60 }
61 
62 static int bnx2x_calc_num_queues(struct bnx2x *bp)
63 {
64 	int nq = bnx2x_num_queues ? : netif_get_num_default_rss_queues();
65 
66 	/* Reduce memory usage in kdump environment by using only one queue */
67 	if (reset_devices)
68 		nq = 1;
69 
70 	nq = clamp(nq, 1, BNX2X_MAX_QUEUES(bp));
71 	return nq;
72 }
73 
74 /**
75  * bnx2x_move_fp - move content of the fastpath structure.
76  *
77  * @bp:		driver handle
78  * @from:	source FP index
79  * @to:		destination FP index
80  *
81  * Makes sure the contents of the bp->fp[to].napi is kept
82  * intact. This is done by first copying the napi struct from
83  * the target to the source, and then mem copying the entire
84  * source onto the target. Update txdata pointers and related
85  * content.
86  */
87 static inline void bnx2x_move_fp(struct bnx2x *bp, int from, int to)
88 {
89 	struct bnx2x_fastpath *from_fp = &bp->fp[from];
90 	struct bnx2x_fastpath *to_fp = &bp->fp[to];
91 	struct bnx2x_sp_objs *from_sp_objs = &bp->sp_objs[from];
92 	struct bnx2x_sp_objs *to_sp_objs = &bp->sp_objs[to];
93 	struct bnx2x_fp_stats *from_fp_stats = &bp->fp_stats[from];
94 	struct bnx2x_fp_stats *to_fp_stats = &bp->fp_stats[to];
95 	int old_max_eth_txqs, new_max_eth_txqs;
96 	int old_txdata_index = 0, new_txdata_index = 0;
97 	struct bnx2x_agg_info *old_tpa_info = to_fp->tpa_info;
98 
99 	/* Copy the NAPI object as it has been already initialized */
100 	from_fp->napi = to_fp->napi;
101 
102 	/* Move bnx2x_fastpath contents */
103 	memcpy(to_fp, from_fp, sizeof(*to_fp));
104 	to_fp->index = to;
105 
106 	/* Retain the tpa_info of the original `to' version as we don't want
107 	 * 2 FPs to contain the same tpa_info pointer.
108 	 */
109 	to_fp->tpa_info = old_tpa_info;
110 
111 	/* move sp_objs contents as well, as their indices match fp ones */
112 	memcpy(to_sp_objs, from_sp_objs, sizeof(*to_sp_objs));
113 
114 	/* move fp_stats contents as well, as their indices match fp ones */
115 	memcpy(to_fp_stats, from_fp_stats, sizeof(*to_fp_stats));
116 
117 	/* Update txdata pointers in fp and move txdata content accordingly:
118 	 * Each fp consumes 'max_cos' txdata structures, so the index should be
119 	 * decremented by max_cos x delta.
120 	 */
121 
122 	old_max_eth_txqs = BNX2X_NUM_ETH_QUEUES(bp) * (bp)->max_cos;
123 	new_max_eth_txqs = (BNX2X_NUM_ETH_QUEUES(bp) - from + to) *
124 				(bp)->max_cos;
125 	if (from == FCOE_IDX(bp)) {
126 		old_txdata_index = old_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
127 		new_txdata_index = new_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
128 	}
129 
130 	memcpy(&bp->bnx2x_txq[new_txdata_index],
131 	       &bp->bnx2x_txq[old_txdata_index],
132 	       sizeof(struct bnx2x_fp_txdata));
133 	to_fp->txdata_ptr[0] = &bp->bnx2x_txq[new_txdata_index];
134 }
135 
136 /**
137  * bnx2x_fill_fw_str - Fill buffer with FW version string.
138  *
139  * @bp:        driver handle
140  * @buf:       character buffer to fill with the fw name
141  * @buf_len:   length of the above buffer
142  *
143  */
144 void bnx2x_fill_fw_str(struct bnx2x *bp, char *buf, size_t buf_len)
145 {
146 	if (IS_PF(bp)) {
147 		u8 phy_fw_ver[PHY_FW_VER_LEN];
148 
149 		phy_fw_ver[0] = '\0';
150 		bnx2x_get_ext_phy_fw_version(&bp->link_params,
151 					     phy_fw_ver, PHY_FW_VER_LEN);
152 		strlcpy(buf, bp->fw_ver, buf_len);
153 		snprintf(buf + strlen(bp->fw_ver), 32 - strlen(bp->fw_ver),
154 			 "bc %d.%d.%d%s%s",
155 			 (bp->common.bc_ver & 0xff0000) >> 16,
156 			 (bp->common.bc_ver & 0xff00) >> 8,
157 			 (bp->common.bc_ver & 0xff),
158 			 ((phy_fw_ver[0] != '\0') ? " phy " : ""), phy_fw_ver);
159 	} else {
160 		bnx2x_vf_fill_fw_str(bp, buf, buf_len);
161 	}
162 }
163 
164 /**
165  * bnx2x_shrink_eth_fp - guarantees fastpath structures stay intact
166  *
167  * @bp:	driver handle
168  * @delta:	number of eth queues which were not allocated
169  */
170 static void bnx2x_shrink_eth_fp(struct bnx2x *bp, int delta)
171 {
172 	int i, cos, old_eth_num = BNX2X_NUM_ETH_QUEUES(bp);
173 
174 	/* Queue pointer cannot be re-set on an fp-basis, as moving pointer
175 	 * backward along the array could cause memory to be overridden
176 	 */
177 	for (cos = 1; cos < bp->max_cos; cos++) {
178 		for (i = 0; i < old_eth_num - delta; i++) {
179 			struct bnx2x_fastpath *fp = &bp->fp[i];
180 			int new_idx = cos * (old_eth_num - delta) + i;
181 
182 			memcpy(&bp->bnx2x_txq[new_idx], fp->txdata_ptr[cos],
183 			       sizeof(struct bnx2x_fp_txdata));
184 			fp->txdata_ptr[cos] = &bp->bnx2x_txq[new_idx];
185 		}
186 	}
187 }
188 
189 int bnx2x_load_count[2][3] = { {0} }; /* per-path: 0-common, 1-port0, 2-port1 */
190 
191 /* free skb in the packet ring at pos idx
192  * return idx of last bd freed
193  */
194 static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata,
195 			     u16 idx, unsigned int *pkts_compl,
196 			     unsigned int *bytes_compl)
197 {
198 	struct sw_tx_bd *tx_buf = &txdata->tx_buf_ring[idx];
199 	struct eth_tx_start_bd *tx_start_bd;
200 	struct eth_tx_bd *tx_data_bd;
201 	struct sk_buff *skb = tx_buf->skb;
202 	u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
203 	int nbd;
204 	u16 split_bd_len = 0;
205 
206 	/* prefetch skb end pointer to speedup dev_kfree_skb() */
207 	prefetch(&skb->end);
208 
209 	DP(NETIF_MSG_TX_DONE, "fp[%d]: pkt_idx %d  buff @(%p)->skb %p\n",
210 	   txdata->txq_index, idx, tx_buf, skb);
211 
212 	tx_start_bd = &txdata->tx_desc_ring[bd_idx].start_bd;
213 
214 	nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
215 #ifdef BNX2X_STOP_ON_ERROR
216 	if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
217 		BNX2X_ERR("BAD nbd!\n");
218 		bnx2x_panic();
219 	}
220 #endif
221 	new_cons = nbd + tx_buf->first_bd;
222 
223 	/* Get the next bd */
224 	bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
225 
226 	/* Skip a parse bd... */
227 	--nbd;
228 	bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
229 
230 	/* TSO headers+data bds share a common mapping. See bnx2x_tx_split() */
231 	if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
232 		tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
233 		split_bd_len = BD_UNMAP_LEN(tx_data_bd);
234 		--nbd;
235 		bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
236 	}
237 
238 	/* unmap first bd */
239 	dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
240 			 BD_UNMAP_LEN(tx_start_bd) + split_bd_len,
241 			 DMA_TO_DEVICE);
242 
243 	/* now free frags */
244 	while (nbd > 0) {
245 
246 		tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
247 		dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
248 			       BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
249 		if (--nbd)
250 			bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
251 	}
252 
253 	/* release skb */
254 	WARN_ON(!skb);
255 	if (likely(skb)) {
256 		(*pkts_compl)++;
257 		(*bytes_compl) += skb->len;
258 	}
259 
260 	dev_kfree_skb_any(skb);
261 	tx_buf->first_bd = 0;
262 	tx_buf->skb = NULL;
263 
264 	return new_cons;
265 }
266 
267 int bnx2x_tx_int(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata)
268 {
269 	struct netdev_queue *txq;
270 	u16 hw_cons, sw_cons, bd_cons = txdata->tx_bd_cons;
271 	unsigned int pkts_compl = 0, bytes_compl = 0;
272 
273 #ifdef BNX2X_STOP_ON_ERROR
274 	if (unlikely(bp->panic))
275 		return -1;
276 #endif
277 
278 	txq = netdev_get_tx_queue(bp->dev, txdata->txq_index);
279 	hw_cons = le16_to_cpu(*txdata->tx_cons_sb);
280 	sw_cons = txdata->tx_pkt_cons;
281 
282 	while (sw_cons != hw_cons) {
283 		u16 pkt_cons;
284 
285 		pkt_cons = TX_BD(sw_cons);
286 
287 		DP(NETIF_MSG_TX_DONE,
288 		   "queue[%d]: hw_cons %u  sw_cons %u  pkt_cons %u\n",
289 		   txdata->txq_index, hw_cons, sw_cons, pkt_cons);
290 
291 		bd_cons = bnx2x_free_tx_pkt(bp, txdata, pkt_cons,
292 					    &pkts_compl, &bytes_compl);
293 
294 		sw_cons++;
295 	}
296 
297 	netdev_tx_completed_queue(txq, pkts_compl, bytes_compl);
298 
299 	txdata->tx_pkt_cons = sw_cons;
300 	txdata->tx_bd_cons = bd_cons;
301 
302 	/* Need to make the tx_bd_cons update visible to start_xmit()
303 	 * before checking for netif_tx_queue_stopped().  Without the
304 	 * memory barrier, there is a small possibility that
305 	 * start_xmit() will miss it and cause the queue to be stopped
306 	 * forever.
307 	 * On the other hand we need an rmb() here to ensure the proper
308 	 * ordering of bit testing in the following
309 	 * netif_tx_queue_stopped(txq) call.
310 	 */
311 	smp_mb();
312 
313 	if (unlikely(netif_tx_queue_stopped(txq))) {
314 		/* Taking tx_lock() is needed to prevent re-enabling the queue
315 		 * while it's empty. This could have happen if rx_action() gets
316 		 * suspended in bnx2x_tx_int() after the condition before
317 		 * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
318 		 *
319 		 * stops the queue->sees fresh tx_bd_cons->releases the queue->
320 		 * sends some packets consuming the whole queue again->
321 		 * stops the queue
322 		 */
323 
324 		__netif_tx_lock(txq, smp_processor_id());
325 
326 		if ((netif_tx_queue_stopped(txq)) &&
327 		    (bp->state == BNX2X_STATE_OPEN) &&
328 		    (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT))
329 			netif_tx_wake_queue(txq);
330 
331 		__netif_tx_unlock(txq);
332 	}
333 	return 0;
334 }
335 
336 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
337 					     u16 idx)
338 {
339 	u16 last_max = fp->last_max_sge;
340 
341 	if (SUB_S16(idx, last_max) > 0)
342 		fp->last_max_sge = idx;
343 }
344 
345 static inline void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
346 					 u16 sge_len,
347 					 struct eth_end_agg_rx_cqe *cqe)
348 {
349 	struct bnx2x *bp = fp->bp;
350 	u16 last_max, last_elem, first_elem;
351 	u16 delta = 0;
352 	u16 i;
353 
354 	if (!sge_len)
355 		return;
356 
357 	/* First mark all used pages */
358 	for (i = 0; i < sge_len; i++)
359 		BIT_VEC64_CLEAR_BIT(fp->sge_mask,
360 			RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[i])));
361 
362 	DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
363 	   sge_len - 1, le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
364 
365 	/* Here we assume that the last SGE index is the biggest */
366 	prefetch((void *)(fp->sge_mask));
367 	bnx2x_update_last_max_sge(fp,
368 		le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
369 
370 	last_max = RX_SGE(fp->last_max_sge);
371 	last_elem = last_max >> BIT_VEC64_ELEM_SHIFT;
372 	first_elem = RX_SGE(fp->rx_sge_prod) >> BIT_VEC64_ELEM_SHIFT;
373 
374 	/* If ring is not full */
375 	if (last_elem + 1 != first_elem)
376 		last_elem++;
377 
378 	/* Now update the prod */
379 	for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
380 		if (likely(fp->sge_mask[i]))
381 			break;
382 
383 		fp->sge_mask[i] = BIT_VEC64_ELEM_ONE_MASK;
384 		delta += BIT_VEC64_ELEM_SZ;
385 	}
386 
387 	if (delta > 0) {
388 		fp->rx_sge_prod += delta;
389 		/* clear page-end entries */
390 		bnx2x_clear_sge_mask_next_elems(fp);
391 	}
392 
393 	DP(NETIF_MSG_RX_STATUS,
394 	   "fp->last_max_sge = %d  fp->rx_sge_prod = %d\n",
395 	   fp->last_max_sge, fp->rx_sge_prod);
396 }
397 
398 /* Get Toeplitz hash value in the skb using the value from the
399  * CQE (calculated by HW).
400  */
401 static u32 bnx2x_get_rxhash(const struct bnx2x *bp,
402 			    const struct eth_fast_path_rx_cqe *cqe,
403 			    enum pkt_hash_types *rxhash_type)
404 {
405 	/* Get Toeplitz hash from CQE */
406 	if ((bp->dev->features & NETIF_F_RXHASH) &&
407 	    (cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG)) {
408 		enum eth_rss_hash_type htype;
409 
410 		htype = cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_TYPE;
411 		*rxhash_type = ((htype == TCP_IPV4_HASH_TYPE) ||
412 				(htype == TCP_IPV6_HASH_TYPE)) ?
413 			       PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3;
414 
415 		return le32_to_cpu(cqe->rss_hash_result);
416 	}
417 	*rxhash_type = PKT_HASH_TYPE_NONE;
418 	return 0;
419 }
420 
421 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
422 			    u16 cons, u16 prod,
423 			    struct eth_fast_path_rx_cqe *cqe)
424 {
425 	struct bnx2x *bp = fp->bp;
426 	struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
427 	struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
428 	struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
429 	dma_addr_t mapping;
430 	struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
431 	struct sw_rx_bd *first_buf = &tpa_info->first_buf;
432 
433 	/* print error if current state != stop */
434 	if (tpa_info->tpa_state != BNX2X_TPA_STOP)
435 		BNX2X_ERR("start of bin not in stop [%d]\n", queue);
436 
437 	/* Try to map an empty data buffer from the aggregation info  */
438 	mapping = dma_map_single(&bp->pdev->dev,
439 				 first_buf->data + NET_SKB_PAD,
440 				 fp->rx_buf_size, DMA_FROM_DEVICE);
441 	/*
442 	 *  ...if it fails - move the skb from the consumer to the producer
443 	 *  and set the current aggregation state as ERROR to drop it
444 	 *  when TPA_STOP arrives.
445 	 */
446 
447 	if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
448 		/* Move the BD from the consumer to the producer */
449 		bnx2x_reuse_rx_data(fp, cons, prod);
450 		tpa_info->tpa_state = BNX2X_TPA_ERROR;
451 		return;
452 	}
453 
454 	/* move empty data from pool to prod */
455 	prod_rx_buf->data = first_buf->data;
456 	dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
457 	/* point prod_bd to new data */
458 	prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
459 	prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
460 
461 	/* move partial skb from cons to pool (don't unmap yet) */
462 	*first_buf = *cons_rx_buf;
463 
464 	/* mark bin state as START */
465 	tpa_info->parsing_flags =
466 		le16_to_cpu(cqe->pars_flags.flags);
467 	tpa_info->vlan_tag = le16_to_cpu(cqe->vlan_tag);
468 	tpa_info->tpa_state = BNX2X_TPA_START;
469 	tpa_info->len_on_bd = le16_to_cpu(cqe->len_on_bd);
470 	tpa_info->placement_offset = cqe->placement_offset;
471 	tpa_info->rxhash = bnx2x_get_rxhash(bp, cqe, &tpa_info->rxhash_type);
472 	if (fp->mode == TPA_MODE_GRO) {
473 		u16 gro_size = le16_to_cpu(cqe->pkt_len_or_gro_seg_len);
474 		tpa_info->full_page = SGE_PAGES / gro_size * gro_size;
475 		tpa_info->gro_size = gro_size;
476 	}
477 
478 #ifdef BNX2X_STOP_ON_ERROR
479 	fp->tpa_queue_used |= (1 << queue);
480 #ifdef _ASM_GENERIC_INT_L64_H
481 	DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n",
482 #else
483 	DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
484 #endif
485 	   fp->tpa_queue_used);
486 #endif
487 }
488 
489 /* Timestamp option length allowed for TPA aggregation:
490  *
491  *		nop nop kind length echo val
492  */
493 #define TPA_TSTAMP_OPT_LEN	12
494 /**
495  * bnx2x_set_gro_params - compute GRO values
496  *
497  * @skb:		packet skb
498  * @parsing_flags:	parsing flags from the START CQE
499  * @len_on_bd:		total length of the first packet for the
500  *			aggregation.
501  * @pkt_len:		length of all segments
502  *
503  * Approximate value of the MSS for this aggregation calculated using
504  * the first packet of it.
505  * Compute number of aggregated segments, and gso_type.
506  */
507 static void bnx2x_set_gro_params(struct sk_buff *skb, u16 parsing_flags,
508 				 u16 len_on_bd, unsigned int pkt_len,
509 				 u16 num_of_coalesced_segs)
510 {
511 	/* TPA aggregation won't have either IP options or TCP options
512 	 * other than timestamp or IPv6 extension headers.
513 	 */
514 	u16 hdrs_len = ETH_HLEN + sizeof(struct tcphdr);
515 
516 	if (GET_FLAG(parsing_flags, PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
517 	    PRS_FLAG_OVERETH_IPV6) {
518 		hdrs_len += sizeof(struct ipv6hdr);
519 		skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
520 	} else {
521 		hdrs_len += sizeof(struct iphdr);
522 		skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
523 	}
524 
525 	/* Check if there was a TCP timestamp, if there is it's will
526 	 * always be 12 bytes length: nop nop kind length echo val.
527 	 *
528 	 * Otherwise FW would close the aggregation.
529 	 */
530 	if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG)
531 		hdrs_len += TPA_TSTAMP_OPT_LEN;
532 
533 	skb_shinfo(skb)->gso_size = len_on_bd - hdrs_len;
534 
535 	/* tcp_gro_complete() will copy NAPI_GRO_CB(skb)->count
536 	 * to skb_shinfo(skb)->gso_segs
537 	 */
538 	NAPI_GRO_CB(skb)->count = num_of_coalesced_segs;
539 }
540 
541 static int bnx2x_alloc_rx_sge(struct bnx2x *bp, struct bnx2x_fastpath *fp,
542 			      u16 index, gfp_t gfp_mask)
543 {
544 	struct page *page = alloc_pages(gfp_mask, PAGES_PER_SGE_SHIFT);
545 	struct sw_rx_page *sw_buf = &fp->rx_page_ring[index];
546 	struct eth_rx_sge *sge = &fp->rx_sge_ring[index];
547 	dma_addr_t mapping;
548 
549 	if (unlikely(page == NULL)) {
550 		BNX2X_ERR("Can't alloc sge\n");
551 		return -ENOMEM;
552 	}
553 
554 	mapping = dma_map_page(&bp->pdev->dev, page, 0,
555 			       SGE_PAGES, DMA_FROM_DEVICE);
556 	if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
557 		__free_pages(page, PAGES_PER_SGE_SHIFT);
558 		BNX2X_ERR("Can't map sge\n");
559 		return -ENOMEM;
560 	}
561 
562 	sw_buf->page = page;
563 	dma_unmap_addr_set(sw_buf, mapping, mapping);
564 
565 	sge->addr_hi = cpu_to_le32(U64_HI(mapping));
566 	sge->addr_lo = cpu_to_le32(U64_LO(mapping));
567 
568 	return 0;
569 }
570 
571 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
572 			       struct bnx2x_agg_info *tpa_info,
573 			       u16 pages,
574 			       struct sk_buff *skb,
575 			       struct eth_end_agg_rx_cqe *cqe,
576 			       u16 cqe_idx)
577 {
578 	struct sw_rx_page *rx_pg, old_rx_pg;
579 	u32 i, frag_len, frag_size;
580 	int err, j, frag_id = 0;
581 	u16 len_on_bd = tpa_info->len_on_bd;
582 	u16 full_page = 0, gro_size = 0;
583 
584 	frag_size = le16_to_cpu(cqe->pkt_len) - len_on_bd;
585 
586 	if (fp->mode == TPA_MODE_GRO) {
587 		gro_size = tpa_info->gro_size;
588 		full_page = tpa_info->full_page;
589 	}
590 
591 	/* This is needed in order to enable forwarding support */
592 	if (frag_size)
593 		bnx2x_set_gro_params(skb, tpa_info->parsing_flags, len_on_bd,
594 				     le16_to_cpu(cqe->pkt_len),
595 				     le16_to_cpu(cqe->num_of_coalesced_segs));
596 
597 #ifdef BNX2X_STOP_ON_ERROR
598 	if (pages > min_t(u32, 8, MAX_SKB_FRAGS) * SGE_PAGES) {
599 		BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
600 			  pages, cqe_idx);
601 		BNX2X_ERR("cqe->pkt_len = %d\n", cqe->pkt_len);
602 		bnx2x_panic();
603 		return -EINVAL;
604 	}
605 #endif
606 
607 	/* Run through the SGL and compose the fragmented skb */
608 	for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
609 		u16 sge_idx = RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[j]));
610 
611 		/* FW gives the indices of the SGE as if the ring is an array
612 		   (meaning that "next" element will consume 2 indices) */
613 		if (fp->mode == TPA_MODE_GRO)
614 			frag_len = min_t(u32, frag_size, (u32)full_page);
615 		else /* LRO */
616 			frag_len = min_t(u32, frag_size, (u32)SGE_PAGES);
617 
618 		rx_pg = &fp->rx_page_ring[sge_idx];
619 		old_rx_pg = *rx_pg;
620 
621 		/* If we fail to allocate a substitute page, we simply stop
622 		   where we are and drop the whole packet */
623 		err = bnx2x_alloc_rx_sge(bp, fp, sge_idx, GFP_ATOMIC);
624 		if (unlikely(err)) {
625 			bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
626 			return err;
627 		}
628 
629 		/* Unmap the page as we're going to pass it to the stack */
630 		dma_unmap_page(&bp->pdev->dev,
631 			       dma_unmap_addr(&old_rx_pg, mapping),
632 			       SGE_PAGES, DMA_FROM_DEVICE);
633 		/* Add one frag and update the appropriate fields in the skb */
634 		if (fp->mode == TPA_MODE_LRO)
635 			skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);
636 		else { /* GRO */
637 			int rem;
638 			int offset = 0;
639 			for (rem = frag_len; rem > 0; rem -= gro_size) {
640 				int len = rem > gro_size ? gro_size : rem;
641 				skb_fill_page_desc(skb, frag_id++,
642 						   old_rx_pg.page, offset, len);
643 				if (offset)
644 					get_page(old_rx_pg.page);
645 				offset += len;
646 			}
647 		}
648 
649 		skb->data_len += frag_len;
650 		skb->truesize += SGE_PAGES;
651 		skb->len += frag_len;
652 
653 		frag_size -= frag_len;
654 	}
655 
656 	return 0;
657 }
658 
659 static void bnx2x_frag_free(const struct bnx2x_fastpath *fp, void *data)
660 {
661 	if (fp->rx_frag_size)
662 		put_page(virt_to_head_page(data));
663 	else
664 		kfree(data);
665 }
666 
667 static void *bnx2x_frag_alloc(const struct bnx2x_fastpath *fp, gfp_t gfp_mask)
668 {
669 	if (fp->rx_frag_size) {
670 		/* GFP_KERNEL allocations are used only during initialization */
671 		if (unlikely(gfp_mask & __GFP_WAIT))
672 			return (void *)__get_free_page(gfp_mask);
673 
674 		return netdev_alloc_frag(fp->rx_frag_size);
675 	}
676 
677 	return kmalloc(fp->rx_buf_size + NET_SKB_PAD, gfp_mask);
678 }
679 
680 #ifdef CONFIG_INET
681 static void bnx2x_gro_ip_csum(struct bnx2x *bp, struct sk_buff *skb)
682 {
683 	const struct iphdr *iph = ip_hdr(skb);
684 	struct tcphdr *th;
685 
686 	skb_set_transport_header(skb, sizeof(struct iphdr));
687 	th = tcp_hdr(skb);
688 
689 	th->check = ~tcp_v4_check(skb->len - skb_transport_offset(skb),
690 				  iph->saddr, iph->daddr, 0);
691 }
692 
693 static void bnx2x_gro_ipv6_csum(struct bnx2x *bp, struct sk_buff *skb)
694 {
695 	struct ipv6hdr *iph = ipv6_hdr(skb);
696 	struct tcphdr *th;
697 
698 	skb_set_transport_header(skb, sizeof(struct ipv6hdr));
699 	th = tcp_hdr(skb);
700 
701 	th->check = ~tcp_v6_check(skb->len - skb_transport_offset(skb),
702 				  &iph->saddr, &iph->daddr, 0);
703 }
704 
705 static void bnx2x_gro_csum(struct bnx2x *bp, struct sk_buff *skb,
706 			    void (*gro_func)(struct bnx2x*, struct sk_buff*))
707 {
708 	skb_set_network_header(skb, 0);
709 	gro_func(bp, skb);
710 	tcp_gro_complete(skb);
711 }
712 #endif
713 
714 static void bnx2x_gro_receive(struct bnx2x *bp, struct bnx2x_fastpath *fp,
715 			       struct sk_buff *skb)
716 {
717 #ifdef CONFIG_INET
718 	if (skb_shinfo(skb)->gso_size) {
719 		switch (be16_to_cpu(skb->protocol)) {
720 		case ETH_P_IP:
721 			bnx2x_gro_csum(bp, skb, bnx2x_gro_ip_csum);
722 			break;
723 		case ETH_P_IPV6:
724 			bnx2x_gro_csum(bp, skb, bnx2x_gro_ipv6_csum);
725 			break;
726 		default:
727 			BNX2X_ERR("Error: FW GRO supports only IPv4/IPv6, not 0x%04x\n",
728 				  be16_to_cpu(skb->protocol));
729 		}
730 	}
731 #endif
732 	skb_record_rx_queue(skb, fp->rx_queue);
733 	napi_gro_receive(&fp->napi, skb);
734 }
735 
736 static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
737 			   struct bnx2x_agg_info *tpa_info,
738 			   u16 pages,
739 			   struct eth_end_agg_rx_cqe *cqe,
740 			   u16 cqe_idx)
741 {
742 	struct sw_rx_bd *rx_buf = &tpa_info->first_buf;
743 	u8 pad = tpa_info->placement_offset;
744 	u16 len = tpa_info->len_on_bd;
745 	struct sk_buff *skb = NULL;
746 	u8 *new_data, *data = rx_buf->data;
747 	u8 old_tpa_state = tpa_info->tpa_state;
748 
749 	tpa_info->tpa_state = BNX2X_TPA_STOP;
750 
751 	/* If we there was an error during the handling of the TPA_START -
752 	 * drop this aggregation.
753 	 */
754 	if (old_tpa_state == BNX2X_TPA_ERROR)
755 		goto drop;
756 
757 	/* Try to allocate the new data */
758 	new_data = bnx2x_frag_alloc(fp, GFP_ATOMIC);
759 	/* Unmap skb in the pool anyway, as we are going to change
760 	   pool entry status to BNX2X_TPA_STOP even if new skb allocation
761 	   fails. */
762 	dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
763 			 fp->rx_buf_size, DMA_FROM_DEVICE);
764 	if (likely(new_data))
765 		skb = build_skb(data, fp->rx_frag_size);
766 
767 	if (likely(skb)) {
768 #ifdef BNX2X_STOP_ON_ERROR
769 		if (pad + len > fp->rx_buf_size) {
770 			BNX2X_ERR("skb_put is about to fail...  pad %d  len %d  rx_buf_size %d\n",
771 				  pad, len, fp->rx_buf_size);
772 			bnx2x_panic();
773 			return;
774 		}
775 #endif
776 
777 		skb_reserve(skb, pad + NET_SKB_PAD);
778 		skb_put(skb, len);
779 		skb_set_hash(skb, tpa_info->rxhash, tpa_info->rxhash_type);
780 
781 		skb->protocol = eth_type_trans(skb, bp->dev);
782 		skb->ip_summed = CHECKSUM_UNNECESSARY;
783 
784 		if (!bnx2x_fill_frag_skb(bp, fp, tpa_info, pages,
785 					 skb, cqe, cqe_idx)) {
786 			if (tpa_info->parsing_flags & PARSING_FLAGS_VLAN)
787 				__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), tpa_info->vlan_tag);
788 			bnx2x_gro_receive(bp, fp, skb);
789 		} else {
790 			DP(NETIF_MSG_RX_STATUS,
791 			   "Failed to allocate new pages - dropping packet!\n");
792 			dev_kfree_skb_any(skb);
793 		}
794 
795 		/* put new data in bin */
796 		rx_buf->data = new_data;
797 
798 		return;
799 	}
800 	bnx2x_frag_free(fp, new_data);
801 drop:
802 	/* drop the packet and keep the buffer in the bin */
803 	DP(NETIF_MSG_RX_STATUS,
804 	   "Failed to allocate or map a new skb - dropping packet!\n");
805 	bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed++;
806 }
807 
808 static int bnx2x_alloc_rx_data(struct bnx2x *bp, struct bnx2x_fastpath *fp,
809 			       u16 index, gfp_t gfp_mask)
810 {
811 	u8 *data;
812 	struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[index];
813 	struct eth_rx_bd *rx_bd = &fp->rx_desc_ring[index];
814 	dma_addr_t mapping;
815 
816 	data = bnx2x_frag_alloc(fp, gfp_mask);
817 	if (unlikely(data == NULL))
818 		return -ENOMEM;
819 
820 	mapping = dma_map_single(&bp->pdev->dev, data + NET_SKB_PAD,
821 				 fp->rx_buf_size,
822 				 DMA_FROM_DEVICE);
823 	if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
824 		bnx2x_frag_free(fp, data);
825 		BNX2X_ERR("Can't map rx data\n");
826 		return -ENOMEM;
827 	}
828 
829 	rx_buf->data = data;
830 	dma_unmap_addr_set(rx_buf, mapping, mapping);
831 
832 	rx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
833 	rx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
834 
835 	return 0;
836 }
837 
838 static
839 void bnx2x_csum_validate(struct sk_buff *skb, union eth_rx_cqe *cqe,
840 				 struct bnx2x_fastpath *fp,
841 				 struct bnx2x_eth_q_stats *qstats)
842 {
843 	/* Do nothing if no L4 csum validation was done.
844 	 * We do not check whether IP csum was validated. For IPv4 we assume
845 	 * that if the card got as far as validating the L4 csum, it also
846 	 * validated the IP csum. IPv6 has no IP csum.
847 	 */
848 	if (cqe->fast_path_cqe.status_flags &
849 	    ETH_FAST_PATH_RX_CQE_L4_XSUM_NO_VALIDATION_FLG)
850 		return;
851 
852 	/* If L4 validation was done, check if an error was found. */
853 
854 	if (cqe->fast_path_cqe.type_error_flags &
855 	    (ETH_FAST_PATH_RX_CQE_IP_BAD_XSUM_FLG |
856 	     ETH_FAST_PATH_RX_CQE_L4_BAD_XSUM_FLG))
857 		qstats->hw_csum_err++;
858 	else
859 		skb->ip_summed = CHECKSUM_UNNECESSARY;
860 }
861 
862 static int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
863 {
864 	struct bnx2x *bp = fp->bp;
865 	u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
866 	u16 sw_comp_cons, sw_comp_prod;
867 	int rx_pkt = 0;
868 	union eth_rx_cqe *cqe;
869 	struct eth_fast_path_rx_cqe *cqe_fp;
870 
871 #ifdef BNX2X_STOP_ON_ERROR
872 	if (unlikely(bp->panic))
873 		return 0;
874 #endif
875 	if (budget <= 0)
876 		return rx_pkt;
877 
878 	bd_cons = fp->rx_bd_cons;
879 	bd_prod = fp->rx_bd_prod;
880 	bd_prod_fw = bd_prod;
881 	sw_comp_cons = fp->rx_comp_cons;
882 	sw_comp_prod = fp->rx_comp_prod;
883 
884 	comp_ring_cons = RCQ_BD(sw_comp_cons);
885 	cqe = &fp->rx_comp_ring[comp_ring_cons];
886 	cqe_fp = &cqe->fast_path_cqe;
887 
888 	DP(NETIF_MSG_RX_STATUS,
889 	   "queue[%d]: sw_comp_cons %u\n", fp->index, sw_comp_cons);
890 
891 	while (BNX2X_IS_CQE_COMPLETED(cqe_fp)) {
892 		struct sw_rx_bd *rx_buf = NULL;
893 		struct sk_buff *skb;
894 		u8 cqe_fp_flags;
895 		enum eth_rx_cqe_type cqe_fp_type;
896 		u16 len, pad, queue;
897 		u8 *data;
898 		u32 rxhash;
899 		enum pkt_hash_types rxhash_type;
900 
901 #ifdef BNX2X_STOP_ON_ERROR
902 		if (unlikely(bp->panic))
903 			return 0;
904 #endif
905 
906 		bd_prod = RX_BD(bd_prod);
907 		bd_cons = RX_BD(bd_cons);
908 
909 		/* A rmb() is required to ensure that the CQE is not read
910 		 * before it is written by the adapter DMA.  PCI ordering
911 		 * rules will make sure the other fields are written before
912 		 * the marker at the end of struct eth_fast_path_rx_cqe
913 		 * but without rmb() a weakly ordered processor can process
914 		 * stale data.  Without the barrier TPA state-machine might
915 		 * enter inconsistent state and kernel stack might be
916 		 * provided with incorrect packet description - these lead
917 		 * to various kernel crashed.
918 		 */
919 		rmb();
920 
921 		cqe_fp_flags = cqe_fp->type_error_flags;
922 		cqe_fp_type = cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE;
923 
924 		DP(NETIF_MSG_RX_STATUS,
925 		   "CQE type %x  err %x  status %x  queue %x  vlan %x  len %u\n",
926 		   CQE_TYPE(cqe_fp_flags),
927 		   cqe_fp_flags, cqe_fp->status_flags,
928 		   le32_to_cpu(cqe_fp->rss_hash_result),
929 		   le16_to_cpu(cqe_fp->vlan_tag),
930 		   le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len));
931 
932 		/* is this a slowpath msg? */
933 		if (unlikely(CQE_TYPE_SLOW(cqe_fp_type))) {
934 			bnx2x_sp_event(fp, cqe);
935 			goto next_cqe;
936 		}
937 
938 		rx_buf = &fp->rx_buf_ring[bd_cons];
939 		data = rx_buf->data;
940 
941 		if (!CQE_TYPE_FAST(cqe_fp_type)) {
942 			struct bnx2x_agg_info *tpa_info;
943 			u16 frag_size, pages;
944 #ifdef BNX2X_STOP_ON_ERROR
945 			/* sanity check */
946 			if (fp->disable_tpa &&
947 			    (CQE_TYPE_START(cqe_fp_type) ||
948 			     CQE_TYPE_STOP(cqe_fp_type)))
949 				BNX2X_ERR("START/STOP packet while disable_tpa type %x\n",
950 					  CQE_TYPE(cqe_fp_type));
951 #endif
952 
953 			if (CQE_TYPE_START(cqe_fp_type)) {
954 				u16 queue = cqe_fp->queue_index;
955 				DP(NETIF_MSG_RX_STATUS,
956 				   "calling tpa_start on queue %d\n",
957 				   queue);
958 
959 				bnx2x_tpa_start(fp, queue,
960 						bd_cons, bd_prod,
961 						cqe_fp);
962 
963 				goto next_rx;
964 			}
965 			queue = cqe->end_agg_cqe.queue_index;
966 			tpa_info = &fp->tpa_info[queue];
967 			DP(NETIF_MSG_RX_STATUS,
968 			   "calling tpa_stop on queue %d\n",
969 			   queue);
970 
971 			frag_size = le16_to_cpu(cqe->end_agg_cqe.pkt_len) -
972 				    tpa_info->len_on_bd;
973 
974 			if (fp->mode == TPA_MODE_GRO)
975 				pages = (frag_size + tpa_info->full_page - 1) /
976 					 tpa_info->full_page;
977 			else
978 				pages = SGE_PAGE_ALIGN(frag_size) >>
979 					SGE_PAGE_SHIFT;
980 
981 			bnx2x_tpa_stop(bp, fp, tpa_info, pages,
982 				       &cqe->end_agg_cqe, comp_ring_cons);
983 #ifdef BNX2X_STOP_ON_ERROR
984 			if (bp->panic)
985 				return 0;
986 #endif
987 
988 			bnx2x_update_sge_prod(fp, pages, &cqe->end_agg_cqe);
989 			goto next_cqe;
990 		}
991 		/* non TPA */
992 		len = le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len);
993 		pad = cqe_fp->placement_offset;
994 		dma_sync_single_for_cpu(&bp->pdev->dev,
995 					dma_unmap_addr(rx_buf, mapping),
996 					pad + RX_COPY_THRESH,
997 					DMA_FROM_DEVICE);
998 		pad += NET_SKB_PAD;
999 		prefetch(data + pad); /* speedup eth_type_trans() */
1000 		/* is this an error packet? */
1001 		if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
1002 			DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
1003 			   "ERROR  flags %x  rx packet %u\n",
1004 			   cqe_fp_flags, sw_comp_cons);
1005 			bnx2x_fp_qstats(bp, fp)->rx_err_discard_pkt++;
1006 			goto reuse_rx;
1007 		}
1008 
1009 		/* Since we don't have a jumbo ring
1010 		 * copy small packets if mtu > 1500
1011 		 */
1012 		if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
1013 		    (len <= RX_COPY_THRESH)) {
1014 			skb = netdev_alloc_skb_ip_align(bp->dev, len);
1015 			if (skb == NULL) {
1016 				DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
1017 				   "ERROR  packet dropped because of alloc failure\n");
1018 				bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
1019 				goto reuse_rx;
1020 			}
1021 			memcpy(skb->data, data + pad, len);
1022 			bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
1023 		} else {
1024 			if (likely(bnx2x_alloc_rx_data(bp, fp, bd_prod,
1025 						       GFP_ATOMIC) == 0)) {
1026 				dma_unmap_single(&bp->pdev->dev,
1027 						 dma_unmap_addr(rx_buf, mapping),
1028 						 fp->rx_buf_size,
1029 						 DMA_FROM_DEVICE);
1030 				skb = build_skb(data, fp->rx_frag_size);
1031 				if (unlikely(!skb)) {
1032 					bnx2x_frag_free(fp, data);
1033 					bnx2x_fp_qstats(bp, fp)->
1034 							rx_skb_alloc_failed++;
1035 					goto next_rx;
1036 				}
1037 				skb_reserve(skb, pad);
1038 			} else {
1039 				DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
1040 				   "ERROR  packet dropped because of alloc failure\n");
1041 				bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
1042 reuse_rx:
1043 				bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
1044 				goto next_rx;
1045 			}
1046 		}
1047 
1048 		skb_put(skb, len);
1049 		skb->protocol = eth_type_trans(skb, bp->dev);
1050 
1051 		/* Set Toeplitz hash for a none-LRO skb */
1052 		rxhash = bnx2x_get_rxhash(bp, cqe_fp, &rxhash_type);
1053 		skb_set_hash(skb, rxhash, rxhash_type);
1054 
1055 		skb_checksum_none_assert(skb);
1056 
1057 		if (bp->dev->features & NETIF_F_RXCSUM)
1058 			bnx2x_csum_validate(skb, cqe, fp,
1059 					    bnx2x_fp_qstats(bp, fp));
1060 
1061 		skb_record_rx_queue(skb, fp->rx_queue);
1062 
1063 		if (le16_to_cpu(cqe_fp->pars_flags.flags) &
1064 		    PARSING_FLAGS_VLAN)
1065 			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
1066 					       le16_to_cpu(cqe_fp->vlan_tag));
1067 
1068 		skb_mark_napi_id(skb, &fp->napi);
1069 
1070 		if (bnx2x_fp_ll_polling(fp))
1071 			netif_receive_skb(skb);
1072 		else
1073 			napi_gro_receive(&fp->napi, skb);
1074 next_rx:
1075 		rx_buf->data = NULL;
1076 
1077 		bd_cons = NEXT_RX_IDX(bd_cons);
1078 		bd_prod = NEXT_RX_IDX(bd_prod);
1079 		bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
1080 		rx_pkt++;
1081 next_cqe:
1082 		sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
1083 		sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
1084 
1085 		/* mark CQE as free */
1086 		BNX2X_SEED_CQE(cqe_fp);
1087 
1088 		if (rx_pkt == budget)
1089 			break;
1090 
1091 		comp_ring_cons = RCQ_BD(sw_comp_cons);
1092 		cqe = &fp->rx_comp_ring[comp_ring_cons];
1093 		cqe_fp = &cqe->fast_path_cqe;
1094 	} /* while */
1095 
1096 	fp->rx_bd_cons = bd_cons;
1097 	fp->rx_bd_prod = bd_prod_fw;
1098 	fp->rx_comp_cons = sw_comp_cons;
1099 	fp->rx_comp_prod = sw_comp_prod;
1100 
1101 	/* Update producers */
1102 	bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
1103 			     fp->rx_sge_prod);
1104 
1105 	fp->rx_pkt += rx_pkt;
1106 	fp->rx_calls++;
1107 
1108 	return rx_pkt;
1109 }
1110 
1111 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
1112 {
1113 	struct bnx2x_fastpath *fp = fp_cookie;
1114 	struct bnx2x *bp = fp->bp;
1115 	u8 cos;
1116 
1117 	DP(NETIF_MSG_INTR,
1118 	   "got an MSI-X interrupt on IDX:SB [fp %d fw_sd %d igusb %d]\n",
1119 	   fp->index, fp->fw_sb_id, fp->igu_sb_id);
1120 
1121 	bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
1122 
1123 #ifdef BNX2X_STOP_ON_ERROR
1124 	if (unlikely(bp->panic))
1125 		return IRQ_HANDLED;
1126 #endif
1127 
1128 	/* Handle Rx and Tx according to MSI-X vector */
1129 	for_each_cos_in_tx_queue(fp, cos)
1130 		prefetch(fp->txdata_ptr[cos]->tx_cons_sb);
1131 
1132 	prefetch(&fp->sb_running_index[SM_RX_ID]);
1133 	napi_schedule(&bnx2x_fp(bp, fp->index, napi));
1134 
1135 	return IRQ_HANDLED;
1136 }
1137 
1138 /* HW Lock for shared dual port PHYs */
1139 void bnx2x_acquire_phy_lock(struct bnx2x *bp)
1140 {
1141 	mutex_lock(&bp->port.phy_mutex);
1142 
1143 	bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1144 }
1145 
1146 void bnx2x_release_phy_lock(struct bnx2x *bp)
1147 {
1148 	bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1149 
1150 	mutex_unlock(&bp->port.phy_mutex);
1151 }
1152 
1153 /* calculates MF speed according to current linespeed and MF configuration */
1154 u16 bnx2x_get_mf_speed(struct bnx2x *bp)
1155 {
1156 	u16 line_speed = bp->link_vars.line_speed;
1157 	if (IS_MF(bp)) {
1158 		u16 maxCfg = bnx2x_extract_max_cfg(bp,
1159 						   bp->mf_config[BP_VN(bp)]);
1160 
1161 		/* Calculate the current MAX line speed limit for the MF
1162 		 * devices
1163 		 */
1164 		if (IS_MF_SI(bp))
1165 			line_speed = (line_speed * maxCfg) / 100;
1166 		else { /* SD mode */
1167 			u16 vn_max_rate = maxCfg * 100;
1168 
1169 			if (vn_max_rate < line_speed)
1170 				line_speed = vn_max_rate;
1171 		}
1172 	}
1173 
1174 	return line_speed;
1175 }
1176 
1177 /**
1178  * bnx2x_fill_report_data - fill link report data to report
1179  *
1180  * @bp:		driver handle
1181  * @data:	link state to update
1182  *
1183  * It uses a none-atomic bit operations because is called under the mutex.
1184  */
1185 static void bnx2x_fill_report_data(struct bnx2x *bp,
1186 				   struct bnx2x_link_report_data *data)
1187 {
1188 	u16 line_speed = bnx2x_get_mf_speed(bp);
1189 
1190 	memset(data, 0, sizeof(*data));
1191 
1192 	/* Fill the report data: effective line speed */
1193 	data->line_speed = line_speed;
1194 
1195 	/* Link is down */
1196 	if (!bp->link_vars.link_up || (bp->flags & MF_FUNC_DIS))
1197 		__set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1198 			  &data->link_report_flags);
1199 
1200 	/* Full DUPLEX */
1201 	if (bp->link_vars.duplex == DUPLEX_FULL)
1202 		__set_bit(BNX2X_LINK_REPORT_FD, &data->link_report_flags);
1203 
1204 	/* Rx Flow Control is ON */
1205 	if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX)
1206 		__set_bit(BNX2X_LINK_REPORT_RX_FC_ON, &data->link_report_flags);
1207 
1208 	/* Tx Flow Control is ON */
1209 	if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX)
1210 		__set_bit(BNX2X_LINK_REPORT_TX_FC_ON, &data->link_report_flags);
1211 }
1212 
1213 /**
1214  * bnx2x_link_report - report link status to OS.
1215  *
1216  * @bp:		driver handle
1217  *
1218  * Calls the __bnx2x_link_report() under the same locking scheme
1219  * as a link/PHY state managing code to ensure a consistent link
1220  * reporting.
1221  */
1222 
1223 void bnx2x_link_report(struct bnx2x *bp)
1224 {
1225 	bnx2x_acquire_phy_lock(bp);
1226 	__bnx2x_link_report(bp);
1227 	bnx2x_release_phy_lock(bp);
1228 }
1229 
1230 /**
1231  * __bnx2x_link_report - report link status to OS.
1232  *
1233  * @bp:		driver handle
1234  *
1235  * None atomic implementation.
1236  * Should be called under the phy_lock.
1237  */
1238 void __bnx2x_link_report(struct bnx2x *bp)
1239 {
1240 	struct bnx2x_link_report_data cur_data;
1241 
1242 	/* reread mf_cfg */
1243 	if (IS_PF(bp) && !CHIP_IS_E1(bp))
1244 		bnx2x_read_mf_cfg(bp);
1245 
1246 	/* Read the current link report info */
1247 	bnx2x_fill_report_data(bp, &cur_data);
1248 
1249 	/* Don't report link down or exactly the same link status twice */
1250 	if (!memcmp(&cur_data, &bp->last_reported_link, sizeof(cur_data)) ||
1251 	    (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1252 		      &bp->last_reported_link.link_report_flags) &&
1253 	     test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1254 		      &cur_data.link_report_flags)))
1255 		return;
1256 
1257 	bp->link_cnt++;
1258 
1259 	/* We are going to report a new link parameters now -
1260 	 * remember the current data for the next time.
1261 	 */
1262 	memcpy(&bp->last_reported_link, &cur_data, sizeof(cur_data));
1263 
1264 	if (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1265 		     &cur_data.link_report_flags)) {
1266 		netif_carrier_off(bp->dev);
1267 		netdev_err(bp->dev, "NIC Link is Down\n");
1268 		return;
1269 	} else {
1270 		const char *duplex;
1271 		const char *flow;
1272 
1273 		netif_carrier_on(bp->dev);
1274 
1275 		if (test_and_clear_bit(BNX2X_LINK_REPORT_FD,
1276 				       &cur_data.link_report_flags))
1277 			duplex = "full";
1278 		else
1279 			duplex = "half";
1280 
1281 		/* Handle the FC at the end so that only these flags would be
1282 		 * possibly set. This way we may easily check if there is no FC
1283 		 * enabled.
1284 		 */
1285 		if (cur_data.link_report_flags) {
1286 			if (test_bit(BNX2X_LINK_REPORT_RX_FC_ON,
1287 				     &cur_data.link_report_flags)) {
1288 				if (test_bit(BNX2X_LINK_REPORT_TX_FC_ON,
1289 				     &cur_data.link_report_flags))
1290 					flow = "ON - receive & transmit";
1291 				else
1292 					flow = "ON - receive";
1293 			} else {
1294 				flow = "ON - transmit";
1295 			}
1296 		} else {
1297 			flow = "none";
1298 		}
1299 		netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n",
1300 			    cur_data.line_speed, duplex, flow);
1301 	}
1302 }
1303 
1304 static void bnx2x_set_next_page_sgl(struct bnx2x_fastpath *fp)
1305 {
1306 	int i;
1307 
1308 	for (i = 1; i <= NUM_RX_SGE_PAGES; i++) {
1309 		struct eth_rx_sge *sge;
1310 
1311 		sge = &fp->rx_sge_ring[RX_SGE_CNT * i - 2];
1312 		sge->addr_hi =
1313 			cpu_to_le32(U64_HI(fp->rx_sge_mapping +
1314 			BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1315 
1316 		sge->addr_lo =
1317 			cpu_to_le32(U64_LO(fp->rx_sge_mapping +
1318 			BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1319 	}
1320 }
1321 
1322 static void bnx2x_free_tpa_pool(struct bnx2x *bp,
1323 				struct bnx2x_fastpath *fp, int last)
1324 {
1325 	int i;
1326 
1327 	for (i = 0; i < last; i++) {
1328 		struct bnx2x_agg_info *tpa_info = &fp->tpa_info[i];
1329 		struct sw_rx_bd *first_buf = &tpa_info->first_buf;
1330 		u8 *data = first_buf->data;
1331 
1332 		if (data == NULL) {
1333 			DP(NETIF_MSG_IFDOWN, "tpa bin %d empty on free\n", i);
1334 			continue;
1335 		}
1336 		if (tpa_info->tpa_state == BNX2X_TPA_START)
1337 			dma_unmap_single(&bp->pdev->dev,
1338 					 dma_unmap_addr(first_buf, mapping),
1339 					 fp->rx_buf_size, DMA_FROM_DEVICE);
1340 		bnx2x_frag_free(fp, data);
1341 		first_buf->data = NULL;
1342 	}
1343 }
1344 
1345 void bnx2x_init_rx_rings_cnic(struct bnx2x *bp)
1346 {
1347 	int j;
1348 
1349 	for_each_rx_queue_cnic(bp, j) {
1350 		struct bnx2x_fastpath *fp = &bp->fp[j];
1351 
1352 		fp->rx_bd_cons = 0;
1353 
1354 		/* Activate BD ring */
1355 		/* Warning!
1356 		 * this will generate an interrupt (to the TSTORM)
1357 		 * must only be done after chip is initialized
1358 		 */
1359 		bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1360 				     fp->rx_sge_prod);
1361 	}
1362 }
1363 
1364 void bnx2x_init_rx_rings(struct bnx2x *bp)
1365 {
1366 	int func = BP_FUNC(bp);
1367 	u16 ring_prod;
1368 	int i, j;
1369 
1370 	/* Allocate TPA resources */
1371 	for_each_eth_queue(bp, j) {
1372 		struct bnx2x_fastpath *fp = &bp->fp[j];
1373 
1374 		DP(NETIF_MSG_IFUP,
1375 		   "mtu %d  rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size);
1376 
1377 		if (!fp->disable_tpa) {
1378 			/* Fill the per-aggregation pool */
1379 			for (i = 0; i < MAX_AGG_QS(bp); i++) {
1380 				struct bnx2x_agg_info *tpa_info =
1381 					&fp->tpa_info[i];
1382 				struct sw_rx_bd *first_buf =
1383 					&tpa_info->first_buf;
1384 
1385 				first_buf->data =
1386 					bnx2x_frag_alloc(fp, GFP_KERNEL);
1387 				if (!first_buf->data) {
1388 					BNX2X_ERR("Failed to allocate TPA skb pool for queue[%d] - disabling TPA on this queue!\n",
1389 						  j);
1390 					bnx2x_free_tpa_pool(bp, fp, i);
1391 					fp->disable_tpa = 1;
1392 					break;
1393 				}
1394 				dma_unmap_addr_set(first_buf, mapping, 0);
1395 				tpa_info->tpa_state = BNX2X_TPA_STOP;
1396 			}
1397 
1398 			/* "next page" elements initialization */
1399 			bnx2x_set_next_page_sgl(fp);
1400 
1401 			/* set SGEs bit mask */
1402 			bnx2x_init_sge_ring_bit_mask(fp);
1403 
1404 			/* Allocate SGEs and initialize the ring elements */
1405 			for (i = 0, ring_prod = 0;
1406 			     i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
1407 
1408 				if (bnx2x_alloc_rx_sge(bp, fp, ring_prod,
1409 						       GFP_KERNEL) < 0) {
1410 					BNX2X_ERR("was only able to allocate %d rx sges\n",
1411 						  i);
1412 					BNX2X_ERR("disabling TPA for queue[%d]\n",
1413 						  j);
1414 					/* Cleanup already allocated elements */
1415 					bnx2x_free_rx_sge_range(bp, fp,
1416 								ring_prod);
1417 					bnx2x_free_tpa_pool(bp, fp,
1418 							    MAX_AGG_QS(bp));
1419 					fp->disable_tpa = 1;
1420 					ring_prod = 0;
1421 					break;
1422 				}
1423 				ring_prod = NEXT_SGE_IDX(ring_prod);
1424 			}
1425 
1426 			fp->rx_sge_prod = ring_prod;
1427 		}
1428 	}
1429 
1430 	for_each_eth_queue(bp, j) {
1431 		struct bnx2x_fastpath *fp = &bp->fp[j];
1432 
1433 		fp->rx_bd_cons = 0;
1434 
1435 		/* Activate BD ring */
1436 		/* Warning!
1437 		 * this will generate an interrupt (to the TSTORM)
1438 		 * must only be done after chip is initialized
1439 		 */
1440 		bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1441 				     fp->rx_sge_prod);
1442 
1443 		if (j != 0)
1444 			continue;
1445 
1446 		if (CHIP_IS_E1(bp)) {
1447 			REG_WR(bp, BAR_USTRORM_INTMEM +
1448 			       USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
1449 			       U64_LO(fp->rx_comp_mapping));
1450 			REG_WR(bp, BAR_USTRORM_INTMEM +
1451 			       USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
1452 			       U64_HI(fp->rx_comp_mapping));
1453 		}
1454 	}
1455 }
1456 
1457 static void bnx2x_free_tx_skbs_queue(struct bnx2x_fastpath *fp)
1458 {
1459 	u8 cos;
1460 	struct bnx2x *bp = fp->bp;
1461 
1462 	for_each_cos_in_tx_queue(fp, cos) {
1463 		struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
1464 		unsigned pkts_compl = 0, bytes_compl = 0;
1465 
1466 		u16 sw_prod = txdata->tx_pkt_prod;
1467 		u16 sw_cons = txdata->tx_pkt_cons;
1468 
1469 		while (sw_cons != sw_prod) {
1470 			bnx2x_free_tx_pkt(bp, txdata, TX_BD(sw_cons),
1471 					  &pkts_compl, &bytes_compl);
1472 			sw_cons++;
1473 		}
1474 
1475 		netdev_tx_reset_queue(
1476 			netdev_get_tx_queue(bp->dev,
1477 					    txdata->txq_index));
1478 	}
1479 }
1480 
1481 static void bnx2x_free_tx_skbs_cnic(struct bnx2x *bp)
1482 {
1483 	int i;
1484 
1485 	for_each_tx_queue_cnic(bp, i) {
1486 		bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1487 	}
1488 }
1489 
1490 static void bnx2x_free_tx_skbs(struct bnx2x *bp)
1491 {
1492 	int i;
1493 
1494 	for_each_eth_queue(bp, i) {
1495 		bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1496 	}
1497 }
1498 
1499 static void bnx2x_free_rx_bds(struct bnx2x_fastpath *fp)
1500 {
1501 	struct bnx2x *bp = fp->bp;
1502 	int i;
1503 
1504 	/* ring wasn't allocated */
1505 	if (fp->rx_buf_ring == NULL)
1506 		return;
1507 
1508 	for (i = 0; i < NUM_RX_BD; i++) {
1509 		struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
1510 		u8 *data = rx_buf->data;
1511 
1512 		if (data == NULL)
1513 			continue;
1514 		dma_unmap_single(&bp->pdev->dev,
1515 				 dma_unmap_addr(rx_buf, mapping),
1516 				 fp->rx_buf_size, DMA_FROM_DEVICE);
1517 
1518 		rx_buf->data = NULL;
1519 		bnx2x_frag_free(fp, data);
1520 	}
1521 }
1522 
1523 static void bnx2x_free_rx_skbs_cnic(struct bnx2x *bp)
1524 {
1525 	int j;
1526 
1527 	for_each_rx_queue_cnic(bp, j) {
1528 		bnx2x_free_rx_bds(&bp->fp[j]);
1529 	}
1530 }
1531 
1532 static void bnx2x_free_rx_skbs(struct bnx2x *bp)
1533 {
1534 	int j;
1535 
1536 	for_each_eth_queue(bp, j) {
1537 		struct bnx2x_fastpath *fp = &bp->fp[j];
1538 
1539 		bnx2x_free_rx_bds(fp);
1540 
1541 		if (!fp->disable_tpa)
1542 			bnx2x_free_tpa_pool(bp, fp, MAX_AGG_QS(bp));
1543 	}
1544 }
1545 
1546 static void bnx2x_free_skbs_cnic(struct bnx2x *bp)
1547 {
1548 	bnx2x_free_tx_skbs_cnic(bp);
1549 	bnx2x_free_rx_skbs_cnic(bp);
1550 }
1551 
1552 void bnx2x_free_skbs(struct bnx2x *bp)
1553 {
1554 	bnx2x_free_tx_skbs(bp);
1555 	bnx2x_free_rx_skbs(bp);
1556 }
1557 
1558 void bnx2x_update_max_mf_config(struct bnx2x *bp, u32 value)
1559 {
1560 	/* load old values */
1561 	u32 mf_cfg = bp->mf_config[BP_VN(bp)];
1562 
1563 	if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) {
1564 		/* leave all but MAX value */
1565 		mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK;
1566 
1567 		/* set new MAX value */
1568 		mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT)
1569 				& FUNC_MF_CFG_MAX_BW_MASK;
1570 
1571 		bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg);
1572 	}
1573 }
1574 
1575 /**
1576  * bnx2x_free_msix_irqs - free previously requested MSI-X IRQ vectors
1577  *
1578  * @bp:		driver handle
1579  * @nvecs:	number of vectors to be released
1580  */
1581 static void bnx2x_free_msix_irqs(struct bnx2x *bp, int nvecs)
1582 {
1583 	int i, offset = 0;
1584 
1585 	if (nvecs == offset)
1586 		return;
1587 
1588 	/* VFs don't have a default SB */
1589 	if (IS_PF(bp)) {
1590 		free_irq(bp->msix_table[offset].vector, bp->dev);
1591 		DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
1592 		   bp->msix_table[offset].vector);
1593 		offset++;
1594 	}
1595 
1596 	if (CNIC_SUPPORT(bp)) {
1597 		if (nvecs == offset)
1598 			return;
1599 		offset++;
1600 	}
1601 
1602 	for_each_eth_queue(bp, i) {
1603 		if (nvecs == offset)
1604 			return;
1605 		DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq\n",
1606 		   i, bp->msix_table[offset].vector);
1607 
1608 		free_irq(bp->msix_table[offset++].vector, &bp->fp[i]);
1609 	}
1610 }
1611 
1612 void bnx2x_free_irq(struct bnx2x *bp)
1613 {
1614 	if (bp->flags & USING_MSIX_FLAG &&
1615 	    !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1616 		int nvecs = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_SUPPORT(bp);
1617 
1618 		/* vfs don't have a default status block */
1619 		if (IS_PF(bp))
1620 			nvecs++;
1621 
1622 		bnx2x_free_msix_irqs(bp, nvecs);
1623 	} else {
1624 		free_irq(bp->dev->irq, bp->dev);
1625 	}
1626 }
1627 
1628 int bnx2x_enable_msix(struct bnx2x *bp)
1629 {
1630 	int msix_vec = 0, i, rc;
1631 
1632 	/* VFs don't have a default status block */
1633 	if (IS_PF(bp)) {
1634 		bp->msix_table[msix_vec].entry = msix_vec;
1635 		BNX2X_DEV_INFO("msix_table[0].entry = %d (slowpath)\n",
1636 			       bp->msix_table[0].entry);
1637 		msix_vec++;
1638 	}
1639 
1640 	/* Cnic requires an msix vector for itself */
1641 	if (CNIC_SUPPORT(bp)) {
1642 		bp->msix_table[msix_vec].entry = msix_vec;
1643 		BNX2X_DEV_INFO("msix_table[%d].entry = %d (CNIC)\n",
1644 			       msix_vec, bp->msix_table[msix_vec].entry);
1645 		msix_vec++;
1646 	}
1647 
1648 	/* We need separate vectors for ETH queues only (not FCoE) */
1649 	for_each_eth_queue(bp, i) {
1650 		bp->msix_table[msix_vec].entry = msix_vec;
1651 		BNX2X_DEV_INFO("msix_table[%d].entry = %d (fastpath #%u)\n",
1652 			       msix_vec, msix_vec, i);
1653 		msix_vec++;
1654 	}
1655 
1656 	DP(BNX2X_MSG_SP, "about to request enable msix with %d vectors\n",
1657 	   msix_vec);
1658 
1659 	rc = pci_enable_msix_range(bp->pdev, &bp->msix_table[0],
1660 				   BNX2X_MIN_MSIX_VEC_CNT(bp), msix_vec);
1661 	/*
1662 	 * reconfigure number of tx/rx queues according to available
1663 	 * MSI-X vectors
1664 	 */
1665 	if (rc == -ENOSPC) {
1666 		/* Get by with single vector */
1667 		rc = pci_enable_msix_range(bp->pdev, &bp->msix_table[0], 1, 1);
1668 		if (rc < 0) {
1669 			BNX2X_DEV_INFO("Single MSI-X is not attainable rc %d\n",
1670 				       rc);
1671 			goto no_msix;
1672 		}
1673 
1674 		BNX2X_DEV_INFO("Using single MSI-X vector\n");
1675 		bp->flags |= USING_SINGLE_MSIX_FLAG;
1676 
1677 		BNX2X_DEV_INFO("set number of queues to 1\n");
1678 		bp->num_ethernet_queues = 1;
1679 		bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1680 	} else if (rc < 0) {
1681 		BNX2X_DEV_INFO("MSI-X is not attainable rc %d\n", rc);
1682 		goto no_msix;
1683 	} else if (rc < msix_vec) {
1684 		/* how less vectors we will have? */
1685 		int diff = msix_vec - rc;
1686 
1687 		BNX2X_DEV_INFO("Trying to use less MSI-X vectors: %d\n", rc);
1688 
1689 		/*
1690 		 * decrease number of queues by number of unallocated entries
1691 		 */
1692 		bp->num_ethernet_queues -= diff;
1693 		bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1694 
1695 		BNX2X_DEV_INFO("New queue configuration set: %d\n",
1696 			       bp->num_queues);
1697 	}
1698 
1699 	bp->flags |= USING_MSIX_FLAG;
1700 
1701 	return 0;
1702 
1703 no_msix:
1704 	/* fall to INTx if not enough memory */
1705 	if (rc == -ENOMEM)
1706 		bp->flags |= DISABLE_MSI_FLAG;
1707 
1708 	return rc;
1709 }
1710 
1711 static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1712 {
1713 	int i, rc, offset = 0;
1714 
1715 	/* no default status block for vf */
1716 	if (IS_PF(bp)) {
1717 		rc = request_irq(bp->msix_table[offset++].vector,
1718 				 bnx2x_msix_sp_int, 0,
1719 				 bp->dev->name, bp->dev);
1720 		if (rc) {
1721 			BNX2X_ERR("request sp irq failed\n");
1722 			return -EBUSY;
1723 		}
1724 	}
1725 
1726 	if (CNIC_SUPPORT(bp))
1727 		offset++;
1728 
1729 	for_each_eth_queue(bp, i) {
1730 		struct bnx2x_fastpath *fp = &bp->fp[i];
1731 		snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1732 			 bp->dev->name, i);
1733 
1734 		rc = request_irq(bp->msix_table[offset].vector,
1735 				 bnx2x_msix_fp_int, 0, fp->name, fp);
1736 		if (rc) {
1737 			BNX2X_ERR("request fp #%d irq (%d) failed  rc %d\n", i,
1738 			      bp->msix_table[offset].vector, rc);
1739 			bnx2x_free_msix_irqs(bp, offset);
1740 			return -EBUSY;
1741 		}
1742 
1743 		offset++;
1744 	}
1745 
1746 	i = BNX2X_NUM_ETH_QUEUES(bp);
1747 	if (IS_PF(bp)) {
1748 		offset = 1 + CNIC_SUPPORT(bp);
1749 		netdev_info(bp->dev,
1750 			    "using MSI-X  IRQs: sp %d  fp[%d] %d ... fp[%d] %d\n",
1751 			    bp->msix_table[0].vector,
1752 			    0, bp->msix_table[offset].vector,
1753 			    i - 1, bp->msix_table[offset + i - 1].vector);
1754 	} else {
1755 		offset = CNIC_SUPPORT(bp);
1756 		netdev_info(bp->dev,
1757 			    "using MSI-X  IRQs: fp[%d] %d ... fp[%d] %d\n",
1758 			    0, bp->msix_table[offset].vector,
1759 			    i - 1, bp->msix_table[offset + i - 1].vector);
1760 	}
1761 	return 0;
1762 }
1763 
1764 int bnx2x_enable_msi(struct bnx2x *bp)
1765 {
1766 	int rc;
1767 
1768 	rc = pci_enable_msi(bp->pdev);
1769 	if (rc) {
1770 		BNX2X_DEV_INFO("MSI is not attainable\n");
1771 		return -1;
1772 	}
1773 	bp->flags |= USING_MSI_FLAG;
1774 
1775 	return 0;
1776 }
1777 
1778 static int bnx2x_req_irq(struct bnx2x *bp)
1779 {
1780 	unsigned long flags;
1781 	unsigned int irq;
1782 
1783 	if (bp->flags & (USING_MSI_FLAG | USING_MSIX_FLAG))
1784 		flags = 0;
1785 	else
1786 		flags = IRQF_SHARED;
1787 
1788 	if (bp->flags & USING_MSIX_FLAG)
1789 		irq = bp->msix_table[0].vector;
1790 	else
1791 		irq = bp->pdev->irq;
1792 
1793 	return request_irq(irq, bnx2x_interrupt, flags, bp->dev->name, bp->dev);
1794 }
1795 
1796 static int bnx2x_setup_irqs(struct bnx2x *bp)
1797 {
1798 	int rc = 0;
1799 	if (bp->flags & USING_MSIX_FLAG &&
1800 	    !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1801 		rc = bnx2x_req_msix_irqs(bp);
1802 		if (rc)
1803 			return rc;
1804 	} else {
1805 		rc = bnx2x_req_irq(bp);
1806 		if (rc) {
1807 			BNX2X_ERR("IRQ request failed  rc %d, aborting\n", rc);
1808 			return rc;
1809 		}
1810 		if (bp->flags & USING_MSI_FLAG) {
1811 			bp->dev->irq = bp->pdev->irq;
1812 			netdev_info(bp->dev, "using MSI IRQ %d\n",
1813 				    bp->dev->irq);
1814 		}
1815 		if (bp->flags & USING_MSIX_FLAG) {
1816 			bp->dev->irq = bp->msix_table[0].vector;
1817 			netdev_info(bp->dev, "using MSIX IRQ %d\n",
1818 				    bp->dev->irq);
1819 		}
1820 	}
1821 
1822 	return 0;
1823 }
1824 
1825 static void bnx2x_napi_enable_cnic(struct bnx2x *bp)
1826 {
1827 	int i;
1828 
1829 	for_each_rx_queue_cnic(bp, i) {
1830 		bnx2x_fp_init_lock(&bp->fp[i]);
1831 		napi_enable(&bnx2x_fp(bp, i, napi));
1832 	}
1833 }
1834 
1835 static void bnx2x_napi_enable(struct bnx2x *bp)
1836 {
1837 	int i;
1838 
1839 	for_each_eth_queue(bp, i) {
1840 		bnx2x_fp_init_lock(&bp->fp[i]);
1841 		napi_enable(&bnx2x_fp(bp, i, napi));
1842 	}
1843 }
1844 
1845 static void bnx2x_napi_disable_cnic(struct bnx2x *bp)
1846 {
1847 	int i;
1848 
1849 	for_each_rx_queue_cnic(bp, i) {
1850 		napi_disable(&bnx2x_fp(bp, i, napi));
1851 		while (!bnx2x_fp_ll_disable(&bp->fp[i]))
1852 			usleep_range(1000, 2000);
1853 	}
1854 }
1855 
1856 static void bnx2x_napi_disable(struct bnx2x *bp)
1857 {
1858 	int i;
1859 
1860 	for_each_eth_queue(bp, i) {
1861 		napi_disable(&bnx2x_fp(bp, i, napi));
1862 		while (!bnx2x_fp_ll_disable(&bp->fp[i]))
1863 			usleep_range(1000, 2000);
1864 	}
1865 }
1866 
1867 void bnx2x_netif_start(struct bnx2x *bp)
1868 {
1869 	if (netif_running(bp->dev)) {
1870 		bnx2x_napi_enable(bp);
1871 		if (CNIC_LOADED(bp))
1872 			bnx2x_napi_enable_cnic(bp);
1873 		bnx2x_int_enable(bp);
1874 		if (bp->state == BNX2X_STATE_OPEN)
1875 			netif_tx_wake_all_queues(bp->dev);
1876 	}
1877 }
1878 
1879 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1880 {
1881 	bnx2x_int_disable_sync(bp, disable_hw);
1882 	bnx2x_napi_disable(bp);
1883 	if (CNIC_LOADED(bp))
1884 		bnx2x_napi_disable_cnic(bp);
1885 }
1886 
1887 u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb,
1888 		       void *accel_priv, select_queue_fallback_t fallback)
1889 {
1890 	struct bnx2x *bp = netdev_priv(dev);
1891 
1892 	if (CNIC_LOADED(bp) && !NO_FCOE(bp)) {
1893 		struct ethhdr *hdr = (struct ethhdr *)skb->data;
1894 		u16 ether_type = ntohs(hdr->h_proto);
1895 
1896 		/* Skip VLAN tag if present */
1897 		if (ether_type == ETH_P_8021Q) {
1898 			struct vlan_ethhdr *vhdr =
1899 				(struct vlan_ethhdr *)skb->data;
1900 
1901 			ether_type = ntohs(vhdr->h_vlan_encapsulated_proto);
1902 		}
1903 
1904 		/* If ethertype is FCoE or FIP - use FCoE ring */
1905 		if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP))
1906 			return bnx2x_fcoe_tx(bp, txq_index);
1907 	}
1908 
1909 	/* select a non-FCoE queue */
1910 	return fallback(dev, skb) % BNX2X_NUM_ETH_QUEUES(bp);
1911 }
1912 
1913 void bnx2x_set_num_queues(struct bnx2x *bp)
1914 {
1915 	/* RSS queues */
1916 	bp->num_ethernet_queues = bnx2x_calc_num_queues(bp);
1917 
1918 	/* override in STORAGE SD modes */
1919 	if (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))
1920 		bp->num_ethernet_queues = 1;
1921 
1922 	/* Add special queues */
1923 	bp->num_cnic_queues = CNIC_SUPPORT(bp); /* For FCOE */
1924 	bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1925 
1926 	BNX2X_DEV_INFO("set number of queues to %d\n", bp->num_queues);
1927 }
1928 
1929 /**
1930  * bnx2x_set_real_num_queues - configure netdev->real_num_[tx,rx]_queues
1931  *
1932  * @bp:		Driver handle
1933  *
1934  * We currently support for at most 16 Tx queues for each CoS thus we will
1935  * allocate a multiple of 16 for ETH L2 rings according to the value of the
1936  * bp->max_cos.
1937  *
1938  * If there is an FCoE L2 queue the appropriate Tx queue will have the next
1939  * index after all ETH L2 indices.
1940  *
1941  * If the actual number of Tx queues (for each CoS) is less than 16 then there
1942  * will be the holes at the end of each group of 16 ETh L2 indices (0..15,
1943  * 16..31,...) with indices that are not coupled with any real Tx queue.
1944  *
1945  * The proper configuration of skb->queue_mapping is handled by
1946  * bnx2x_select_queue() and __skb_tx_hash().
1947  *
1948  * bnx2x_setup_tc() takes care of the proper TC mappings so that __skb_tx_hash()
1949  * will return a proper Tx index if TC is enabled (netdev->num_tc > 0).
1950  */
1951 static int bnx2x_set_real_num_queues(struct bnx2x *bp, int include_cnic)
1952 {
1953 	int rc, tx, rx;
1954 
1955 	tx = BNX2X_NUM_ETH_QUEUES(bp) * bp->max_cos;
1956 	rx = BNX2X_NUM_ETH_QUEUES(bp);
1957 
1958 /* account for fcoe queue */
1959 	if (include_cnic && !NO_FCOE(bp)) {
1960 		rx++;
1961 		tx++;
1962 	}
1963 
1964 	rc = netif_set_real_num_tx_queues(bp->dev, tx);
1965 	if (rc) {
1966 		BNX2X_ERR("Failed to set real number of Tx queues: %d\n", rc);
1967 		return rc;
1968 	}
1969 	rc = netif_set_real_num_rx_queues(bp->dev, rx);
1970 	if (rc) {
1971 		BNX2X_ERR("Failed to set real number of Rx queues: %d\n", rc);
1972 		return rc;
1973 	}
1974 
1975 	DP(NETIF_MSG_IFUP, "Setting real num queues to (tx, rx) (%d, %d)\n",
1976 			  tx, rx);
1977 
1978 	return rc;
1979 }
1980 
1981 static void bnx2x_set_rx_buf_size(struct bnx2x *bp)
1982 {
1983 	int i;
1984 
1985 	for_each_queue(bp, i) {
1986 		struct bnx2x_fastpath *fp = &bp->fp[i];
1987 		u32 mtu;
1988 
1989 		/* Always use a mini-jumbo MTU for the FCoE L2 ring */
1990 		if (IS_FCOE_IDX(i))
1991 			/*
1992 			 * Although there are no IP frames expected to arrive to
1993 			 * this ring we still want to add an
1994 			 * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer
1995 			 * overrun attack.
1996 			 */
1997 			mtu = BNX2X_FCOE_MINI_JUMBO_MTU;
1998 		else
1999 			mtu = bp->dev->mtu;
2000 		fp->rx_buf_size = BNX2X_FW_RX_ALIGN_START +
2001 				  IP_HEADER_ALIGNMENT_PADDING +
2002 				  ETH_OVREHEAD +
2003 				  mtu +
2004 				  BNX2X_FW_RX_ALIGN_END;
2005 		/* Note : rx_buf_size doesn't take into account NET_SKB_PAD */
2006 		if (fp->rx_buf_size + NET_SKB_PAD <= PAGE_SIZE)
2007 			fp->rx_frag_size = fp->rx_buf_size + NET_SKB_PAD;
2008 		else
2009 			fp->rx_frag_size = 0;
2010 	}
2011 }
2012 
2013 static int bnx2x_init_rss(struct bnx2x *bp)
2014 {
2015 	int i;
2016 	u8 num_eth_queues = BNX2X_NUM_ETH_QUEUES(bp);
2017 
2018 	/* Prepare the initial contents for the indirection table if RSS is
2019 	 * enabled
2020 	 */
2021 	for (i = 0; i < sizeof(bp->rss_conf_obj.ind_table); i++)
2022 		bp->rss_conf_obj.ind_table[i] =
2023 			bp->fp->cl_id +
2024 			ethtool_rxfh_indir_default(i, num_eth_queues);
2025 
2026 	/*
2027 	 * For 57710 and 57711 SEARCHER configuration (rss_keys) is
2028 	 * per-port, so if explicit configuration is needed , do it only
2029 	 * for a PMF.
2030 	 *
2031 	 * For 57712 and newer on the other hand it's a per-function
2032 	 * configuration.
2033 	 */
2034 	return bnx2x_config_rss_eth(bp, bp->port.pmf || !CHIP_IS_E1x(bp));
2035 }
2036 
2037 int bnx2x_rss(struct bnx2x *bp, struct bnx2x_rss_config_obj *rss_obj,
2038 	      bool config_hash, bool enable)
2039 {
2040 	struct bnx2x_config_rss_params params = {NULL};
2041 
2042 	/* Although RSS is meaningless when there is a single HW queue we
2043 	 * still need it enabled in order to have HW Rx hash generated.
2044 	 *
2045 	 * if (!is_eth_multi(bp))
2046 	 *      bp->multi_mode = ETH_RSS_MODE_DISABLED;
2047 	 */
2048 
2049 	params.rss_obj = rss_obj;
2050 
2051 	__set_bit(RAMROD_COMP_WAIT, &params.ramrod_flags);
2052 
2053 	if (enable) {
2054 		__set_bit(BNX2X_RSS_MODE_REGULAR, &params.rss_flags);
2055 
2056 		/* RSS configuration */
2057 		__set_bit(BNX2X_RSS_IPV4, &params.rss_flags);
2058 		__set_bit(BNX2X_RSS_IPV4_TCP, &params.rss_flags);
2059 		__set_bit(BNX2X_RSS_IPV6, &params.rss_flags);
2060 		__set_bit(BNX2X_RSS_IPV6_TCP, &params.rss_flags);
2061 		if (rss_obj->udp_rss_v4)
2062 			__set_bit(BNX2X_RSS_IPV4_UDP, &params.rss_flags);
2063 		if (rss_obj->udp_rss_v6)
2064 			__set_bit(BNX2X_RSS_IPV6_UDP, &params.rss_flags);
2065 	} else {
2066 		__set_bit(BNX2X_RSS_MODE_DISABLED, &params.rss_flags);
2067 	}
2068 
2069 	/* Hash bits */
2070 	params.rss_result_mask = MULTI_MASK;
2071 
2072 	memcpy(params.ind_table, rss_obj->ind_table, sizeof(params.ind_table));
2073 
2074 	if (config_hash) {
2075 		/* RSS keys */
2076 		prandom_bytes(params.rss_key, T_ETH_RSS_KEY * 4);
2077 		__set_bit(BNX2X_RSS_SET_SRCH, &params.rss_flags);
2078 	}
2079 
2080 	if (IS_PF(bp))
2081 		return bnx2x_config_rss(bp, &params);
2082 	else
2083 		return bnx2x_vfpf_config_rss(bp, &params);
2084 }
2085 
2086 static int bnx2x_init_hw(struct bnx2x *bp, u32 load_code)
2087 {
2088 	struct bnx2x_func_state_params func_params = {NULL};
2089 
2090 	/* Prepare parameters for function state transitions */
2091 	__set_bit(RAMROD_COMP_WAIT, &func_params.ramrod_flags);
2092 
2093 	func_params.f_obj = &bp->func_obj;
2094 	func_params.cmd = BNX2X_F_CMD_HW_INIT;
2095 
2096 	func_params.params.hw_init.load_phase = load_code;
2097 
2098 	return bnx2x_func_state_change(bp, &func_params);
2099 }
2100 
2101 /*
2102  * Cleans the object that have internal lists without sending
2103  * ramrods. Should be run when interrupts are disabled.
2104  */
2105 void bnx2x_squeeze_objects(struct bnx2x *bp)
2106 {
2107 	int rc;
2108 	unsigned long ramrod_flags = 0, vlan_mac_flags = 0;
2109 	struct bnx2x_mcast_ramrod_params rparam = {NULL};
2110 	struct bnx2x_vlan_mac_obj *mac_obj = &bp->sp_objs->mac_obj;
2111 
2112 	/***************** Cleanup MACs' object first *************************/
2113 
2114 	/* Wait for completion of requested */
2115 	__set_bit(RAMROD_COMP_WAIT, &ramrod_flags);
2116 	/* Perform a dry cleanup */
2117 	__set_bit(RAMROD_DRV_CLR_ONLY, &ramrod_flags);
2118 
2119 	/* Clean ETH primary MAC */
2120 	__set_bit(BNX2X_ETH_MAC, &vlan_mac_flags);
2121 	rc = mac_obj->delete_all(bp, &bp->sp_objs->mac_obj, &vlan_mac_flags,
2122 				 &ramrod_flags);
2123 	if (rc != 0)
2124 		BNX2X_ERR("Failed to clean ETH MACs: %d\n", rc);
2125 
2126 	/* Cleanup UC list */
2127 	vlan_mac_flags = 0;
2128 	__set_bit(BNX2X_UC_LIST_MAC, &vlan_mac_flags);
2129 	rc = mac_obj->delete_all(bp, mac_obj, &vlan_mac_flags,
2130 				 &ramrod_flags);
2131 	if (rc != 0)
2132 		BNX2X_ERR("Failed to clean UC list MACs: %d\n", rc);
2133 
2134 	/***************** Now clean mcast object *****************************/
2135 	rparam.mcast_obj = &bp->mcast_obj;
2136 	__set_bit(RAMROD_DRV_CLR_ONLY, &rparam.ramrod_flags);
2137 
2138 	/* Add a DEL command... - Since we're doing a driver cleanup only,
2139 	 * we take a lock surrounding both the initial send and the CONTs,
2140 	 * as we don't want a true completion to disrupt us in the middle.
2141 	 */
2142 	netif_addr_lock_bh(bp->dev);
2143 	rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_DEL);
2144 	if (rc < 0)
2145 		BNX2X_ERR("Failed to add a new DEL command to a multi-cast object: %d\n",
2146 			  rc);
2147 
2148 	/* ...and wait until all pending commands are cleared */
2149 	rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2150 	while (rc != 0) {
2151 		if (rc < 0) {
2152 			BNX2X_ERR("Failed to clean multi-cast object: %d\n",
2153 				  rc);
2154 			netif_addr_unlock_bh(bp->dev);
2155 			return;
2156 		}
2157 
2158 		rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2159 	}
2160 	netif_addr_unlock_bh(bp->dev);
2161 }
2162 
2163 #ifndef BNX2X_STOP_ON_ERROR
2164 #define LOAD_ERROR_EXIT(bp, label) \
2165 	do { \
2166 		(bp)->state = BNX2X_STATE_ERROR; \
2167 		goto label; \
2168 	} while (0)
2169 
2170 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
2171 	do { \
2172 		bp->cnic_loaded = false; \
2173 		goto label; \
2174 	} while (0)
2175 #else /*BNX2X_STOP_ON_ERROR*/
2176 #define LOAD_ERROR_EXIT(bp, label) \
2177 	do { \
2178 		(bp)->state = BNX2X_STATE_ERROR; \
2179 		(bp)->panic = 1; \
2180 		return -EBUSY; \
2181 	} while (0)
2182 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
2183 	do { \
2184 		bp->cnic_loaded = false; \
2185 		(bp)->panic = 1; \
2186 		return -EBUSY; \
2187 	} while (0)
2188 #endif /*BNX2X_STOP_ON_ERROR*/
2189 
2190 static void bnx2x_free_fw_stats_mem(struct bnx2x *bp)
2191 {
2192 	BNX2X_PCI_FREE(bp->fw_stats, bp->fw_stats_mapping,
2193 		       bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2194 	return;
2195 }
2196 
2197 static int bnx2x_alloc_fw_stats_mem(struct bnx2x *bp)
2198 {
2199 	int num_groups, vf_headroom = 0;
2200 	int is_fcoe_stats = NO_FCOE(bp) ? 0 : 1;
2201 
2202 	/* number of queues for statistics is number of eth queues + FCoE */
2203 	u8 num_queue_stats = BNX2X_NUM_ETH_QUEUES(bp) + is_fcoe_stats;
2204 
2205 	/* Total number of FW statistics requests =
2206 	 * 1 for port stats + 1 for PF stats + potential 2 for FCoE (fcoe proper
2207 	 * and fcoe l2 queue) stats + num of queues (which includes another 1
2208 	 * for fcoe l2 queue if applicable)
2209 	 */
2210 	bp->fw_stats_num = 2 + is_fcoe_stats + num_queue_stats;
2211 
2212 	/* vf stats appear in the request list, but their data is allocated by
2213 	 * the VFs themselves. We don't include them in the bp->fw_stats_num as
2214 	 * it is used to determine where to place the vf stats queries in the
2215 	 * request struct
2216 	 */
2217 	if (IS_SRIOV(bp))
2218 		vf_headroom = bnx2x_vf_headroom(bp);
2219 
2220 	/* Request is built from stats_query_header and an array of
2221 	 * stats_query_cmd_group each of which contains
2222 	 * STATS_QUERY_CMD_COUNT rules. The real number or requests is
2223 	 * configured in the stats_query_header.
2224 	 */
2225 	num_groups =
2226 		(((bp->fw_stats_num + vf_headroom) / STATS_QUERY_CMD_COUNT) +
2227 		 (((bp->fw_stats_num + vf_headroom) % STATS_QUERY_CMD_COUNT) ?
2228 		 1 : 0));
2229 
2230 	DP(BNX2X_MSG_SP, "stats fw_stats_num %d, vf headroom %d, num_groups %d\n",
2231 	   bp->fw_stats_num, vf_headroom, num_groups);
2232 	bp->fw_stats_req_sz = sizeof(struct stats_query_header) +
2233 		num_groups * sizeof(struct stats_query_cmd_group);
2234 
2235 	/* Data for statistics requests + stats_counter
2236 	 * stats_counter holds per-STORM counters that are incremented
2237 	 * when STORM has finished with the current request.
2238 	 * memory for FCoE offloaded statistics are counted anyway,
2239 	 * even if they will not be sent.
2240 	 * VF stats are not accounted for here as the data of VF stats is stored
2241 	 * in memory allocated by the VF, not here.
2242 	 */
2243 	bp->fw_stats_data_sz = sizeof(struct per_port_stats) +
2244 		sizeof(struct per_pf_stats) +
2245 		sizeof(struct fcoe_statistics_params) +
2246 		sizeof(struct per_queue_stats) * num_queue_stats +
2247 		sizeof(struct stats_counter);
2248 
2249 	bp->fw_stats = BNX2X_PCI_ALLOC(&bp->fw_stats_mapping,
2250 				       bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2251 	if (!bp->fw_stats)
2252 		goto alloc_mem_err;
2253 
2254 	/* Set shortcuts */
2255 	bp->fw_stats_req = (struct bnx2x_fw_stats_req *)bp->fw_stats;
2256 	bp->fw_stats_req_mapping = bp->fw_stats_mapping;
2257 	bp->fw_stats_data = (struct bnx2x_fw_stats_data *)
2258 		((u8 *)bp->fw_stats + bp->fw_stats_req_sz);
2259 	bp->fw_stats_data_mapping = bp->fw_stats_mapping +
2260 		bp->fw_stats_req_sz;
2261 
2262 	DP(BNX2X_MSG_SP, "statistics request base address set to %x %x\n",
2263 	   U64_HI(bp->fw_stats_req_mapping),
2264 	   U64_LO(bp->fw_stats_req_mapping));
2265 	DP(BNX2X_MSG_SP, "statistics data base address set to %x %x\n",
2266 	   U64_HI(bp->fw_stats_data_mapping),
2267 	   U64_LO(bp->fw_stats_data_mapping));
2268 	return 0;
2269 
2270 alloc_mem_err:
2271 	bnx2x_free_fw_stats_mem(bp);
2272 	BNX2X_ERR("Can't allocate FW stats memory\n");
2273 	return -ENOMEM;
2274 }
2275 
2276 /* send load request to mcp and analyze response */
2277 static int bnx2x_nic_load_request(struct bnx2x *bp, u32 *load_code)
2278 {
2279 	u32 param;
2280 
2281 	/* init fw_seq */
2282 	bp->fw_seq =
2283 		(SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_mb_header) &
2284 		 DRV_MSG_SEQ_NUMBER_MASK);
2285 	BNX2X_DEV_INFO("fw_seq 0x%08x\n", bp->fw_seq);
2286 
2287 	/* Get current FW pulse sequence */
2288 	bp->fw_drv_pulse_wr_seq =
2289 		(SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_pulse_mb) &
2290 		 DRV_PULSE_SEQ_MASK);
2291 	BNX2X_DEV_INFO("drv_pulse 0x%x\n", bp->fw_drv_pulse_wr_seq);
2292 
2293 	param = DRV_MSG_CODE_LOAD_REQ_WITH_LFA;
2294 
2295 	if (IS_MF_SD(bp) && bnx2x_port_after_undi(bp))
2296 		param |= DRV_MSG_CODE_LOAD_REQ_FORCE_LFA;
2297 
2298 	/* load request */
2299 	(*load_code) = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ, param);
2300 
2301 	/* if mcp fails to respond we must abort */
2302 	if (!(*load_code)) {
2303 		BNX2X_ERR("MCP response failure, aborting\n");
2304 		return -EBUSY;
2305 	}
2306 
2307 	/* If mcp refused (e.g. other port is in diagnostic mode) we
2308 	 * must abort
2309 	 */
2310 	if ((*load_code) == FW_MSG_CODE_DRV_LOAD_REFUSED) {
2311 		BNX2X_ERR("MCP refused load request, aborting\n");
2312 		return -EBUSY;
2313 	}
2314 	return 0;
2315 }
2316 
2317 /* check whether another PF has already loaded FW to chip. In
2318  * virtualized environments a pf from another VM may have already
2319  * initialized the device including loading FW
2320  */
2321 int bnx2x_compare_fw_ver(struct bnx2x *bp, u32 load_code, bool print_err)
2322 {
2323 	/* is another pf loaded on this engine? */
2324 	if (load_code != FW_MSG_CODE_DRV_LOAD_COMMON_CHIP &&
2325 	    load_code != FW_MSG_CODE_DRV_LOAD_COMMON) {
2326 		/* build my FW version dword */
2327 		u32 my_fw = (BCM_5710_FW_MAJOR_VERSION) +
2328 			(BCM_5710_FW_MINOR_VERSION << 8) +
2329 			(BCM_5710_FW_REVISION_VERSION << 16) +
2330 			(BCM_5710_FW_ENGINEERING_VERSION << 24);
2331 
2332 		/* read loaded FW from chip */
2333 		u32 loaded_fw = REG_RD(bp, XSEM_REG_PRAM);
2334 
2335 		DP(BNX2X_MSG_SP, "loaded fw %x, my fw %x\n",
2336 		   loaded_fw, my_fw);
2337 
2338 		/* abort nic load if version mismatch */
2339 		if (my_fw != loaded_fw) {
2340 			if (print_err)
2341 				BNX2X_ERR("bnx2x with FW %x was already loaded which mismatches my %x FW. Aborting\n",
2342 					  loaded_fw, my_fw);
2343 			else
2344 				BNX2X_DEV_INFO("bnx2x with FW %x was already loaded which mismatches my %x FW, possibly due to MF UNDI\n",
2345 					       loaded_fw, my_fw);
2346 			return -EBUSY;
2347 		}
2348 	}
2349 	return 0;
2350 }
2351 
2352 /* returns the "mcp load_code" according to global load_count array */
2353 static int bnx2x_nic_load_no_mcp(struct bnx2x *bp, int port)
2354 {
2355 	int path = BP_PATH(bp);
2356 
2357 	DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d]      %d, %d, %d\n",
2358 	   path, bnx2x_load_count[path][0], bnx2x_load_count[path][1],
2359 	   bnx2x_load_count[path][2]);
2360 	bnx2x_load_count[path][0]++;
2361 	bnx2x_load_count[path][1 + port]++;
2362 	DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d]  %d, %d, %d\n",
2363 	   path, bnx2x_load_count[path][0], bnx2x_load_count[path][1],
2364 	   bnx2x_load_count[path][2]);
2365 	if (bnx2x_load_count[path][0] == 1)
2366 		return FW_MSG_CODE_DRV_LOAD_COMMON;
2367 	else if (bnx2x_load_count[path][1 + port] == 1)
2368 		return FW_MSG_CODE_DRV_LOAD_PORT;
2369 	else
2370 		return FW_MSG_CODE_DRV_LOAD_FUNCTION;
2371 }
2372 
2373 /* mark PMF if applicable */
2374 static void bnx2x_nic_load_pmf(struct bnx2x *bp, u32 load_code)
2375 {
2376 	if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2377 	    (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
2378 	    (load_code == FW_MSG_CODE_DRV_LOAD_PORT)) {
2379 		bp->port.pmf = 1;
2380 		/* We need the barrier to ensure the ordering between the
2381 		 * writing to bp->port.pmf here and reading it from the
2382 		 * bnx2x_periodic_task().
2383 		 */
2384 		smp_mb();
2385 	} else {
2386 		bp->port.pmf = 0;
2387 	}
2388 
2389 	DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf);
2390 }
2391 
2392 static void bnx2x_nic_load_afex_dcc(struct bnx2x *bp, int load_code)
2393 {
2394 	if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2395 	     (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
2396 	    (bp->common.shmem2_base)) {
2397 		if (SHMEM2_HAS(bp, dcc_support))
2398 			SHMEM2_WR(bp, dcc_support,
2399 				  (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
2400 				   SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
2401 		if (SHMEM2_HAS(bp, afex_driver_support))
2402 			SHMEM2_WR(bp, afex_driver_support,
2403 				  SHMEM_AFEX_SUPPORTED_VERSION_ONE);
2404 	}
2405 
2406 	/* Set AFEX default VLAN tag to an invalid value */
2407 	bp->afex_def_vlan_tag = -1;
2408 }
2409 
2410 /**
2411  * bnx2x_bz_fp - zero content of the fastpath structure.
2412  *
2413  * @bp:		driver handle
2414  * @index:	fastpath index to be zeroed
2415  *
2416  * Makes sure the contents of the bp->fp[index].napi is kept
2417  * intact.
2418  */
2419 static void bnx2x_bz_fp(struct bnx2x *bp, int index)
2420 {
2421 	struct bnx2x_fastpath *fp = &bp->fp[index];
2422 	int cos;
2423 	struct napi_struct orig_napi = fp->napi;
2424 	struct bnx2x_agg_info *orig_tpa_info = fp->tpa_info;
2425 
2426 	/* bzero bnx2x_fastpath contents */
2427 	if (fp->tpa_info)
2428 		memset(fp->tpa_info, 0, ETH_MAX_AGGREGATION_QUEUES_E1H_E2 *
2429 		       sizeof(struct bnx2x_agg_info));
2430 	memset(fp, 0, sizeof(*fp));
2431 
2432 	/* Restore the NAPI object as it has been already initialized */
2433 	fp->napi = orig_napi;
2434 	fp->tpa_info = orig_tpa_info;
2435 	fp->bp = bp;
2436 	fp->index = index;
2437 	if (IS_ETH_FP(fp))
2438 		fp->max_cos = bp->max_cos;
2439 	else
2440 		/* Special queues support only one CoS */
2441 		fp->max_cos = 1;
2442 
2443 	/* Init txdata pointers */
2444 	if (IS_FCOE_FP(fp))
2445 		fp->txdata_ptr[0] = &bp->bnx2x_txq[FCOE_TXQ_IDX(bp)];
2446 	if (IS_ETH_FP(fp))
2447 		for_each_cos_in_tx_queue(fp, cos)
2448 			fp->txdata_ptr[cos] = &bp->bnx2x_txq[cos *
2449 				BNX2X_NUM_ETH_QUEUES(bp) + index];
2450 
2451 	/* set the tpa flag for each queue. The tpa flag determines the queue
2452 	 * minimal size so it must be set prior to queue memory allocation
2453 	 */
2454 	fp->disable_tpa = !(bp->flags & TPA_ENABLE_FLAG ||
2455 				  (bp->flags & GRO_ENABLE_FLAG &&
2456 				   bnx2x_mtu_allows_gro(bp->dev->mtu)));
2457 	if (bp->flags & TPA_ENABLE_FLAG)
2458 		fp->mode = TPA_MODE_LRO;
2459 	else if (bp->flags & GRO_ENABLE_FLAG)
2460 		fp->mode = TPA_MODE_GRO;
2461 
2462 	/* We don't want TPA on an FCoE L2 ring */
2463 	if (IS_FCOE_FP(fp))
2464 		fp->disable_tpa = 1;
2465 }
2466 
2467 int bnx2x_load_cnic(struct bnx2x *bp)
2468 {
2469 	int i, rc, port = BP_PORT(bp);
2470 
2471 	DP(NETIF_MSG_IFUP, "Starting CNIC-related load\n");
2472 
2473 	mutex_init(&bp->cnic_mutex);
2474 
2475 	if (IS_PF(bp)) {
2476 		rc = bnx2x_alloc_mem_cnic(bp);
2477 		if (rc) {
2478 			BNX2X_ERR("Unable to allocate bp memory for cnic\n");
2479 			LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2480 		}
2481 	}
2482 
2483 	rc = bnx2x_alloc_fp_mem_cnic(bp);
2484 	if (rc) {
2485 		BNX2X_ERR("Unable to allocate memory for cnic fps\n");
2486 		LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2487 	}
2488 
2489 	/* Update the number of queues with the cnic queues */
2490 	rc = bnx2x_set_real_num_queues(bp, 1);
2491 	if (rc) {
2492 		BNX2X_ERR("Unable to set real_num_queues including cnic\n");
2493 		LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2494 	}
2495 
2496 	/* Add all CNIC NAPI objects */
2497 	bnx2x_add_all_napi_cnic(bp);
2498 	DP(NETIF_MSG_IFUP, "cnic napi added\n");
2499 	bnx2x_napi_enable_cnic(bp);
2500 
2501 	rc = bnx2x_init_hw_func_cnic(bp);
2502 	if (rc)
2503 		LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic1);
2504 
2505 	bnx2x_nic_init_cnic(bp);
2506 
2507 	if (IS_PF(bp)) {
2508 		/* Enable Timer scan */
2509 		REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 1);
2510 
2511 		/* setup cnic queues */
2512 		for_each_cnic_queue(bp, i) {
2513 			rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
2514 			if (rc) {
2515 				BNX2X_ERR("Queue setup failed\n");
2516 				LOAD_ERROR_EXIT(bp, load_error_cnic2);
2517 			}
2518 		}
2519 	}
2520 
2521 	/* Initialize Rx filter. */
2522 	bnx2x_set_rx_mode_inner(bp);
2523 
2524 	/* re-read iscsi info */
2525 	bnx2x_get_iscsi_info(bp);
2526 	bnx2x_setup_cnic_irq_info(bp);
2527 	bnx2x_setup_cnic_info(bp);
2528 	bp->cnic_loaded = true;
2529 	if (bp->state == BNX2X_STATE_OPEN)
2530 		bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
2531 
2532 	DP(NETIF_MSG_IFUP, "Ending successfully CNIC-related load\n");
2533 
2534 	return 0;
2535 
2536 #ifndef BNX2X_STOP_ON_ERROR
2537 load_error_cnic2:
2538 	/* Disable Timer scan */
2539 	REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 0);
2540 
2541 load_error_cnic1:
2542 	bnx2x_napi_disable_cnic(bp);
2543 	/* Update the number of queues without the cnic queues */
2544 	if (bnx2x_set_real_num_queues(bp, 0))
2545 		BNX2X_ERR("Unable to set real_num_queues not including cnic\n");
2546 load_error_cnic0:
2547 	BNX2X_ERR("CNIC-related load failed\n");
2548 	bnx2x_free_fp_mem_cnic(bp);
2549 	bnx2x_free_mem_cnic(bp);
2550 	return rc;
2551 #endif /* ! BNX2X_STOP_ON_ERROR */
2552 }
2553 
2554 /* must be called with rtnl_lock */
2555 int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
2556 {
2557 	int port = BP_PORT(bp);
2558 	int i, rc = 0, load_code = 0;
2559 
2560 	DP(NETIF_MSG_IFUP, "Starting NIC load\n");
2561 	DP(NETIF_MSG_IFUP,
2562 	   "CNIC is %s\n", CNIC_ENABLED(bp) ? "enabled" : "disabled");
2563 
2564 #ifdef BNX2X_STOP_ON_ERROR
2565 	if (unlikely(bp->panic)) {
2566 		BNX2X_ERR("Can't load NIC when there is panic\n");
2567 		return -EPERM;
2568 	}
2569 #endif
2570 
2571 	bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
2572 
2573 	/* zero the structure w/o any lock, before SP handler is initialized */
2574 	memset(&bp->last_reported_link, 0, sizeof(bp->last_reported_link));
2575 	__set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
2576 		&bp->last_reported_link.link_report_flags);
2577 
2578 	if (IS_PF(bp))
2579 		/* must be called before memory allocation and HW init */
2580 		bnx2x_ilt_set_info(bp);
2581 
2582 	/*
2583 	 * Zero fastpath structures preserving invariants like napi, which are
2584 	 * allocated only once, fp index, max_cos, bp pointer.
2585 	 * Also set fp->disable_tpa and txdata_ptr.
2586 	 */
2587 	DP(NETIF_MSG_IFUP, "num queues: %d", bp->num_queues);
2588 	for_each_queue(bp, i)
2589 		bnx2x_bz_fp(bp, i);
2590 	memset(bp->bnx2x_txq, 0, (BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS +
2591 				  bp->num_cnic_queues) *
2592 				  sizeof(struct bnx2x_fp_txdata));
2593 
2594 	bp->fcoe_init = false;
2595 
2596 	/* Set the receive queues buffer size */
2597 	bnx2x_set_rx_buf_size(bp);
2598 
2599 	if (IS_PF(bp)) {
2600 		rc = bnx2x_alloc_mem(bp);
2601 		if (rc) {
2602 			BNX2X_ERR("Unable to allocate bp memory\n");
2603 			return rc;
2604 		}
2605 	}
2606 
2607 	/* need to be done after alloc mem, since it's self adjusting to amount
2608 	 * of memory available for RSS queues
2609 	 */
2610 	rc = bnx2x_alloc_fp_mem(bp);
2611 	if (rc) {
2612 		BNX2X_ERR("Unable to allocate memory for fps\n");
2613 		LOAD_ERROR_EXIT(bp, load_error0);
2614 	}
2615 
2616 	/* Allocated memory for FW statistics  */
2617 	if (bnx2x_alloc_fw_stats_mem(bp))
2618 		LOAD_ERROR_EXIT(bp, load_error0);
2619 
2620 	/* request pf to initialize status blocks */
2621 	if (IS_VF(bp)) {
2622 		rc = bnx2x_vfpf_init(bp);
2623 		if (rc)
2624 			LOAD_ERROR_EXIT(bp, load_error0);
2625 	}
2626 
2627 	/* As long as bnx2x_alloc_mem() may possibly update
2628 	 * bp->num_queues, bnx2x_set_real_num_queues() should always
2629 	 * come after it. At this stage cnic queues are not counted.
2630 	 */
2631 	rc = bnx2x_set_real_num_queues(bp, 0);
2632 	if (rc) {
2633 		BNX2X_ERR("Unable to set real_num_queues\n");
2634 		LOAD_ERROR_EXIT(bp, load_error0);
2635 	}
2636 
2637 	/* configure multi cos mappings in kernel.
2638 	 * this configuration may be overridden by a multi class queue
2639 	 * discipline or by a dcbx negotiation result.
2640 	 */
2641 	bnx2x_setup_tc(bp->dev, bp->max_cos);
2642 
2643 	/* Add all NAPI objects */
2644 	bnx2x_add_all_napi(bp);
2645 	DP(NETIF_MSG_IFUP, "napi added\n");
2646 	bnx2x_napi_enable(bp);
2647 
2648 	if (IS_PF(bp)) {
2649 		/* set pf load just before approaching the MCP */
2650 		bnx2x_set_pf_load(bp);
2651 
2652 		/* if mcp exists send load request and analyze response */
2653 		if (!BP_NOMCP(bp)) {
2654 			/* attempt to load pf */
2655 			rc = bnx2x_nic_load_request(bp, &load_code);
2656 			if (rc)
2657 				LOAD_ERROR_EXIT(bp, load_error1);
2658 
2659 			/* what did mcp say? */
2660 			rc = bnx2x_compare_fw_ver(bp, load_code, true);
2661 			if (rc) {
2662 				bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2663 				LOAD_ERROR_EXIT(bp, load_error2);
2664 			}
2665 		} else {
2666 			load_code = bnx2x_nic_load_no_mcp(bp, port);
2667 		}
2668 
2669 		/* mark pmf if applicable */
2670 		bnx2x_nic_load_pmf(bp, load_code);
2671 
2672 		/* Init Function state controlling object */
2673 		bnx2x__init_func_obj(bp);
2674 
2675 		/* Initialize HW */
2676 		rc = bnx2x_init_hw(bp, load_code);
2677 		if (rc) {
2678 			BNX2X_ERR("HW init failed, aborting\n");
2679 			bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2680 			LOAD_ERROR_EXIT(bp, load_error2);
2681 		}
2682 	}
2683 
2684 	bnx2x_pre_irq_nic_init(bp);
2685 
2686 	/* Connect to IRQs */
2687 	rc = bnx2x_setup_irqs(bp);
2688 	if (rc) {
2689 		BNX2X_ERR("setup irqs failed\n");
2690 		if (IS_PF(bp))
2691 			bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2692 		LOAD_ERROR_EXIT(bp, load_error2);
2693 	}
2694 
2695 	/* Init per-function objects */
2696 	if (IS_PF(bp)) {
2697 		/* Setup NIC internals and enable interrupts */
2698 		bnx2x_post_irq_nic_init(bp, load_code);
2699 
2700 		bnx2x_init_bp_objs(bp);
2701 		bnx2x_iov_nic_init(bp);
2702 
2703 		/* Set AFEX default VLAN tag to an invalid value */
2704 		bp->afex_def_vlan_tag = -1;
2705 		bnx2x_nic_load_afex_dcc(bp, load_code);
2706 		bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
2707 		rc = bnx2x_func_start(bp);
2708 		if (rc) {
2709 			BNX2X_ERR("Function start failed!\n");
2710 			bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2711 
2712 			LOAD_ERROR_EXIT(bp, load_error3);
2713 		}
2714 
2715 		/* Send LOAD_DONE command to MCP */
2716 		if (!BP_NOMCP(bp)) {
2717 			load_code = bnx2x_fw_command(bp,
2718 						     DRV_MSG_CODE_LOAD_DONE, 0);
2719 			if (!load_code) {
2720 				BNX2X_ERR("MCP response failure, aborting\n");
2721 				rc = -EBUSY;
2722 				LOAD_ERROR_EXIT(bp, load_error3);
2723 			}
2724 		}
2725 
2726 		/* initialize FW coalescing state machines in RAM */
2727 		bnx2x_update_coalesce(bp);
2728 	}
2729 
2730 	/* setup the leading queue */
2731 	rc = bnx2x_setup_leading(bp);
2732 	if (rc) {
2733 		BNX2X_ERR("Setup leading failed!\n");
2734 		LOAD_ERROR_EXIT(bp, load_error3);
2735 	}
2736 
2737 	/* set up the rest of the queues */
2738 	for_each_nondefault_eth_queue(bp, i) {
2739 		if (IS_PF(bp))
2740 			rc = bnx2x_setup_queue(bp, &bp->fp[i], false);
2741 		else /* VF */
2742 			rc = bnx2x_vfpf_setup_q(bp, &bp->fp[i], false);
2743 		if (rc) {
2744 			BNX2X_ERR("Queue %d setup failed\n", i);
2745 			LOAD_ERROR_EXIT(bp, load_error3);
2746 		}
2747 	}
2748 
2749 	/* setup rss */
2750 	rc = bnx2x_init_rss(bp);
2751 	if (rc) {
2752 		BNX2X_ERR("PF RSS init failed\n");
2753 		LOAD_ERROR_EXIT(bp, load_error3);
2754 	}
2755 
2756 	/* Now when Clients are configured we are ready to work */
2757 	bp->state = BNX2X_STATE_OPEN;
2758 
2759 	/* Configure a ucast MAC */
2760 	if (IS_PF(bp))
2761 		rc = bnx2x_set_eth_mac(bp, true);
2762 	else /* vf */
2763 		rc = bnx2x_vfpf_config_mac(bp, bp->dev->dev_addr, bp->fp->index,
2764 					   true);
2765 	if (rc) {
2766 		BNX2X_ERR("Setting Ethernet MAC failed\n");
2767 		LOAD_ERROR_EXIT(bp, load_error3);
2768 	}
2769 
2770 	if (IS_PF(bp) && bp->pending_max) {
2771 		bnx2x_update_max_mf_config(bp, bp->pending_max);
2772 		bp->pending_max = 0;
2773 	}
2774 
2775 	if (bp->port.pmf) {
2776 		rc = bnx2x_initial_phy_init(bp, load_mode);
2777 		if (rc)
2778 			LOAD_ERROR_EXIT(bp, load_error3);
2779 	}
2780 	bp->link_params.feature_config_flags &= ~FEATURE_CONFIG_BOOT_FROM_SAN;
2781 
2782 	/* Start fast path */
2783 
2784 	/* Initialize Rx filter. */
2785 	bnx2x_set_rx_mode_inner(bp);
2786 
2787 	/* Start the Tx */
2788 	switch (load_mode) {
2789 	case LOAD_NORMAL:
2790 		/* Tx queue should be only re-enabled */
2791 		netif_tx_wake_all_queues(bp->dev);
2792 		break;
2793 
2794 	case LOAD_OPEN:
2795 		netif_tx_start_all_queues(bp->dev);
2796 		smp_mb__after_atomic();
2797 		break;
2798 
2799 	case LOAD_DIAG:
2800 	case LOAD_LOOPBACK_EXT:
2801 		bp->state = BNX2X_STATE_DIAG;
2802 		break;
2803 
2804 	default:
2805 		break;
2806 	}
2807 
2808 	if (bp->port.pmf)
2809 		bnx2x_update_drv_flags(bp, 1 << DRV_FLAGS_PORT_MASK, 0);
2810 	else
2811 		bnx2x__link_status_update(bp);
2812 
2813 	/* start the timer */
2814 	mod_timer(&bp->timer, jiffies + bp->current_interval);
2815 
2816 	if (CNIC_ENABLED(bp))
2817 		bnx2x_load_cnic(bp);
2818 
2819 	if (IS_PF(bp))
2820 		bnx2x_schedule_sp_rtnl(bp, BNX2X_SP_RTNL_GET_DRV_VERSION, 0);
2821 
2822 	if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2823 		/* mark driver is loaded in shmem2 */
2824 		u32 val;
2825 		val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2826 		SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2827 			  val | DRV_FLAGS_CAPABILITIES_LOADED_SUPPORTED |
2828 			  DRV_FLAGS_CAPABILITIES_LOADED_L2);
2829 	}
2830 
2831 	/* Wait for all pending SP commands to complete */
2832 	if (IS_PF(bp) && !bnx2x_wait_sp_comp(bp, ~0x0UL)) {
2833 		BNX2X_ERR("Timeout waiting for SP elements to complete\n");
2834 		bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
2835 		return -EBUSY;
2836 	}
2837 
2838 	/* If PMF - send ADMIN DCBX msg to MFW to initiate DCBX FSM */
2839 	if (bp->port.pmf && (bp->state != BNX2X_STATE_DIAG))
2840 		bnx2x_dcbx_init(bp, false);
2841 
2842 	DP(NETIF_MSG_IFUP, "Ending successfully NIC load\n");
2843 
2844 	return 0;
2845 
2846 #ifndef BNX2X_STOP_ON_ERROR
2847 load_error3:
2848 	if (IS_PF(bp)) {
2849 		bnx2x_int_disable_sync(bp, 1);
2850 
2851 		/* Clean queueable objects */
2852 		bnx2x_squeeze_objects(bp);
2853 	}
2854 
2855 	/* Free SKBs, SGEs, TPA pool and driver internals */
2856 	bnx2x_free_skbs(bp);
2857 	for_each_rx_queue(bp, i)
2858 		bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2859 
2860 	/* Release IRQs */
2861 	bnx2x_free_irq(bp);
2862 load_error2:
2863 	if (IS_PF(bp) && !BP_NOMCP(bp)) {
2864 		bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
2865 		bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
2866 	}
2867 
2868 	bp->port.pmf = 0;
2869 load_error1:
2870 	bnx2x_napi_disable(bp);
2871 	bnx2x_del_all_napi(bp);
2872 
2873 	/* clear pf_load status, as it was already set */
2874 	if (IS_PF(bp))
2875 		bnx2x_clear_pf_load(bp);
2876 load_error0:
2877 	bnx2x_free_fw_stats_mem(bp);
2878 	bnx2x_free_fp_mem(bp);
2879 	bnx2x_free_mem(bp);
2880 
2881 	return rc;
2882 #endif /* ! BNX2X_STOP_ON_ERROR */
2883 }
2884 
2885 int bnx2x_drain_tx_queues(struct bnx2x *bp)
2886 {
2887 	u8 rc = 0, cos, i;
2888 
2889 	/* Wait until tx fastpath tasks complete */
2890 	for_each_tx_queue(bp, i) {
2891 		struct bnx2x_fastpath *fp = &bp->fp[i];
2892 
2893 		for_each_cos_in_tx_queue(fp, cos)
2894 			rc = bnx2x_clean_tx_queue(bp, fp->txdata_ptr[cos]);
2895 		if (rc)
2896 			return rc;
2897 	}
2898 	return 0;
2899 }
2900 
2901 /* must be called with rtnl_lock */
2902 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode, bool keep_link)
2903 {
2904 	int i;
2905 	bool global = false;
2906 
2907 	DP(NETIF_MSG_IFUP, "Starting NIC unload\n");
2908 
2909 	/* mark driver is unloaded in shmem2 */
2910 	if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2911 		u32 val;
2912 		val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2913 		SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2914 			  val & ~DRV_FLAGS_CAPABILITIES_LOADED_L2);
2915 	}
2916 
2917 	if (IS_PF(bp) && bp->recovery_state != BNX2X_RECOVERY_DONE &&
2918 	    (bp->state == BNX2X_STATE_CLOSED ||
2919 	     bp->state == BNX2X_STATE_ERROR)) {
2920 		/* We can get here if the driver has been unloaded
2921 		 * during parity error recovery and is either waiting for a
2922 		 * leader to complete or for other functions to unload and
2923 		 * then ifdown has been issued. In this case we want to
2924 		 * unload and let other functions to complete a recovery
2925 		 * process.
2926 		 */
2927 		bp->recovery_state = BNX2X_RECOVERY_DONE;
2928 		bp->is_leader = 0;
2929 		bnx2x_release_leader_lock(bp);
2930 		smp_mb();
2931 
2932 		DP(NETIF_MSG_IFDOWN, "Releasing a leadership...\n");
2933 		BNX2X_ERR("Can't unload in closed or error state\n");
2934 		return -EINVAL;
2935 	}
2936 
2937 	/* Nothing to do during unload if previous bnx2x_nic_load()
2938 	 * have not completed successfully - all resources are released.
2939 	 *
2940 	 * we can get here only after unsuccessful ndo_* callback, during which
2941 	 * dev->IFF_UP flag is still on.
2942 	 */
2943 	if (bp->state == BNX2X_STATE_CLOSED || bp->state == BNX2X_STATE_ERROR)
2944 		return 0;
2945 
2946 	/* It's important to set the bp->state to the value different from
2947 	 * BNX2X_STATE_OPEN and only then stop the Tx. Otherwise bnx2x_tx_int()
2948 	 * may restart the Tx from the NAPI context (see bnx2x_tx_int()).
2949 	 */
2950 	bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
2951 	smp_mb();
2952 
2953 	/* indicate to VFs that the PF is going down */
2954 	bnx2x_iov_channel_down(bp);
2955 
2956 	if (CNIC_LOADED(bp))
2957 		bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
2958 
2959 	/* Stop Tx */
2960 	bnx2x_tx_disable(bp);
2961 	netdev_reset_tc(bp->dev);
2962 
2963 	bp->rx_mode = BNX2X_RX_MODE_NONE;
2964 
2965 	del_timer_sync(&bp->timer);
2966 
2967 	if (IS_PF(bp)) {
2968 		/* Set ALWAYS_ALIVE bit in shmem */
2969 		bp->fw_drv_pulse_wr_seq |= DRV_PULSE_ALWAYS_ALIVE;
2970 		bnx2x_drv_pulse(bp);
2971 		bnx2x_stats_handle(bp, STATS_EVENT_STOP);
2972 		bnx2x_save_statistics(bp);
2973 	}
2974 
2975 	/* wait till consumers catch up with producers in all queues */
2976 	bnx2x_drain_tx_queues(bp);
2977 
2978 	/* if VF indicate to PF this function is going down (PF will delete sp
2979 	 * elements and clear initializations
2980 	 */
2981 	if (IS_VF(bp))
2982 		bnx2x_vfpf_close_vf(bp);
2983 	else if (unload_mode != UNLOAD_RECOVERY)
2984 		/* if this is a normal/close unload need to clean up chip*/
2985 		bnx2x_chip_cleanup(bp, unload_mode, keep_link);
2986 	else {
2987 		/* Send the UNLOAD_REQUEST to the MCP */
2988 		bnx2x_send_unload_req(bp, unload_mode);
2989 
2990 		/* Prevent transactions to host from the functions on the
2991 		 * engine that doesn't reset global blocks in case of global
2992 		 * attention once global blocks are reset and gates are opened
2993 		 * (the engine which leader will perform the recovery
2994 		 * last).
2995 		 */
2996 		if (!CHIP_IS_E1x(bp))
2997 			bnx2x_pf_disable(bp);
2998 
2999 		/* Disable HW interrupts, NAPI */
3000 		bnx2x_netif_stop(bp, 1);
3001 		/* Delete all NAPI objects */
3002 		bnx2x_del_all_napi(bp);
3003 		if (CNIC_LOADED(bp))
3004 			bnx2x_del_all_napi_cnic(bp);
3005 		/* Release IRQs */
3006 		bnx2x_free_irq(bp);
3007 
3008 		/* Report UNLOAD_DONE to MCP */
3009 		bnx2x_send_unload_done(bp, false);
3010 	}
3011 
3012 	/*
3013 	 * At this stage no more interrupts will arrive so we may safely clean
3014 	 * the queueable objects here in case they failed to get cleaned so far.
3015 	 */
3016 	if (IS_PF(bp))
3017 		bnx2x_squeeze_objects(bp);
3018 
3019 	/* There should be no more pending SP commands at this stage */
3020 	bp->sp_state = 0;
3021 
3022 	bp->port.pmf = 0;
3023 
3024 	/* clear pending work in rtnl task */
3025 	bp->sp_rtnl_state = 0;
3026 	smp_mb();
3027 
3028 	/* Free SKBs, SGEs, TPA pool and driver internals */
3029 	bnx2x_free_skbs(bp);
3030 	if (CNIC_LOADED(bp))
3031 		bnx2x_free_skbs_cnic(bp);
3032 	for_each_rx_queue(bp, i)
3033 		bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
3034 
3035 	bnx2x_free_fp_mem(bp);
3036 	if (CNIC_LOADED(bp))
3037 		bnx2x_free_fp_mem_cnic(bp);
3038 
3039 	if (IS_PF(bp)) {
3040 		if (CNIC_LOADED(bp))
3041 			bnx2x_free_mem_cnic(bp);
3042 	}
3043 	bnx2x_free_mem(bp);
3044 
3045 	bp->state = BNX2X_STATE_CLOSED;
3046 	bp->cnic_loaded = false;
3047 
3048 	/* Clear driver version indication in shmem */
3049 	if (IS_PF(bp))
3050 		bnx2x_update_mng_version(bp);
3051 
3052 	/* Check if there are pending parity attentions. If there are - set
3053 	 * RECOVERY_IN_PROGRESS.
3054 	 */
3055 	if (IS_PF(bp) && bnx2x_chk_parity_attn(bp, &global, false)) {
3056 		bnx2x_set_reset_in_progress(bp);
3057 
3058 		/* Set RESET_IS_GLOBAL if needed */
3059 		if (global)
3060 			bnx2x_set_reset_global(bp);
3061 	}
3062 
3063 	/* The last driver must disable a "close the gate" if there is no
3064 	 * parity attention or "process kill" pending.
3065 	 */
3066 	if (IS_PF(bp) &&
3067 	    !bnx2x_clear_pf_load(bp) &&
3068 	    bnx2x_reset_is_done(bp, BP_PATH(bp)))
3069 		bnx2x_disable_close_the_gate(bp);
3070 
3071 	DP(NETIF_MSG_IFUP, "Ending NIC unload\n");
3072 
3073 	return 0;
3074 }
3075 
3076 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
3077 {
3078 	u16 pmcsr;
3079 
3080 	/* If there is no power capability, silently succeed */
3081 	if (!bp->pdev->pm_cap) {
3082 		BNX2X_DEV_INFO("No power capability. Breaking.\n");
3083 		return 0;
3084 	}
3085 
3086 	pci_read_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL, &pmcsr);
3087 
3088 	switch (state) {
3089 	case PCI_D0:
3090 		pci_write_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL,
3091 				      ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
3092 				       PCI_PM_CTRL_PME_STATUS));
3093 
3094 		if (pmcsr & PCI_PM_CTRL_STATE_MASK)
3095 			/* delay required during transition out of D3hot */
3096 			msleep(20);
3097 		break;
3098 
3099 	case PCI_D3hot:
3100 		/* If there are other clients above don't
3101 		   shut down the power */
3102 		if (atomic_read(&bp->pdev->enable_cnt) != 1)
3103 			return 0;
3104 		/* Don't shut down the power for emulation and FPGA */
3105 		if (CHIP_REV_IS_SLOW(bp))
3106 			return 0;
3107 
3108 		pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
3109 		pmcsr |= 3;
3110 
3111 		if (bp->wol)
3112 			pmcsr |= PCI_PM_CTRL_PME_ENABLE;
3113 
3114 		pci_write_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL,
3115 				      pmcsr);
3116 
3117 		/* No more memory access after this point until
3118 		* device is brought back to D0.
3119 		*/
3120 		break;
3121 
3122 	default:
3123 		dev_err(&bp->pdev->dev, "Can't support state = %d\n", state);
3124 		return -EINVAL;
3125 	}
3126 	return 0;
3127 }
3128 
3129 /*
3130  * net_device service functions
3131  */
3132 static int bnx2x_poll(struct napi_struct *napi, int budget)
3133 {
3134 	int work_done = 0;
3135 	u8 cos;
3136 	struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
3137 						 napi);
3138 	struct bnx2x *bp = fp->bp;
3139 
3140 	while (1) {
3141 #ifdef BNX2X_STOP_ON_ERROR
3142 		if (unlikely(bp->panic)) {
3143 			napi_complete(napi);
3144 			return 0;
3145 		}
3146 #endif
3147 		if (!bnx2x_fp_lock_napi(fp))
3148 			return work_done;
3149 
3150 		for_each_cos_in_tx_queue(fp, cos)
3151 			if (bnx2x_tx_queue_has_work(fp->txdata_ptr[cos]))
3152 				bnx2x_tx_int(bp, fp->txdata_ptr[cos]);
3153 
3154 		if (bnx2x_has_rx_work(fp)) {
3155 			work_done += bnx2x_rx_int(fp, budget - work_done);
3156 
3157 			/* must not complete if we consumed full budget */
3158 			if (work_done >= budget) {
3159 				bnx2x_fp_unlock_napi(fp);
3160 				break;
3161 			}
3162 		}
3163 
3164 		/* Fall out from the NAPI loop if needed */
3165 		if (!bnx2x_fp_unlock_napi(fp) &&
3166 		    !(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
3167 
3168 			/* No need to update SB for FCoE L2 ring as long as
3169 			 * it's connected to the default SB and the SB
3170 			 * has been updated when NAPI was scheduled.
3171 			 */
3172 			if (IS_FCOE_FP(fp)) {
3173 				napi_complete(napi);
3174 				break;
3175 			}
3176 			bnx2x_update_fpsb_idx(fp);
3177 			/* bnx2x_has_rx_work() reads the status block,
3178 			 * thus we need to ensure that status block indices
3179 			 * have been actually read (bnx2x_update_fpsb_idx)
3180 			 * prior to this check (bnx2x_has_rx_work) so that
3181 			 * we won't write the "newer" value of the status block
3182 			 * to IGU (if there was a DMA right after
3183 			 * bnx2x_has_rx_work and if there is no rmb, the memory
3184 			 * reading (bnx2x_update_fpsb_idx) may be postponed
3185 			 * to right before bnx2x_ack_sb). In this case there
3186 			 * will never be another interrupt until there is
3187 			 * another update of the status block, while there
3188 			 * is still unhandled work.
3189 			 */
3190 			rmb();
3191 
3192 			if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
3193 				napi_complete(napi);
3194 				/* Re-enable interrupts */
3195 				DP(NETIF_MSG_RX_STATUS,
3196 				   "Update index to %d\n", fp->fp_hc_idx);
3197 				bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
3198 					     le16_to_cpu(fp->fp_hc_idx),
3199 					     IGU_INT_ENABLE, 1);
3200 				break;
3201 			}
3202 		}
3203 	}
3204 
3205 	return work_done;
3206 }
3207 
3208 #ifdef CONFIG_NET_RX_BUSY_POLL
3209 /* must be called with local_bh_disable()d */
3210 int bnx2x_low_latency_recv(struct napi_struct *napi)
3211 {
3212 	struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
3213 						 napi);
3214 	struct bnx2x *bp = fp->bp;
3215 	int found = 0;
3216 
3217 	if ((bp->state == BNX2X_STATE_CLOSED) ||
3218 	    (bp->state == BNX2X_STATE_ERROR) ||
3219 	    (bp->flags & (TPA_ENABLE_FLAG | GRO_ENABLE_FLAG)))
3220 		return LL_FLUSH_FAILED;
3221 
3222 	if (!bnx2x_fp_lock_poll(fp))
3223 		return LL_FLUSH_BUSY;
3224 
3225 	if (bnx2x_has_rx_work(fp))
3226 		found = bnx2x_rx_int(fp, 4);
3227 
3228 	bnx2x_fp_unlock_poll(fp);
3229 
3230 	return found;
3231 }
3232 #endif
3233 
3234 /* we split the first BD into headers and data BDs
3235  * to ease the pain of our fellow microcode engineers
3236  * we use one mapping for both BDs
3237  */
3238 static u16 bnx2x_tx_split(struct bnx2x *bp,
3239 			  struct bnx2x_fp_txdata *txdata,
3240 			  struct sw_tx_bd *tx_buf,
3241 			  struct eth_tx_start_bd **tx_bd, u16 hlen,
3242 			  u16 bd_prod)
3243 {
3244 	struct eth_tx_start_bd *h_tx_bd = *tx_bd;
3245 	struct eth_tx_bd *d_tx_bd;
3246 	dma_addr_t mapping;
3247 	int old_len = le16_to_cpu(h_tx_bd->nbytes);
3248 
3249 	/* first fix first BD */
3250 	h_tx_bd->nbytes = cpu_to_le16(hlen);
3251 
3252 	DP(NETIF_MSG_TX_QUEUED,	"TSO split header size is %d (%x:%x)\n",
3253 	   h_tx_bd->nbytes, h_tx_bd->addr_hi, h_tx_bd->addr_lo);
3254 
3255 	/* now get a new data BD
3256 	 * (after the pbd) and fill it */
3257 	bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3258 	d_tx_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3259 
3260 	mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
3261 			   le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
3262 
3263 	d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3264 	d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3265 	d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
3266 
3267 	/* this marks the BD as one that has no individual mapping */
3268 	tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
3269 
3270 	DP(NETIF_MSG_TX_QUEUED,
3271 	   "TSO split data size is %d (%x:%x)\n",
3272 	   d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
3273 
3274 	/* update tx_bd */
3275 	*tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
3276 
3277 	return bd_prod;
3278 }
3279 
3280 #define bswab32(b32) ((__force __le32) swab32((__force __u32) (b32)))
3281 #define bswab16(b16) ((__force __le16) swab16((__force __u16) (b16)))
3282 static __le16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
3283 {
3284 	__sum16 tsum = (__force __sum16) csum;
3285 
3286 	if (fix > 0)
3287 		tsum = ~csum_fold(csum_sub((__force __wsum) csum,
3288 				  csum_partial(t_header - fix, fix, 0)));
3289 
3290 	else if (fix < 0)
3291 		tsum = ~csum_fold(csum_add((__force __wsum) csum,
3292 				  csum_partial(t_header, -fix, 0)));
3293 
3294 	return bswab16(tsum);
3295 }
3296 
3297 static u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
3298 {
3299 	u32 rc;
3300 	__u8 prot = 0;
3301 	__be16 protocol;
3302 
3303 	if (skb->ip_summed != CHECKSUM_PARTIAL)
3304 		return XMIT_PLAIN;
3305 
3306 	protocol = vlan_get_protocol(skb);
3307 	if (protocol == htons(ETH_P_IPV6)) {
3308 		rc = XMIT_CSUM_V6;
3309 		prot = ipv6_hdr(skb)->nexthdr;
3310 	} else {
3311 		rc = XMIT_CSUM_V4;
3312 		prot = ip_hdr(skb)->protocol;
3313 	}
3314 
3315 	if (!CHIP_IS_E1x(bp) && skb->encapsulation) {
3316 		if (inner_ip_hdr(skb)->version == 6) {
3317 			rc |= XMIT_CSUM_ENC_V6;
3318 			if (inner_ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3319 				rc |= XMIT_CSUM_TCP;
3320 		} else {
3321 			rc |= XMIT_CSUM_ENC_V4;
3322 			if (inner_ip_hdr(skb)->protocol == IPPROTO_TCP)
3323 				rc |= XMIT_CSUM_TCP;
3324 		}
3325 	}
3326 	if (prot == IPPROTO_TCP)
3327 		rc |= XMIT_CSUM_TCP;
3328 
3329 	if (skb_is_gso(skb)) {
3330 		if (skb_is_gso_v6(skb)) {
3331 			rc |= (XMIT_GSO_V6 | XMIT_CSUM_TCP);
3332 			if (rc & XMIT_CSUM_ENC)
3333 				rc |= XMIT_GSO_ENC_V6;
3334 		} else {
3335 			rc |= (XMIT_GSO_V4 | XMIT_CSUM_TCP);
3336 			if (rc & XMIT_CSUM_ENC)
3337 				rc |= XMIT_GSO_ENC_V4;
3338 		}
3339 	}
3340 
3341 	return rc;
3342 }
3343 
3344 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
3345 /* check if packet requires linearization (packet is too fragmented)
3346    no need to check fragmentation if page size > 8K (there will be no
3347    violation to FW restrictions) */
3348 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
3349 			     u32 xmit_type)
3350 {
3351 	int to_copy = 0;
3352 	int hlen = 0;
3353 	int first_bd_sz = 0;
3354 
3355 	/* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
3356 	if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
3357 
3358 		if (xmit_type & XMIT_GSO) {
3359 			unsigned short lso_mss = skb_shinfo(skb)->gso_size;
3360 			/* Check if LSO packet needs to be copied:
3361 			   3 = 1 (for headers BD) + 2 (for PBD and last BD) */
3362 			int wnd_size = MAX_FETCH_BD - 3;
3363 			/* Number of windows to check */
3364 			int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
3365 			int wnd_idx = 0;
3366 			int frag_idx = 0;
3367 			u32 wnd_sum = 0;
3368 
3369 			/* Headers length */
3370 			hlen = (int)(skb_transport_header(skb) - skb->data) +
3371 				tcp_hdrlen(skb);
3372 
3373 			/* Amount of data (w/o headers) on linear part of SKB*/
3374 			first_bd_sz = skb_headlen(skb) - hlen;
3375 
3376 			wnd_sum  = first_bd_sz;
3377 
3378 			/* Calculate the first sum - it's special */
3379 			for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
3380 				wnd_sum +=
3381 					skb_frag_size(&skb_shinfo(skb)->frags[frag_idx]);
3382 
3383 			/* If there was data on linear skb data - check it */
3384 			if (first_bd_sz > 0) {
3385 				if (unlikely(wnd_sum < lso_mss)) {
3386 					to_copy = 1;
3387 					goto exit_lbl;
3388 				}
3389 
3390 				wnd_sum -= first_bd_sz;
3391 			}
3392 
3393 			/* Others are easier: run through the frag list and
3394 			   check all windows */
3395 			for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
3396 				wnd_sum +=
3397 			  skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1]);
3398 
3399 				if (unlikely(wnd_sum < lso_mss)) {
3400 					to_copy = 1;
3401 					break;
3402 				}
3403 				wnd_sum -=
3404 					skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx]);
3405 			}
3406 		} else {
3407 			/* in non-LSO too fragmented packet should always
3408 			   be linearized */
3409 			to_copy = 1;
3410 		}
3411 	}
3412 
3413 exit_lbl:
3414 	if (unlikely(to_copy))
3415 		DP(NETIF_MSG_TX_QUEUED,
3416 		   "Linearization IS REQUIRED for %s packet. num_frags %d  hlen %d  first_bd_sz %d\n",
3417 		   (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
3418 		   skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
3419 
3420 	return to_copy;
3421 }
3422 #endif
3423 
3424 static void bnx2x_set_pbd_gso_e2(struct sk_buff *skb, u32 *parsing_data,
3425 				 u32 xmit_type)
3426 {
3427 	struct ipv6hdr *ipv6;
3428 
3429 	*parsing_data |= (skb_shinfo(skb)->gso_size <<
3430 			      ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT) &
3431 			      ETH_TX_PARSE_BD_E2_LSO_MSS;
3432 
3433 	if (xmit_type & XMIT_GSO_ENC_V6)
3434 		ipv6 = inner_ipv6_hdr(skb);
3435 	else if (xmit_type & XMIT_GSO_V6)
3436 		ipv6 = ipv6_hdr(skb);
3437 	else
3438 		ipv6 = NULL;
3439 
3440 	if (ipv6 && ipv6->nexthdr == NEXTHDR_IPV6)
3441 		*parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR;
3442 }
3443 
3444 /**
3445  * bnx2x_set_pbd_gso - update PBD in GSO case.
3446  *
3447  * @skb:	packet skb
3448  * @pbd:	parse BD
3449  * @xmit_type:	xmit flags
3450  */
3451 static void bnx2x_set_pbd_gso(struct sk_buff *skb,
3452 			      struct eth_tx_parse_bd_e1x *pbd,
3453 			      struct eth_tx_start_bd *tx_start_bd,
3454 			      u32 xmit_type)
3455 {
3456 	pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
3457 	pbd->tcp_send_seq = bswab32(tcp_hdr(skb)->seq);
3458 	pbd->tcp_flags = pbd_tcp_flags(tcp_hdr(skb));
3459 
3460 	if (xmit_type & XMIT_GSO_V4) {
3461 		pbd->ip_id = bswab16(ip_hdr(skb)->id);
3462 		pbd->tcp_pseudo_csum =
3463 			bswab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
3464 						   ip_hdr(skb)->daddr,
3465 						   0, IPPROTO_TCP, 0));
3466 
3467 		/* GSO on 57710/57711 needs FW to calculate IP checksum */
3468 		tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IP_CSUM;
3469 	} else {
3470 		pbd->tcp_pseudo_csum =
3471 			bswab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3472 						 &ipv6_hdr(skb)->daddr,
3473 						 0, IPPROTO_TCP, 0));
3474 	}
3475 
3476 	pbd->global_data |=
3477 		cpu_to_le16(ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN);
3478 }
3479 
3480 /**
3481  * bnx2x_set_pbd_csum_enc - update PBD with checksum and return header length
3482  *
3483  * @bp:			driver handle
3484  * @skb:		packet skb
3485  * @parsing_data:	data to be updated
3486  * @xmit_type:		xmit flags
3487  *
3488  * 57712/578xx related, when skb has encapsulation
3489  */
3490 static u8 bnx2x_set_pbd_csum_enc(struct bnx2x *bp, struct sk_buff *skb,
3491 				 u32 *parsing_data, u32 xmit_type)
3492 {
3493 	*parsing_data |=
3494 		((((u8 *)skb_inner_transport_header(skb) - skb->data) >> 1) <<
3495 		ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3496 		ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3497 
3498 	if (xmit_type & XMIT_CSUM_TCP) {
3499 		*parsing_data |= ((inner_tcp_hdrlen(skb) / 4) <<
3500 			ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3501 			ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3502 
3503 		return skb_inner_transport_header(skb) +
3504 			inner_tcp_hdrlen(skb) - skb->data;
3505 	}
3506 
3507 	/* We support checksum offload for TCP and UDP only.
3508 	 * No need to pass the UDP header length - it's a constant.
3509 	 */
3510 	return skb_inner_transport_header(skb) +
3511 		sizeof(struct udphdr) - skb->data;
3512 }
3513 
3514 /**
3515  * bnx2x_set_pbd_csum_e2 - update PBD with checksum and return header length
3516  *
3517  * @bp:			driver handle
3518  * @skb:		packet skb
3519  * @parsing_data:	data to be updated
3520  * @xmit_type:		xmit flags
3521  *
3522  * 57712/578xx related
3523  */
3524 static u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
3525 				u32 *parsing_data, u32 xmit_type)
3526 {
3527 	*parsing_data |=
3528 		((((u8 *)skb_transport_header(skb) - skb->data) >> 1) <<
3529 		ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3530 		ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3531 
3532 	if (xmit_type & XMIT_CSUM_TCP) {
3533 		*parsing_data |= ((tcp_hdrlen(skb) / 4) <<
3534 			ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3535 			ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3536 
3537 		return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data;
3538 	}
3539 	/* We support checksum offload for TCP and UDP only.
3540 	 * No need to pass the UDP header length - it's a constant.
3541 	 */
3542 	return skb_transport_header(skb) + sizeof(struct udphdr) - skb->data;
3543 }
3544 
3545 /* set FW indication according to inner or outer protocols if tunneled */
3546 static void bnx2x_set_sbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3547 			       struct eth_tx_start_bd *tx_start_bd,
3548 			       u32 xmit_type)
3549 {
3550 	tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
3551 
3552 	if (xmit_type & (XMIT_CSUM_ENC_V6 | XMIT_CSUM_V6))
3553 		tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IPV6;
3554 
3555 	if (!(xmit_type & XMIT_CSUM_TCP))
3556 		tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IS_UDP;
3557 }
3558 
3559 /**
3560  * bnx2x_set_pbd_csum - update PBD with checksum and return header length
3561  *
3562  * @bp:		driver handle
3563  * @skb:	packet skb
3564  * @pbd:	parse BD to be updated
3565  * @xmit_type:	xmit flags
3566  */
3567 static u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3568 			     struct eth_tx_parse_bd_e1x *pbd,
3569 			     u32 xmit_type)
3570 {
3571 	u8 hlen = (skb_network_header(skb) - skb->data) >> 1;
3572 
3573 	/* for now NS flag is not used in Linux */
3574 	pbd->global_data =
3575 		cpu_to_le16(hlen |
3576 			    ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3577 			     ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT));
3578 
3579 	pbd->ip_hlen_w = (skb_transport_header(skb) -
3580 			skb_network_header(skb)) >> 1;
3581 
3582 	hlen += pbd->ip_hlen_w;
3583 
3584 	/* We support checksum offload for TCP and UDP only */
3585 	if (xmit_type & XMIT_CSUM_TCP)
3586 		hlen += tcp_hdrlen(skb) / 2;
3587 	else
3588 		hlen += sizeof(struct udphdr) / 2;
3589 
3590 	pbd->total_hlen_w = cpu_to_le16(hlen);
3591 	hlen = hlen*2;
3592 
3593 	if (xmit_type & XMIT_CSUM_TCP) {
3594 		pbd->tcp_pseudo_csum = bswab16(tcp_hdr(skb)->check);
3595 
3596 	} else {
3597 		s8 fix = SKB_CS_OFF(skb); /* signed! */
3598 
3599 		DP(NETIF_MSG_TX_QUEUED,
3600 		   "hlen %d  fix %d  csum before fix %x\n",
3601 		   le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb));
3602 
3603 		/* HW bug: fixup the CSUM */
3604 		pbd->tcp_pseudo_csum =
3605 			bnx2x_csum_fix(skb_transport_header(skb),
3606 				       SKB_CS(skb), fix);
3607 
3608 		DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
3609 		   pbd->tcp_pseudo_csum);
3610 	}
3611 
3612 	return hlen;
3613 }
3614 
3615 static void bnx2x_update_pbds_gso_enc(struct sk_buff *skb,
3616 				      struct eth_tx_parse_bd_e2 *pbd_e2,
3617 				      struct eth_tx_parse_2nd_bd *pbd2,
3618 				      u16 *global_data,
3619 				      u32 xmit_type)
3620 {
3621 	u16 hlen_w = 0;
3622 	u8 outerip_off, outerip_len = 0;
3623 
3624 	/* from outer IP to transport */
3625 	hlen_w = (skb_inner_transport_header(skb) -
3626 		  skb_network_header(skb)) >> 1;
3627 
3628 	/* transport len */
3629 	hlen_w += inner_tcp_hdrlen(skb) >> 1;
3630 
3631 	pbd2->fw_ip_hdr_to_payload_w = hlen_w;
3632 
3633 	/* outer IP header info */
3634 	if (xmit_type & XMIT_CSUM_V4) {
3635 		struct iphdr *iph = ip_hdr(skb);
3636 		u32 csum = (__force u32)(~iph->check) -
3637 			   (__force u32)iph->tot_len -
3638 			   (__force u32)iph->frag_off;
3639 
3640 		pbd2->fw_ip_csum_wo_len_flags_frag =
3641 			bswab16(csum_fold((__force __wsum)csum));
3642 	} else {
3643 		pbd2->fw_ip_hdr_to_payload_w =
3644 			hlen_w - ((sizeof(struct ipv6hdr)) >> 1);
3645 	}
3646 
3647 	pbd2->tcp_send_seq = bswab32(inner_tcp_hdr(skb)->seq);
3648 
3649 	pbd2->tcp_flags = pbd_tcp_flags(inner_tcp_hdr(skb));
3650 
3651 	if (xmit_type & XMIT_GSO_V4) {
3652 		pbd2->hw_ip_id = bswab16(inner_ip_hdr(skb)->id);
3653 
3654 		pbd_e2->data.tunnel_data.pseudo_csum =
3655 			bswab16(~csum_tcpudp_magic(
3656 					inner_ip_hdr(skb)->saddr,
3657 					inner_ip_hdr(skb)->daddr,
3658 					0, IPPROTO_TCP, 0));
3659 
3660 		outerip_len = ip_hdr(skb)->ihl << 1;
3661 	} else {
3662 		pbd_e2->data.tunnel_data.pseudo_csum =
3663 			bswab16(~csum_ipv6_magic(
3664 					&inner_ipv6_hdr(skb)->saddr,
3665 					&inner_ipv6_hdr(skb)->daddr,
3666 					0, IPPROTO_TCP, 0));
3667 	}
3668 
3669 	outerip_off = (skb_network_header(skb) - skb->data) >> 1;
3670 
3671 	*global_data |=
3672 		outerip_off |
3673 		(!!(xmit_type & XMIT_CSUM_V6) <<
3674 			ETH_TX_PARSE_2ND_BD_IP_HDR_TYPE_OUTER_SHIFT) |
3675 		(outerip_len <<
3676 			ETH_TX_PARSE_2ND_BD_IP_HDR_LEN_OUTER_W_SHIFT) |
3677 		((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3678 			ETH_TX_PARSE_2ND_BD_LLC_SNAP_EN_SHIFT);
3679 
3680 	if (ip_hdr(skb)->protocol == IPPROTO_UDP) {
3681 		SET_FLAG(*global_data, ETH_TX_PARSE_2ND_BD_TUNNEL_UDP_EXIST, 1);
3682 		pbd2->tunnel_udp_hdr_start_w = skb_transport_offset(skb) >> 1;
3683 	}
3684 }
3685 
3686 /* called with netif_tx_lock
3687  * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
3688  * netif_wake_queue()
3689  */
3690 netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
3691 {
3692 	struct bnx2x *bp = netdev_priv(dev);
3693 
3694 	struct netdev_queue *txq;
3695 	struct bnx2x_fp_txdata *txdata;
3696 	struct sw_tx_bd *tx_buf;
3697 	struct eth_tx_start_bd *tx_start_bd, *first_bd;
3698 	struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
3699 	struct eth_tx_parse_bd_e1x *pbd_e1x = NULL;
3700 	struct eth_tx_parse_bd_e2 *pbd_e2 = NULL;
3701 	struct eth_tx_parse_2nd_bd *pbd2 = NULL;
3702 	u32 pbd_e2_parsing_data = 0;
3703 	u16 pkt_prod, bd_prod;
3704 	int nbd, txq_index;
3705 	dma_addr_t mapping;
3706 	u32 xmit_type = bnx2x_xmit_type(bp, skb);
3707 	int i;
3708 	u8 hlen = 0;
3709 	__le16 pkt_size = 0;
3710 	struct ethhdr *eth;
3711 	u8 mac_type = UNICAST_ADDRESS;
3712 
3713 #ifdef BNX2X_STOP_ON_ERROR
3714 	if (unlikely(bp->panic))
3715 		return NETDEV_TX_BUSY;
3716 #endif
3717 
3718 	txq_index = skb_get_queue_mapping(skb);
3719 	txq = netdev_get_tx_queue(dev, txq_index);
3720 
3721 	BUG_ON(txq_index >= MAX_ETH_TXQ_IDX(bp) + (CNIC_LOADED(bp) ? 1 : 0));
3722 
3723 	txdata = &bp->bnx2x_txq[txq_index];
3724 
3725 	/* enable this debug print to view the transmission queue being used
3726 	DP(NETIF_MSG_TX_QUEUED, "indices: txq %d, fp %d, txdata %d\n",
3727 	   txq_index, fp_index, txdata_index); */
3728 
3729 	/* enable this debug print to view the transmission details
3730 	DP(NETIF_MSG_TX_QUEUED,
3731 	   "transmitting packet cid %d fp index %d txdata_index %d tx_data ptr %p fp pointer %p\n",
3732 	   txdata->cid, fp_index, txdata_index, txdata, fp); */
3733 
3734 	if (unlikely(bnx2x_tx_avail(bp, txdata) <
3735 			skb_shinfo(skb)->nr_frags +
3736 			BDS_PER_TX_PKT +
3737 			NEXT_CNT_PER_TX_PKT(MAX_BDS_PER_TX_PKT))) {
3738 		/* Handle special storage cases separately */
3739 		if (txdata->tx_ring_size == 0) {
3740 			struct bnx2x_eth_q_stats *q_stats =
3741 				bnx2x_fp_qstats(bp, txdata->parent_fp);
3742 			q_stats->driver_filtered_tx_pkt++;
3743 			dev_kfree_skb(skb);
3744 			return NETDEV_TX_OK;
3745 		}
3746 		bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
3747 		netif_tx_stop_queue(txq);
3748 		BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
3749 
3750 		return NETDEV_TX_BUSY;
3751 	}
3752 
3753 	DP(NETIF_MSG_TX_QUEUED,
3754 	   "queue[%d]: SKB: summed %x  protocol %x protocol(%x,%x) gso type %x  xmit_type %x len %d\n",
3755 	   txq_index, skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
3756 	   ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type,
3757 	   skb->len);
3758 
3759 	eth = (struct ethhdr *)skb->data;
3760 
3761 	/* set flag according to packet type (UNICAST_ADDRESS is default)*/
3762 	if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
3763 		if (is_broadcast_ether_addr(eth->h_dest))
3764 			mac_type = BROADCAST_ADDRESS;
3765 		else
3766 			mac_type = MULTICAST_ADDRESS;
3767 	}
3768 
3769 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - BDS_PER_TX_PKT)
3770 	/* First, check if we need to linearize the skb (due to FW
3771 	   restrictions). No need to check fragmentation if page size > 8K
3772 	   (there will be no violation to FW restrictions) */
3773 	if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
3774 		/* Statistics of linearization */
3775 		bp->lin_cnt++;
3776 		if (skb_linearize(skb) != 0) {
3777 			DP(NETIF_MSG_TX_QUEUED,
3778 			   "SKB linearization failed - silently dropping this SKB\n");
3779 			dev_kfree_skb_any(skb);
3780 			return NETDEV_TX_OK;
3781 		}
3782 	}
3783 #endif
3784 	/* Map skb linear data for DMA */
3785 	mapping = dma_map_single(&bp->pdev->dev, skb->data,
3786 				 skb_headlen(skb), DMA_TO_DEVICE);
3787 	if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
3788 		DP(NETIF_MSG_TX_QUEUED,
3789 		   "SKB mapping failed - silently dropping this SKB\n");
3790 		dev_kfree_skb_any(skb);
3791 		return NETDEV_TX_OK;
3792 	}
3793 	/*
3794 	Please read carefully. First we use one BD which we mark as start,
3795 	then we have a parsing info BD (used for TSO or xsum),
3796 	and only then we have the rest of the TSO BDs.
3797 	(don't forget to mark the last one as last,
3798 	and to unmap only AFTER you write to the BD ...)
3799 	And above all, all pdb sizes are in words - NOT DWORDS!
3800 	*/
3801 
3802 	/* get current pkt produced now - advance it just before sending packet
3803 	 * since mapping of pages may fail and cause packet to be dropped
3804 	 */
3805 	pkt_prod = txdata->tx_pkt_prod;
3806 	bd_prod = TX_BD(txdata->tx_bd_prod);
3807 
3808 	/* get a tx_buf and first BD
3809 	 * tx_start_bd may be changed during SPLIT,
3810 	 * but first_bd will always stay first
3811 	 */
3812 	tx_buf = &txdata->tx_buf_ring[TX_BD(pkt_prod)];
3813 	tx_start_bd = &txdata->tx_desc_ring[bd_prod].start_bd;
3814 	first_bd = tx_start_bd;
3815 
3816 	tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
3817 
3818 	/* header nbd: indirectly zero other flags! */
3819 	tx_start_bd->general_data = 1 << ETH_TX_START_BD_HDR_NBDS_SHIFT;
3820 
3821 	/* remember the first BD of the packet */
3822 	tx_buf->first_bd = txdata->tx_bd_prod;
3823 	tx_buf->skb = skb;
3824 	tx_buf->flags = 0;
3825 
3826 	DP(NETIF_MSG_TX_QUEUED,
3827 	   "sending pkt %u @%p  next_idx %u  bd %u @%p\n",
3828 	   pkt_prod, tx_buf, txdata->tx_pkt_prod, bd_prod, tx_start_bd);
3829 
3830 	if (vlan_tx_tag_present(skb)) {
3831 		tx_start_bd->vlan_or_ethertype =
3832 		    cpu_to_le16(vlan_tx_tag_get(skb));
3833 		tx_start_bd->bd_flags.as_bitfield |=
3834 		    (X_ETH_OUTBAND_VLAN << ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
3835 	} else {
3836 		/* when transmitting in a vf, start bd must hold the ethertype
3837 		 * for fw to enforce it
3838 		 */
3839 		if (IS_VF(bp))
3840 			tx_start_bd->vlan_or_ethertype =
3841 				cpu_to_le16(ntohs(eth->h_proto));
3842 		else
3843 			/* used by FW for packet accounting */
3844 			tx_start_bd->vlan_or_ethertype = cpu_to_le16(pkt_prod);
3845 	}
3846 
3847 	nbd = 2; /* start_bd + pbd + frags (updated when pages are mapped) */
3848 
3849 	/* turn on parsing and get a BD */
3850 	bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3851 
3852 	if (xmit_type & XMIT_CSUM)
3853 		bnx2x_set_sbd_csum(bp, skb, tx_start_bd, xmit_type);
3854 
3855 	if (!CHIP_IS_E1x(bp)) {
3856 		pbd_e2 = &txdata->tx_desc_ring[bd_prod].parse_bd_e2;
3857 		memset(pbd_e2, 0, sizeof(struct eth_tx_parse_bd_e2));
3858 
3859 		if (xmit_type & XMIT_CSUM_ENC) {
3860 			u16 global_data = 0;
3861 
3862 			/* Set PBD in enc checksum offload case */
3863 			hlen = bnx2x_set_pbd_csum_enc(bp, skb,
3864 						      &pbd_e2_parsing_data,
3865 						      xmit_type);
3866 
3867 			/* turn on 2nd parsing and get a BD */
3868 			bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3869 
3870 			pbd2 = &txdata->tx_desc_ring[bd_prod].parse_2nd_bd;
3871 
3872 			memset(pbd2, 0, sizeof(*pbd2));
3873 
3874 			pbd_e2->data.tunnel_data.ip_hdr_start_inner_w =
3875 				(skb_inner_network_header(skb) -
3876 				 skb->data) >> 1;
3877 
3878 			if (xmit_type & XMIT_GSO_ENC)
3879 				bnx2x_update_pbds_gso_enc(skb, pbd_e2, pbd2,
3880 							  &global_data,
3881 							  xmit_type);
3882 
3883 			pbd2->global_data = cpu_to_le16(global_data);
3884 
3885 			/* add addition parse BD indication to start BD */
3886 			SET_FLAG(tx_start_bd->general_data,
3887 				 ETH_TX_START_BD_PARSE_NBDS, 1);
3888 			/* set encapsulation flag in start BD */
3889 			SET_FLAG(tx_start_bd->general_data,
3890 				 ETH_TX_START_BD_TUNNEL_EXIST, 1);
3891 			nbd++;
3892 		} else if (xmit_type & XMIT_CSUM) {
3893 			/* Set PBD in checksum offload case w/o encapsulation */
3894 			hlen = bnx2x_set_pbd_csum_e2(bp, skb,
3895 						     &pbd_e2_parsing_data,
3896 						     xmit_type);
3897 		}
3898 
3899 		/* Add the macs to the parsing BD if this is a vf or if
3900 		 * Tx Switching is enabled.
3901 		 */
3902 		if (IS_VF(bp)) {
3903 			/* override GRE parameters in BD */
3904 			bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.src_hi,
3905 					      &pbd_e2->data.mac_addr.src_mid,
3906 					      &pbd_e2->data.mac_addr.src_lo,
3907 					      eth->h_source);
3908 
3909 			bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.dst_hi,
3910 					      &pbd_e2->data.mac_addr.dst_mid,
3911 					      &pbd_e2->data.mac_addr.dst_lo,
3912 					      eth->h_dest);
3913 		} else if (bp->flags & TX_SWITCHING) {
3914 			bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.dst_hi,
3915 					      &pbd_e2->data.mac_addr.dst_mid,
3916 					      &pbd_e2->data.mac_addr.dst_lo,
3917 					      eth->h_dest);
3918 		}
3919 
3920 		SET_FLAG(pbd_e2_parsing_data,
3921 			 ETH_TX_PARSE_BD_E2_ETH_ADDR_TYPE, mac_type);
3922 	} else {
3923 		u16 global_data = 0;
3924 		pbd_e1x = &txdata->tx_desc_ring[bd_prod].parse_bd_e1x;
3925 		memset(pbd_e1x, 0, sizeof(struct eth_tx_parse_bd_e1x));
3926 		/* Set PBD in checksum offload case */
3927 		if (xmit_type & XMIT_CSUM)
3928 			hlen = bnx2x_set_pbd_csum(bp, skb, pbd_e1x, xmit_type);
3929 
3930 		SET_FLAG(global_data,
3931 			 ETH_TX_PARSE_BD_E1X_ETH_ADDR_TYPE, mac_type);
3932 		pbd_e1x->global_data |= cpu_to_le16(global_data);
3933 	}
3934 
3935 	/* Setup the data pointer of the first BD of the packet */
3936 	tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3937 	tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3938 	tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
3939 	pkt_size = tx_start_bd->nbytes;
3940 
3941 	DP(NETIF_MSG_TX_QUEUED,
3942 	   "first bd @%p  addr (%x:%x)  nbytes %d  flags %x  vlan %x\n",
3943 	   tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
3944 	   le16_to_cpu(tx_start_bd->nbytes),
3945 	   tx_start_bd->bd_flags.as_bitfield,
3946 	   le16_to_cpu(tx_start_bd->vlan_or_ethertype));
3947 
3948 	if (xmit_type & XMIT_GSO) {
3949 
3950 		DP(NETIF_MSG_TX_QUEUED,
3951 		   "TSO packet len %d  hlen %d  total len %d  tso size %d\n",
3952 		   skb->len, hlen, skb_headlen(skb),
3953 		   skb_shinfo(skb)->gso_size);
3954 
3955 		tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
3956 
3957 		if (unlikely(skb_headlen(skb) > hlen)) {
3958 			nbd++;
3959 			bd_prod = bnx2x_tx_split(bp, txdata, tx_buf,
3960 						 &tx_start_bd, hlen,
3961 						 bd_prod);
3962 		}
3963 		if (!CHIP_IS_E1x(bp))
3964 			bnx2x_set_pbd_gso_e2(skb, &pbd_e2_parsing_data,
3965 					     xmit_type);
3966 		else
3967 			bnx2x_set_pbd_gso(skb, pbd_e1x, first_bd, xmit_type);
3968 	}
3969 
3970 	/* Set the PBD's parsing_data field if not zero
3971 	 * (for the chips newer than 57711).
3972 	 */
3973 	if (pbd_e2_parsing_data)
3974 		pbd_e2->parsing_data = cpu_to_le32(pbd_e2_parsing_data);
3975 
3976 	tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
3977 
3978 	/* Handle fragmented skb */
3979 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3980 		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3981 
3982 		mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 0,
3983 					   skb_frag_size(frag), DMA_TO_DEVICE);
3984 		if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
3985 			unsigned int pkts_compl = 0, bytes_compl = 0;
3986 
3987 			DP(NETIF_MSG_TX_QUEUED,
3988 			   "Unable to map page - dropping packet...\n");
3989 
3990 			/* we need unmap all buffers already mapped
3991 			 * for this SKB;
3992 			 * first_bd->nbd need to be properly updated
3993 			 * before call to bnx2x_free_tx_pkt
3994 			 */
3995 			first_bd->nbd = cpu_to_le16(nbd);
3996 			bnx2x_free_tx_pkt(bp, txdata,
3997 					  TX_BD(txdata->tx_pkt_prod),
3998 					  &pkts_compl, &bytes_compl);
3999 			return NETDEV_TX_OK;
4000 		}
4001 
4002 		bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
4003 		tx_data_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
4004 		if (total_pkt_bd == NULL)
4005 			total_pkt_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
4006 
4007 		tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
4008 		tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
4009 		tx_data_bd->nbytes = cpu_to_le16(skb_frag_size(frag));
4010 		le16_add_cpu(&pkt_size, skb_frag_size(frag));
4011 		nbd++;
4012 
4013 		DP(NETIF_MSG_TX_QUEUED,
4014 		   "frag %d  bd @%p  addr (%x:%x)  nbytes %d\n",
4015 		   i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
4016 		   le16_to_cpu(tx_data_bd->nbytes));
4017 	}
4018 
4019 	DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
4020 
4021 	/* update with actual num BDs */
4022 	first_bd->nbd = cpu_to_le16(nbd);
4023 
4024 	bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
4025 
4026 	/* now send a tx doorbell, counting the next BD
4027 	 * if the packet contains or ends with it
4028 	 */
4029 	if (TX_BD_POFF(bd_prod) < nbd)
4030 		nbd++;
4031 
4032 	/* total_pkt_bytes should be set on the first data BD if
4033 	 * it's not an LSO packet and there is more than one
4034 	 * data BD. In this case pkt_size is limited by an MTU value.
4035 	 * However we prefer to set it for an LSO packet (while we don't
4036 	 * have to) in order to save some CPU cycles in a none-LSO
4037 	 * case, when we much more care about them.
4038 	 */
4039 	if (total_pkt_bd != NULL)
4040 		total_pkt_bd->total_pkt_bytes = pkt_size;
4041 
4042 	if (pbd_e1x)
4043 		DP(NETIF_MSG_TX_QUEUED,
4044 		   "PBD (E1X) @%p  ip_data %x  ip_hlen %u  ip_id %u  lso_mss %u  tcp_flags %x  xsum %x  seq %u  hlen %u\n",
4045 		   pbd_e1x, pbd_e1x->global_data, pbd_e1x->ip_hlen_w,
4046 		   pbd_e1x->ip_id, pbd_e1x->lso_mss, pbd_e1x->tcp_flags,
4047 		   pbd_e1x->tcp_pseudo_csum, pbd_e1x->tcp_send_seq,
4048 		    le16_to_cpu(pbd_e1x->total_hlen_w));
4049 	if (pbd_e2)
4050 		DP(NETIF_MSG_TX_QUEUED,
4051 		   "PBD (E2) @%p  dst %x %x %x src %x %x %x parsing_data %x\n",
4052 		   pbd_e2,
4053 		   pbd_e2->data.mac_addr.dst_hi,
4054 		   pbd_e2->data.mac_addr.dst_mid,
4055 		   pbd_e2->data.mac_addr.dst_lo,
4056 		   pbd_e2->data.mac_addr.src_hi,
4057 		   pbd_e2->data.mac_addr.src_mid,
4058 		   pbd_e2->data.mac_addr.src_lo,
4059 		   pbd_e2->parsing_data);
4060 	DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d  bd %u\n", nbd, bd_prod);
4061 
4062 	netdev_tx_sent_queue(txq, skb->len);
4063 
4064 	skb_tx_timestamp(skb);
4065 
4066 	txdata->tx_pkt_prod++;
4067 	/*
4068 	 * Make sure that the BD data is updated before updating the producer
4069 	 * since FW might read the BD right after the producer is updated.
4070 	 * This is only applicable for weak-ordered memory model archs such
4071 	 * as IA-64. The following barrier is also mandatory since FW will
4072 	 * assumes packets must have BDs.
4073 	 */
4074 	wmb();
4075 
4076 	txdata->tx_db.data.prod += nbd;
4077 	barrier();
4078 
4079 	DOORBELL(bp, txdata->cid, txdata->tx_db.raw);
4080 
4081 	mmiowb();
4082 
4083 	txdata->tx_bd_prod += nbd;
4084 
4085 	if (unlikely(bnx2x_tx_avail(bp, txdata) < MAX_DESC_PER_TX_PKT)) {
4086 		netif_tx_stop_queue(txq);
4087 
4088 		/* paired memory barrier is in bnx2x_tx_int(), we have to keep
4089 		 * ordering of set_bit() in netif_tx_stop_queue() and read of
4090 		 * fp->bd_tx_cons */
4091 		smp_mb();
4092 
4093 		bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
4094 		if (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT)
4095 			netif_tx_wake_queue(txq);
4096 	}
4097 	txdata->tx_pkt++;
4098 
4099 	return NETDEV_TX_OK;
4100 }
4101 
4102 /**
4103  * bnx2x_setup_tc - routine to configure net_device for multi tc
4104  *
4105  * @netdev: net device to configure
4106  * @tc: number of traffic classes to enable
4107  *
4108  * callback connected to the ndo_setup_tc function pointer
4109  */
4110 int bnx2x_setup_tc(struct net_device *dev, u8 num_tc)
4111 {
4112 	int cos, prio, count, offset;
4113 	struct bnx2x *bp = netdev_priv(dev);
4114 
4115 	/* setup tc must be called under rtnl lock */
4116 	ASSERT_RTNL();
4117 
4118 	/* no traffic classes requested. Aborting */
4119 	if (!num_tc) {
4120 		netdev_reset_tc(dev);
4121 		return 0;
4122 	}
4123 
4124 	/* requested to support too many traffic classes */
4125 	if (num_tc > bp->max_cos) {
4126 		BNX2X_ERR("support for too many traffic classes requested: %d. Max supported is %d\n",
4127 			  num_tc, bp->max_cos);
4128 		return -EINVAL;
4129 	}
4130 
4131 	/* declare amount of supported traffic classes */
4132 	if (netdev_set_num_tc(dev, num_tc)) {
4133 		BNX2X_ERR("failed to declare %d traffic classes\n", num_tc);
4134 		return -EINVAL;
4135 	}
4136 
4137 	/* configure priority to traffic class mapping */
4138 	for (prio = 0; prio < BNX2X_MAX_PRIORITY; prio++) {
4139 		netdev_set_prio_tc_map(dev, prio, bp->prio_to_cos[prio]);
4140 		DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
4141 		   "mapping priority %d to tc %d\n",
4142 		   prio, bp->prio_to_cos[prio]);
4143 	}
4144 
4145 	/* Use this configuration to differentiate tc0 from other COSes
4146 	   This can be used for ets or pfc, and save the effort of setting
4147 	   up a multio class queue disc or negotiating DCBX with a switch
4148 	netdev_set_prio_tc_map(dev, 0, 0);
4149 	DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", 0, 0);
4150 	for (prio = 1; prio < 16; prio++) {
4151 		netdev_set_prio_tc_map(dev, prio, 1);
4152 		DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", prio, 1);
4153 	} */
4154 
4155 	/* configure traffic class to transmission queue mapping */
4156 	for (cos = 0; cos < bp->max_cos; cos++) {
4157 		count = BNX2X_NUM_ETH_QUEUES(bp);
4158 		offset = cos * BNX2X_NUM_NON_CNIC_QUEUES(bp);
4159 		netdev_set_tc_queue(dev, cos, count, offset);
4160 		DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
4161 		   "mapping tc %d to offset %d count %d\n",
4162 		   cos, offset, count);
4163 	}
4164 
4165 	return 0;
4166 }
4167 
4168 /* called with rtnl_lock */
4169 int bnx2x_change_mac_addr(struct net_device *dev, void *p)
4170 {
4171 	struct sockaddr *addr = p;
4172 	struct bnx2x *bp = netdev_priv(dev);
4173 	int rc = 0;
4174 
4175 	if (!bnx2x_is_valid_ether_addr(bp, addr->sa_data)) {
4176 		BNX2X_ERR("Requested MAC address is not valid\n");
4177 		return -EINVAL;
4178 	}
4179 
4180 	if ((IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp)) &&
4181 	    !is_zero_ether_addr(addr->sa_data)) {
4182 		BNX2X_ERR("Can't configure non-zero address on iSCSI or FCoE functions in MF-SD mode\n");
4183 		return -EINVAL;
4184 	}
4185 
4186 	if (netif_running(dev))  {
4187 		rc = bnx2x_set_eth_mac(bp, false);
4188 		if (rc)
4189 			return rc;
4190 	}
4191 
4192 	memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
4193 
4194 	if (netif_running(dev))
4195 		rc = bnx2x_set_eth_mac(bp, true);
4196 
4197 	return rc;
4198 }
4199 
4200 static void bnx2x_free_fp_mem_at(struct bnx2x *bp, int fp_index)
4201 {
4202 	union host_hc_status_block *sb = &bnx2x_fp(bp, fp_index, status_blk);
4203 	struct bnx2x_fastpath *fp = &bp->fp[fp_index];
4204 	u8 cos;
4205 
4206 	/* Common */
4207 
4208 	if (IS_FCOE_IDX(fp_index)) {
4209 		memset(sb, 0, sizeof(union host_hc_status_block));
4210 		fp->status_blk_mapping = 0;
4211 	} else {
4212 		/* status blocks */
4213 		if (!CHIP_IS_E1x(bp))
4214 			BNX2X_PCI_FREE(sb->e2_sb,
4215 				       bnx2x_fp(bp, fp_index,
4216 						status_blk_mapping),
4217 				       sizeof(struct host_hc_status_block_e2));
4218 		else
4219 			BNX2X_PCI_FREE(sb->e1x_sb,
4220 				       bnx2x_fp(bp, fp_index,
4221 						status_blk_mapping),
4222 				       sizeof(struct host_hc_status_block_e1x));
4223 	}
4224 
4225 	/* Rx */
4226 	if (!skip_rx_queue(bp, fp_index)) {
4227 		bnx2x_free_rx_bds(fp);
4228 
4229 		/* fastpath rx rings: rx_buf rx_desc rx_comp */
4230 		BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_buf_ring));
4231 		BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_desc_ring),
4232 			       bnx2x_fp(bp, fp_index, rx_desc_mapping),
4233 			       sizeof(struct eth_rx_bd) * NUM_RX_BD);
4234 
4235 		BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_comp_ring),
4236 			       bnx2x_fp(bp, fp_index, rx_comp_mapping),
4237 			       sizeof(struct eth_fast_path_rx_cqe) *
4238 			       NUM_RCQ_BD);
4239 
4240 		/* SGE ring */
4241 		BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_page_ring));
4242 		BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_sge_ring),
4243 			       bnx2x_fp(bp, fp_index, rx_sge_mapping),
4244 			       BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4245 	}
4246 
4247 	/* Tx */
4248 	if (!skip_tx_queue(bp, fp_index)) {
4249 		/* fastpath tx rings: tx_buf tx_desc */
4250 		for_each_cos_in_tx_queue(fp, cos) {
4251 			struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
4252 
4253 			DP(NETIF_MSG_IFDOWN,
4254 			   "freeing tx memory of fp %d cos %d cid %d\n",
4255 			   fp_index, cos, txdata->cid);
4256 
4257 			BNX2X_FREE(txdata->tx_buf_ring);
4258 			BNX2X_PCI_FREE(txdata->tx_desc_ring,
4259 				txdata->tx_desc_mapping,
4260 				sizeof(union eth_tx_bd_types) * NUM_TX_BD);
4261 		}
4262 	}
4263 	/* end of fastpath */
4264 }
4265 
4266 static void bnx2x_free_fp_mem_cnic(struct bnx2x *bp)
4267 {
4268 	int i;
4269 	for_each_cnic_queue(bp, i)
4270 		bnx2x_free_fp_mem_at(bp, i);
4271 }
4272 
4273 void bnx2x_free_fp_mem(struct bnx2x *bp)
4274 {
4275 	int i;
4276 	for_each_eth_queue(bp, i)
4277 		bnx2x_free_fp_mem_at(bp, i);
4278 }
4279 
4280 static void set_sb_shortcuts(struct bnx2x *bp, int index)
4281 {
4282 	union host_hc_status_block status_blk = bnx2x_fp(bp, index, status_blk);
4283 	if (!CHIP_IS_E1x(bp)) {
4284 		bnx2x_fp(bp, index, sb_index_values) =
4285 			(__le16 *)status_blk.e2_sb->sb.index_values;
4286 		bnx2x_fp(bp, index, sb_running_index) =
4287 			(__le16 *)status_blk.e2_sb->sb.running_index;
4288 	} else {
4289 		bnx2x_fp(bp, index, sb_index_values) =
4290 			(__le16 *)status_blk.e1x_sb->sb.index_values;
4291 		bnx2x_fp(bp, index, sb_running_index) =
4292 			(__le16 *)status_blk.e1x_sb->sb.running_index;
4293 	}
4294 }
4295 
4296 /* Returns the number of actually allocated BDs */
4297 static int bnx2x_alloc_rx_bds(struct bnx2x_fastpath *fp,
4298 			      int rx_ring_size)
4299 {
4300 	struct bnx2x *bp = fp->bp;
4301 	u16 ring_prod, cqe_ring_prod;
4302 	int i, failure_cnt = 0;
4303 
4304 	fp->rx_comp_cons = 0;
4305 	cqe_ring_prod = ring_prod = 0;
4306 
4307 	/* This routine is called only during fo init so
4308 	 * fp->eth_q_stats.rx_skb_alloc_failed = 0
4309 	 */
4310 	for (i = 0; i < rx_ring_size; i++) {
4311 		if (bnx2x_alloc_rx_data(bp, fp, ring_prod, GFP_KERNEL) < 0) {
4312 			failure_cnt++;
4313 			continue;
4314 		}
4315 		ring_prod = NEXT_RX_IDX(ring_prod);
4316 		cqe_ring_prod = NEXT_RCQ_IDX(cqe_ring_prod);
4317 		WARN_ON(ring_prod <= (i - failure_cnt));
4318 	}
4319 
4320 	if (failure_cnt)
4321 		BNX2X_ERR("was only able to allocate %d rx skbs on queue[%d]\n",
4322 			  i - failure_cnt, fp->index);
4323 
4324 	fp->rx_bd_prod = ring_prod;
4325 	/* Limit the CQE producer by the CQE ring size */
4326 	fp->rx_comp_prod = min_t(u16, NUM_RCQ_RINGS*RCQ_DESC_CNT,
4327 			       cqe_ring_prod);
4328 	fp->rx_pkt = fp->rx_calls = 0;
4329 
4330 	bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed += failure_cnt;
4331 
4332 	return i - failure_cnt;
4333 }
4334 
4335 static void bnx2x_set_next_page_rx_cq(struct bnx2x_fastpath *fp)
4336 {
4337 	int i;
4338 
4339 	for (i = 1; i <= NUM_RCQ_RINGS; i++) {
4340 		struct eth_rx_cqe_next_page *nextpg;
4341 
4342 		nextpg = (struct eth_rx_cqe_next_page *)
4343 			&fp->rx_comp_ring[RCQ_DESC_CNT * i - 1];
4344 		nextpg->addr_hi =
4345 			cpu_to_le32(U64_HI(fp->rx_comp_mapping +
4346 				   BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4347 		nextpg->addr_lo =
4348 			cpu_to_le32(U64_LO(fp->rx_comp_mapping +
4349 				   BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4350 	}
4351 }
4352 
4353 static int bnx2x_alloc_fp_mem_at(struct bnx2x *bp, int index)
4354 {
4355 	union host_hc_status_block *sb;
4356 	struct bnx2x_fastpath *fp = &bp->fp[index];
4357 	int ring_size = 0;
4358 	u8 cos;
4359 	int rx_ring_size = 0;
4360 
4361 	if (!bp->rx_ring_size &&
4362 	    (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))) {
4363 		rx_ring_size = MIN_RX_SIZE_NONTPA;
4364 		bp->rx_ring_size = rx_ring_size;
4365 	} else if (!bp->rx_ring_size) {
4366 		rx_ring_size = MAX_RX_AVAIL/BNX2X_NUM_RX_QUEUES(bp);
4367 
4368 		if (CHIP_IS_E3(bp)) {
4369 			u32 cfg = SHMEM_RD(bp,
4370 					   dev_info.port_hw_config[BP_PORT(bp)].
4371 					   default_cfg);
4372 
4373 			/* Decrease ring size for 1G functions */
4374 			if ((cfg & PORT_HW_CFG_NET_SERDES_IF_MASK) ==
4375 			    PORT_HW_CFG_NET_SERDES_IF_SGMII)
4376 				rx_ring_size /= 10;
4377 		}
4378 
4379 		/* allocate at least number of buffers required by FW */
4380 		rx_ring_size = max_t(int, bp->disable_tpa ? MIN_RX_SIZE_NONTPA :
4381 				     MIN_RX_SIZE_TPA, rx_ring_size);
4382 
4383 		bp->rx_ring_size = rx_ring_size;
4384 	} else /* if rx_ring_size specified - use it */
4385 		rx_ring_size = bp->rx_ring_size;
4386 
4387 	DP(BNX2X_MSG_SP, "calculated rx_ring_size %d\n", rx_ring_size);
4388 
4389 	/* Common */
4390 	sb = &bnx2x_fp(bp, index, status_blk);
4391 
4392 	if (!IS_FCOE_IDX(index)) {
4393 		/* status blocks */
4394 		if (!CHIP_IS_E1x(bp)) {
4395 			sb->e2_sb = BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, status_blk_mapping),
4396 						    sizeof(struct host_hc_status_block_e2));
4397 			if (!sb->e2_sb)
4398 				goto alloc_mem_err;
4399 		} else {
4400 			sb->e1x_sb = BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, status_blk_mapping),
4401 						     sizeof(struct host_hc_status_block_e1x));
4402 			if (!sb->e1x_sb)
4403 				goto alloc_mem_err;
4404 		}
4405 	}
4406 
4407 	/* FCoE Queue uses Default SB and doesn't ACK the SB, thus no need to
4408 	 * set shortcuts for it.
4409 	 */
4410 	if (!IS_FCOE_IDX(index))
4411 		set_sb_shortcuts(bp, index);
4412 
4413 	/* Tx */
4414 	if (!skip_tx_queue(bp, index)) {
4415 		/* fastpath tx rings: tx_buf tx_desc */
4416 		for_each_cos_in_tx_queue(fp, cos) {
4417 			struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
4418 
4419 			DP(NETIF_MSG_IFUP,
4420 			   "allocating tx memory of fp %d cos %d\n",
4421 			   index, cos);
4422 
4423 			txdata->tx_buf_ring = kcalloc(NUM_TX_BD,
4424 						      sizeof(struct sw_tx_bd),
4425 						      GFP_KERNEL);
4426 			if (!txdata->tx_buf_ring)
4427 				goto alloc_mem_err;
4428 			txdata->tx_desc_ring = BNX2X_PCI_ALLOC(&txdata->tx_desc_mapping,
4429 							       sizeof(union eth_tx_bd_types) * NUM_TX_BD);
4430 			if (!txdata->tx_desc_ring)
4431 				goto alloc_mem_err;
4432 		}
4433 	}
4434 
4435 	/* Rx */
4436 	if (!skip_rx_queue(bp, index)) {
4437 		/* fastpath rx rings: rx_buf rx_desc rx_comp */
4438 		bnx2x_fp(bp, index, rx_buf_ring) =
4439 			kcalloc(NUM_RX_BD, sizeof(struct sw_rx_bd), GFP_KERNEL);
4440 		if (!bnx2x_fp(bp, index, rx_buf_ring))
4441 			goto alloc_mem_err;
4442 		bnx2x_fp(bp, index, rx_desc_ring) =
4443 			BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, rx_desc_mapping),
4444 					sizeof(struct eth_rx_bd) * NUM_RX_BD);
4445 		if (!bnx2x_fp(bp, index, rx_desc_ring))
4446 			goto alloc_mem_err;
4447 
4448 		/* Seed all CQEs by 1s */
4449 		bnx2x_fp(bp, index, rx_comp_ring) =
4450 			BNX2X_PCI_FALLOC(&bnx2x_fp(bp, index, rx_comp_mapping),
4451 					 sizeof(struct eth_fast_path_rx_cqe) * NUM_RCQ_BD);
4452 		if (!bnx2x_fp(bp, index, rx_comp_ring))
4453 			goto alloc_mem_err;
4454 
4455 		/* SGE ring */
4456 		bnx2x_fp(bp, index, rx_page_ring) =
4457 			kcalloc(NUM_RX_SGE, sizeof(struct sw_rx_page),
4458 				GFP_KERNEL);
4459 		if (!bnx2x_fp(bp, index, rx_page_ring))
4460 			goto alloc_mem_err;
4461 		bnx2x_fp(bp, index, rx_sge_ring) =
4462 			BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, rx_sge_mapping),
4463 					BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4464 		if (!bnx2x_fp(bp, index, rx_sge_ring))
4465 			goto alloc_mem_err;
4466 		/* RX BD ring */
4467 		bnx2x_set_next_page_rx_bd(fp);
4468 
4469 		/* CQ ring */
4470 		bnx2x_set_next_page_rx_cq(fp);
4471 
4472 		/* BDs */
4473 		ring_size = bnx2x_alloc_rx_bds(fp, rx_ring_size);
4474 		if (ring_size < rx_ring_size)
4475 			goto alloc_mem_err;
4476 	}
4477 
4478 	return 0;
4479 
4480 /* handles low memory cases */
4481 alloc_mem_err:
4482 	BNX2X_ERR("Unable to allocate full memory for queue %d (size %d)\n",
4483 						index, ring_size);
4484 	/* FW will drop all packets if queue is not big enough,
4485 	 * In these cases we disable the queue
4486 	 * Min size is different for OOO, TPA and non-TPA queues
4487 	 */
4488 	if (ring_size < (fp->disable_tpa ?
4489 				MIN_RX_SIZE_NONTPA : MIN_RX_SIZE_TPA)) {
4490 			/* release memory allocated for this queue */
4491 			bnx2x_free_fp_mem_at(bp, index);
4492 			return -ENOMEM;
4493 	}
4494 	return 0;
4495 }
4496 
4497 static int bnx2x_alloc_fp_mem_cnic(struct bnx2x *bp)
4498 {
4499 	if (!NO_FCOE(bp))
4500 		/* FCoE */
4501 		if (bnx2x_alloc_fp_mem_at(bp, FCOE_IDX(bp)))
4502 			/* we will fail load process instead of mark
4503 			 * NO_FCOE_FLAG
4504 			 */
4505 			return -ENOMEM;
4506 
4507 	return 0;
4508 }
4509 
4510 static int bnx2x_alloc_fp_mem(struct bnx2x *bp)
4511 {
4512 	int i;
4513 
4514 	/* 1. Allocate FP for leading - fatal if error
4515 	 * 2. Allocate RSS - fix number of queues if error
4516 	 */
4517 
4518 	/* leading */
4519 	if (bnx2x_alloc_fp_mem_at(bp, 0))
4520 		return -ENOMEM;
4521 
4522 	/* RSS */
4523 	for_each_nondefault_eth_queue(bp, i)
4524 		if (bnx2x_alloc_fp_mem_at(bp, i))
4525 			break;
4526 
4527 	/* handle memory failures */
4528 	if (i != BNX2X_NUM_ETH_QUEUES(bp)) {
4529 		int delta = BNX2X_NUM_ETH_QUEUES(bp) - i;
4530 
4531 		WARN_ON(delta < 0);
4532 		bnx2x_shrink_eth_fp(bp, delta);
4533 		if (CNIC_SUPPORT(bp))
4534 			/* move non eth FPs next to last eth FP
4535 			 * must be done in that order
4536 			 * FCOE_IDX < FWD_IDX < OOO_IDX
4537 			 */
4538 
4539 			/* move FCoE fp even NO_FCOE_FLAG is on */
4540 			bnx2x_move_fp(bp, FCOE_IDX(bp), FCOE_IDX(bp) - delta);
4541 		bp->num_ethernet_queues -= delta;
4542 		bp->num_queues = bp->num_ethernet_queues +
4543 				 bp->num_cnic_queues;
4544 		BNX2X_ERR("Adjusted num of queues from %d to %d\n",
4545 			  bp->num_queues + delta, bp->num_queues);
4546 	}
4547 
4548 	return 0;
4549 }
4550 
4551 void bnx2x_free_mem_bp(struct bnx2x *bp)
4552 {
4553 	int i;
4554 
4555 	for (i = 0; i < bp->fp_array_size; i++)
4556 		kfree(bp->fp[i].tpa_info);
4557 	kfree(bp->fp);
4558 	kfree(bp->sp_objs);
4559 	kfree(bp->fp_stats);
4560 	kfree(bp->bnx2x_txq);
4561 	kfree(bp->msix_table);
4562 	kfree(bp->ilt);
4563 }
4564 
4565 int bnx2x_alloc_mem_bp(struct bnx2x *bp)
4566 {
4567 	struct bnx2x_fastpath *fp;
4568 	struct msix_entry *tbl;
4569 	struct bnx2x_ilt *ilt;
4570 	int msix_table_size = 0;
4571 	int fp_array_size, txq_array_size;
4572 	int i;
4573 
4574 	/*
4575 	 * The biggest MSI-X table we might need is as a maximum number of fast
4576 	 * path IGU SBs plus default SB (for PF only).
4577 	 */
4578 	msix_table_size = bp->igu_sb_cnt;
4579 	if (IS_PF(bp))
4580 		msix_table_size++;
4581 	BNX2X_DEV_INFO("msix_table_size %d\n", msix_table_size);
4582 
4583 	/* fp array: RSS plus CNIC related L2 queues */
4584 	fp_array_size = BNX2X_MAX_RSS_COUNT(bp) + CNIC_SUPPORT(bp);
4585 	bp->fp_array_size = fp_array_size;
4586 	BNX2X_DEV_INFO("fp_array_size %d\n", bp->fp_array_size);
4587 
4588 	fp = kcalloc(bp->fp_array_size, sizeof(*fp), GFP_KERNEL);
4589 	if (!fp)
4590 		goto alloc_err;
4591 	for (i = 0; i < bp->fp_array_size; i++) {
4592 		fp[i].tpa_info =
4593 			kcalloc(ETH_MAX_AGGREGATION_QUEUES_E1H_E2,
4594 				sizeof(struct bnx2x_agg_info), GFP_KERNEL);
4595 		if (!(fp[i].tpa_info))
4596 			goto alloc_err;
4597 	}
4598 
4599 	bp->fp = fp;
4600 
4601 	/* allocate sp objs */
4602 	bp->sp_objs = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_sp_objs),
4603 			      GFP_KERNEL);
4604 	if (!bp->sp_objs)
4605 		goto alloc_err;
4606 
4607 	/* allocate fp_stats */
4608 	bp->fp_stats = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_fp_stats),
4609 			       GFP_KERNEL);
4610 	if (!bp->fp_stats)
4611 		goto alloc_err;
4612 
4613 	/* Allocate memory for the transmission queues array */
4614 	txq_array_size =
4615 		BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS + CNIC_SUPPORT(bp);
4616 	BNX2X_DEV_INFO("txq_array_size %d", txq_array_size);
4617 
4618 	bp->bnx2x_txq = kcalloc(txq_array_size, sizeof(struct bnx2x_fp_txdata),
4619 				GFP_KERNEL);
4620 	if (!bp->bnx2x_txq)
4621 		goto alloc_err;
4622 
4623 	/* msix table */
4624 	tbl = kcalloc(msix_table_size, sizeof(*tbl), GFP_KERNEL);
4625 	if (!tbl)
4626 		goto alloc_err;
4627 	bp->msix_table = tbl;
4628 
4629 	/* ilt */
4630 	ilt = kzalloc(sizeof(*ilt), GFP_KERNEL);
4631 	if (!ilt)
4632 		goto alloc_err;
4633 	bp->ilt = ilt;
4634 
4635 	return 0;
4636 alloc_err:
4637 	bnx2x_free_mem_bp(bp);
4638 	return -ENOMEM;
4639 }
4640 
4641 int bnx2x_reload_if_running(struct net_device *dev)
4642 {
4643 	struct bnx2x *bp = netdev_priv(dev);
4644 
4645 	if (unlikely(!netif_running(dev)))
4646 		return 0;
4647 
4648 	bnx2x_nic_unload(bp, UNLOAD_NORMAL, true);
4649 	return bnx2x_nic_load(bp, LOAD_NORMAL);
4650 }
4651 
4652 int bnx2x_get_cur_phy_idx(struct bnx2x *bp)
4653 {
4654 	u32 sel_phy_idx = 0;
4655 	if (bp->link_params.num_phys <= 1)
4656 		return INT_PHY;
4657 
4658 	if (bp->link_vars.link_up) {
4659 		sel_phy_idx = EXT_PHY1;
4660 		/* In case link is SERDES, check if the EXT_PHY2 is the one */
4661 		if ((bp->link_vars.link_status & LINK_STATUS_SERDES_LINK) &&
4662 		    (bp->link_params.phy[EXT_PHY2].supported & SUPPORTED_FIBRE))
4663 			sel_phy_idx = EXT_PHY2;
4664 	} else {
4665 
4666 		switch (bnx2x_phy_selection(&bp->link_params)) {
4667 		case PORT_HW_CFG_PHY_SELECTION_HARDWARE_DEFAULT:
4668 		case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY:
4669 		case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY_PRIORITY:
4670 		       sel_phy_idx = EXT_PHY1;
4671 		       break;
4672 		case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY:
4673 		case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY_PRIORITY:
4674 		       sel_phy_idx = EXT_PHY2;
4675 		       break;
4676 		}
4677 	}
4678 
4679 	return sel_phy_idx;
4680 }
4681 int bnx2x_get_link_cfg_idx(struct bnx2x *bp)
4682 {
4683 	u32 sel_phy_idx = bnx2x_get_cur_phy_idx(bp);
4684 	/*
4685 	 * The selected activated PHY is always after swapping (in case PHY
4686 	 * swapping is enabled). So when swapping is enabled, we need to reverse
4687 	 * the configuration
4688 	 */
4689 
4690 	if (bp->link_params.multi_phy_config &
4691 	    PORT_HW_CFG_PHY_SWAPPED_ENABLED) {
4692 		if (sel_phy_idx == EXT_PHY1)
4693 			sel_phy_idx = EXT_PHY2;
4694 		else if (sel_phy_idx == EXT_PHY2)
4695 			sel_phy_idx = EXT_PHY1;
4696 	}
4697 	return LINK_CONFIG_IDX(sel_phy_idx);
4698 }
4699 
4700 #ifdef NETDEV_FCOE_WWNN
4701 int bnx2x_fcoe_get_wwn(struct net_device *dev, u64 *wwn, int type)
4702 {
4703 	struct bnx2x *bp = netdev_priv(dev);
4704 	struct cnic_eth_dev *cp = &bp->cnic_eth_dev;
4705 
4706 	switch (type) {
4707 	case NETDEV_FCOE_WWNN:
4708 		*wwn = HILO_U64(cp->fcoe_wwn_node_name_hi,
4709 				cp->fcoe_wwn_node_name_lo);
4710 		break;
4711 	case NETDEV_FCOE_WWPN:
4712 		*wwn = HILO_U64(cp->fcoe_wwn_port_name_hi,
4713 				cp->fcoe_wwn_port_name_lo);
4714 		break;
4715 	default:
4716 		BNX2X_ERR("Wrong WWN type requested - %d\n", type);
4717 		return -EINVAL;
4718 	}
4719 
4720 	return 0;
4721 }
4722 #endif
4723 
4724 /* called with rtnl_lock */
4725 int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
4726 {
4727 	struct bnx2x *bp = netdev_priv(dev);
4728 
4729 	if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
4730 		BNX2X_ERR("Can't perform change MTU during parity recovery\n");
4731 		return -EAGAIN;
4732 	}
4733 
4734 	if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) ||
4735 	    ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE)) {
4736 		BNX2X_ERR("Can't support requested MTU size\n");
4737 		return -EINVAL;
4738 	}
4739 
4740 	/* This does not race with packet allocation
4741 	 * because the actual alloc size is
4742 	 * only updated as part of load
4743 	 */
4744 	dev->mtu = new_mtu;
4745 
4746 	return bnx2x_reload_if_running(dev);
4747 }
4748 
4749 netdev_features_t bnx2x_fix_features(struct net_device *dev,
4750 				     netdev_features_t features)
4751 {
4752 	struct bnx2x *bp = netdev_priv(dev);
4753 
4754 	/* TPA requires Rx CSUM offloading */
4755 	if (!(features & NETIF_F_RXCSUM) || bp->disable_tpa) {
4756 		features &= ~NETIF_F_LRO;
4757 		features &= ~NETIF_F_GRO;
4758 	}
4759 
4760 	return features;
4761 }
4762 
4763 int bnx2x_set_features(struct net_device *dev, netdev_features_t features)
4764 {
4765 	struct bnx2x *bp = netdev_priv(dev);
4766 	u32 flags = bp->flags;
4767 	u32 changes;
4768 	bool bnx2x_reload = false;
4769 
4770 	if (features & NETIF_F_LRO)
4771 		flags |= TPA_ENABLE_FLAG;
4772 	else
4773 		flags &= ~TPA_ENABLE_FLAG;
4774 
4775 	if (features & NETIF_F_GRO)
4776 		flags |= GRO_ENABLE_FLAG;
4777 	else
4778 		flags &= ~GRO_ENABLE_FLAG;
4779 
4780 	if (features & NETIF_F_LOOPBACK) {
4781 		if (bp->link_params.loopback_mode != LOOPBACK_BMAC) {
4782 			bp->link_params.loopback_mode = LOOPBACK_BMAC;
4783 			bnx2x_reload = true;
4784 		}
4785 	} else {
4786 		if (bp->link_params.loopback_mode != LOOPBACK_NONE) {
4787 			bp->link_params.loopback_mode = LOOPBACK_NONE;
4788 			bnx2x_reload = true;
4789 		}
4790 	}
4791 
4792 	changes = flags ^ bp->flags;
4793 
4794 	/* if GRO is changed while LRO is enabled, don't force a reload */
4795 	if ((changes & GRO_ENABLE_FLAG) && (flags & TPA_ENABLE_FLAG))
4796 		changes &= ~GRO_ENABLE_FLAG;
4797 
4798 	if (changes)
4799 		bnx2x_reload = true;
4800 
4801 	bp->flags = flags;
4802 
4803 	if (bnx2x_reload) {
4804 		if (bp->recovery_state == BNX2X_RECOVERY_DONE)
4805 			return bnx2x_reload_if_running(dev);
4806 		/* else: bnx2x_nic_load() will be called at end of recovery */
4807 	}
4808 
4809 	return 0;
4810 }
4811 
4812 void bnx2x_tx_timeout(struct net_device *dev)
4813 {
4814 	struct bnx2x *bp = netdev_priv(dev);
4815 
4816 #ifdef BNX2X_STOP_ON_ERROR
4817 	if (!bp->panic)
4818 		bnx2x_panic();
4819 #endif
4820 
4821 	/* This allows the netif to be shutdown gracefully before resetting */
4822 	bnx2x_schedule_sp_rtnl(bp, BNX2X_SP_RTNL_TX_TIMEOUT, 0);
4823 }
4824 
4825 int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state)
4826 {
4827 	struct net_device *dev = pci_get_drvdata(pdev);
4828 	struct bnx2x *bp;
4829 
4830 	if (!dev) {
4831 		dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
4832 		return -ENODEV;
4833 	}
4834 	bp = netdev_priv(dev);
4835 
4836 	rtnl_lock();
4837 
4838 	pci_save_state(pdev);
4839 
4840 	if (!netif_running(dev)) {
4841 		rtnl_unlock();
4842 		return 0;
4843 	}
4844 
4845 	netif_device_detach(dev);
4846 
4847 	bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
4848 
4849 	bnx2x_set_power_state(bp, pci_choose_state(pdev, state));
4850 
4851 	rtnl_unlock();
4852 
4853 	return 0;
4854 }
4855 
4856 int bnx2x_resume(struct pci_dev *pdev)
4857 {
4858 	struct net_device *dev = pci_get_drvdata(pdev);
4859 	struct bnx2x *bp;
4860 	int rc;
4861 
4862 	if (!dev) {
4863 		dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
4864 		return -ENODEV;
4865 	}
4866 	bp = netdev_priv(dev);
4867 
4868 	if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
4869 		BNX2X_ERR("Handling parity error recovery. Try again later\n");
4870 		return -EAGAIN;
4871 	}
4872 
4873 	rtnl_lock();
4874 
4875 	pci_restore_state(pdev);
4876 
4877 	if (!netif_running(dev)) {
4878 		rtnl_unlock();
4879 		return 0;
4880 	}
4881 
4882 	bnx2x_set_power_state(bp, PCI_D0);
4883 	netif_device_attach(dev);
4884 
4885 	rc = bnx2x_nic_load(bp, LOAD_OPEN);
4886 
4887 	rtnl_unlock();
4888 
4889 	return rc;
4890 }
4891 
4892 void bnx2x_set_ctx_validation(struct bnx2x *bp, struct eth_context *cxt,
4893 			      u32 cid)
4894 {
4895 	if (!cxt) {
4896 		BNX2X_ERR("bad context pointer %p\n", cxt);
4897 		return;
4898 	}
4899 
4900 	/* ustorm cxt validation */
4901 	cxt->ustorm_ag_context.cdu_usage =
4902 		CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
4903 			CDU_REGION_NUMBER_UCM_AG, ETH_CONNECTION_TYPE);
4904 	/* xcontext validation */
4905 	cxt->xstorm_ag_context.cdu_reserved =
4906 		CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
4907 			CDU_REGION_NUMBER_XCM_AG, ETH_CONNECTION_TYPE);
4908 }
4909 
4910 static void storm_memset_hc_timeout(struct bnx2x *bp, u8 port,
4911 				    u8 fw_sb_id, u8 sb_index,
4912 				    u8 ticks)
4913 {
4914 	u32 addr = BAR_CSTRORM_INTMEM +
4915 		   CSTORM_STATUS_BLOCK_DATA_TIMEOUT_OFFSET(fw_sb_id, sb_index);
4916 	REG_WR8(bp, addr, ticks);
4917 	DP(NETIF_MSG_IFUP,
4918 	   "port %x fw_sb_id %d sb_index %d ticks %d\n",
4919 	   port, fw_sb_id, sb_index, ticks);
4920 }
4921 
4922 static void storm_memset_hc_disable(struct bnx2x *bp, u8 port,
4923 				    u16 fw_sb_id, u8 sb_index,
4924 				    u8 disable)
4925 {
4926 	u32 enable_flag = disable ? 0 : (1 << HC_INDEX_DATA_HC_ENABLED_SHIFT);
4927 	u32 addr = BAR_CSTRORM_INTMEM +
4928 		   CSTORM_STATUS_BLOCK_DATA_FLAGS_OFFSET(fw_sb_id, sb_index);
4929 	u8 flags = REG_RD8(bp, addr);
4930 	/* clear and set */
4931 	flags &= ~HC_INDEX_DATA_HC_ENABLED;
4932 	flags |= enable_flag;
4933 	REG_WR8(bp, addr, flags);
4934 	DP(NETIF_MSG_IFUP,
4935 	   "port %x fw_sb_id %d sb_index %d disable %d\n",
4936 	   port, fw_sb_id, sb_index, disable);
4937 }
4938 
4939 void bnx2x_update_coalesce_sb_index(struct bnx2x *bp, u8 fw_sb_id,
4940 				    u8 sb_index, u8 disable, u16 usec)
4941 {
4942 	int port = BP_PORT(bp);
4943 	u8 ticks = usec / BNX2X_BTR;
4944 
4945 	storm_memset_hc_timeout(bp, port, fw_sb_id, sb_index, ticks);
4946 
4947 	disable = disable ? 1 : (usec ? 0 : 1);
4948 	storm_memset_hc_disable(bp, port, fw_sb_id, sb_index, disable);
4949 }
4950 
4951 void bnx2x_schedule_sp_rtnl(struct bnx2x *bp, enum sp_rtnl_flag flag,
4952 			    u32 verbose)
4953 {
4954 	smp_mb__before_atomic();
4955 	set_bit(flag, &bp->sp_rtnl_state);
4956 	smp_mb__after_atomic();
4957 	DP((BNX2X_MSG_SP | verbose), "Scheduling sp_rtnl task [Flag: %d]\n",
4958 	   flag);
4959 	schedule_delayed_work(&bp->sp_rtnl_task, 0);
4960 }
4961 EXPORT_SYMBOL(bnx2x_schedule_sp_rtnl);
4962