1 /* bnx2x_cmn.c: Broadcom Everest network driver. 2 * 3 * Copyright (c) 2007-2013 Broadcom Corporation 4 * 5 * This program is free software; you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License as published by 7 * the Free Software Foundation. 8 * 9 * Maintained by: Eilon Greenstein <eilong@broadcom.com> 10 * Written by: Eliezer Tamir 11 * Based on code from Michael Chan's bnx2 driver 12 * UDP CSUM errata workaround by Arik Gendelman 13 * Slowpath and fastpath rework by Vladislav Zolotarov 14 * Statistics and Link management by Yitchak Gertner 15 * 16 */ 17 18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 19 20 #include <linux/etherdevice.h> 21 #include <linux/if_vlan.h> 22 #include <linux/interrupt.h> 23 #include <linux/ip.h> 24 #include <net/tcp.h> 25 #include <net/ipv6.h> 26 #include <net/ip6_checksum.h> 27 #include <net/busy_poll.h> 28 #include <linux/prefetch.h> 29 #include "bnx2x_cmn.h" 30 #include "bnx2x_init.h" 31 #include "bnx2x_sp.h" 32 33 /** 34 * bnx2x_move_fp - move content of the fastpath structure. 35 * 36 * @bp: driver handle 37 * @from: source FP index 38 * @to: destination FP index 39 * 40 * Makes sure the contents of the bp->fp[to].napi is kept 41 * intact. This is done by first copying the napi struct from 42 * the target to the source, and then mem copying the entire 43 * source onto the target. Update txdata pointers and related 44 * content. 45 */ 46 static inline void bnx2x_move_fp(struct bnx2x *bp, int from, int to) 47 { 48 struct bnx2x_fastpath *from_fp = &bp->fp[from]; 49 struct bnx2x_fastpath *to_fp = &bp->fp[to]; 50 struct bnx2x_sp_objs *from_sp_objs = &bp->sp_objs[from]; 51 struct bnx2x_sp_objs *to_sp_objs = &bp->sp_objs[to]; 52 struct bnx2x_fp_stats *from_fp_stats = &bp->fp_stats[from]; 53 struct bnx2x_fp_stats *to_fp_stats = &bp->fp_stats[to]; 54 int old_max_eth_txqs, new_max_eth_txqs; 55 int old_txdata_index = 0, new_txdata_index = 0; 56 struct bnx2x_agg_info *old_tpa_info = to_fp->tpa_info; 57 58 /* Copy the NAPI object as it has been already initialized */ 59 from_fp->napi = to_fp->napi; 60 61 /* Move bnx2x_fastpath contents */ 62 memcpy(to_fp, from_fp, sizeof(*to_fp)); 63 to_fp->index = to; 64 65 /* Retain the tpa_info of the original `to' version as we don't want 66 * 2 FPs to contain the same tpa_info pointer. 67 */ 68 to_fp->tpa_info = old_tpa_info; 69 70 /* move sp_objs contents as well, as their indices match fp ones */ 71 memcpy(to_sp_objs, from_sp_objs, sizeof(*to_sp_objs)); 72 73 /* move fp_stats contents as well, as their indices match fp ones */ 74 memcpy(to_fp_stats, from_fp_stats, sizeof(*to_fp_stats)); 75 76 /* Update txdata pointers in fp and move txdata content accordingly: 77 * Each fp consumes 'max_cos' txdata structures, so the index should be 78 * decremented by max_cos x delta. 79 */ 80 81 old_max_eth_txqs = BNX2X_NUM_ETH_QUEUES(bp) * (bp)->max_cos; 82 new_max_eth_txqs = (BNX2X_NUM_ETH_QUEUES(bp) - from + to) * 83 (bp)->max_cos; 84 if (from == FCOE_IDX(bp)) { 85 old_txdata_index = old_max_eth_txqs + FCOE_TXQ_IDX_OFFSET; 86 new_txdata_index = new_max_eth_txqs + FCOE_TXQ_IDX_OFFSET; 87 } 88 89 memcpy(&bp->bnx2x_txq[new_txdata_index], 90 &bp->bnx2x_txq[old_txdata_index], 91 sizeof(struct bnx2x_fp_txdata)); 92 to_fp->txdata_ptr[0] = &bp->bnx2x_txq[new_txdata_index]; 93 } 94 95 /** 96 * bnx2x_fill_fw_str - Fill buffer with FW version string. 97 * 98 * @bp: driver handle 99 * @buf: character buffer to fill with the fw name 100 * @buf_len: length of the above buffer 101 * 102 */ 103 void bnx2x_fill_fw_str(struct bnx2x *bp, char *buf, size_t buf_len) 104 { 105 if (IS_PF(bp)) { 106 u8 phy_fw_ver[PHY_FW_VER_LEN]; 107 108 phy_fw_ver[0] = '\0'; 109 bnx2x_get_ext_phy_fw_version(&bp->link_params, 110 phy_fw_ver, PHY_FW_VER_LEN); 111 strlcpy(buf, bp->fw_ver, buf_len); 112 snprintf(buf + strlen(bp->fw_ver), 32 - strlen(bp->fw_ver), 113 "bc %d.%d.%d%s%s", 114 (bp->common.bc_ver & 0xff0000) >> 16, 115 (bp->common.bc_ver & 0xff00) >> 8, 116 (bp->common.bc_ver & 0xff), 117 ((phy_fw_ver[0] != '\0') ? " phy " : ""), phy_fw_ver); 118 } else { 119 bnx2x_vf_fill_fw_str(bp, buf, buf_len); 120 } 121 } 122 123 /** 124 * bnx2x_shrink_eth_fp - guarantees fastpath structures stay intact 125 * 126 * @bp: driver handle 127 * @delta: number of eth queues which were not allocated 128 */ 129 static void bnx2x_shrink_eth_fp(struct bnx2x *bp, int delta) 130 { 131 int i, cos, old_eth_num = BNX2X_NUM_ETH_QUEUES(bp); 132 133 /* Queue pointer cannot be re-set on an fp-basis, as moving pointer 134 * backward along the array could cause memory to be overridden 135 */ 136 for (cos = 1; cos < bp->max_cos; cos++) { 137 for (i = 0; i < old_eth_num - delta; i++) { 138 struct bnx2x_fastpath *fp = &bp->fp[i]; 139 int new_idx = cos * (old_eth_num - delta) + i; 140 141 memcpy(&bp->bnx2x_txq[new_idx], fp->txdata_ptr[cos], 142 sizeof(struct bnx2x_fp_txdata)); 143 fp->txdata_ptr[cos] = &bp->bnx2x_txq[new_idx]; 144 } 145 } 146 } 147 148 int load_count[2][3] = { {0} }; /* per-path: 0-common, 1-port0, 2-port1 */ 149 150 /* free skb in the packet ring at pos idx 151 * return idx of last bd freed 152 */ 153 static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata, 154 u16 idx, unsigned int *pkts_compl, 155 unsigned int *bytes_compl) 156 { 157 struct sw_tx_bd *tx_buf = &txdata->tx_buf_ring[idx]; 158 struct eth_tx_start_bd *tx_start_bd; 159 struct eth_tx_bd *tx_data_bd; 160 struct sk_buff *skb = tx_buf->skb; 161 u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons; 162 int nbd; 163 164 /* prefetch skb end pointer to speedup dev_kfree_skb() */ 165 prefetch(&skb->end); 166 167 DP(NETIF_MSG_TX_DONE, "fp[%d]: pkt_idx %d buff @(%p)->skb %p\n", 168 txdata->txq_index, idx, tx_buf, skb); 169 170 /* unmap first bd */ 171 tx_start_bd = &txdata->tx_desc_ring[bd_idx].start_bd; 172 dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd), 173 BD_UNMAP_LEN(tx_start_bd), DMA_TO_DEVICE); 174 175 nbd = le16_to_cpu(tx_start_bd->nbd) - 1; 176 #ifdef BNX2X_STOP_ON_ERROR 177 if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) { 178 BNX2X_ERR("BAD nbd!\n"); 179 bnx2x_panic(); 180 } 181 #endif 182 new_cons = nbd + tx_buf->first_bd; 183 184 /* Get the next bd */ 185 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx)); 186 187 /* Skip a parse bd... */ 188 --nbd; 189 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx)); 190 191 /* ...and the TSO split header bd since they have no mapping */ 192 if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) { 193 --nbd; 194 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx)); 195 } 196 197 /* now free frags */ 198 while (nbd > 0) { 199 200 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd; 201 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd), 202 BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE); 203 if (--nbd) 204 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx)); 205 } 206 207 /* release skb */ 208 WARN_ON(!skb); 209 if (likely(skb)) { 210 (*pkts_compl)++; 211 (*bytes_compl) += skb->len; 212 } 213 214 dev_kfree_skb_any(skb); 215 tx_buf->first_bd = 0; 216 tx_buf->skb = NULL; 217 218 return new_cons; 219 } 220 221 int bnx2x_tx_int(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata) 222 { 223 struct netdev_queue *txq; 224 u16 hw_cons, sw_cons, bd_cons = txdata->tx_bd_cons; 225 unsigned int pkts_compl = 0, bytes_compl = 0; 226 227 #ifdef BNX2X_STOP_ON_ERROR 228 if (unlikely(bp->panic)) 229 return -1; 230 #endif 231 232 txq = netdev_get_tx_queue(bp->dev, txdata->txq_index); 233 hw_cons = le16_to_cpu(*txdata->tx_cons_sb); 234 sw_cons = txdata->tx_pkt_cons; 235 236 while (sw_cons != hw_cons) { 237 u16 pkt_cons; 238 239 pkt_cons = TX_BD(sw_cons); 240 241 DP(NETIF_MSG_TX_DONE, 242 "queue[%d]: hw_cons %u sw_cons %u pkt_cons %u\n", 243 txdata->txq_index, hw_cons, sw_cons, pkt_cons); 244 245 bd_cons = bnx2x_free_tx_pkt(bp, txdata, pkt_cons, 246 &pkts_compl, &bytes_compl); 247 248 sw_cons++; 249 } 250 251 netdev_tx_completed_queue(txq, pkts_compl, bytes_compl); 252 253 txdata->tx_pkt_cons = sw_cons; 254 txdata->tx_bd_cons = bd_cons; 255 256 /* Need to make the tx_bd_cons update visible to start_xmit() 257 * before checking for netif_tx_queue_stopped(). Without the 258 * memory barrier, there is a small possibility that 259 * start_xmit() will miss it and cause the queue to be stopped 260 * forever. 261 * On the other hand we need an rmb() here to ensure the proper 262 * ordering of bit testing in the following 263 * netif_tx_queue_stopped(txq) call. 264 */ 265 smp_mb(); 266 267 if (unlikely(netif_tx_queue_stopped(txq))) { 268 /* Taking tx_lock() is needed to prevent re-enabling the queue 269 * while it's empty. This could have happen if rx_action() gets 270 * suspended in bnx2x_tx_int() after the condition before 271 * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()): 272 * 273 * stops the queue->sees fresh tx_bd_cons->releases the queue-> 274 * sends some packets consuming the whole queue again-> 275 * stops the queue 276 */ 277 278 __netif_tx_lock(txq, smp_processor_id()); 279 280 if ((netif_tx_queue_stopped(txq)) && 281 (bp->state == BNX2X_STATE_OPEN) && 282 (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT)) 283 netif_tx_wake_queue(txq); 284 285 __netif_tx_unlock(txq); 286 } 287 return 0; 288 } 289 290 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp, 291 u16 idx) 292 { 293 u16 last_max = fp->last_max_sge; 294 295 if (SUB_S16(idx, last_max) > 0) 296 fp->last_max_sge = idx; 297 } 298 299 static inline void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp, 300 u16 sge_len, 301 struct eth_end_agg_rx_cqe *cqe) 302 { 303 struct bnx2x *bp = fp->bp; 304 u16 last_max, last_elem, first_elem; 305 u16 delta = 0; 306 u16 i; 307 308 if (!sge_len) 309 return; 310 311 /* First mark all used pages */ 312 for (i = 0; i < sge_len; i++) 313 BIT_VEC64_CLEAR_BIT(fp->sge_mask, 314 RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[i]))); 315 316 DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n", 317 sge_len - 1, le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1])); 318 319 /* Here we assume that the last SGE index is the biggest */ 320 prefetch((void *)(fp->sge_mask)); 321 bnx2x_update_last_max_sge(fp, 322 le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1])); 323 324 last_max = RX_SGE(fp->last_max_sge); 325 last_elem = last_max >> BIT_VEC64_ELEM_SHIFT; 326 first_elem = RX_SGE(fp->rx_sge_prod) >> BIT_VEC64_ELEM_SHIFT; 327 328 /* If ring is not full */ 329 if (last_elem + 1 != first_elem) 330 last_elem++; 331 332 /* Now update the prod */ 333 for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) { 334 if (likely(fp->sge_mask[i])) 335 break; 336 337 fp->sge_mask[i] = BIT_VEC64_ELEM_ONE_MASK; 338 delta += BIT_VEC64_ELEM_SZ; 339 } 340 341 if (delta > 0) { 342 fp->rx_sge_prod += delta; 343 /* clear page-end entries */ 344 bnx2x_clear_sge_mask_next_elems(fp); 345 } 346 347 DP(NETIF_MSG_RX_STATUS, 348 "fp->last_max_sge = %d fp->rx_sge_prod = %d\n", 349 fp->last_max_sge, fp->rx_sge_prod); 350 } 351 352 /* Get Toeplitz hash value in the skb using the value from the 353 * CQE (calculated by HW). 354 */ 355 static u32 bnx2x_get_rxhash(const struct bnx2x *bp, 356 const struct eth_fast_path_rx_cqe *cqe, 357 bool *l4_rxhash) 358 { 359 /* Get Toeplitz hash from CQE */ 360 if ((bp->dev->features & NETIF_F_RXHASH) && 361 (cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG)) { 362 enum eth_rss_hash_type htype; 363 364 htype = cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_TYPE; 365 *l4_rxhash = (htype == TCP_IPV4_HASH_TYPE) || 366 (htype == TCP_IPV6_HASH_TYPE); 367 return le32_to_cpu(cqe->rss_hash_result); 368 } 369 *l4_rxhash = false; 370 return 0; 371 } 372 373 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue, 374 u16 cons, u16 prod, 375 struct eth_fast_path_rx_cqe *cqe) 376 { 377 struct bnx2x *bp = fp->bp; 378 struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons]; 379 struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod]; 380 struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod]; 381 dma_addr_t mapping; 382 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue]; 383 struct sw_rx_bd *first_buf = &tpa_info->first_buf; 384 385 /* print error if current state != stop */ 386 if (tpa_info->tpa_state != BNX2X_TPA_STOP) 387 BNX2X_ERR("start of bin not in stop [%d]\n", queue); 388 389 /* Try to map an empty data buffer from the aggregation info */ 390 mapping = dma_map_single(&bp->pdev->dev, 391 first_buf->data + NET_SKB_PAD, 392 fp->rx_buf_size, DMA_FROM_DEVICE); 393 /* 394 * ...if it fails - move the skb from the consumer to the producer 395 * and set the current aggregation state as ERROR to drop it 396 * when TPA_STOP arrives. 397 */ 398 399 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) { 400 /* Move the BD from the consumer to the producer */ 401 bnx2x_reuse_rx_data(fp, cons, prod); 402 tpa_info->tpa_state = BNX2X_TPA_ERROR; 403 return; 404 } 405 406 /* move empty data from pool to prod */ 407 prod_rx_buf->data = first_buf->data; 408 dma_unmap_addr_set(prod_rx_buf, mapping, mapping); 409 /* point prod_bd to new data */ 410 prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping)); 411 prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping)); 412 413 /* move partial skb from cons to pool (don't unmap yet) */ 414 *first_buf = *cons_rx_buf; 415 416 /* mark bin state as START */ 417 tpa_info->parsing_flags = 418 le16_to_cpu(cqe->pars_flags.flags); 419 tpa_info->vlan_tag = le16_to_cpu(cqe->vlan_tag); 420 tpa_info->tpa_state = BNX2X_TPA_START; 421 tpa_info->len_on_bd = le16_to_cpu(cqe->len_on_bd); 422 tpa_info->placement_offset = cqe->placement_offset; 423 tpa_info->rxhash = bnx2x_get_rxhash(bp, cqe, &tpa_info->l4_rxhash); 424 if (fp->mode == TPA_MODE_GRO) { 425 u16 gro_size = le16_to_cpu(cqe->pkt_len_or_gro_seg_len); 426 tpa_info->full_page = SGE_PAGES / gro_size * gro_size; 427 tpa_info->gro_size = gro_size; 428 } 429 430 #ifdef BNX2X_STOP_ON_ERROR 431 fp->tpa_queue_used |= (1 << queue); 432 #ifdef _ASM_GENERIC_INT_L64_H 433 DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n", 434 #else 435 DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n", 436 #endif 437 fp->tpa_queue_used); 438 #endif 439 } 440 441 /* Timestamp option length allowed for TPA aggregation: 442 * 443 * nop nop kind length echo val 444 */ 445 #define TPA_TSTAMP_OPT_LEN 12 446 /** 447 * bnx2x_set_gro_params - compute GRO values 448 * 449 * @skb: packet skb 450 * @parsing_flags: parsing flags from the START CQE 451 * @len_on_bd: total length of the first packet for the 452 * aggregation. 453 * @pkt_len: length of all segments 454 * 455 * Approximate value of the MSS for this aggregation calculated using 456 * the first packet of it. 457 * Compute number of aggregated segments, and gso_type. 458 */ 459 static void bnx2x_set_gro_params(struct sk_buff *skb, u16 parsing_flags, 460 u16 len_on_bd, unsigned int pkt_len, 461 u16 num_of_coalesced_segs) 462 { 463 /* TPA aggregation won't have either IP options or TCP options 464 * other than timestamp or IPv6 extension headers. 465 */ 466 u16 hdrs_len = ETH_HLEN + sizeof(struct tcphdr); 467 468 if (GET_FLAG(parsing_flags, PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) == 469 PRS_FLAG_OVERETH_IPV6) { 470 hdrs_len += sizeof(struct ipv6hdr); 471 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6; 472 } else { 473 hdrs_len += sizeof(struct iphdr); 474 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4; 475 } 476 477 /* Check if there was a TCP timestamp, if there is it's will 478 * always be 12 bytes length: nop nop kind length echo val. 479 * 480 * Otherwise FW would close the aggregation. 481 */ 482 if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG) 483 hdrs_len += TPA_TSTAMP_OPT_LEN; 484 485 skb_shinfo(skb)->gso_size = len_on_bd - hdrs_len; 486 487 /* tcp_gro_complete() will copy NAPI_GRO_CB(skb)->count 488 * to skb_shinfo(skb)->gso_segs 489 */ 490 NAPI_GRO_CB(skb)->count = num_of_coalesced_segs; 491 } 492 493 static int bnx2x_alloc_rx_sge(struct bnx2x *bp, struct bnx2x_fastpath *fp, 494 u16 index, gfp_t gfp_mask) 495 { 496 struct page *page = alloc_pages(gfp_mask, PAGES_PER_SGE_SHIFT); 497 struct sw_rx_page *sw_buf = &fp->rx_page_ring[index]; 498 struct eth_rx_sge *sge = &fp->rx_sge_ring[index]; 499 dma_addr_t mapping; 500 501 if (unlikely(page == NULL)) { 502 BNX2X_ERR("Can't alloc sge\n"); 503 return -ENOMEM; 504 } 505 506 mapping = dma_map_page(&bp->pdev->dev, page, 0, 507 SGE_PAGES, DMA_FROM_DEVICE); 508 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) { 509 __free_pages(page, PAGES_PER_SGE_SHIFT); 510 BNX2X_ERR("Can't map sge\n"); 511 return -ENOMEM; 512 } 513 514 sw_buf->page = page; 515 dma_unmap_addr_set(sw_buf, mapping, mapping); 516 517 sge->addr_hi = cpu_to_le32(U64_HI(mapping)); 518 sge->addr_lo = cpu_to_le32(U64_LO(mapping)); 519 520 return 0; 521 } 522 523 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp, 524 struct bnx2x_agg_info *tpa_info, 525 u16 pages, 526 struct sk_buff *skb, 527 struct eth_end_agg_rx_cqe *cqe, 528 u16 cqe_idx) 529 { 530 struct sw_rx_page *rx_pg, old_rx_pg; 531 u32 i, frag_len, frag_size; 532 int err, j, frag_id = 0; 533 u16 len_on_bd = tpa_info->len_on_bd; 534 u16 full_page = 0, gro_size = 0; 535 536 frag_size = le16_to_cpu(cqe->pkt_len) - len_on_bd; 537 538 if (fp->mode == TPA_MODE_GRO) { 539 gro_size = tpa_info->gro_size; 540 full_page = tpa_info->full_page; 541 } 542 543 /* This is needed in order to enable forwarding support */ 544 if (frag_size) 545 bnx2x_set_gro_params(skb, tpa_info->parsing_flags, len_on_bd, 546 le16_to_cpu(cqe->pkt_len), 547 le16_to_cpu(cqe->num_of_coalesced_segs)); 548 549 #ifdef BNX2X_STOP_ON_ERROR 550 if (pages > min_t(u32, 8, MAX_SKB_FRAGS) * SGE_PAGES) { 551 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n", 552 pages, cqe_idx); 553 BNX2X_ERR("cqe->pkt_len = %d\n", cqe->pkt_len); 554 bnx2x_panic(); 555 return -EINVAL; 556 } 557 #endif 558 559 /* Run through the SGL and compose the fragmented skb */ 560 for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) { 561 u16 sge_idx = RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[j])); 562 563 /* FW gives the indices of the SGE as if the ring is an array 564 (meaning that "next" element will consume 2 indices) */ 565 if (fp->mode == TPA_MODE_GRO) 566 frag_len = min_t(u32, frag_size, (u32)full_page); 567 else /* LRO */ 568 frag_len = min_t(u32, frag_size, (u32)SGE_PAGES); 569 570 rx_pg = &fp->rx_page_ring[sge_idx]; 571 old_rx_pg = *rx_pg; 572 573 /* If we fail to allocate a substitute page, we simply stop 574 where we are and drop the whole packet */ 575 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx, GFP_ATOMIC); 576 if (unlikely(err)) { 577 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++; 578 return err; 579 } 580 581 /* Unmap the page as we're going to pass it to the stack */ 582 dma_unmap_page(&bp->pdev->dev, 583 dma_unmap_addr(&old_rx_pg, mapping), 584 SGE_PAGES, DMA_FROM_DEVICE); 585 /* Add one frag and update the appropriate fields in the skb */ 586 if (fp->mode == TPA_MODE_LRO) 587 skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len); 588 else { /* GRO */ 589 int rem; 590 int offset = 0; 591 for (rem = frag_len; rem > 0; rem -= gro_size) { 592 int len = rem > gro_size ? gro_size : rem; 593 skb_fill_page_desc(skb, frag_id++, 594 old_rx_pg.page, offset, len); 595 if (offset) 596 get_page(old_rx_pg.page); 597 offset += len; 598 } 599 } 600 601 skb->data_len += frag_len; 602 skb->truesize += SGE_PAGES; 603 skb->len += frag_len; 604 605 frag_size -= frag_len; 606 } 607 608 return 0; 609 } 610 611 static void bnx2x_frag_free(const struct bnx2x_fastpath *fp, void *data) 612 { 613 if (fp->rx_frag_size) 614 put_page(virt_to_head_page(data)); 615 else 616 kfree(data); 617 } 618 619 static void *bnx2x_frag_alloc(const struct bnx2x_fastpath *fp, gfp_t gfp_mask) 620 { 621 if (fp->rx_frag_size) { 622 /* GFP_KERNEL allocations are used only during initialization */ 623 if (unlikely(gfp_mask & __GFP_WAIT)) 624 return (void *)__get_free_page(gfp_mask); 625 626 return netdev_alloc_frag(fp->rx_frag_size); 627 } 628 629 return kmalloc(fp->rx_buf_size + NET_SKB_PAD, gfp_mask); 630 } 631 632 #ifdef CONFIG_INET 633 static void bnx2x_gro_ip_csum(struct bnx2x *bp, struct sk_buff *skb) 634 { 635 const struct iphdr *iph = ip_hdr(skb); 636 struct tcphdr *th; 637 638 skb_set_transport_header(skb, sizeof(struct iphdr)); 639 th = tcp_hdr(skb); 640 641 th->check = ~tcp_v4_check(skb->len - skb_transport_offset(skb), 642 iph->saddr, iph->daddr, 0); 643 } 644 645 static void bnx2x_gro_ipv6_csum(struct bnx2x *bp, struct sk_buff *skb) 646 { 647 struct ipv6hdr *iph = ipv6_hdr(skb); 648 struct tcphdr *th; 649 650 skb_set_transport_header(skb, sizeof(struct ipv6hdr)); 651 th = tcp_hdr(skb); 652 653 th->check = ~tcp_v6_check(skb->len - skb_transport_offset(skb), 654 &iph->saddr, &iph->daddr, 0); 655 } 656 657 static void bnx2x_gro_csum(struct bnx2x *bp, struct sk_buff *skb, 658 void (*gro_func)(struct bnx2x*, struct sk_buff*)) 659 { 660 skb_set_network_header(skb, 0); 661 gro_func(bp, skb); 662 tcp_gro_complete(skb); 663 } 664 #endif 665 666 static void bnx2x_gro_receive(struct bnx2x *bp, struct bnx2x_fastpath *fp, 667 struct sk_buff *skb) 668 { 669 #ifdef CONFIG_INET 670 if (skb_shinfo(skb)->gso_size) { 671 switch (be16_to_cpu(skb->protocol)) { 672 case ETH_P_IP: 673 bnx2x_gro_csum(bp, skb, bnx2x_gro_ip_csum); 674 break; 675 case ETH_P_IPV6: 676 bnx2x_gro_csum(bp, skb, bnx2x_gro_ipv6_csum); 677 break; 678 default: 679 BNX2X_ERR("Error: FW GRO supports only IPv4/IPv6, not 0x%04x\n", 680 be16_to_cpu(skb->protocol)); 681 } 682 } 683 #endif 684 napi_gro_receive(&fp->napi, skb); 685 } 686 687 static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp, 688 struct bnx2x_agg_info *tpa_info, 689 u16 pages, 690 struct eth_end_agg_rx_cqe *cqe, 691 u16 cqe_idx) 692 { 693 struct sw_rx_bd *rx_buf = &tpa_info->first_buf; 694 u8 pad = tpa_info->placement_offset; 695 u16 len = tpa_info->len_on_bd; 696 struct sk_buff *skb = NULL; 697 u8 *new_data, *data = rx_buf->data; 698 u8 old_tpa_state = tpa_info->tpa_state; 699 700 tpa_info->tpa_state = BNX2X_TPA_STOP; 701 702 /* If we there was an error during the handling of the TPA_START - 703 * drop this aggregation. 704 */ 705 if (old_tpa_state == BNX2X_TPA_ERROR) 706 goto drop; 707 708 /* Try to allocate the new data */ 709 new_data = bnx2x_frag_alloc(fp, GFP_ATOMIC); 710 /* Unmap skb in the pool anyway, as we are going to change 711 pool entry status to BNX2X_TPA_STOP even if new skb allocation 712 fails. */ 713 dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping), 714 fp->rx_buf_size, DMA_FROM_DEVICE); 715 if (likely(new_data)) 716 skb = build_skb(data, fp->rx_frag_size); 717 718 if (likely(skb)) { 719 #ifdef BNX2X_STOP_ON_ERROR 720 if (pad + len > fp->rx_buf_size) { 721 BNX2X_ERR("skb_put is about to fail... pad %d len %d rx_buf_size %d\n", 722 pad, len, fp->rx_buf_size); 723 bnx2x_panic(); 724 return; 725 } 726 #endif 727 728 skb_reserve(skb, pad + NET_SKB_PAD); 729 skb_put(skb, len); 730 skb->rxhash = tpa_info->rxhash; 731 skb->l4_rxhash = tpa_info->l4_rxhash; 732 733 skb->protocol = eth_type_trans(skb, bp->dev); 734 skb->ip_summed = CHECKSUM_UNNECESSARY; 735 736 if (!bnx2x_fill_frag_skb(bp, fp, tpa_info, pages, 737 skb, cqe, cqe_idx)) { 738 if (tpa_info->parsing_flags & PARSING_FLAGS_VLAN) 739 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), tpa_info->vlan_tag); 740 bnx2x_gro_receive(bp, fp, skb); 741 } else { 742 DP(NETIF_MSG_RX_STATUS, 743 "Failed to allocate new pages - dropping packet!\n"); 744 dev_kfree_skb_any(skb); 745 } 746 747 /* put new data in bin */ 748 rx_buf->data = new_data; 749 750 return; 751 } 752 bnx2x_frag_free(fp, new_data); 753 drop: 754 /* drop the packet and keep the buffer in the bin */ 755 DP(NETIF_MSG_RX_STATUS, 756 "Failed to allocate or map a new skb - dropping packet!\n"); 757 bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed++; 758 } 759 760 static int bnx2x_alloc_rx_data(struct bnx2x *bp, struct bnx2x_fastpath *fp, 761 u16 index, gfp_t gfp_mask) 762 { 763 u8 *data; 764 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[index]; 765 struct eth_rx_bd *rx_bd = &fp->rx_desc_ring[index]; 766 dma_addr_t mapping; 767 768 data = bnx2x_frag_alloc(fp, gfp_mask); 769 if (unlikely(data == NULL)) 770 return -ENOMEM; 771 772 mapping = dma_map_single(&bp->pdev->dev, data + NET_SKB_PAD, 773 fp->rx_buf_size, 774 DMA_FROM_DEVICE); 775 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) { 776 bnx2x_frag_free(fp, data); 777 BNX2X_ERR("Can't map rx data\n"); 778 return -ENOMEM; 779 } 780 781 rx_buf->data = data; 782 dma_unmap_addr_set(rx_buf, mapping, mapping); 783 784 rx_bd->addr_hi = cpu_to_le32(U64_HI(mapping)); 785 rx_bd->addr_lo = cpu_to_le32(U64_LO(mapping)); 786 787 return 0; 788 } 789 790 static 791 void bnx2x_csum_validate(struct sk_buff *skb, union eth_rx_cqe *cqe, 792 struct bnx2x_fastpath *fp, 793 struct bnx2x_eth_q_stats *qstats) 794 { 795 /* Do nothing if no L4 csum validation was done. 796 * We do not check whether IP csum was validated. For IPv4 we assume 797 * that if the card got as far as validating the L4 csum, it also 798 * validated the IP csum. IPv6 has no IP csum. 799 */ 800 if (cqe->fast_path_cqe.status_flags & 801 ETH_FAST_PATH_RX_CQE_L4_XSUM_NO_VALIDATION_FLG) 802 return; 803 804 /* If L4 validation was done, check if an error was found. */ 805 806 if (cqe->fast_path_cqe.type_error_flags & 807 (ETH_FAST_PATH_RX_CQE_IP_BAD_XSUM_FLG | 808 ETH_FAST_PATH_RX_CQE_L4_BAD_XSUM_FLG)) 809 qstats->hw_csum_err++; 810 else 811 skb->ip_summed = CHECKSUM_UNNECESSARY; 812 } 813 814 int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget) 815 { 816 struct bnx2x *bp = fp->bp; 817 u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons; 818 u16 sw_comp_cons, sw_comp_prod; 819 int rx_pkt = 0; 820 union eth_rx_cqe *cqe; 821 struct eth_fast_path_rx_cqe *cqe_fp; 822 823 #ifdef BNX2X_STOP_ON_ERROR 824 if (unlikely(bp->panic)) 825 return 0; 826 #endif 827 828 bd_cons = fp->rx_bd_cons; 829 bd_prod = fp->rx_bd_prod; 830 bd_prod_fw = bd_prod; 831 sw_comp_cons = fp->rx_comp_cons; 832 sw_comp_prod = fp->rx_comp_prod; 833 834 comp_ring_cons = RCQ_BD(sw_comp_cons); 835 cqe = &fp->rx_comp_ring[comp_ring_cons]; 836 cqe_fp = &cqe->fast_path_cqe; 837 838 DP(NETIF_MSG_RX_STATUS, 839 "queue[%d]: sw_comp_cons %u\n", fp->index, sw_comp_cons); 840 841 while (BNX2X_IS_CQE_COMPLETED(cqe_fp)) { 842 struct sw_rx_bd *rx_buf = NULL; 843 struct sk_buff *skb; 844 u8 cqe_fp_flags; 845 enum eth_rx_cqe_type cqe_fp_type; 846 u16 len, pad, queue; 847 u8 *data; 848 bool l4_rxhash; 849 850 #ifdef BNX2X_STOP_ON_ERROR 851 if (unlikely(bp->panic)) 852 return 0; 853 #endif 854 855 bd_prod = RX_BD(bd_prod); 856 bd_cons = RX_BD(bd_cons); 857 858 cqe_fp_flags = cqe_fp->type_error_flags; 859 cqe_fp_type = cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE; 860 861 DP(NETIF_MSG_RX_STATUS, 862 "CQE type %x err %x status %x queue %x vlan %x len %u\n", 863 CQE_TYPE(cqe_fp_flags), 864 cqe_fp_flags, cqe_fp->status_flags, 865 le32_to_cpu(cqe_fp->rss_hash_result), 866 le16_to_cpu(cqe_fp->vlan_tag), 867 le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len)); 868 869 /* is this a slowpath msg? */ 870 if (unlikely(CQE_TYPE_SLOW(cqe_fp_type))) { 871 bnx2x_sp_event(fp, cqe); 872 goto next_cqe; 873 } 874 875 rx_buf = &fp->rx_buf_ring[bd_cons]; 876 data = rx_buf->data; 877 878 if (!CQE_TYPE_FAST(cqe_fp_type)) { 879 struct bnx2x_agg_info *tpa_info; 880 u16 frag_size, pages; 881 #ifdef BNX2X_STOP_ON_ERROR 882 /* sanity check */ 883 if (fp->disable_tpa && 884 (CQE_TYPE_START(cqe_fp_type) || 885 CQE_TYPE_STOP(cqe_fp_type))) 886 BNX2X_ERR("START/STOP packet while disable_tpa type %x\n", 887 CQE_TYPE(cqe_fp_type)); 888 #endif 889 890 if (CQE_TYPE_START(cqe_fp_type)) { 891 u16 queue = cqe_fp->queue_index; 892 DP(NETIF_MSG_RX_STATUS, 893 "calling tpa_start on queue %d\n", 894 queue); 895 896 bnx2x_tpa_start(fp, queue, 897 bd_cons, bd_prod, 898 cqe_fp); 899 900 goto next_rx; 901 } 902 queue = cqe->end_agg_cqe.queue_index; 903 tpa_info = &fp->tpa_info[queue]; 904 DP(NETIF_MSG_RX_STATUS, 905 "calling tpa_stop on queue %d\n", 906 queue); 907 908 frag_size = le16_to_cpu(cqe->end_agg_cqe.pkt_len) - 909 tpa_info->len_on_bd; 910 911 if (fp->mode == TPA_MODE_GRO) 912 pages = (frag_size + tpa_info->full_page - 1) / 913 tpa_info->full_page; 914 else 915 pages = SGE_PAGE_ALIGN(frag_size) >> 916 SGE_PAGE_SHIFT; 917 918 bnx2x_tpa_stop(bp, fp, tpa_info, pages, 919 &cqe->end_agg_cqe, comp_ring_cons); 920 #ifdef BNX2X_STOP_ON_ERROR 921 if (bp->panic) 922 return 0; 923 #endif 924 925 bnx2x_update_sge_prod(fp, pages, &cqe->end_agg_cqe); 926 goto next_cqe; 927 } 928 /* non TPA */ 929 len = le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len); 930 pad = cqe_fp->placement_offset; 931 dma_sync_single_for_cpu(&bp->pdev->dev, 932 dma_unmap_addr(rx_buf, mapping), 933 pad + RX_COPY_THRESH, 934 DMA_FROM_DEVICE); 935 pad += NET_SKB_PAD; 936 prefetch(data + pad); /* speedup eth_type_trans() */ 937 /* is this an error packet? */ 938 if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) { 939 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS, 940 "ERROR flags %x rx packet %u\n", 941 cqe_fp_flags, sw_comp_cons); 942 bnx2x_fp_qstats(bp, fp)->rx_err_discard_pkt++; 943 goto reuse_rx; 944 } 945 946 /* Since we don't have a jumbo ring 947 * copy small packets if mtu > 1500 948 */ 949 if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) && 950 (len <= RX_COPY_THRESH)) { 951 skb = netdev_alloc_skb_ip_align(bp->dev, len); 952 if (skb == NULL) { 953 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS, 954 "ERROR packet dropped because of alloc failure\n"); 955 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++; 956 goto reuse_rx; 957 } 958 memcpy(skb->data, data + pad, len); 959 bnx2x_reuse_rx_data(fp, bd_cons, bd_prod); 960 } else { 961 if (likely(bnx2x_alloc_rx_data(bp, fp, bd_prod, 962 GFP_ATOMIC) == 0)) { 963 dma_unmap_single(&bp->pdev->dev, 964 dma_unmap_addr(rx_buf, mapping), 965 fp->rx_buf_size, 966 DMA_FROM_DEVICE); 967 skb = build_skb(data, fp->rx_frag_size); 968 if (unlikely(!skb)) { 969 bnx2x_frag_free(fp, data); 970 bnx2x_fp_qstats(bp, fp)-> 971 rx_skb_alloc_failed++; 972 goto next_rx; 973 } 974 skb_reserve(skb, pad); 975 } else { 976 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS, 977 "ERROR packet dropped because of alloc failure\n"); 978 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++; 979 reuse_rx: 980 bnx2x_reuse_rx_data(fp, bd_cons, bd_prod); 981 goto next_rx; 982 } 983 } 984 985 skb_put(skb, len); 986 skb->protocol = eth_type_trans(skb, bp->dev); 987 988 /* Set Toeplitz hash for a none-LRO skb */ 989 skb->rxhash = bnx2x_get_rxhash(bp, cqe_fp, &l4_rxhash); 990 skb->l4_rxhash = l4_rxhash; 991 992 skb_checksum_none_assert(skb); 993 994 if (bp->dev->features & NETIF_F_RXCSUM) 995 bnx2x_csum_validate(skb, cqe, fp, 996 bnx2x_fp_qstats(bp, fp)); 997 998 skb_record_rx_queue(skb, fp->rx_queue); 999 1000 if (le16_to_cpu(cqe_fp->pars_flags.flags) & 1001 PARSING_FLAGS_VLAN) 1002 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), 1003 le16_to_cpu(cqe_fp->vlan_tag)); 1004 1005 skb_mark_napi_id(skb, &fp->napi); 1006 1007 if (bnx2x_fp_ll_polling(fp)) 1008 netif_receive_skb(skb); 1009 else 1010 napi_gro_receive(&fp->napi, skb); 1011 next_rx: 1012 rx_buf->data = NULL; 1013 1014 bd_cons = NEXT_RX_IDX(bd_cons); 1015 bd_prod = NEXT_RX_IDX(bd_prod); 1016 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw); 1017 rx_pkt++; 1018 next_cqe: 1019 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod); 1020 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons); 1021 1022 /* mark CQE as free */ 1023 BNX2X_SEED_CQE(cqe_fp); 1024 1025 if (rx_pkt == budget) 1026 break; 1027 1028 comp_ring_cons = RCQ_BD(sw_comp_cons); 1029 cqe = &fp->rx_comp_ring[comp_ring_cons]; 1030 cqe_fp = &cqe->fast_path_cqe; 1031 } /* while */ 1032 1033 fp->rx_bd_cons = bd_cons; 1034 fp->rx_bd_prod = bd_prod_fw; 1035 fp->rx_comp_cons = sw_comp_cons; 1036 fp->rx_comp_prod = sw_comp_prod; 1037 1038 /* Update producers */ 1039 bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod, 1040 fp->rx_sge_prod); 1041 1042 fp->rx_pkt += rx_pkt; 1043 fp->rx_calls++; 1044 1045 return rx_pkt; 1046 } 1047 1048 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie) 1049 { 1050 struct bnx2x_fastpath *fp = fp_cookie; 1051 struct bnx2x *bp = fp->bp; 1052 u8 cos; 1053 1054 DP(NETIF_MSG_INTR, 1055 "got an MSI-X interrupt on IDX:SB [fp %d fw_sd %d igusb %d]\n", 1056 fp->index, fp->fw_sb_id, fp->igu_sb_id); 1057 1058 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0); 1059 1060 #ifdef BNX2X_STOP_ON_ERROR 1061 if (unlikely(bp->panic)) 1062 return IRQ_HANDLED; 1063 #endif 1064 1065 /* Handle Rx and Tx according to MSI-X vector */ 1066 for_each_cos_in_tx_queue(fp, cos) 1067 prefetch(fp->txdata_ptr[cos]->tx_cons_sb); 1068 1069 prefetch(&fp->sb_running_index[SM_RX_ID]); 1070 napi_schedule(&bnx2x_fp(bp, fp->index, napi)); 1071 1072 return IRQ_HANDLED; 1073 } 1074 1075 /* HW Lock for shared dual port PHYs */ 1076 void bnx2x_acquire_phy_lock(struct bnx2x *bp) 1077 { 1078 mutex_lock(&bp->port.phy_mutex); 1079 1080 bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO); 1081 } 1082 1083 void bnx2x_release_phy_lock(struct bnx2x *bp) 1084 { 1085 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO); 1086 1087 mutex_unlock(&bp->port.phy_mutex); 1088 } 1089 1090 /* calculates MF speed according to current linespeed and MF configuration */ 1091 u16 bnx2x_get_mf_speed(struct bnx2x *bp) 1092 { 1093 u16 line_speed = bp->link_vars.line_speed; 1094 if (IS_MF(bp)) { 1095 u16 maxCfg = bnx2x_extract_max_cfg(bp, 1096 bp->mf_config[BP_VN(bp)]); 1097 1098 /* Calculate the current MAX line speed limit for the MF 1099 * devices 1100 */ 1101 if (IS_MF_SI(bp)) 1102 line_speed = (line_speed * maxCfg) / 100; 1103 else { /* SD mode */ 1104 u16 vn_max_rate = maxCfg * 100; 1105 1106 if (vn_max_rate < line_speed) 1107 line_speed = vn_max_rate; 1108 } 1109 } 1110 1111 return line_speed; 1112 } 1113 1114 /** 1115 * bnx2x_fill_report_data - fill link report data to report 1116 * 1117 * @bp: driver handle 1118 * @data: link state to update 1119 * 1120 * It uses a none-atomic bit operations because is called under the mutex. 1121 */ 1122 static void bnx2x_fill_report_data(struct bnx2x *bp, 1123 struct bnx2x_link_report_data *data) 1124 { 1125 u16 line_speed = bnx2x_get_mf_speed(bp); 1126 1127 memset(data, 0, sizeof(*data)); 1128 1129 /* Fill the report data: effective line speed */ 1130 data->line_speed = line_speed; 1131 1132 /* Link is down */ 1133 if (!bp->link_vars.link_up || (bp->flags & MF_FUNC_DIS)) 1134 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN, 1135 &data->link_report_flags); 1136 1137 /* Full DUPLEX */ 1138 if (bp->link_vars.duplex == DUPLEX_FULL) 1139 __set_bit(BNX2X_LINK_REPORT_FD, &data->link_report_flags); 1140 1141 /* Rx Flow Control is ON */ 1142 if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX) 1143 __set_bit(BNX2X_LINK_REPORT_RX_FC_ON, &data->link_report_flags); 1144 1145 /* Tx Flow Control is ON */ 1146 if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX) 1147 __set_bit(BNX2X_LINK_REPORT_TX_FC_ON, &data->link_report_flags); 1148 } 1149 1150 /** 1151 * bnx2x_link_report - report link status to OS. 1152 * 1153 * @bp: driver handle 1154 * 1155 * Calls the __bnx2x_link_report() under the same locking scheme 1156 * as a link/PHY state managing code to ensure a consistent link 1157 * reporting. 1158 */ 1159 1160 void bnx2x_link_report(struct bnx2x *bp) 1161 { 1162 bnx2x_acquire_phy_lock(bp); 1163 __bnx2x_link_report(bp); 1164 bnx2x_release_phy_lock(bp); 1165 } 1166 1167 /** 1168 * __bnx2x_link_report - report link status to OS. 1169 * 1170 * @bp: driver handle 1171 * 1172 * None atomic implementation. 1173 * Should be called under the phy_lock. 1174 */ 1175 void __bnx2x_link_report(struct bnx2x *bp) 1176 { 1177 struct bnx2x_link_report_data cur_data; 1178 1179 /* reread mf_cfg */ 1180 if (IS_PF(bp) && !CHIP_IS_E1(bp)) 1181 bnx2x_read_mf_cfg(bp); 1182 1183 /* Read the current link report info */ 1184 bnx2x_fill_report_data(bp, &cur_data); 1185 1186 /* Don't report link down or exactly the same link status twice */ 1187 if (!memcmp(&cur_data, &bp->last_reported_link, sizeof(cur_data)) || 1188 (test_bit(BNX2X_LINK_REPORT_LINK_DOWN, 1189 &bp->last_reported_link.link_report_flags) && 1190 test_bit(BNX2X_LINK_REPORT_LINK_DOWN, 1191 &cur_data.link_report_flags))) 1192 return; 1193 1194 bp->link_cnt++; 1195 1196 /* We are going to report a new link parameters now - 1197 * remember the current data for the next time. 1198 */ 1199 memcpy(&bp->last_reported_link, &cur_data, sizeof(cur_data)); 1200 1201 if (test_bit(BNX2X_LINK_REPORT_LINK_DOWN, 1202 &cur_data.link_report_flags)) { 1203 netif_carrier_off(bp->dev); 1204 netdev_err(bp->dev, "NIC Link is Down\n"); 1205 return; 1206 } else { 1207 const char *duplex; 1208 const char *flow; 1209 1210 netif_carrier_on(bp->dev); 1211 1212 if (test_and_clear_bit(BNX2X_LINK_REPORT_FD, 1213 &cur_data.link_report_flags)) 1214 duplex = "full"; 1215 else 1216 duplex = "half"; 1217 1218 /* Handle the FC at the end so that only these flags would be 1219 * possibly set. This way we may easily check if there is no FC 1220 * enabled. 1221 */ 1222 if (cur_data.link_report_flags) { 1223 if (test_bit(BNX2X_LINK_REPORT_RX_FC_ON, 1224 &cur_data.link_report_flags)) { 1225 if (test_bit(BNX2X_LINK_REPORT_TX_FC_ON, 1226 &cur_data.link_report_flags)) 1227 flow = "ON - receive & transmit"; 1228 else 1229 flow = "ON - receive"; 1230 } else { 1231 flow = "ON - transmit"; 1232 } 1233 } else { 1234 flow = "none"; 1235 } 1236 netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n", 1237 cur_data.line_speed, duplex, flow); 1238 } 1239 } 1240 1241 static void bnx2x_set_next_page_sgl(struct bnx2x_fastpath *fp) 1242 { 1243 int i; 1244 1245 for (i = 1; i <= NUM_RX_SGE_PAGES; i++) { 1246 struct eth_rx_sge *sge; 1247 1248 sge = &fp->rx_sge_ring[RX_SGE_CNT * i - 2]; 1249 sge->addr_hi = 1250 cpu_to_le32(U64_HI(fp->rx_sge_mapping + 1251 BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES))); 1252 1253 sge->addr_lo = 1254 cpu_to_le32(U64_LO(fp->rx_sge_mapping + 1255 BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES))); 1256 } 1257 } 1258 1259 static void bnx2x_free_tpa_pool(struct bnx2x *bp, 1260 struct bnx2x_fastpath *fp, int last) 1261 { 1262 int i; 1263 1264 for (i = 0; i < last; i++) { 1265 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[i]; 1266 struct sw_rx_bd *first_buf = &tpa_info->first_buf; 1267 u8 *data = first_buf->data; 1268 1269 if (data == NULL) { 1270 DP(NETIF_MSG_IFDOWN, "tpa bin %d empty on free\n", i); 1271 continue; 1272 } 1273 if (tpa_info->tpa_state == BNX2X_TPA_START) 1274 dma_unmap_single(&bp->pdev->dev, 1275 dma_unmap_addr(first_buf, mapping), 1276 fp->rx_buf_size, DMA_FROM_DEVICE); 1277 bnx2x_frag_free(fp, data); 1278 first_buf->data = NULL; 1279 } 1280 } 1281 1282 void bnx2x_init_rx_rings_cnic(struct bnx2x *bp) 1283 { 1284 int j; 1285 1286 for_each_rx_queue_cnic(bp, j) { 1287 struct bnx2x_fastpath *fp = &bp->fp[j]; 1288 1289 fp->rx_bd_cons = 0; 1290 1291 /* Activate BD ring */ 1292 /* Warning! 1293 * this will generate an interrupt (to the TSTORM) 1294 * must only be done after chip is initialized 1295 */ 1296 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod, 1297 fp->rx_sge_prod); 1298 } 1299 } 1300 1301 void bnx2x_init_rx_rings(struct bnx2x *bp) 1302 { 1303 int func = BP_FUNC(bp); 1304 u16 ring_prod; 1305 int i, j; 1306 1307 /* Allocate TPA resources */ 1308 for_each_eth_queue(bp, j) { 1309 struct bnx2x_fastpath *fp = &bp->fp[j]; 1310 1311 DP(NETIF_MSG_IFUP, 1312 "mtu %d rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size); 1313 1314 if (!fp->disable_tpa) { 1315 /* Fill the per-aggregation pool */ 1316 for (i = 0; i < MAX_AGG_QS(bp); i++) { 1317 struct bnx2x_agg_info *tpa_info = 1318 &fp->tpa_info[i]; 1319 struct sw_rx_bd *first_buf = 1320 &tpa_info->first_buf; 1321 1322 first_buf->data = 1323 bnx2x_frag_alloc(fp, GFP_KERNEL); 1324 if (!first_buf->data) { 1325 BNX2X_ERR("Failed to allocate TPA skb pool for queue[%d] - disabling TPA on this queue!\n", 1326 j); 1327 bnx2x_free_tpa_pool(bp, fp, i); 1328 fp->disable_tpa = 1; 1329 break; 1330 } 1331 dma_unmap_addr_set(first_buf, mapping, 0); 1332 tpa_info->tpa_state = BNX2X_TPA_STOP; 1333 } 1334 1335 /* "next page" elements initialization */ 1336 bnx2x_set_next_page_sgl(fp); 1337 1338 /* set SGEs bit mask */ 1339 bnx2x_init_sge_ring_bit_mask(fp); 1340 1341 /* Allocate SGEs and initialize the ring elements */ 1342 for (i = 0, ring_prod = 0; 1343 i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) { 1344 1345 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod, 1346 GFP_KERNEL) < 0) { 1347 BNX2X_ERR("was only able to allocate %d rx sges\n", 1348 i); 1349 BNX2X_ERR("disabling TPA for queue[%d]\n", 1350 j); 1351 /* Cleanup already allocated elements */ 1352 bnx2x_free_rx_sge_range(bp, fp, 1353 ring_prod); 1354 bnx2x_free_tpa_pool(bp, fp, 1355 MAX_AGG_QS(bp)); 1356 fp->disable_tpa = 1; 1357 ring_prod = 0; 1358 break; 1359 } 1360 ring_prod = NEXT_SGE_IDX(ring_prod); 1361 } 1362 1363 fp->rx_sge_prod = ring_prod; 1364 } 1365 } 1366 1367 for_each_eth_queue(bp, j) { 1368 struct bnx2x_fastpath *fp = &bp->fp[j]; 1369 1370 fp->rx_bd_cons = 0; 1371 1372 /* Activate BD ring */ 1373 /* Warning! 1374 * this will generate an interrupt (to the TSTORM) 1375 * must only be done after chip is initialized 1376 */ 1377 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod, 1378 fp->rx_sge_prod); 1379 1380 if (j != 0) 1381 continue; 1382 1383 if (CHIP_IS_E1(bp)) { 1384 REG_WR(bp, BAR_USTRORM_INTMEM + 1385 USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func), 1386 U64_LO(fp->rx_comp_mapping)); 1387 REG_WR(bp, BAR_USTRORM_INTMEM + 1388 USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4, 1389 U64_HI(fp->rx_comp_mapping)); 1390 } 1391 } 1392 } 1393 1394 static void bnx2x_free_tx_skbs_queue(struct bnx2x_fastpath *fp) 1395 { 1396 u8 cos; 1397 struct bnx2x *bp = fp->bp; 1398 1399 for_each_cos_in_tx_queue(fp, cos) { 1400 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos]; 1401 unsigned pkts_compl = 0, bytes_compl = 0; 1402 1403 u16 sw_prod = txdata->tx_pkt_prod; 1404 u16 sw_cons = txdata->tx_pkt_cons; 1405 1406 while (sw_cons != sw_prod) { 1407 bnx2x_free_tx_pkt(bp, txdata, TX_BD(sw_cons), 1408 &pkts_compl, &bytes_compl); 1409 sw_cons++; 1410 } 1411 1412 netdev_tx_reset_queue( 1413 netdev_get_tx_queue(bp->dev, 1414 txdata->txq_index)); 1415 } 1416 } 1417 1418 static void bnx2x_free_tx_skbs_cnic(struct bnx2x *bp) 1419 { 1420 int i; 1421 1422 for_each_tx_queue_cnic(bp, i) { 1423 bnx2x_free_tx_skbs_queue(&bp->fp[i]); 1424 } 1425 } 1426 1427 static void bnx2x_free_tx_skbs(struct bnx2x *bp) 1428 { 1429 int i; 1430 1431 for_each_eth_queue(bp, i) { 1432 bnx2x_free_tx_skbs_queue(&bp->fp[i]); 1433 } 1434 } 1435 1436 static void bnx2x_free_rx_bds(struct bnx2x_fastpath *fp) 1437 { 1438 struct bnx2x *bp = fp->bp; 1439 int i; 1440 1441 /* ring wasn't allocated */ 1442 if (fp->rx_buf_ring == NULL) 1443 return; 1444 1445 for (i = 0; i < NUM_RX_BD; i++) { 1446 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i]; 1447 u8 *data = rx_buf->data; 1448 1449 if (data == NULL) 1450 continue; 1451 dma_unmap_single(&bp->pdev->dev, 1452 dma_unmap_addr(rx_buf, mapping), 1453 fp->rx_buf_size, DMA_FROM_DEVICE); 1454 1455 rx_buf->data = NULL; 1456 bnx2x_frag_free(fp, data); 1457 } 1458 } 1459 1460 static void bnx2x_free_rx_skbs_cnic(struct bnx2x *bp) 1461 { 1462 int j; 1463 1464 for_each_rx_queue_cnic(bp, j) { 1465 bnx2x_free_rx_bds(&bp->fp[j]); 1466 } 1467 } 1468 1469 static void bnx2x_free_rx_skbs(struct bnx2x *bp) 1470 { 1471 int j; 1472 1473 for_each_eth_queue(bp, j) { 1474 struct bnx2x_fastpath *fp = &bp->fp[j]; 1475 1476 bnx2x_free_rx_bds(fp); 1477 1478 if (!fp->disable_tpa) 1479 bnx2x_free_tpa_pool(bp, fp, MAX_AGG_QS(bp)); 1480 } 1481 } 1482 1483 void bnx2x_free_skbs_cnic(struct bnx2x *bp) 1484 { 1485 bnx2x_free_tx_skbs_cnic(bp); 1486 bnx2x_free_rx_skbs_cnic(bp); 1487 } 1488 1489 void bnx2x_free_skbs(struct bnx2x *bp) 1490 { 1491 bnx2x_free_tx_skbs(bp); 1492 bnx2x_free_rx_skbs(bp); 1493 } 1494 1495 void bnx2x_update_max_mf_config(struct bnx2x *bp, u32 value) 1496 { 1497 /* load old values */ 1498 u32 mf_cfg = bp->mf_config[BP_VN(bp)]; 1499 1500 if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) { 1501 /* leave all but MAX value */ 1502 mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK; 1503 1504 /* set new MAX value */ 1505 mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT) 1506 & FUNC_MF_CFG_MAX_BW_MASK; 1507 1508 bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg); 1509 } 1510 } 1511 1512 /** 1513 * bnx2x_free_msix_irqs - free previously requested MSI-X IRQ vectors 1514 * 1515 * @bp: driver handle 1516 * @nvecs: number of vectors to be released 1517 */ 1518 static void bnx2x_free_msix_irqs(struct bnx2x *bp, int nvecs) 1519 { 1520 int i, offset = 0; 1521 1522 if (nvecs == offset) 1523 return; 1524 1525 /* VFs don't have a default SB */ 1526 if (IS_PF(bp)) { 1527 free_irq(bp->msix_table[offset].vector, bp->dev); 1528 DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n", 1529 bp->msix_table[offset].vector); 1530 offset++; 1531 } 1532 1533 if (CNIC_SUPPORT(bp)) { 1534 if (nvecs == offset) 1535 return; 1536 offset++; 1537 } 1538 1539 for_each_eth_queue(bp, i) { 1540 if (nvecs == offset) 1541 return; 1542 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq\n", 1543 i, bp->msix_table[offset].vector); 1544 1545 free_irq(bp->msix_table[offset++].vector, &bp->fp[i]); 1546 } 1547 } 1548 1549 void bnx2x_free_irq(struct bnx2x *bp) 1550 { 1551 if (bp->flags & USING_MSIX_FLAG && 1552 !(bp->flags & USING_SINGLE_MSIX_FLAG)) { 1553 int nvecs = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_SUPPORT(bp); 1554 1555 /* vfs don't have a default status block */ 1556 if (IS_PF(bp)) 1557 nvecs++; 1558 1559 bnx2x_free_msix_irqs(bp, nvecs); 1560 } else { 1561 free_irq(bp->dev->irq, bp->dev); 1562 } 1563 } 1564 1565 int bnx2x_enable_msix(struct bnx2x *bp) 1566 { 1567 int msix_vec = 0, i, rc; 1568 1569 /* VFs don't have a default status block */ 1570 if (IS_PF(bp)) { 1571 bp->msix_table[msix_vec].entry = msix_vec; 1572 BNX2X_DEV_INFO("msix_table[0].entry = %d (slowpath)\n", 1573 bp->msix_table[0].entry); 1574 msix_vec++; 1575 } 1576 1577 /* Cnic requires an msix vector for itself */ 1578 if (CNIC_SUPPORT(bp)) { 1579 bp->msix_table[msix_vec].entry = msix_vec; 1580 BNX2X_DEV_INFO("msix_table[%d].entry = %d (CNIC)\n", 1581 msix_vec, bp->msix_table[msix_vec].entry); 1582 msix_vec++; 1583 } 1584 1585 /* We need separate vectors for ETH queues only (not FCoE) */ 1586 for_each_eth_queue(bp, i) { 1587 bp->msix_table[msix_vec].entry = msix_vec; 1588 BNX2X_DEV_INFO("msix_table[%d].entry = %d (fastpath #%u)\n", 1589 msix_vec, msix_vec, i); 1590 msix_vec++; 1591 } 1592 1593 DP(BNX2X_MSG_SP, "about to request enable msix with %d vectors\n", 1594 msix_vec); 1595 1596 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], msix_vec); 1597 1598 /* 1599 * reconfigure number of tx/rx queues according to available 1600 * MSI-X vectors 1601 */ 1602 if (rc >= BNX2X_MIN_MSIX_VEC_CNT(bp)) { 1603 /* how less vectors we will have? */ 1604 int diff = msix_vec - rc; 1605 1606 BNX2X_DEV_INFO("Trying to use less MSI-X vectors: %d\n", rc); 1607 1608 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], rc); 1609 1610 if (rc) { 1611 BNX2X_DEV_INFO("MSI-X is not attainable rc %d\n", rc); 1612 goto no_msix; 1613 } 1614 /* 1615 * decrease number of queues by number of unallocated entries 1616 */ 1617 bp->num_ethernet_queues -= diff; 1618 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues; 1619 1620 BNX2X_DEV_INFO("New queue configuration set: %d\n", 1621 bp->num_queues); 1622 } else if (rc > 0) { 1623 /* Get by with single vector */ 1624 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], 1); 1625 if (rc) { 1626 BNX2X_DEV_INFO("Single MSI-X is not attainable rc %d\n", 1627 rc); 1628 goto no_msix; 1629 } 1630 1631 BNX2X_DEV_INFO("Using single MSI-X vector\n"); 1632 bp->flags |= USING_SINGLE_MSIX_FLAG; 1633 1634 BNX2X_DEV_INFO("set number of queues to 1\n"); 1635 bp->num_ethernet_queues = 1; 1636 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues; 1637 } else if (rc < 0) { 1638 BNX2X_DEV_INFO("MSI-X is not attainable rc %d\n", rc); 1639 goto no_msix; 1640 } 1641 1642 bp->flags |= USING_MSIX_FLAG; 1643 1644 return 0; 1645 1646 no_msix: 1647 /* fall to INTx if not enough memory */ 1648 if (rc == -ENOMEM) 1649 bp->flags |= DISABLE_MSI_FLAG; 1650 1651 return rc; 1652 } 1653 1654 static int bnx2x_req_msix_irqs(struct bnx2x *bp) 1655 { 1656 int i, rc, offset = 0; 1657 1658 /* no default status block for vf */ 1659 if (IS_PF(bp)) { 1660 rc = request_irq(bp->msix_table[offset++].vector, 1661 bnx2x_msix_sp_int, 0, 1662 bp->dev->name, bp->dev); 1663 if (rc) { 1664 BNX2X_ERR("request sp irq failed\n"); 1665 return -EBUSY; 1666 } 1667 } 1668 1669 if (CNIC_SUPPORT(bp)) 1670 offset++; 1671 1672 for_each_eth_queue(bp, i) { 1673 struct bnx2x_fastpath *fp = &bp->fp[i]; 1674 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d", 1675 bp->dev->name, i); 1676 1677 rc = request_irq(bp->msix_table[offset].vector, 1678 bnx2x_msix_fp_int, 0, fp->name, fp); 1679 if (rc) { 1680 BNX2X_ERR("request fp #%d irq (%d) failed rc %d\n", i, 1681 bp->msix_table[offset].vector, rc); 1682 bnx2x_free_msix_irqs(bp, offset); 1683 return -EBUSY; 1684 } 1685 1686 offset++; 1687 } 1688 1689 i = BNX2X_NUM_ETH_QUEUES(bp); 1690 if (IS_PF(bp)) { 1691 offset = 1 + CNIC_SUPPORT(bp); 1692 netdev_info(bp->dev, 1693 "using MSI-X IRQs: sp %d fp[%d] %d ... fp[%d] %d\n", 1694 bp->msix_table[0].vector, 1695 0, bp->msix_table[offset].vector, 1696 i - 1, bp->msix_table[offset + i - 1].vector); 1697 } else { 1698 offset = CNIC_SUPPORT(bp); 1699 netdev_info(bp->dev, 1700 "using MSI-X IRQs: fp[%d] %d ... fp[%d] %d\n", 1701 0, bp->msix_table[offset].vector, 1702 i - 1, bp->msix_table[offset + i - 1].vector); 1703 } 1704 return 0; 1705 } 1706 1707 int bnx2x_enable_msi(struct bnx2x *bp) 1708 { 1709 int rc; 1710 1711 rc = pci_enable_msi(bp->pdev); 1712 if (rc) { 1713 BNX2X_DEV_INFO("MSI is not attainable\n"); 1714 return -1; 1715 } 1716 bp->flags |= USING_MSI_FLAG; 1717 1718 return 0; 1719 } 1720 1721 static int bnx2x_req_irq(struct bnx2x *bp) 1722 { 1723 unsigned long flags; 1724 unsigned int irq; 1725 1726 if (bp->flags & (USING_MSI_FLAG | USING_MSIX_FLAG)) 1727 flags = 0; 1728 else 1729 flags = IRQF_SHARED; 1730 1731 if (bp->flags & USING_MSIX_FLAG) 1732 irq = bp->msix_table[0].vector; 1733 else 1734 irq = bp->pdev->irq; 1735 1736 return request_irq(irq, bnx2x_interrupt, flags, bp->dev->name, bp->dev); 1737 } 1738 1739 static int bnx2x_setup_irqs(struct bnx2x *bp) 1740 { 1741 int rc = 0; 1742 if (bp->flags & USING_MSIX_FLAG && 1743 !(bp->flags & USING_SINGLE_MSIX_FLAG)) { 1744 rc = bnx2x_req_msix_irqs(bp); 1745 if (rc) 1746 return rc; 1747 } else { 1748 rc = bnx2x_req_irq(bp); 1749 if (rc) { 1750 BNX2X_ERR("IRQ request failed rc %d, aborting\n", rc); 1751 return rc; 1752 } 1753 if (bp->flags & USING_MSI_FLAG) { 1754 bp->dev->irq = bp->pdev->irq; 1755 netdev_info(bp->dev, "using MSI IRQ %d\n", 1756 bp->dev->irq); 1757 } 1758 if (bp->flags & USING_MSIX_FLAG) { 1759 bp->dev->irq = bp->msix_table[0].vector; 1760 netdev_info(bp->dev, "using MSIX IRQ %d\n", 1761 bp->dev->irq); 1762 } 1763 } 1764 1765 return 0; 1766 } 1767 1768 static void bnx2x_napi_enable_cnic(struct bnx2x *bp) 1769 { 1770 int i; 1771 1772 for_each_rx_queue_cnic(bp, i) { 1773 bnx2x_fp_init_lock(&bp->fp[i]); 1774 napi_enable(&bnx2x_fp(bp, i, napi)); 1775 } 1776 } 1777 1778 static void bnx2x_napi_enable(struct bnx2x *bp) 1779 { 1780 int i; 1781 1782 for_each_eth_queue(bp, i) { 1783 bnx2x_fp_init_lock(&bp->fp[i]); 1784 napi_enable(&bnx2x_fp(bp, i, napi)); 1785 } 1786 } 1787 1788 static void bnx2x_napi_disable_cnic(struct bnx2x *bp) 1789 { 1790 int i; 1791 1792 local_bh_disable(); 1793 for_each_rx_queue_cnic(bp, i) { 1794 napi_disable(&bnx2x_fp(bp, i, napi)); 1795 while (!bnx2x_fp_lock_napi(&bp->fp[i])) 1796 mdelay(1); 1797 } 1798 local_bh_enable(); 1799 } 1800 1801 static void bnx2x_napi_disable(struct bnx2x *bp) 1802 { 1803 int i; 1804 1805 local_bh_disable(); 1806 for_each_eth_queue(bp, i) { 1807 napi_disable(&bnx2x_fp(bp, i, napi)); 1808 while (!bnx2x_fp_lock_napi(&bp->fp[i])) 1809 mdelay(1); 1810 } 1811 local_bh_enable(); 1812 } 1813 1814 void bnx2x_netif_start(struct bnx2x *bp) 1815 { 1816 if (netif_running(bp->dev)) { 1817 bnx2x_napi_enable(bp); 1818 if (CNIC_LOADED(bp)) 1819 bnx2x_napi_enable_cnic(bp); 1820 bnx2x_int_enable(bp); 1821 if (bp->state == BNX2X_STATE_OPEN) 1822 netif_tx_wake_all_queues(bp->dev); 1823 } 1824 } 1825 1826 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw) 1827 { 1828 bnx2x_int_disable_sync(bp, disable_hw); 1829 bnx2x_napi_disable(bp); 1830 if (CNIC_LOADED(bp)) 1831 bnx2x_napi_disable_cnic(bp); 1832 } 1833 1834 u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb) 1835 { 1836 struct bnx2x *bp = netdev_priv(dev); 1837 1838 if (CNIC_LOADED(bp) && !NO_FCOE(bp)) { 1839 struct ethhdr *hdr = (struct ethhdr *)skb->data; 1840 u16 ether_type = ntohs(hdr->h_proto); 1841 1842 /* Skip VLAN tag if present */ 1843 if (ether_type == ETH_P_8021Q) { 1844 struct vlan_ethhdr *vhdr = 1845 (struct vlan_ethhdr *)skb->data; 1846 1847 ether_type = ntohs(vhdr->h_vlan_encapsulated_proto); 1848 } 1849 1850 /* If ethertype is FCoE or FIP - use FCoE ring */ 1851 if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP)) 1852 return bnx2x_fcoe_tx(bp, txq_index); 1853 } 1854 1855 /* select a non-FCoE queue */ 1856 return __netdev_pick_tx(dev, skb) % BNX2X_NUM_ETH_QUEUES(bp); 1857 } 1858 1859 void bnx2x_set_num_queues(struct bnx2x *bp) 1860 { 1861 /* RSS queues */ 1862 bp->num_ethernet_queues = bnx2x_calc_num_queues(bp); 1863 1864 /* override in STORAGE SD modes */ 1865 if (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp)) 1866 bp->num_ethernet_queues = 1; 1867 1868 /* Add special queues */ 1869 bp->num_cnic_queues = CNIC_SUPPORT(bp); /* For FCOE */ 1870 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues; 1871 1872 BNX2X_DEV_INFO("set number of queues to %d\n", bp->num_queues); 1873 } 1874 1875 /** 1876 * bnx2x_set_real_num_queues - configure netdev->real_num_[tx,rx]_queues 1877 * 1878 * @bp: Driver handle 1879 * 1880 * We currently support for at most 16 Tx queues for each CoS thus we will 1881 * allocate a multiple of 16 for ETH L2 rings according to the value of the 1882 * bp->max_cos. 1883 * 1884 * If there is an FCoE L2 queue the appropriate Tx queue will have the next 1885 * index after all ETH L2 indices. 1886 * 1887 * If the actual number of Tx queues (for each CoS) is less than 16 then there 1888 * will be the holes at the end of each group of 16 ETh L2 indices (0..15, 1889 * 16..31,...) with indices that are not coupled with any real Tx queue. 1890 * 1891 * The proper configuration of skb->queue_mapping is handled by 1892 * bnx2x_select_queue() and __skb_tx_hash(). 1893 * 1894 * bnx2x_setup_tc() takes care of the proper TC mappings so that __skb_tx_hash() 1895 * will return a proper Tx index if TC is enabled (netdev->num_tc > 0). 1896 */ 1897 static int bnx2x_set_real_num_queues(struct bnx2x *bp, int include_cnic) 1898 { 1899 int rc, tx, rx; 1900 1901 tx = BNX2X_NUM_ETH_QUEUES(bp) * bp->max_cos; 1902 rx = BNX2X_NUM_ETH_QUEUES(bp); 1903 1904 /* account for fcoe queue */ 1905 if (include_cnic && !NO_FCOE(bp)) { 1906 rx++; 1907 tx++; 1908 } 1909 1910 rc = netif_set_real_num_tx_queues(bp->dev, tx); 1911 if (rc) { 1912 BNX2X_ERR("Failed to set real number of Tx queues: %d\n", rc); 1913 return rc; 1914 } 1915 rc = netif_set_real_num_rx_queues(bp->dev, rx); 1916 if (rc) { 1917 BNX2X_ERR("Failed to set real number of Rx queues: %d\n", rc); 1918 return rc; 1919 } 1920 1921 DP(NETIF_MSG_IFUP, "Setting real num queues to (tx, rx) (%d, %d)\n", 1922 tx, rx); 1923 1924 return rc; 1925 } 1926 1927 static void bnx2x_set_rx_buf_size(struct bnx2x *bp) 1928 { 1929 int i; 1930 1931 for_each_queue(bp, i) { 1932 struct bnx2x_fastpath *fp = &bp->fp[i]; 1933 u32 mtu; 1934 1935 /* Always use a mini-jumbo MTU for the FCoE L2 ring */ 1936 if (IS_FCOE_IDX(i)) 1937 /* 1938 * Although there are no IP frames expected to arrive to 1939 * this ring we still want to add an 1940 * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer 1941 * overrun attack. 1942 */ 1943 mtu = BNX2X_FCOE_MINI_JUMBO_MTU; 1944 else 1945 mtu = bp->dev->mtu; 1946 fp->rx_buf_size = BNX2X_FW_RX_ALIGN_START + 1947 IP_HEADER_ALIGNMENT_PADDING + 1948 ETH_OVREHEAD + 1949 mtu + 1950 BNX2X_FW_RX_ALIGN_END; 1951 /* Note : rx_buf_size doesn't take into account NET_SKB_PAD */ 1952 if (fp->rx_buf_size + NET_SKB_PAD <= PAGE_SIZE) 1953 fp->rx_frag_size = fp->rx_buf_size + NET_SKB_PAD; 1954 else 1955 fp->rx_frag_size = 0; 1956 } 1957 } 1958 1959 static int bnx2x_init_rss(struct bnx2x *bp) 1960 { 1961 int i; 1962 u8 num_eth_queues = BNX2X_NUM_ETH_QUEUES(bp); 1963 1964 /* Prepare the initial contents for the indirection table if RSS is 1965 * enabled 1966 */ 1967 for (i = 0; i < sizeof(bp->rss_conf_obj.ind_table); i++) 1968 bp->rss_conf_obj.ind_table[i] = 1969 bp->fp->cl_id + 1970 ethtool_rxfh_indir_default(i, num_eth_queues); 1971 1972 /* 1973 * For 57710 and 57711 SEARCHER configuration (rss_keys) is 1974 * per-port, so if explicit configuration is needed , do it only 1975 * for a PMF. 1976 * 1977 * For 57712 and newer on the other hand it's a per-function 1978 * configuration. 1979 */ 1980 return bnx2x_config_rss_eth(bp, bp->port.pmf || !CHIP_IS_E1x(bp)); 1981 } 1982 1983 int bnx2x_rss(struct bnx2x *bp, struct bnx2x_rss_config_obj *rss_obj, 1984 bool config_hash, bool enable) 1985 { 1986 struct bnx2x_config_rss_params params = {NULL}; 1987 1988 /* Although RSS is meaningless when there is a single HW queue we 1989 * still need it enabled in order to have HW Rx hash generated. 1990 * 1991 * if (!is_eth_multi(bp)) 1992 * bp->multi_mode = ETH_RSS_MODE_DISABLED; 1993 */ 1994 1995 params.rss_obj = rss_obj; 1996 1997 __set_bit(RAMROD_COMP_WAIT, ¶ms.ramrod_flags); 1998 1999 if (enable) { 2000 __set_bit(BNX2X_RSS_MODE_REGULAR, ¶ms.rss_flags); 2001 2002 /* RSS configuration */ 2003 __set_bit(BNX2X_RSS_IPV4, ¶ms.rss_flags); 2004 __set_bit(BNX2X_RSS_IPV4_TCP, ¶ms.rss_flags); 2005 __set_bit(BNX2X_RSS_IPV6, ¶ms.rss_flags); 2006 __set_bit(BNX2X_RSS_IPV6_TCP, ¶ms.rss_flags); 2007 if (rss_obj->udp_rss_v4) 2008 __set_bit(BNX2X_RSS_IPV4_UDP, ¶ms.rss_flags); 2009 if (rss_obj->udp_rss_v6) 2010 __set_bit(BNX2X_RSS_IPV6_UDP, ¶ms.rss_flags); 2011 } else { 2012 __set_bit(BNX2X_RSS_MODE_DISABLED, ¶ms.rss_flags); 2013 } 2014 2015 /* Hash bits */ 2016 params.rss_result_mask = MULTI_MASK; 2017 2018 memcpy(params.ind_table, rss_obj->ind_table, sizeof(params.ind_table)); 2019 2020 if (config_hash) { 2021 /* RSS keys */ 2022 prandom_bytes(params.rss_key, T_ETH_RSS_KEY * 4); 2023 __set_bit(BNX2X_RSS_SET_SRCH, ¶ms.rss_flags); 2024 } 2025 2026 if (IS_PF(bp)) 2027 return bnx2x_config_rss(bp, ¶ms); 2028 else 2029 return bnx2x_vfpf_config_rss(bp, ¶ms); 2030 } 2031 2032 static int bnx2x_init_hw(struct bnx2x *bp, u32 load_code) 2033 { 2034 struct bnx2x_func_state_params func_params = {NULL}; 2035 2036 /* Prepare parameters for function state transitions */ 2037 __set_bit(RAMROD_COMP_WAIT, &func_params.ramrod_flags); 2038 2039 func_params.f_obj = &bp->func_obj; 2040 func_params.cmd = BNX2X_F_CMD_HW_INIT; 2041 2042 func_params.params.hw_init.load_phase = load_code; 2043 2044 return bnx2x_func_state_change(bp, &func_params); 2045 } 2046 2047 /* 2048 * Cleans the object that have internal lists without sending 2049 * ramrods. Should be run when interrupts are disabled. 2050 */ 2051 void bnx2x_squeeze_objects(struct bnx2x *bp) 2052 { 2053 int rc; 2054 unsigned long ramrod_flags = 0, vlan_mac_flags = 0; 2055 struct bnx2x_mcast_ramrod_params rparam = {NULL}; 2056 struct bnx2x_vlan_mac_obj *mac_obj = &bp->sp_objs->mac_obj; 2057 2058 /***************** Cleanup MACs' object first *************************/ 2059 2060 /* Wait for completion of requested */ 2061 __set_bit(RAMROD_COMP_WAIT, &ramrod_flags); 2062 /* Perform a dry cleanup */ 2063 __set_bit(RAMROD_DRV_CLR_ONLY, &ramrod_flags); 2064 2065 /* Clean ETH primary MAC */ 2066 __set_bit(BNX2X_ETH_MAC, &vlan_mac_flags); 2067 rc = mac_obj->delete_all(bp, &bp->sp_objs->mac_obj, &vlan_mac_flags, 2068 &ramrod_flags); 2069 if (rc != 0) 2070 BNX2X_ERR("Failed to clean ETH MACs: %d\n", rc); 2071 2072 /* Cleanup UC list */ 2073 vlan_mac_flags = 0; 2074 __set_bit(BNX2X_UC_LIST_MAC, &vlan_mac_flags); 2075 rc = mac_obj->delete_all(bp, mac_obj, &vlan_mac_flags, 2076 &ramrod_flags); 2077 if (rc != 0) 2078 BNX2X_ERR("Failed to clean UC list MACs: %d\n", rc); 2079 2080 /***************** Now clean mcast object *****************************/ 2081 rparam.mcast_obj = &bp->mcast_obj; 2082 __set_bit(RAMROD_DRV_CLR_ONLY, &rparam.ramrod_flags); 2083 2084 /* Add a DEL command... - Since we're doing a driver cleanup only, 2085 * we take a lock surrounding both the initial send and the CONTs, 2086 * as we don't want a true completion to disrupt us in the middle. 2087 */ 2088 netif_addr_lock_bh(bp->dev); 2089 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_DEL); 2090 if (rc < 0) 2091 BNX2X_ERR("Failed to add a new DEL command to a multi-cast object: %d\n", 2092 rc); 2093 2094 /* ...and wait until all pending commands are cleared */ 2095 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT); 2096 while (rc != 0) { 2097 if (rc < 0) { 2098 BNX2X_ERR("Failed to clean multi-cast object: %d\n", 2099 rc); 2100 netif_addr_unlock_bh(bp->dev); 2101 return; 2102 } 2103 2104 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT); 2105 } 2106 netif_addr_unlock_bh(bp->dev); 2107 } 2108 2109 #ifndef BNX2X_STOP_ON_ERROR 2110 #define LOAD_ERROR_EXIT(bp, label) \ 2111 do { \ 2112 (bp)->state = BNX2X_STATE_ERROR; \ 2113 goto label; \ 2114 } while (0) 2115 2116 #define LOAD_ERROR_EXIT_CNIC(bp, label) \ 2117 do { \ 2118 bp->cnic_loaded = false; \ 2119 goto label; \ 2120 } while (0) 2121 #else /*BNX2X_STOP_ON_ERROR*/ 2122 #define LOAD_ERROR_EXIT(bp, label) \ 2123 do { \ 2124 (bp)->state = BNX2X_STATE_ERROR; \ 2125 (bp)->panic = 1; \ 2126 return -EBUSY; \ 2127 } while (0) 2128 #define LOAD_ERROR_EXIT_CNIC(bp, label) \ 2129 do { \ 2130 bp->cnic_loaded = false; \ 2131 (bp)->panic = 1; \ 2132 return -EBUSY; \ 2133 } while (0) 2134 #endif /*BNX2X_STOP_ON_ERROR*/ 2135 2136 static void bnx2x_free_fw_stats_mem(struct bnx2x *bp) 2137 { 2138 BNX2X_PCI_FREE(bp->fw_stats, bp->fw_stats_mapping, 2139 bp->fw_stats_data_sz + bp->fw_stats_req_sz); 2140 return; 2141 } 2142 2143 static int bnx2x_alloc_fw_stats_mem(struct bnx2x *bp) 2144 { 2145 int num_groups, vf_headroom = 0; 2146 int is_fcoe_stats = NO_FCOE(bp) ? 0 : 1; 2147 2148 /* number of queues for statistics is number of eth queues + FCoE */ 2149 u8 num_queue_stats = BNX2X_NUM_ETH_QUEUES(bp) + is_fcoe_stats; 2150 2151 /* Total number of FW statistics requests = 2152 * 1 for port stats + 1 for PF stats + potential 2 for FCoE (fcoe proper 2153 * and fcoe l2 queue) stats + num of queues (which includes another 1 2154 * for fcoe l2 queue if applicable) 2155 */ 2156 bp->fw_stats_num = 2 + is_fcoe_stats + num_queue_stats; 2157 2158 /* vf stats appear in the request list, but their data is allocated by 2159 * the VFs themselves. We don't include them in the bp->fw_stats_num as 2160 * it is used to determine where to place the vf stats queries in the 2161 * request struct 2162 */ 2163 if (IS_SRIOV(bp)) 2164 vf_headroom = bnx2x_vf_headroom(bp); 2165 2166 /* Request is built from stats_query_header and an array of 2167 * stats_query_cmd_group each of which contains 2168 * STATS_QUERY_CMD_COUNT rules. The real number or requests is 2169 * configured in the stats_query_header. 2170 */ 2171 num_groups = 2172 (((bp->fw_stats_num + vf_headroom) / STATS_QUERY_CMD_COUNT) + 2173 (((bp->fw_stats_num + vf_headroom) % STATS_QUERY_CMD_COUNT) ? 2174 1 : 0)); 2175 2176 DP(BNX2X_MSG_SP, "stats fw_stats_num %d, vf headroom %d, num_groups %d\n", 2177 bp->fw_stats_num, vf_headroom, num_groups); 2178 bp->fw_stats_req_sz = sizeof(struct stats_query_header) + 2179 num_groups * sizeof(struct stats_query_cmd_group); 2180 2181 /* Data for statistics requests + stats_counter 2182 * stats_counter holds per-STORM counters that are incremented 2183 * when STORM has finished with the current request. 2184 * memory for FCoE offloaded statistics are counted anyway, 2185 * even if they will not be sent. 2186 * VF stats are not accounted for here as the data of VF stats is stored 2187 * in memory allocated by the VF, not here. 2188 */ 2189 bp->fw_stats_data_sz = sizeof(struct per_port_stats) + 2190 sizeof(struct per_pf_stats) + 2191 sizeof(struct fcoe_statistics_params) + 2192 sizeof(struct per_queue_stats) * num_queue_stats + 2193 sizeof(struct stats_counter); 2194 2195 BNX2X_PCI_ALLOC(bp->fw_stats, &bp->fw_stats_mapping, 2196 bp->fw_stats_data_sz + bp->fw_stats_req_sz); 2197 2198 /* Set shortcuts */ 2199 bp->fw_stats_req = (struct bnx2x_fw_stats_req *)bp->fw_stats; 2200 bp->fw_stats_req_mapping = bp->fw_stats_mapping; 2201 bp->fw_stats_data = (struct bnx2x_fw_stats_data *) 2202 ((u8 *)bp->fw_stats + bp->fw_stats_req_sz); 2203 bp->fw_stats_data_mapping = bp->fw_stats_mapping + 2204 bp->fw_stats_req_sz; 2205 2206 DP(BNX2X_MSG_SP, "statistics request base address set to %x %x\n", 2207 U64_HI(bp->fw_stats_req_mapping), 2208 U64_LO(bp->fw_stats_req_mapping)); 2209 DP(BNX2X_MSG_SP, "statistics data base address set to %x %x\n", 2210 U64_HI(bp->fw_stats_data_mapping), 2211 U64_LO(bp->fw_stats_data_mapping)); 2212 return 0; 2213 2214 alloc_mem_err: 2215 bnx2x_free_fw_stats_mem(bp); 2216 BNX2X_ERR("Can't allocate FW stats memory\n"); 2217 return -ENOMEM; 2218 } 2219 2220 /* send load request to mcp and analyze response */ 2221 static int bnx2x_nic_load_request(struct bnx2x *bp, u32 *load_code) 2222 { 2223 u32 param; 2224 2225 /* init fw_seq */ 2226 bp->fw_seq = 2227 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_mb_header) & 2228 DRV_MSG_SEQ_NUMBER_MASK); 2229 BNX2X_DEV_INFO("fw_seq 0x%08x\n", bp->fw_seq); 2230 2231 /* Get current FW pulse sequence */ 2232 bp->fw_drv_pulse_wr_seq = 2233 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_pulse_mb) & 2234 DRV_PULSE_SEQ_MASK); 2235 BNX2X_DEV_INFO("drv_pulse 0x%x\n", bp->fw_drv_pulse_wr_seq); 2236 2237 param = DRV_MSG_CODE_LOAD_REQ_WITH_LFA; 2238 2239 if (IS_MF_SD(bp) && bnx2x_port_after_undi(bp)) 2240 param |= DRV_MSG_CODE_LOAD_REQ_FORCE_LFA; 2241 2242 /* load request */ 2243 (*load_code) = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ, param); 2244 2245 /* if mcp fails to respond we must abort */ 2246 if (!(*load_code)) { 2247 BNX2X_ERR("MCP response failure, aborting\n"); 2248 return -EBUSY; 2249 } 2250 2251 /* If mcp refused (e.g. other port is in diagnostic mode) we 2252 * must abort 2253 */ 2254 if ((*load_code) == FW_MSG_CODE_DRV_LOAD_REFUSED) { 2255 BNX2X_ERR("MCP refused load request, aborting\n"); 2256 return -EBUSY; 2257 } 2258 return 0; 2259 } 2260 2261 /* check whether another PF has already loaded FW to chip. In 2262 * virtualized environments a pf from another VM may have already 2263 * initialized the device including loading FW 2264 */ 2265 int bnx2x_nic_load_analyze_req(struct bnx2x *bp, u32 load_code) 2266 { 2267 /* is another pf loaded on this engine? */ 2268 if (load_code != FW_MSG_CODE_DRV_LOAD_COMMON_CHIP && 2269 load_code != FW_MSG_CODE_DRV_LOAD_COMMON) { 2270 /* build my FW version dword */ 2271 u32 my_fw = (BCM_5710_FW_MAJOR_VERSION) + 2272 (BCM_5710_FW_MINOR_VERSION << 8) + 2273 (BCM_5710_FW_REVISION_VERSION << 16) + 2274 (BCM_5710_FW_ENGINEERING_VERSION << 24); 2275 2276 /* read loaded FW from chip */ 2277 u32 loaded_fw = REG_RD(bp, XSEM_REG_PRAM); 2278 2279 DP(BNX2X_MSG_SP, "loaded fw %x, my fw %x\n", 2280 loaded_fw, my_fw); 2281 2282 /* abort nic load if version mismatch */ 2283 if (my_fw != loaded_fw) { 2284 BNX2X_ERR("bnx2x with FW %x was already loaded which mismatches my %x FW. Aborting\n", 2285 loaded_fw, my_fw); 2286 return -EBUSY; 2287 } 2288 } 2289 return 0; 2290 } 2291 2292 /* returns the "mcp load_code" according to global load_count array */ 2293 static int bnx2x_nic_load_no_mcp(struct bnx2x *bp, int port) 2294 { 2295 int path = BP_PATH(bp); 2296 2297 DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d] %d, %d, %d\n", 2298 path, load_count[path][0], load_count[path][1], 2299 load_count[path][2]); 2300 load_count[path][0]++; 2301 load_count[path][1 + port]++; 2302 DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d] %d, %d, %d\n", 2303 path, load_count[path][0], load_count[path][1], 2304 load_count[path][2]); 2305 if (load_count[path][0] == 1) 2306 return FW_MSG_CODE_DRV_LOAD_COMMON; 2307 else if (load_count[path][1 + port] == 1) 2308 return FW_MSG_CODE_DRV_LOAD_PORT; 2309 else 2310 return FW_MSG_CODE_DRV_LOAD_FUNCTION; 2311 } 2312 2313 /* mark PMF if applicable */ 2314 static void bnx2x_nic_load_pmf(struct bnx2x *bp, u32 load_code) 2315 { 2316 if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) || 2317 (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) || 2318 (load_code == FW_MSG_CODE_DRV_LOAD_PORT)) { 2319 bp->port.pmf = 1; 2320 /* We need the barrier to ensure the ordering between the 2321 * writing to bp->port.pmf here and reading it from the 2322 * bnx2x_periodic_task(). 2323 */ 2324 smp_mb(); 2325 } else { 2326 bp->port.pmf = 0; 2327 } 2328 2329 DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf); 2330 } 2331 2332 static void bnx2x_nic_load_afex_dcc(struct bnx2x *bp, int load_code) 2333 { 2334 if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) || 2335 (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) && 2336 (bp->common.shmem2_base)) { 2337 if (SHMEM2_HAS(bp, dcc_support)) 2338 SHMEM2_WR(bp, dcc_support, 2339 (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV | 2340 SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV)); 2341 if (SHMEM2_HAS(bp, afex_driver_support)) 2342 SHMEM2_WR(bp, afex_driver_support, 2343 SHMEM_AFEX_SUPPORTED_VERSION_ONE); 2344 } 2345 2346 /* Set AFEX default VLAN tag to an invalid value */ 2347 bp->afex_def_vlan_tag = -1; 2348 } 2349 2350 /** 2351 * bnx2x_bz_fp - zero content of the fastpath structure. 2352 * 2353 * @bp: driver handle 2354 * @index: fastpath index to be zeroed 2355 * 2356 * Makes sure the contents of the bp->fp[index].napi is kept 2357 * intact. 2358 */ 2359 static void bnx2x_bz_fp(struct bnx2x *bp, int index) 2360 { 2361 struct bnx2x_fastpath *fp = &bp->fp[index]; 2362 int cos; 2363 struct napi_struct orig_napi = fp->napi; 2364 struct bnx2x_agg_info *orig_tpa_info = fp->tpa_info; 2365 2366 /* bzero bnx2x_fastpath contents */ 2367 if (fp->tpa_info) 2368 memset(fp->tpa_info, 0, ETH_MAX_AGGREGATION_QUEUES_E1H_E2 * 2369 sizeof(struct bnx2x_agg_info)); 2370 memset(fp, 0, sizeof(*fp)); 2371 2372 /* Restore the NAPI object as it has been already initialized */ 2373 fp->napi = orig_napi; 2374 fp->tpa_info = orig_tpa_info; 2375 fp->bp = bp; 2376 fp->index = index; 2377 if (IS_ETH_FP(fp)) 2378 fp->max_cos = bp->max_cos; 2379 else 2380 /* Special queues support only one CoS */ 2381 fp->max_cos = 1; 2382 2383 /* Init txdata pointers */ 2384 if (IS_FCOE_FP(fp)) 2385 fp->txdata_ptr[0] = &bp->bnx2x_txq[FCOE_TXQ_IDX(bp)]; 2386 if (IS_ETH_FP(fp)) 2387 for_each_cos_in_tx_queue(fp, cos) 2388 fp->txdata_ptr[cos] = &bp->bnx2x_txq[cos * 2389 BNX2X_NUM_ETH_QUEUES(bp) + index]; 2390 2391 /* set the tpa flag for each queue. The tpa flag determines the queue 2392 * minimal size so it must be set prior to queue memory allocation 2393 */ 2394 fp->disable_tpa = !(bp->flags & TPA_ENABLE_FLAG || 2395 (bp->flags & GRO_ENABLE_FLAG && 2396 bnx2x_mtu_allows_gro(bp->dev->mtu))); 2397 if (bp->flags & TPA_ENABLE_FLAG) 2398 fp->mode = TPA_MODE_LRO; 2399 else if (bp->flags & GRO_ENABLE_FLAG) 2400 fp->mode = TPA_MODE_GRO; 2401 2402 /* We don't want TPA on an FCoE L2 ring */ 2403 if (IS_FCOE_FP(fp)) 2404 fp->disable_tpa = 1; 2405 } 2406 2407 int bnx2x_load_cnic(struct bnx2x *bp) 2408 { 2409 int i, rc, port = BP_PORT(bp); 2410 2411 DP(NETIF_MSG_IFUP, "Starting CNIC-related load\n"); 2412 2413 mutex_init(&bp->cnic_mutex); 2414 2415 if (IS_PF(bp)) { 2416 rc = bnx2x_alloc_mem_cnic(bp); 2417 if (rc) { 2418 BNX2X_ERR("Unable to allocate bp memory for cnic\n"); 2419 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0); 2420 } 2421 } 2422 2423 rc = bnx2x_alloc_fp_mem_cnic(bp); 2424 if (rc) { 2425 BNX2X_ERR("Unable to allocate memory for cnic fps\n"); 2426 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0); 2427 } 2428 2429 /* Update the number of queues with the cnic queues */ 2430 rc = bnx2x_set_real_num_queues(bp, 1); 2431 if (rc) { 2432 BNX2X_ERR("Unable to set real_num_queues including cnic\n"); 2433 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0); 2434 } 2435 2436 /* Add all CNIC NAPI objects */ 2437 bnx2x_add_all_napi_cnic(bp); 2438 DP(NETIF_MSG_IFUP, "cnic napi added\n"); 2439 bnx2x_napi_enable_cnic(bp); 2440 2441 rc = bnx2x_init_hw_func_cnic(bp); 2442 if (rc) 2443 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic1); 2444 2445 bnx2x_nic_init_cnic(bp); 2446 2447 if (IS_PF(bp)) { 2448 /* Enable Timer scan */ 2449 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 1); 2450 2451 /* setup cnic queues */ 2452 for_each_cnic_queue(bp, i) { 2453 rc = bnx2x_setup_queue(bp, &bp->fp[i], 0); 2454 if (rc) { 2455 BNX2X_ERR("Queue setup failed\n"); 2456 LOAD_ERROR_EXIT(bp, load_error_cnic2); 2457 } 2458 } 2459 } 2460 2461 /* Initialize Rx filter. */ 2462 bnx2x_set_rx_mode_inner(bp); 2463 2464 /* re-read iscsi info */ 2465 bnx2x_get_iscsi_info(bp); 2466 bnx2x_setup_cnic_irq_info(bp); 2467 bnx2x_setup_cnic_info(bp); 2468 bp->cnic_loaded = true; 2469 if (bp->state == BNX2X_STATE_OPEN) 2470 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD); 2471 2472 DP(NETIF_MSG_IFUP, "Ending successfully CNIC-related load\n"); 2473 2474 return 0; 2475 2476 #ifndef BNX2X_STOP_ON_ERROR 2477 load_error_cnic2: 2478 /* Disable Timer scan */ 2479 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 0); 2480 2481 load_error_cnic1: 2482 bnx2x_napi_disable_cnic(bp); 2483 /* Update the number of queues without the cnic queues */ 2484 if (bnx2x_set_real_num_queues(bp, 0)) 2485 BNX2X_ERR("Unable to set real_num_queues not including cnic\n"); 2486 load_error_cnic0: 2487 BNX2X_ERR("CNIC-related load failed\n"); 2488 bnx2x_free_fp_mem_cnic(bp); 2489 bnx2x_free_mem_cnic(bp); 2490 return rc; 2491 #endif /* ! BNX2X_STOP_ON_ERROR */ 2492 } 2493 2494 /* must be called with rtnl_lock */ 2495 int bnx2x_nic_load(struct bnx2x *bp, int load_mode) 2496 { 2497 int port = BP_PORT(bp); 2498 int i, rc = 0, load_code = 0; 2499 2500 DP(NETIF_MSG_IFUP, "Starting NIC load\n"); 2501 DP(NETIF_MSG_IFUP, 2502 "CNIC is %s\n", CNIC_ENABLED(bp) ? "enabled" : "disabled"); 2503 2504 #ifdef BNX2X_STOP_ON_ERROR 2505 if (unlikely(bp->panic)) { 2506 BNX2X_ERR("Can't load NIC when there is panic\n"); 2507 return -EPERM; 2508 } 2509 #endif 2510 2511 bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD; 2512 2513 /* zero the structure w/o any lock, before SP handler is initialized */ 2514 memset(&bp->last_reported_link, 0, sizeof(bp->last_reported_link)); 2515 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN, 2516 &bp->last_reported_link.link_report_flags); 2517 2518 if (IS_PF(bp)) 2519 /* must be called before memory allocation and HW init */ 2520 bnx2x_ilt_set_info(bp); 2521 2522 /* 2523 * Zero fastpath structures preserving invariants like napi, which are 2524 * allocated only once, fp index, max_cos, bp pointer. 2525 * Also set fp->disable_tpa and txdata_ptr. 2526 */ 2527 DP(NETIF_MSG_IFUP, "num queues: %d", bp->num_queues); 2528 for_each_queue(bp, i) 2529 bnx2x_bz_fp(bp, i); 2530 memset(bp->bnx2x_txq, 0, (BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS + 2531 bp->num_cnic_queues) * 2532 sizeof(struct bnx2x_fp_txdata)); 2533 2534 bp->fcoe_init = false; 2535 2536 /* Set the receive queues buffer size */ 2537 bnx2x_set_rx_buf_size(bp); 2538 2539 if (IS_PF(bp)) { 2540 rc = bnx2x_alloc_mem(bp); 2541 if (rc) { 2542 BNX2X_ERR("Unable to allocate bp memory\n"); 2543 return rc; 2544 } 2545 } 2546 2547 /* Allocated memory for FW statistics */ 2548 if (bnx2x_alloc_fw_stats_mem(bp)) 2549 LOAD_ERROR_EXIT(bp, load_error0); 2550 2551 /* need to be done after alloc mem, since it's self adjusting to amount 2552 * of memory available for RSS queues 2553 */ 2554 rc = bnx2x_alloc_fp_mem(bp); 2555 if (rc) { 2556 BNX2X_ERR("Unable to allocate memory for fps\n"); 2557 LOAD_ERROR_EXIT(bp, load_error0); 2558 } 2559 2560 /* request pf to initialize status blocks */ 2561 if (IS_VF(bp)) { 2562 rc = bnx2x_vfpf_init(bp); 2563 if (rc) 2564 LOAD_ERROR_EXIT(bp, load_error0); 2565 } 2566 2567 /* As long as bnx2x_alloc_mem() may possibly update 2568 * bp->num_queues, bnx2x_set_real_num_queues() should always 2569 * come after it. At this stage cnic queues are not counted. 2570 */ 2571 rc = bnx2x_set_real_num_queues(bp, 0); 2572 if (rc) { 2573 BNX2X_ERR("Unable to set real_num_queues\n"); 2574 LOAD_ERROR_EXIT(bp, load_error0); 2575 } 2576 2577 /* configure multi cos mappings in kernel. 2578 * this configuration may be overridden by a multi class queue 2579 * discipline or by a dcbx negotiation result. 2580 */ 2581 bnx2x_setup_tc(bp->dev, bp->max_cos); 2582 2583 /* Add all NAPI objects */ 2584 bnx2x_add_all_napi(bp); 2585 DP(NETIF_MSG_IFUP, "napi added\n"); 2586 bnx2x_napi_enable(bp); 2587 2588 if (IS_PF(bp)) { 2589 /* set pf load just before approaching the MCP */ 2590 bnx2x_set_pf_load(bp); 2591 2592 /* if mcp exists send load request and analyze response */ 2593 if (!BP_NOMCP(bp)) { 2594 /* attempt to load pf */ 2595 rc = bnx2x_nic_load_request(bp, &load_code); 2596 if (rc) 2597 LOAD_ERROR_EXIT(bp, load_error1); 2598 2599 /* what did mcp say? */ 2600 rc = bnx2x_nic_load_analyze_req(bp, load_code); 2601 if (rc) { 2602 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0); 2603 LOAD_ERROR_EXIT(bp, load_error2); 2604 } 2605 } else { 2606 load_code = bnx2x_nic_load_no_mcp(bp, port); 2607 } 2608 2609 /* mark pmf if applicable */ 2610 bnx2x_nic_load_pmf(bp, load_code); 2611 2612 /* Init Function state controlling object */ 2613 bnx2x__init_func_obj(bp); 2614 2615 /* Initialize HW */ 2616 rc = bnx2x_init_hw(bp, load_code); 2617 if (rc) { 2618 BNX2X_ERR("HW init failed, aborting\n"); 2619 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0); 2620 LOAD_ERROR_EXIT(bp, load_error2); 2621 } 2622 } 2623 2624 bnx2x_pre_irq_nic_init(bp); 2625 2626 /* Connect to IRQs */ 2627 rc = bnx2x_setup_irqs(bp); 2628 if (rc) { 2629 BNX2X_ERR("setup irqs failed\n"); 2630 if (IS_PF(bp)) 2631 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0); 2632 LOAD_ERROR_EXIT(bp, load_error2); 2633 } 2634 2635 /* Init per-function objects */ 2636 if (IS_PF(bp)) { 2637 /* Setup NIC internals and enable interrupts */ 2638 bnx2x_post_irq_nic_init(bp, load_code); 2639 2640 bnx2x_init_bp_objs(bp); 2641 bnx2x_iov_nic_init(bp); 2642 2643 /* Set AFEX default VLAN tag to an invalid value */ 2644 bp->afex_def_vlan_tag = -1; 2645 bnx2x_nic_load_afex_dcc(bp, load_code); 2646 bp->state = BNX2X_STATE_OPENING_WAIT4_PORT; 2647 rc = bnx2x_func_start(bp); 2648 if (rc) { 2649 BNX2X_ERR("Function start failed!\n"); 2650 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0); 2651 2652 LOAD_ERROR_EXIT(bp, load_error3); 2653 } 2654 2655 /* Send LOAD_DONE command to MCP */ 2656 if (!BP_NOMCP(bp)) { 2657 load_code = bnx2x_fw_command(bp, 2658 DRV_MSG_CODE_LOAD_DONE, 0); 2659 if (!load_code) { 2660 BNX2X_ERR("MCP response failure, aborting\n"); 2661 rc = -EBUSY; 2662 LOAD_ERROR_EXIT(bp, load_error3); 2663 } 2664 } 2665 2666 /* initialize FW coalescing state machines in RAM */ 2667 bnx2x_update_coalesce(bp); 2668 } 2669 2670 /* setup the leading queue */ 2671 rc = bnx2x_setup_leading(bp); 2672 if (rc) { 2673 BNX2X_ERR("Setup leading failed!\n"); 2674 LOAD_ERROR_EXIT(bp, load_error3); 2675 } 2676 2677 /* set up the rest of the queues */ 2678 for_each_nondefault_eth_queue(bp, i) { 2679 if (IS_PF(bp)) 2680 rc = bnx2x_setup_queue(bp, &bp->fp[i], false); 2681 else /* VF */ 2682 rc = bnx2x_vfpf_setup_q(bp, &bp->fp[i], false); 2683 if (rc) { 2684 BNX2X_ERR("Queue %d setup failed\n", i); 2685 LOAD_ERROR_EXIT(bp, load_error3); 2686 } 2687 } 2688 2689 /* setup rss */ 2690 rc = bnx2x_init_rss(bp); 2691 if (rc) { 2692 BNX2X_ERR("PF RSS init failed\n"); 2693 LOAD_ERROR_EXIT(bp, load_error3); 2694 } 2695 2696 /* Now when Clients are configured we are ready to work */ 2697 bp->state = BNX2X_STATE_OPEN; 2698 2699 /* Configure a ucast MAC */ 2700 if (IS_PF(bp)) 2701 rc = bnx2x_set_eth_mac(bp, true); 2702 else /* vf */ 2703 rc = bnx2x_vfpf_config_mac(bp, bp->dev->dev_addr, bp->fp->index, 2704 true); 2705 if (rc) { 2706 BNX2X_ERR("Setting Ethernet MAC failed\n"); 2707 LOAD_ERROR_EXIT(bp, load_error3); 2708 } 2709 2710 if (IS_PF(bp) && bp->pending_max) { 2711 bnx2x_update_max_mf_config(bp, bp->pending_max); 2712 bp->pending_max = 0; 2713 } 2714 2715 if (bp->port.pmf) { 2716 rc = bnx2x_initial_phy_init(bp, load_mode); 2717 if (rc) 2718 LOAD_ERROR_EXIT(bp, load_error3); 2719 } 2720 bp->link_params.feature_config_flags &= ~FEATURE_CONFIG_BOOT_FROM_SAN; 2721 2722 /* Start fast path */ 2723 2724 /* Initialize Rx filter. */ 2725 bnx2x_set_rx_mode_inner(bp); 2726 2727 /* Start the Tx */ 2728 switch (load_mode) { 2729 case LOAD_NORMAL: 2730 /* Tx queue should be only re-enabled */ 2731 netif_tx_wake_all_queues(bp->dev); 2732 break; 2733 2734 case LOAD_OPEN: 2735 netif_tx_start_all_queues(bp->dev); 2736 smp_mb__after_clear_bit(); 2737 break; 2738 2739 case LOAD_DIAG: 2740 case LOAD_LOOPBACK_EXT: 2741 bp->state = BNX2X_STATE_DIAG; 2742 break; 2743 2744 default: 2745 break; 2746 } 2747 2748 if (bp->port.pmf) 2749 bnx2x_update_drv_flags(bp, 1 << DRV_FLAGS_PORT_MASK, 0); 2750 else 2751 bnx2x__link_status_update(bp); 2752 2753 /* start the timer */ 2754 mod_timer(&bp->timer, jiffies + bp->current_interval); 2755 2756 if (CNIC_ENABLED(bp)) 2757 bnx2x_load_cnic(bp); 2758 2759 if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) { 2760 /* mark driver is loaded in shmem2 */ 2761 u32 val; 2762 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]); 2763 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)], 2764 val | DRV_FLAGS_CAPABILITIES_LOADED_SUPPORTED | 2765 DRV_FLAGS_CAPABILITIES_LOADED_L2); 2766 } 2767 2768 /* Wait for all pending SP commands to complete */ 2769 if (IS_PF(bp) && !bnx2x_wait_sp_comp(bp, ~0x0UL)) { 2770 BNX2X_ERR("Timeout waiting for SP elements to complete\n"); 2771 bnx2x_nic_unload(bp, UNLOAD_CLOSE, false); 2772 return -EBUSY; 2773 } 2774 2775 /* If PMF - send ADMIN DCBX msg to MFW to initiate DCBX FSM */ 2776 if (bp->port.pmf && (bp->state != BNX2X_STATE_DIAG)) 2777 bnx2x_dcbx_init(bp, false); 2778 2779 DP(NETIF_MSG_IFUP, "Ending successfully NIC load\n"); 2780 2781 return 0; 2782 2783 #ifndef BNX2X_STOP_ON_ERROR 2784 load_error3: 2785 if (IS_PF(bp)) { 2786 bnx2x_int_disable_sync(bp, 1); 2787 2788 /* Clean queueable objects */ 2789 bnx2x_squeeze_objects(bp); 2790 } 2791 2792 /* Free SKBs, SGEs, TPA pool and driver internals */ 2793 bnx2x_free_skbs(bp); 2794 for_each_rx_queue(bp, i) 2795 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE); 2796 2797 /* Release IRQs */ 2798 bnx2x_free_irq(bp); 2799 load_error2: 2800 if (IS_PF(bp) && !BP_NOMCP(bp)) { 2801 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0); 2802 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0); 2803 } 2804 2805 bp->port.pmf = 0; 2806 load_error1: 2807 bnx2x_napi_disable(bp); 2808 bnx2x_del_all_napi(bp); 2809 2810 /* clear pf_load status, as it was already set */ 2811 if (IS_PF(bp)) 2812 bnx2x_clear_pf_load(bp); 2813 load_error0: 2814 bnx2x_free_fp_mem(bp); 2815 bnx2x_free_fw_stats_mem(bp); 2816 bnx2x_free_mem(bp); 2817 2818 return rc; 2819 #endif /* ! BNX2X_STOP_ON_ERROR */ 2820 } 2821 2822 int bnx2x_drain_tx_queues(struct bnx2x *bp) 2823 { 2824 u8 rc = 0, cos, i; 2825 2826 /* Wait until tx fastpath tasks complete */ 2827 for_each_tx_queue(bp, i) { 2828 struct bnx2x_fastpath *fp = &bp->fp[i]; 2829 2830 for_each_cos_in_tx_queue(fp, cos) 2831 rc = bnx2x_clean_tx_queue(bp, fp->txdata_ptr[cos]); 2832 if (rc) 2833 return rc; 2834 } 2835 return 0; 2836 } 2837 2838 /* must be called with rtnl_lock */ 2839 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode, bool keep_link) 2840 { 2841 int i; 2842 bool global = false; 2843 2844 DP(NETIF_MSG_IFUP, "Starting NIC unload\n"); 2845 2846 /* mark driver is unloaded in shmem2 */ 2847 if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) { 2848 u32 val; 2849 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]); 2850 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)], 2851 val & ~DRV_FLAGS_CAPABILITIES_LOADED_L2); 2852 } 2853 2854 if (IS_PF(bp) && bp->recovery_state != BNX2X_RECOVERY_DONE && 2855 (bp->state == BNX2X_STATE_CLOSED || 2856 bp->state == BNX2X_STATE_ERROR)) { 2857 /* We can get here if the driver has been unloaded 2858 * during parity error recovery and is either waiting for a 2859 * leader to complete or for other functions to unload and 2860 * then ifdown has been issued. In this case we want to 2861 * unload and let other functions to complete a recovery 2862 * process. 2863 */ 2864 bp->recovery_state = BNX2X_RECOVERY_DONE; 2865 bp->is_leader = 0; 2866 bnx2x_release_leader_lock(bp); 2867 smp_mb(); 2868 2869 DP(NETIF_MSG_IFDOWN, "Releasing a leadership...\n"); 2870 BNX2X_ERR("Can't unload in closed or error state\n"); 2871 return -EINVAL; 2872 } 2873 2874 /* Nothing to do during unload if previous bnx2x_nic_load() 2875 * have not completed successfully - all resources are released. 2876 * 2877 * we can get here only after unsuccessful ndo_* callback, during which 2878 * dev->IFF_UP flag is still on. 2879 */ 2880 if (bp->state == BNX2X_STATE_CLOSED || bp->state == BNX2X_STATE_ERROR) 2881 return 0; 2882 2883 /* It's important to set the bp->state to the value different from 2884 * BNX2X_STATE_OPEN and only then stop the Tx. Otherwise bnx2x_tx_int() 2885 * may restart the Tx from the NAPI context (see bnx2x_tx_int()). 2886 */ 2887 bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT; 2888 smp_mb(); 2889 2890 /* indicate to VFs that the PF is going down */ 2891 bnx2x_iov_channel_down(bp); 2892 2893 if (CNIC_LOADED(bp)) 2894 bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD); 2895 2896 /* Stop Tx */ 2897 bnx2x_tx_disable(bp); 2898 netdev_reset_tc(bp->dev); 2899 2900 bp->rx_mode = BNX2X_RX_MODE_NONE; 2901 2902 del_timer_sync(&bp->timer); 2903 2904 if (IS_PF(bp)) { 2905 /* Set ALWAYS_ALIVE bit in shmem */ 2906 bp->fw_drv_pulse_wr_seq |= DRV_PULSE_ALWAYS_ALIVE; 2907 bnx2x_drv_pulse(bp); 2908 bnx2x_stats_handle(bp, STATS_EVENT_STOP); 2909 bnx2x_save_statistics(bp); 2910 } 2911 2912 /* wait till consumers catch up with producers in all queues */ 2913 bnx2x_drain_tx_queues(bp); 2914 2915 /* if VF indicate to PF this function is going down (PF will delete sp 2916 * elements and clear initializations 2917 */ 2918 if (IS_VF(bp)) 2919 bnx2x_vfpf_close_vf(bp); 2920 else if (unload_mode != UNLOAD_RECOVERY) 2921 /* if this is a normal/close unload need to clean up chip*/ 2922 bnx2x_chip_cleanup(bp, unload_mode, keep_link); 2923 else { 2924 /* Send the UNLOAD_REQUEST to the MCP */ 2925 bnx2x_send_unload_req(bp, unload_mode); 2926 2927 /* Prevent transactions to host from the functions on the 2928 * engine that doesn't reset global blocks in case of global 2929 * attention once global blocks are reset and gates are opened 2930 * (the engine which leader will perform the recovery 2931 * last). 2932 */ 2933 if (!CHIP_IS_E1x(bp)) 2934 bnx2x_pf_disable(bp); 2935 2936 /* Disable HW interrupts, NAPI */ 2937 bnx2x_netif_stop(bp, 1); 2938 /* Delete all NAPI objects */ 2939 bnx2x_del_all_napi(bp); 2940 if (CNIC_LOADED(bp)) 2941 bnx2x_del_all_napi_cnic(bp); 2942 /* Release IRQs */ 2943 bnx2x_free_irq(bp); 2944 2945 /* Report UNLOAD_DONE to MCP */ 2946 bnx2x_send_unload_done(bp, false); 2947 } 2948 2949 /* 2950 * At this stage no more interrupts will arrive so we may safely clean 2951 * the queueable objects here in case they failed to get cleaned so far. 2952 */ 2953 if (IS_PF(bp)) 2954 bnx2x_squeeze_objects(bp); 2955 2956 /* There should be no more pending SP commands at this stage */ 2957 bp->sp_state = 0; 2958 2959 bp->port.pmf = 0; 2960 2961 /* Free SKBs, SGEs, TPA pool and driver internals */ 2962 bnx2x_free_skbs(bp); 2963 if (CNIC_LOADED(bp)) 2964 bnx2x_free_skbs_cnic(bp); 2965 for_each_rx_queue(bp, i) 2966 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE); 2967 2968 bnx2x_free_fp_mem(bp); 2969 if (CNIC_LOADED(bp)) 2970 bnx2x_free_fp_mem_cnic(bp); 2971 2972 if (IS_PF(bp)) { 2973 if (CNIC_LOADED(bp)) 2974 bnx2x_free_mem_cnic(bp); 2975 } 2976 bnx2x_free_mem(bp); 2977 2978 bp->state = BNX2X_STATE_CLOSED; 2979 bp->cnic_loaded = false; 2980 2981 /* Check if there are pending parity attentions. If there are - set 2982 * RECOVERY_IN_PROGRESS. 2983 */ 2984 if (IS_PF(bp) && bnx2x_chk_parity_attn(bp, &global, false)) { 2985 bnx2x_set_reset_in_progress(bp); 2986 2987 /* Set RESET_IS_GLOBAL if needed */ 2988 if (global) 2989 bnx2x_set_reset_global(bp); 2990 } 2991 2992 /* The last driver must disable a "close the gate" if there is no 2993 * parity attention or "process kill" pending. 2994 */ 2995 if (IS_PF(bp) && 2996 !bnx2x_clear_pf_load(bp) && 2997 bnx2x_reset_is_done(bp, BP_PATH(bp))) 2998 bnx2x_disable_close_the_gate(bp); 2999 3000 DP(NETIF_MSG_IFUP, "Ending NIC unload\n"); 3001 3002 return 0; 3003 } 3004 3005 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state) 3006 { 3007 u16 pmcsr; 3008 3009 /* If there is no power capability, silently succeed */ 3010 if (!bp->pdev->pm_cap) { 3011 BNX2X_DEV_INFO("No power capability. Breaking.\n"); 3012 return 0; 3013 } 3014 3015 pci_read_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL, &pmcsr); 3016 3017 switch (state) { 3018 case PCI_D0: 3019 pci_write_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL, 3020 ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) | 3021 PCI_PM_CTRL_PME_STATUS)); 3022 3023 if (pmcsr & PCI_PM_CTRL_STATE_MASK) 3024 /* delay required during transition out of D3hot */ 3025 msleep(20); 3026 break; 3027 3028 case PCI_D3hot: 3029 /* If there are other clients above don't 3030 shut down the power */ 3031 if (atomic_read(&bp->pdev->enable_cnt) != 1) 3032 return 0; 3033 /* Don't shut down the power for emulation and FPGA */ 3034 if (CHIP_REV_IS_SLOW(bp)) 3035 return 0; 3036 3037 pmcsr &= ~PCI_PM_CTRL_STATE_MASK; 3038 pmcsr |= 3; 3039 3040 if (bp->wol) 3041 pmcsr |= PCI_PM_CTRL_PME_ENABLE; 3042 3043 pci_write_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL, 3044 pmcsr); 3045 3046 /* No more memory access after this point until 3047 * device is brought back to D0. 3048 */ 3049 break; 3050 3051 default: 3052 dev_err(&bp->pdev->dev, "Can't support state = %d\n", state); 3053 return -EINVAL; 3054 } 3055 return 0; 3056 } 3057 3058 /* 3059 * net_device service functions 3060 */ 3061 int bnx2x_poll(struct napi_struct *napi, int budget) 3062 { 3063 int work_done = 0; 3064 u8 cos; 3065 struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath, 3066 napi); 3067 struct bnx2x *bp = fp->bp; 3068 3069 while (1) { 3070 #ifdef BNX2X_STOP_ON_ERROR 3071 if (unlikely(bp->panic)) { 3072 napi_complete(napi); 3073 return 0; 3074 } 3075 #endif 3076 if (!bnx2x_fp_lock_napi(fp)) 3077 return work_done; 3078 3079 for_each_cos_in_tx_queue(fp, cos) 3080 if (bnx2x_tx_queue_has_work(fp->txdata_ptr[cos])) 3081 bnx2x_tx_int(bp, fp->txdata_ptr[cos]); 3082 3083 if (bnx2x_has_rx_work(fp)) { 3084 work_done += bnx2x_rx_int(fp, budget - work_done); 3085 3086 /* must not complete if we consumed full budget */ 3087 if (work_done >= budget) { 3088 bnx2x_fp_unlock_napi(fp); 3089 break; 3090 } 3091 } 3092 3093 /* Fall out from the NAPI loop if needed */ 3094 if (!bnx2x_fp_unlock_napi(fp) && 3095 !(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) { 3096 3097 /* No need to update SB for FCoE L2 ring as long as 3098 * it's connected to the default SB and the SB 3099 * has been updated when NAPI was scheduled. 3100 */ 3101 if (IS_FCOE_FP(fp)) { 3102 napi_complete(napi); 3103 break; 3104 } 3105 bnx2x_update_fpsb_idx(fp); 3106 /* bnx2x_has_rx_work() reads the status block, 3107 * thus we need to ensure that status block indices 3108 * have been actually read (bnx2x_update_fpsb_idx) 3109 * prior to this check (bnx2x_has_rx_work) so that 3110 * we won't write the "newer" value of the status block 3111 * to IGU (if there was a DMA right after 3112 * bnx2x_has_rx_work and if there is no rmb, the memory 3113 * reading (bnx2x_update_fpsb_idx) may be postponed 3114 * to right before bnx2x_ack_sb). In this case there 3115 * will never be another interrupt until there is 3116 * another update of the status block, while there 3117 * is still unhandled work. 3118 */ 3119 rmb(); 3120 3121 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) { 3122 napi_complete(napi); 3123 /* Re-enable interrupts */ 3124 DP(NETIF_MSG_RX_STATUS, 3125 "Update index to %d\n", fp->fp_hc_idx); 3126 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 3127 le16_to_cpu(fp->fp_hc_idx), 3128 IGU_INT_ENABLE, 1); 3129 break; 3130 } 3131 } 3132 } 3133 3134 return work_done; 3135 } 3136 3137 #ifdef CONFIG_NET_RX_BUSY_POLL 3138 /* must be called with local_bh_disable()d */ 3139 int bnx2x_low_latency_recv(struct napi_struct *napi) 3140 { 3141 struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath, 3142 napi); 3143 struct bnx2x *bp = fp->bp; 3144 int found = 0; 3145 3146 if ((bp->state == BNX2X_STATE_CLOSED) || 3147 (bp->state == BNX2X_STATE_ERROR) || 3148 (bp->flags & (TPA_ENABLE_FLAG | GRO_ENABLE_FLAG))) 3149 return LL_FLUSH_FAILED; 3150 3151 if (!bnx2x_fp_lock_poll(fp)) 3152 return LL_FLUSH_BUSY; 3153 3154 if (bnx2x_has_rx_work(fp)) 3155 found = bnx2x_rx_int(fp, 4); 3156 3157 bnx2x_fp_unlock_poll(fp); 3158 3159 return found; 3160 } 3161 #endif 3162 3163 /* we split the first BD into headers and data BDs 3164 * to ease the pain of our fellow microcode engineers 3165 * we use one mapping for both BDs 3166 */ 3167 static u16 bnx2x_tx_split(struct bnx2x *bp, 3168 struct bnx2x_fp_txdata *txdata, 3169 struct sw_tx_bd *tx_buf, 3170 struct eth_tx_start_bd **tx_bd, u16 hlen, 3171 u16 bd_prod) 3172 { 3173 struct eth_tx_start_bd *h_tx_bd = *tx_bd; 3174 struct eth_tx_bd *d_tx_bd; 3175 dma_addr_t mapping; 3176 int old_len = le16_to_cpu(h_tx_bd->nbytes); 3177 3178 /* first fix first BD */ 3179 h_tx_bd->nbytes = cpu_to_le16(hlen); 3180 3181 DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d (%x:%x)\n", 3182 h_tx_bd->nbytes, h_tx_bd->addr_hi, h_tx_bd->addr_lo); 3183 3184 /* now get a new data BD 3185 * (after the pbd) and fill it */ 3186 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod)); 3187 d_tx_bd = &txdata->tx_desc_ring[bd_prod].reg_bd; 3188 3189 mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi), 3190 le32_to_cpu(h_tx_bd->addr_lo)) + hlen; 3191 3192 d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping)); 3193 d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping)); 3194 d_tx_bd->nbytes = cpu_to_le16(old_len - hlen); 3195 3196 /* this marks the BD as one that has no individual mapping */ 3197 tx_buf->flags |= BNX2X_TSO_SPLIT_BD; 3198 3199 DP(NETIF_MSG_TX_QUEUED, 3200 "TSO split data size is %d (%x:%x)\n", 3201 d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo); 3202 3203 /* update tx_bd */ 3204 *tx_bd = (struct eth_tx_start_bd *)d_tx_bd; 3205 3206 return bd_prod; 3207 } 3208 3209 #define bswab32(b32) ((__force __le32) swab32((__force __u32) (b32))) 3210 #define bswab16(b16) ((__force __le16) swab16((__force __u16) (b16))) 3211 static __le16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix) 3212 { 3213 __sum16 tsum = (__force __sum16) csum; 3214 3215 if (fix > 0) 3216 tsum = ~csum_fold(csum_sub((__force __wsum) csum, 3217 csum_partial(t_header - fix, fix, 0))); 3218 3219 else if (fix < 0) 3220 tsum = ~csum_fold(csum_add((__force __wsum) csum, 3221 csum_partial(t_header, -fix, 0))); 3222 3223 return bswab16(tsum); 3224 } 3225 3226 static u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb) 3227 { 3228 u32 rc; 3229 __u8 prot = 0; 3230 __be16 protocol; 3231 3232 if (skb->ip_summed != CHECKSUM_PARTIAL) 3233 return XMIT_PLAIN; 3234 3235 protocol = vlan_get_protocol(skb); 3236 if (protocol == htons(ETH_P_IPV6)) { 3237 rc = XMIT_CSUM_V6; 3238 prot = ipv6_hdr(skb)->nexthdr; 3239 } else { 3240 rc = XMIT_CSUM_V4; 3241 prot = ip_hdr(skb)->protocol; 3242 } 3243 3244 if (!CHIP_IS_E1x(bp) && skb->encapsulation) { 3245 if (inner_ip_hdr(skb)->version == 6) { 3246 rc |= XMIT_CSUM_ENC_V6; 3247 if (inner_ipv6_hdr(skb)->nexthdr == IPPROTO_TCP) 3248 rc |= XMIT_CSUM_TCP; 3249 } else { 3250 rc |= XMIT_CSUM_ENC_V4; 3251 if (inner_ip_hdr(skb)->protocol == IPPROTO_TCP) 3252 rc |= XMIT_CSUM_TCP; 3253 } 3254 } 3255 if (prot == IPPROTO_TCP) 3256 rc |= XMIT_CSUM_TCP; 3257 3258 if (skb_is_gso_v6(skb)) { 3259 rc |= (XMIT_GSO_V6 | XMIT_CSUM_TCP); 3260 if (rc & XMIT_CSUM_ENC) 3261 rc |= XMIT_GSO_ENC_V6; 3262 } else if (skb_is_gso(skb)) { 3263 rc |= (XMIT_GSO_V4 | XMIT_CSUM_TCP); 3264 if (rc & XMIT_CSUM_ENC) 3265 rc |= XMIT_GSO_ENC_V4; 3266 } 3267 3268 return rc; 3269 } 3270 3271 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3) 3272 /* check if packet requires linearization (packet is too fragmented) 3273 no need to check fragmentation if page size > 8K (there will be no 3274 violation to FW restrictions) */ 3275 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb, 3276 u32 xmit_type) 3277 { 3278 int to_copy = 0; 3279 int hlen = 0; 3280 int first_bd_sz = 0; 3281 3282 /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */ 3283 if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) { 3284 3285 if (xmit_type & XMIT_GSO) { 3286 unsigned short lso_mss = skb_shinfo(skb)->gso_size; 3287 /* Check if LSO packet needs to be copied: 3288 3 = 1 (for headers BD) + 2 (for PBD and last BD) */ 3289 int wnd_size = MAX_FETCH_BD - 3; 3290 /* Number of windows to check */ 3291 int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size; 3292 int wnd_idx = 0; 3293 int frag_idx = 0; 3294 u32 wnd_sum = 0; 3295 3296 /* Headers length */ 3297 hlen = (int)(skb_transport_header(skb) - skb->data) + 3298 tcp_hdrlen(skb); 3299 3300 /* Amount of data (w/o headers) on linear part of SKB*/ 3301 first_bd_sz = skb_headlen(skb) - hlen; 3302 3303 wnd_sum = first_bd_sz; 3304 3305 /* Calculate the first sum - it's special */ 3306 for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++) 3307 wnd_sum += 3308 skb_frag_size(&skb_shinfo(skb)->frags[frag_idx]); 3309 3310 /* If there was data on linear skb data - check it */ 3311 if (first_bd_sz > 0) { 3312 if (unlikely(wnd_sum < lso_mss)) { 3313 to_copy = 1; 3314 goto exit_lbl; 3315 } 3316 3317 wnd_sum -= first_bd_sz; 3318 } 3319 3320 /* Others are easier: run through the frag list and 3321 check all windows */ 3322 for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) { 3323 wnd_sum += 3324 skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1]); 3325 3326 if (unlikely(wnd_sum < lso_mss)) { 3327 to_copy = 1; 3328 break; 3329 } 3330 wnd_sum -= 3331 skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx]); 3332 } 3333 } else { 3334 /* in non-LSO too fragmented packet should always 3335 be linearized */ 3336 to_copy = 1; 3337 } 3338 } 3339 3340 exit_lbl: 3341 if (unlikely(to_copy)) 3342 DP(NETIF_MSG_TX_QUEUED, 3343 "Linearization IS REQUIRED for %s packet. num_frags %d hlen %d first_bd_sz %d\n", 3344 (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO", 3345 skb_shinfo(skb)->nr_frags, hlen, first_bd_sz); 3346 3347 return to_copy; 3348 } 3349 #endif 3350 3351 static void bnx2x_set_pbd_gso_e2(struct sk_buff *skb, u32 *parsing_data, 3352 u32 xmit_type) 3353 { 3354 struct ipv6hdr *ipv6; 3355 3356 *parsing_data |= (skb_shinfo(skb)->gso_size << 3357 ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT) & 3358 ETH_TX_PARSE_BD_E2_LSO_MSS; 3359 3360 if (xmit_type & XMIT_GSO_ENC_V6) 3361 ipv6 = inner_ipv6_hdr(skb); 3362 else if (xmit_type & XMIT_GSO_V6) 3363 ipv6 = ipv6_hdr(skb); 3364 else 3365 ipv6 = NULL; 3366 3367 if (ipv6 && ipv6->nexthdr == NEXTHDR_IPV6) 3368 *parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR; 3369 } 3370 3371 /** 3372 * bnx2x_set_pbd_gso - update PBD in GSO case. 3373 * 3374 * @skb: packet skb 3375 * @pbd: parse BD 3376 * @xmit_type: xmit flags 3377 */ 3378 static void bnx2x_set_pbd_gso(struct sk_buff *skb, 3379 struct eth_tx_parse_bd_e1x *pbd, 3380 struct eth_tx_start_bd *tx_start_bd, 3381 u32 xmit_type) 3382 { 3383 pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size); 3384 pbd->tcp_send_seq = bswab32(tcp_hdr(skb)->seq); 3385 pbd->tcp_flags = pbd_tcp_flags(tcp_hdr(skb)); 3386 3387 if (xmit_type & XMIT_GSO_V4) { 3388 pbd->ip_id = bswab16(ip_hdr(skb)->id); 3389 pbd->tcp_pseudo_csum = 3390 bswab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr, 3391 ip_hdr(skb)->daddr, 3392 0, IPPROTO_TCP, 0)); 3393 3394 /* GSO on 57710/57711 needs FW to calculate IP checksum */ 3395 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IP_CSUM; 3396 } else { 3397 pbd->tcp_pseudo_csum = 3398 bswab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 3399 &ipv6_hdr(skb)->daddr, 3400 0, IPPROTO_TCP, 0)); 3401 } 3402 3403 pbd->global_data |= 3404 cpu_to_le16(ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN); 3405 } 3406 3407 /** 3408 * bnx2x_set_pbd_csum_enc - update PBD with checksum and return header length 3409 * 3410 * @bp: driver handle 3411 * @skb: packet skb 3412 * @parsing_data: data to be updated 3413 * @xmit_type: xmit flags 3414 * 3415 * 57712/578xx related, when skb has encapsulation 3416 */ 3417 static u8 bnx2x_set_pbd_csum_enc(struct bnx2x *bp, struct sk_buff *skb, 3418 u32 *parsing_data, u32 xmit_type) 3419 { 3420 *parsing_data |= 3421 ((((u8 *)skb_inner_transport_header(skb) - skb->data) >> 1) << 3422 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) & 3423 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W; 3424 3425 if (xmit_type & XMIT_CSUM_TCP) { 3426 *parsing_data |= ((inner_tcp_hdrlen(skb) / 4) << 3427 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) & 3428 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW; 3429 3430 return skb_inner_transport_header(skb) + 3431 inner_tcp_hdrlen(skb) - skb->data; 3432 } 3433 3434 /* We support checksum offload for TCP and UDP only. 3435 * No need to pass the UDP header length - it's a constant. 3436 */ 3437 return skb_inner_transport_header(skb) + 3438 sizeof(struct udphdr) - skb->data; 3439 } 3440 3441 /** 3442 * bnx2x_set_pbd_csum_e2 - update PBD with checksum and return header length 3443 * 3444 * @bp: driver handle 3445 * @skb: packet skb 3446 * @parsing_data: data to be updated 3447 * @xmit_type: xmit flags 3448 * 3449 * 57712/578xx related 3450 */ 3451 static u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb, 3452 u32 *parsing_data, u32 xmit_type) 3453 { 3454 *parsing_data |= 3455 ((((u8 *)skb_transport_header(skb) - skb->data) >> 1) << 3456 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) & 3457 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W; 3458 3459 if (xmit_type & XMIT_CSUM_TCP) { 3460 *parsing_data |= ((tcp_hdrlen(skb) / 4) << 3461 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) & 3462 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW; 3463 3464 return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data; 3465 } 3466 /* We support checksum offload for TCP and UDP only. 3467 * No need to pass the UDP header length - it's a constant. 3468 */ 3469 return skb_transport_header(skb) + sizeof(struct udphdr) - skb->data; 3470 } 3471 3472 /* set FW indication according to inner or outer protocols if tunneled */ 3473 static void bnx2x_set_sbd_csum(struct bnx2x *bp, struct sk_buff *skb, 3474 struct eth_tx_start_bd *tx_start_bd, 3475 u32 xmit_type) 3476 { 3477 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM; 3478 3479 if (xmit_type & (XMIT_CSUM_ENC_V6 | XMIT_CSUM_V6)) 3480 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IPV6; 3481 3482 if (!(xmit_type & XMIT_CSUM_TCP)) 3483 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IS_UDP; 3484 } 3485 3486 /** 3487 * bnx2x_set_pbd_csum - update PBD with checksum and return header length 3488 * 3489 * @bp: driver handle 3490 * @skb: packet skb 3491 * @pbd: parse BD to be updated 3492 * @xmit_type: xmit flags 3493 */ 3494 static u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb, 3495 struct eth_tx_parse_bd_e1x *pbd, 3496 u32 xmit_type) 3497 { 3498 u8 hlen = (skb_network_header(skb) - skb->data) >> 1; 3499 3500 /* for now NS flag is not used in Linux */ 3501 pbd->global_data = 3502 cpu_to_le16(hlen | 3503 ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) << 3504 ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT)); 3505 3506 pbd->ip_hlen_w = (skb_transport_header(skb) - 3507 skb_network_header(skb)) >> 1; 3508 3509 hlen += pbd->ip_hlen_w; 3510 3511 /* We support checksum offload for TCP and UDP only */ 3512 if (xmit_type & XMIT_CSUM_TCP) 3513 hlen += tcp_hdrlen(skb) / 2; 3514 else 3515 hlen += sizeof(struct udphdr) / 2; 3516 3517 pbd->total_hlen_w = cpu_to_le16(hlen); 3518 hlen = hlen*2; 3519 3520 if (xmit_type & XMIT_CSUM_TCP) { 3521 pbd->tcp_pseudo_csum = bswab16(tcp_hdr(skb)->check); 3522 3523 } else { 3524 s8 fix = SKB_CS_OFF(skb); /* signed! */ 3525 3526 DP(NETIF_MSG_TX_QUEUED, 3527 "hlen %d fix %d csum before fix %x\n", 3528 le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb)); 3529 3530 /* HW bug: fixup the CSUM */ 3531 pbd->tcp_pseudo_csum = 3532 bnx2x_csum_fix(skb_transport_header(skb), 3533 SKB_CS(skb), fix); 3534 3535 DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n", 3536 pbd->tcp_pseudo_csum); 3537 } 3538 3539 return hlen; 3540 } 3541 3542 static void bnx2x_update_pbds_gso_enc(struct sk_buff *skb, 3543 struct eth_tx_parse_bd_e2 *pbd_e2, 3544 struct eth_tx_parse_2nd_bd *pbd2, 3545 u16 *global_data, 3546 u32 xmit_type) 3547 { 3548 u16 hlen_w = 0; 3549 u8 outerip_off, outerip_len = 0; 3550 3551 /* from outer IP to transport */ 3552 hlen_w = (skb_inner_transport_header(skb) - 3553 skb_network_header(skb)) >> 1; 3554 3555 /* transport len */ 3556 hlen_w += inner_tcp_hdrlen(skb) >> 1; 3557 3558 pbd2->fw_ip_hdr_to_payload_w = hlen_w; 3559 3560 /* outer IP header info */ 3561 if (xmit_type & XMIT_CSUM_V4) { 3562 struct iphdr *iph = ip_hdr(skb); 3563 u32 csum = (__force u32)(~iph->check) - 3564 (__force u32)iph->tot_len - 3565 (__force u32)iph->frag_off; 3566 3567 pbd2->fw_ip_csum_wo_len_flags_frag = 3568 bswab16(csum_fold((__force __wsum)csum)); 3569 } else { 3570 pbd2->fw_ip_hdr_to_payload_w = 3571 hlen_w - ((sizeof(struct ipv6hdr)) >> 1); 3572 } 3573 3574 pbd2->tcp_send_seq = bswab32(inner_tcp_hdr(skb)->seq); 3575 3576 pbd2->tcp_flags = pbd_tcp_flags(inner_tcp_hdr(skb)); 3577 3578 if (xmit_type & XMIT_GSO_V4) { 3579 pbd2->hw_ip_id = bswab16(inner_ip_hdr(skb)->id); 3580 3581 pbd_e2->data.tunnel_data.pseudo_csum = 3582 bswab16(~csum_tcpudp_magic( 3583 inner_ip_hdr(skb)->saddr, 3584 inner_ip_hdr(skb)->daddr, 3585 0, IPPROTO_TCP, 0)); 3586 3587 outerip_len = ip_hdr(skb)->ihl << 1; 3588 } else { 3589 pbd_e2->data.tunnel_data.pseudo_csum = 3590 bswab16(~csum_ipv6_magic( 3591 &inner_ipv6_hdr(skb)->saddr, 3592 &inner_ipv6_hdr(skb)->daddr, 3593 0, IPPROTO_TCP, 0)); 3594 } 3595 3596 outerip_off = (skb_network_header(skb) - skb->data) >> 1; 3597 3598 *global_data |= 3599 outerip_off | 3600 (!!(xmit_type & XMIT_CSUM_V6) << 3601 ETH_TX_PARSE_2ND_BD_IP_HDR_TYPE_OUTER_SHIFT) | 3602 (outerip_len << 3603 ETH_TX_PARSE_2ND_BD_IP_HDR_LEN_OUTER_W_SHIFT) | 3604 ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) << 3605 ETH_TX_PARSE_2ND_BD_LLC_SNAP_EN_SHIFT); 3606 3607 if (ip_hdr(skb)->protocol == IPPROTO_UDP) { 3608 SET_FLAG(*global_data, ETH_TX_PARSE_2ND_BD_TUNNEL_UDP_EXIST, 1); 3609 pbd2->tunnel_udp_hdr_start_w = skb_transport_offset(skb) >> 1; 3610 } 3611 } 3612 3613 /* called with netif_tx_lock 3614 * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call 3615 * netif_wake_queue() 3616 */ 3617 netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev) 3618 { 3619 struct bnx2x *bp = netdev_priv(dev); 3620 3621 struct netdev_queue *txq; 3622 struct bnx2x_fp_txdata *txdata; 3623 struct sw_tx_bd *tx_buf; 3624 struct eth_tx_start_bd *tx_start_bd, *first_bd; 3625 struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL; 3626 struct eth_tx_parse_bd_e1x *pbd_e1x = NULL; 3627 struct eth_tx_parse_bd_e2 *pbd_e2 = NULL; 3628 struct eth_tx_parse_2nd_bd *pbd2 = NULL; 3629 u32 pbd_e2_parsing_data = 0; 3630 u16 pkt_prod, bd_prod; 3631 int nbd, txq_index; 3632 dma_addr_t mapping; 3633 u32 xmit_type = bnx2x_xmit_type(bp, skb); 3634 int i; 3635 u8 hlen = 0; 3636 __le16 pkt_size = 0; 3637 struct ethhdr *eth; 3638 u8 mac_type = UNICAST_ADDRESS; 3639 3640 #ifdef BNX2X_STOP_ON_ERROR 3641 if (unlikely(bp->panic)) 3642 return NETDEV_TX_BUSY; 3643 #endif 3644 3645 txq_index = skb_get_queue_mapping(skb); 3646 txq = netdev_get_tx_queue(dev, txq_index); 3647 3648 BUG_ON(txq_index >= MAX_ETH_TXQ_IDX(bp) + (CNIC_LOADED(bp) ? 1 : 0)); 3649 3650 txdata = &bp->bnx2x_txq[txq_index]; 3651 3652 /* enable this debug print to view the transmission queue being used 3653 DP(NETIF_MSG_TX_QUEUED, "indices: txq %d, fp %d, txdata %d\n", 3654 txq_index, fp_index, txdata_index); */ 3655 3656 /* enable this debug print to view the transmission details 3657 DP(NETIF_MSG_TX_QUEUED, 3658 "transmitting packet cid %d fp index %d txdata_index %d tx_data ptr %p fp pointer %p\n", 3659 txdata->cid, fp_index, txdata_index, txdata, fp); */ 3660 3661 if (unlikely(bnx2x_tx_avail(bp, txdata) < 3662 skb_shinfo(skb)->nr_frags + 3663 BDS_PER_TX_PKT + 3664 NEXT_CNT_PER_TX_PKT(MAX_BDS_PER_TX_PKT))) { 3665 /* Handle special storage cases separately */ 3666 if (txdata->tx_ring_size == 0) { 3667 struct bnx2x_eth_q_stats *q_stats = 3668 bnx2x_fp_qstats(bp, txdata->parent_fp); 3669 q_stats->driver_filtered_tx_pkt++; 3670 dev_kfree_skb(skb); 3671 return NETDEV_TX_OK; 3672 } 3673 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++; 3674 netif_tx_stop_queue(txq); 3675 BNX2X_ERR("BUG! Tx ring full when queue awake!\n"); 3676 3677 return NETDEV_TX_BUSY; 3678 } 3679 3680 DP(NETIF_MSG_TX_QUEUED, 3681 "queue[%d]: SKB: summed %x protocol %x protocol(%x,%x) gso type %x xmit_type %x len %d\n", 3682 txq_index, skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr, 3683 ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type, 3684 skb->len); 3685 3686 eth = (struct ethhdr *)skb->data; 3687 3688 /* set flag according to packet type (UNICAST_ADDRESS is default)*/ 3689 if (unlikely(is_multicast_ether_addr(eth->h_dest))) { 3690 if (is_broadcast_ether_addr(eth->h_dest)) 3691 mac_type = BROADCAST_ADDRESS; 3692 else 3693 mac_type = MULTICAST_ADDRESS; 3694 } 3695 3696 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - BDS_PER_TX_PKT) 3697 /* First, check if we need to linearize the skb (due to FW 3698 restrictions). No need to check fragmentation if page size > 8K 3699 (there will be no violation to FW restrictions) */ 3700 if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) { 3701 /* Statistics of linearization */ 3702 bp->lin_cnt++; 3703 if (skb_linearize(skb) != 0) { 3704 DP(NETIF_MSG_TX_QUEUED, 3705 "SKB linearization failed - silently dropping this SKB\n"); 3706 dev_kfree_skb_any(skb); 3707 return NETDEV_TX_OK; 3708 } 3709 } 3710 #endif 3711 /* Map skb linear data for DMA */ 3712 mapping = dma_map_single(&bp->pdev->dev, skb->data, 3713 skb_headlen(skb), DMA_TO_DEVICE); 3714 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) { 3715 DP(NETIF_MSG_TX_QUEUED, 3716 "SKB mapping failed - silently dropping this SKB\n"); 3717 dev_kfree_skb_any(skb); 3718 return NETDEV_TX_OK; 3719 } 3720 /* 3721 Please read carefully. First we use one BD which we mark as start, 3722 then we have a parsing info BD (used for TSO or xsum), 3723 and only then we have the rest of the TSO BDs. 3724 (don't forget to mark the last one as last, 3725 and to unmap only AFTER you write to the BD ...) 3726 And above all, all pdb sizes are in words - NOT DWORDS! 3727 */ 3728 3729 /* get current pkt produced now - advance it just before sending packet 3730 * since mapping of pages may fail and cause packet to be dropped 3731 */ 3732 pkt_prod = txdata->tx_pkt_prod; 3733 bd_prod = TX_BD(txdata->tx_bd_prod); 3734 3735 /* get a tx_buf and first BD 3736 * tx_start_bd may be changed during SPLIT, 3737 * but first_bd will always stay first 3738 */ 3739 tx_buf = &txdata->tx_buf_ring[TX_BD(pkt_prod)]; 3740 tx_start_bd = &txdata->tx_desc_ring[bd_prod].start_bd; 3741 first_bd = tx_start_bd; 3742 3743 tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD; 3744 3745 /* header nbd: indirectly zero other flags! */ 3746 tx_start_bd->general_data = 1 << ETH_TX_START_BD_HDR_NBDS_SHIFT; 3747 3748 /* remember the first BD of the packet */ 3749 tx_buf->first_bd = txdata->tx_bd_prod; 3750 tx_buf->skb = skb; 3751 tx_buf->flags = 0; 3752 3753 DP(NETIF_MSG_TX_QUEUED, 3754 "sending pkt %u @%p next_idx %u bd %u @%p\n", 3755 pkt_prod, tx_buf, txdata->tx_pkt_prod, bd_prod, tx_start_bd); 3756 3757 if (vlan_tx_tag_present(skb)) { 3758 tx_start_bd->vlan_or_ethertype = 3759 cpu_to_le16(vlan_tx_tag_get(skb)); 3760 tx_start_bd->bd_flags.as_bitfield |= 3761 (X_ETH_OUTBAND_VLAN << ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT); 3762 } else { 3763 /* when transmitting in a vf, start bd must hold the ethertype 3764 * for fw to enforce it 3765 */ 3766 if (IS_VF(bp)) 3767 tx_start_bd->vlan_or_ethertype = 3768 cpu_to_le16(ntohs(eth->h_proto)); 3769 else 3770 /* used by FW for packet accounting */ 3771 tx_start_bd->vlan_or_ethertype = cpu_to_le16(pkt_prod); 3772 } 3773 3774 nbd = 2; /* start_bd + pbd + frags (updated when pages are mapped) */ 3775 3776 /* turn on parsing and get a BD */ 3777 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod)); 3778 3779 if (xmit_type & XMIT_CSUM) 3780 bnx2x_set_sbd_csum(bp, skb, tx_start_bd, xmit_type); 3781 3782 if (!CHIP_IS_E1x(bp)) { 3783 pbd_e2 = &txdata->tx_desc_ring[bd_prod].parse_bd_e2; 3784 memset(pbd_e2, 0, sizeof(struct eth_tx_parse_bd_e2)); 3785 3786 if (xmit_type & XMIT_CSUM_ENC) { 3787 u16 global_data = 0; 3788 3789 /* Set PBD in enc checksum offload case */ 3790 hlen = bnx2x_set_pbd_csum_enc(bp, skb, 3791 &pbd_e2_parsing_data, 3792 xmit_type); 3793 3794 /* turn on 2nd parsing and get a BD */ 3795 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod)); 3796 3797 pbd2 = &txdata->tx_desc_ring[bd_prod].parse_2nd_bd; 3798 3799 memset(pbd2, 0, sizeof(*pbd2)); 3800 3801 pbd_e2->data.tunnel_data.ip_hdr_start_inner_w = 3802 (skb_inner_network_header(skb) - 3803 skb->data) >> 1; 3804 3805 if (xmit_type & XMIT_GSO_ENC) 3806 bnx2x_update_pbds_gso_enc(skb, pbd_e2, pbd2, 3807 &global_data, 3808 xmit_type); 3809 3810 pbd2->global_data = cpu_to_le16(global_data); 3811 3812 /* add addition parse BD indication to start BD */ 3813 SET_FLAG(tx_start_bd->general_data, 3814 ETH_TX_START_BD_PARSE_NBDS, 1); 3815 /* set encapsulation flag in start BD */ 3816 SET_FLAG(tx_start_bd->general_data, 3817 ETH_TX_START_BD_TUNNEL_EXIST, 1); 3818 nbd++; 3819 } else if (xmit_type & XMIT_CSUM) { 3820 /* Set PBD in checksum offload case w/o encapsulation */ 3821 hlen = bnx2x_set_pbd_csum_e2(bp, skb, 3822 &pbd_e2_parsing_data, 3823 xmit_type); 3824 } 3825 3826 /* Add the macs to the parsing BD this is a vf */ 3827 if (IS_VF(bp)) { 3828 /* override GRE parameters in BD */ 3829 bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.src_hi, 3830 &pbd_e2->data.mac_addr.src_mid, 3831 &pbd_e2->data.mac_addr.src_lo, 3832 eth->h_source); 3833 3834 bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.dst_hi, 3835 &pbd_e2->data.mac_addr.dst_mid, 3836 &pbd_e2->data.mac_addr.dst_lo, 3837 eth->h_dest); 3838 } 3839 3840 SET_FLAG(pbd_e2_parsing_data, 3841 ETH_TX_PARSE_BD_E2_ETH_ADDR_TYPE, mac_type); 3842 } else { 3843 u16 global_data = 0; 3844 pbd_e1x = &txdata->tx_desc_ring[bd_prod].parse_bd_e1x; 3845 memset(pbd_e1x, 0, sizeof(struct eth_tx_parse_bd_e1x)); 3846 /* Set PBD in checksum offload case */ 3847 if (xmit_type & XMIT_CSUM) 3848 hlen = bnx2x_set_pbd_csum(bp, skb, pbd_e1x, xmit_type); 3849 3850 SET_FLAG(global_data, 3851 ETH_TX_PARSE_BD_E1X_ETH_ADDR_TYPE, mac_type); 3852 pbd_e1x->global_data |= cpu_to_le16(global_data); 3853 } 3854 3855 /* Setup the data pointer of the first BD of the packet */ 3856 tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping)); 3857 tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping)); 3858 tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb)); 3859 pkt_size = tx_start_bd->nbytes; 3860 3861 DP(NETIF_MSG_TX_QUEUED, 3862 "first bd @%p addr (%x:%x) nbytes %d flags %x vlan %x\n", 3863 tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo, 3864 le16_to_cpu(tx_start_bd->nbytes), 3865 tx_start_bd->bd_flags.as_bitfield, 3866 le16_to_cpu(tx_start_bd->vlan_or_ethertype)); 3867 3868 if (xmit_type & XMIT_GSO) { 3869 3870 DP(NETIF_MSG_TX_QUEUED, 3871 "TSO packet len %d hlen %d total len %d tso size %d\n", 3872 skb->len, hlen, skb_headlen(skb), 3873 skb_shinfo(skb)->gso_size); 3874 3875 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO; 3876 3877 if (unlikely(skb_headlen(skb) > hlen)) { 3878 nbd++; 3879 bd_prod = bnx2x_tx_split(bp, txdata, tx_buf, 3880 &tx_start_bd, hlen, 3881 bd_prod); 3882 } 3883 if (!CHIP_IS_E1x(bp)) 3884 bnx2x_set_pbd_gso_e2(skb, &pbd_e2_parsing_data, 3885 xmit_type); 3886 else 3887 bnx2x_set_pbd_gso(skb, pbd_e1x, first_bd, xmit_type); 3888 } 3889 3890 /* Set the PBD's parsing_data field if not zero 3891 * (for the chips newer than 57711). 3892 */ 3893 if (pbd_e2_parsing_data) 3894 pbd_e2->parsing_data = cpu_to_le32(pbd_e2_parsing_data); 3895 3896 tx_data_bd = (struct eth_tx_bd *)tx_start_bd; 3897 3898 /* Handle fragmented skb */ 3899 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 3900 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 3901 3902 mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 0, 3903 skb_frag_size(frag), DMA_TO_DEVICE); 3904 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) { 3905 unsigned int pkts_compl = 0, bytes_compl = 0; 3906 3907 DP(NETIF_MSG_TX_QUEUED, 3908 "Unable to map page - dropping packet...\n"); 3909 3910 /* we need unmap all buffers already mapped 3911 * for this SKB; 3912 * first_bd->nbd need to be properly updated 3913 * before call to bnx2x_free_tx_pkt 3914 */ 3915 first_bd->nbd = cpu_to_le16(nbd); 3916 bnx2x_free_tx_pkt(bp, txdata, 3917 TX_BD(txdata->tx_pkt_prod), 3918 &pkts_compl, &bytes_compl); 3919 return NETDEV_TX_OK; 3920 } 3921 3922 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod)); 3923 tx_data_bd = &txdata->tx_desc_ring[bd_prod].reg_bd; 3924 if (total_pkt_bd == NULL) 3925 total_pkt_bd = &txdata->tx_desc_ring[bd_prod].reg_bd; 3926 3927 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping)); 3928 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping)); 3929 tx_data_bd->nbytes = cpu_to_le16(skb_frag_size(frag)); 3930 le16_add_cpu(&pkt_size, skb_frag_size(frag)); 3931 nbd++; 3932 3933 DP(NETIF_MSG_TX_QUEUED, 3934 "frag %d bd @%p addr (%x:%x) nbytes %d\n", 3935 i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo, 3936 le16_to_cpu(tx_data_bd->nbytes)); 3937 } 3938 3939 DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd); 3940 3941 /* update with actual num BDs */ 3942 first_bd->nbd = cpu_to_le16(nbd); 3943 3944 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod)); 3945 3946 /* now send a tx doorbell, counting the next BD 3947 * if the packet contains or ends with it 3948 */ 3949 if (TX_BD_POFF(bd_prod) < nbd) 3950 nbd++; 3951 3952 /* total_pkt_bytes should be set on the first data BD if 3953 * it's not an LSO packet and there is more than one 3954 * data BD. In this case pkt_size is limited by an MTU value. 3955 * However we prefer to set it for an LSO packet (while we don't 3956 * have to) in order to save some CPU cycles in a none-LSO 3957 * case, when we much more care about them. 3958 */ 3959 if (total_pkt_bd != NULL) 3960 total_pkt_bd->total_pkt_bytes = pkt_size; 3961 3962 if (pbd_e1x) 3963 DP(NETIF_MSG_TX_QUEUED, 3964 "PBD (E1X) @%p ip_data %x ip_hlen %u ip_id %u lso_mss %u tcp_flags %x xsum %x seq %u hlen %u\n", 3965 pbd_e1x, pbd_e1x->global_data, pbd_e1x->ip_hlen_w, 3966 pbd_e1x->ip_id, pbd_e1x->lso_mss, pbd_e1x->tcp_flags, 3967 pbd_e1x->tcp_pseudo_csum, pbd_e1x->tcp_send_seq, 3968 le16_to_cpu(pbd_e1x->total_hlen_w)); 3969 if (pbd_e2) 3970 DP(NETIF_MSG_TX_QUEUED, 3971 "PBD (E2) @%p dst %x %x %x src %x %x %x parsing_data %x\n", 3972 pbd_e2, 3973 pbd_e2->data.mac_addr.dst_hi, 3974 pbd_e2->data.mac_addr.dst_mid, 3975 pbd_e2->data.mac_addr.dst_lo, 3976 pbd_e2->data.mac_addr.src_hi, 3977 pbd_e2->data.mac_addr.src_mid, 3978 pbd_e2->data.mac_addr.src_lo, 3979 pbd_e2->parsing_data); 3980 DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d bd %u\n", nbd, bd_prod); 3981 3982 netdev_tx_sent_queue(txq, skb->len); 3983 3984 skb_tx_timestamp(skb); 3985 3986 txdata->tx_pkt_prod++; 3987 /* 3988 * Make sure that the BD data is updated before updating the producer 3989 * since FW might read the BD right after the producer is updated. 3990 * This is only applicable for weak-ordered memory model archs such 3991 * as IA-64. The following barrier is also mandatory since FW will 3992 * assumes packets must have BDs. 3993 */ 3994 wmb(); 3995 3996 txdata->tx_db.data.prod += nbd; 3997 barrier(); 3998 3999 DOORBELL(bp, txdata->cid, txdata->tx_db.raw); 4000 4001 mmiowb(); 4002 4003 txdata->tx_bd_prod += nbd; 4004 4005 if (unlikely(bnx2x_tx_avail(bp, txdata) < MAX_DESC_PER_TX_PKT)) { 4006 netif_tx_stop_queue(txq); 4007 4008 /* paired memory barrier is in bnx2x_tx_int(), we have to keep 4009 * ordering of set_bit() in netif_tx_stop_queue() and read of 4010 * fp->bd_tx_cons */ 4011 smp_mb(); 4012 4013 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++; 4014 if (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT) 4015 netif_tx_wake_queue(txq); 4016 } 4017 txdata->tx_pkt++; 4018 4019 return NETDEV_TX_OK; 4020 } 4021 4022 /** 4023 * bnx2x_setup_tc - routine to configure net_device for multi tc 4024 * 4025 * @netdev: net device to configure 4026 * @tc: number of traffic classes to enable 4027 * 4028 * callback connected to the ndo_setup_tc function pointer 4029 */ 4030 int bnx2x_setup_tc(struct net_device *dev, u8 num_tc) 4031 { 4032 int cos, prio, count, offset; 4033 struct bnx2x *bp = netdev_priv(dev); 4034 4035 /* setup tc must be called under rtnl lock */ 4036 ASSERT_RTNL(); 4037 4038 /* no traffic classes requested. Aborting */ 4039 if (!num_tc) { 4040 netdev_reset_tc(dev); 4041 return 0; 4042 } 4043 4044 /* requested to support too many traffic classes */ 4045 if (num_tc > bp->max_cos) { 4046 BNX2X_ERR("support for too many traffic classes requested: %d. Max supported is %d\n", 4047 num_tc, bp->max_cos); 4048 return -EINVAL; 4049 } 4050 4051 /* declare amount of supported traffic classes */ 4052 if (netdev_set_num_tc(dev, num_tc)) { 4053 BNX2X_ERR("failed to declare %d traffic classes\n", num_tc); 4054 return -EINVAL; 4055 } 4056 4057 /* configure priority to traffic class mapping */ 4058 for (prio = 0; prio < BNX2X_MAX_PRIORITY; prio++) { 4059 netdev_set_prio_tc_map(dev, prio, bp->prio_to_cos[prio]); 4060 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP, 4061 "mapping priority %d to tc %d\n", 4062 prio, bp->prio_to_cos[prio]); 4063 } 4064 4065 /* Use this configuration to differentiate tc0 from other COSes 4066 This can be used for ets or pfc, and save the effort of setting 4067 up a multio class queue disc or negotiating DCBX with a switch 4068 netdev_set_prio_tc_map(dev, 0, 0); 4069 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", 0, 0); 4070 for (prio = 1; prio < 16; prio++) { 4071 netdev_set_prio_tc_map(dev, prio, 1); 4072 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", prio, 1); 4073 } */ 4074 4075 /* configure traffic class to transmission queue mapping */ 4076 for (cos = 0; cos < bp->max_cos; cos++) { 4077 count = BNX2X_NUM_ETH_QUEUES(bp); 4078 offset = cos * BNX2X_NUM_NON_CNIC_QUEUES(bp); 4079 netdev_set_tc_queue(dev, cos, count, offset); 4080 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP, 4081 "mapping tc %d to offset %d count %d\n", 4082 cos, offset, count); 4083 } 4084 4085 return 0; 4086 } 4087 4088 /* called with rtnl_lock */ 4089 int bnx2x_change_mac_addr(struct net_device *dev, void *p) 4090 { 4091 struct sockaddr *addr = p; 4092 struct bnx2x *bp = netdev_priv(dev); 4093 int rc = 0; 4094 4095 if (!bnx2x_is_valid_ether_addr(bp, addr->sa_data)) { 4096 BNX2X_ERR("Requested MAC address is not valid\n"); 4097 return -EINVAL; 4098 } 4099 4100 if ((IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp)) && 4101 !is_zero_ether_addr(addr->sa_data)) { 4102 BNX2X_ERR("Can't configure non-zero address on iSCSI or FCoE functions in MF-SD mode\n"); 4103 return -EINVAL; 4104 } 4105 4106 if (netif_running(dev)) { 4107 rc = bnx2x_set_eth_mac(bp, false); 4108 if (rc) 4109 return rc; 4110 } 4111 4112 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len); 4113 4114 if (netif_running(dev)) 4115 rc = bnx2x_set_eth_mac(bp, true); 4116 4117 return rc; 4118 } 4119 4120 static void bnx2x_free_fp_mem_at(struct bnx2x *bp, int fp_index) 4121 { 4122 union host_hc_status_block *sb = &bnx2x_fp(bp, fp_index, status_blk); 4123 struct bnx2x_fastpath *fp = &bp->fp[fp_index]; 4124 u8 cos; 4125 4126 /* Common */ 4127 4128 if (IS_FCOE_IDX(fp_index)) { 4129 memset(sb, 0, sizeof(union host_hc_status_block)); 4130 fp->status_blk_mapping = 0; 4131 } else { 4132 /* status blocks */ 4133 if (!CHIP_IS_E1x(bp)) 4134 BNX2X_PCI_FREE(sb->e2_sb, 4135 bnx2x_fp(bp, fp_index, 4136 status_blk_mapping), 4137 sizeof(struct host_hc_status_block_e2)); 4138 else 4139 BNX2X_PCI_FREE(sb->e1x_sb, 4140 bnx2x_fp(bp, fp_index, 4141 status_blk_mapping), 4142 sizeof(struct host_hc_status_block_e1x)); 4143 } 4144 4145 /* Rx */ 4146 if (!skip_rx_queue(bp, fp_index)) { 4147 bnx2x_free_rx_bds(fp); 4148 4149 /* fastpath rx rings: rx_buf rx_desc rx_comp */ 4150 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_buf_ring)); 4151 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_desc_ring), 4152 bnx2x_fp(bp, fp_index, rx_desc_mapping), 4153 sizeof(struct eth_rx_bd) * NUM_RX_BD); 4154 4155 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_comp_ring), 4156 bnx2x_fp(bp, fp_index, rx_comp_mapping), 4157 sizeof(struct eth_fast_path_rx_cqe) * 4158 NUM_RCQ_BD); 4159 4160 /* SGE ring */ 4161 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_page_ring)); 4162 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_sge_ring), 4163 bnx2x_fp(bp, fp_index, rx_sge_mapping), 4164 BCM_PAGE_SIZE * NUM_RX_SGE_PAGES); 4165 } 4166 4167 /* Tx */ 4168 if (!skip_tx_queue(bp, fp_index)) { 4169 /* fastpath tx rings: tx_buf tx_desc */ 4170 for_each_cos_in_tx_queue(fp, cos) { 4171 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos]; 4172 4173 DP(NETIF_MSG_IFDOWN, 4174 "freeing tx memory of fp %d cos %d cid %d\n", 4175 fp_index, cos, txdata->cid); 4176 4177 BNX2X_FREE(txdata->tx_buf_ring); 4178 BNX2X_PCI_FREE(txdata->tx_desc_ring, 4179 txdata->tx_desc_mapping, 4180 sizeof(union eth_tx_bd_types) * NUM_TX_BD); 4181 } 4182 } 4183 /* end of fastpath */ 4184 } 4185 4186 void bnx2x_free_fp_mem_cnic(struct bnx2x *bp) 4187 { 4188 int i; 4189 for_each_cnic_queue(bp, i) 4190 bnx2x_free_fp_mem_at(bp, i); 4191 } 4192 4193 void bnx2x_free_fp_mem(struct bnx2x *bp) 4194 { 4195 int i; 4196 for_each_eth_queue(bp, i) 4197 bnx2x_free_fp_mem_at(bp, i); 4198 } 4199 4200 static void set_sb_shortcuts(struct bnx2x *bp, int index) 4201 { 4202 union host_hc_status_block status_blk = bnx2x_fp(bp, index, status_blk); 4203 if (!CHIP_IS_E1x(bp)) { 4204 bnx2x_fp(bp, index, sb_index_values) = 4205 (__le16 *)status_blk.e2_sb->sb.index_values; 4206 bnx2x_fp(bp, index, sb_running_index) = 4207 (__le16 *)status_blk.e2_sb->sb.running_index; 4208 } else { 4209 bnx2x_fp(bp, index, sb_index_values) = 4210 (__le16 *)status_blk.e1x_sb->sb.index_values; 4211 bnx2x_fp(bp, index, sb_running_index) = 4212 (__le16 *)status_blk.e1x_sb->sb.running_index; 4213 } 4214 } 4215 4216 /* Returns the number of actually allocated BDs */ 4217 static int bnx2x_alloc_rx_bds(struct bnx2x_fastpath *fp, 4218 int rx_ring_size) 4219 { 4220 struct bnx2x *bp = fp->bp; 4221 u16 ring_prod, cqe_ring_prod; 4222 int i, failure_cnt = 0; 4223 4224 fp->rx_comp_cons = 0; 4225 cqe_ring_prod = ring_prod = 0; 4226 4227 /* This routine is called only during fo init so 4228 * fp->eth_q_stats.rx_skb_alloc_failed = 0 4229 */ 4230 for (i = 0; i < rx_ring_size; i++) { 4231 if (bnx2x_alloc_rx_data(bp, fp, ring_prod, GFP_KERNEL) < 0) { 4232 failure_cnt++; 4233 continue; 4234 } 4235 ring_prod = NEXT_RX_IDX(ring_prod); 4236 cqe_ring_prod = NEXT_RCQ_IDX(cqe_ring_prod); 4237 WARN_ON(ring_prod <= (i - failure_cnt)); 4238 } 4239 4240 if (failure_cnt) 4241 BNX2X_ERR("was only able to allocate %d rx skbs on queue[%d]\n", 4242 i - failure_cnt, fp->index); 4243 4244 fp->rx_bd_prod = ring_prod; 4245 /* Limit the CQE producer by the CQE ring size */ 4246 fp->rx_comp_prod = min_t(u16, NUM_RCQ_RINGS*RCQ_DESC_CNT, 4247 cqe_ring_prod); 4248 fp->rx_pkt = fp->rx_calls = 0; 4249 4250 bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed += failure_cnt; 4251 4252 return i - failure_cnt; 4253 } 4254 4255 static void bnx2x_set_next_page_rx_cq(struct bnx2x_fastpath *fp) 4256 { 4257 int i; 4258 4259 for (i = 1; i <= NUM_RCQ_RINGS; i++) { 4260 struct eth_rx_cqe_next_page *nextpg; 4261 4262 nextpg = (struct eth_rx_cqe_next_page *) 4263 &fp->rx_comp_ring[RCQ_DESC_CNT * i - 1]; 4264 nextpg->addr_hi = 4265 cpu_to_le32(U64_HI(fp->rx_comp_mapping + 4266 BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS))); 4267 nextpg->addr_lo = 4268 cpu_to_le32(U64_LO(fp->rx_comp_mapping + 4269 BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS))); 4270 } 4271 } 4272 4273 static int bnx2x_alloc_fp_mem_at(struct bnx2x *bp, int index) 4274 { 4275 union host_hc_status_block *sb; 4276 struct bnx2x_fastpath *fp = &bp->fp[index]; 4277 int ring_size = 0; 4278 u8 cos; 4279 int rx_ring_size = 0; 4280 4281 if (!bp->rx_ring_size && 4282 (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))) { 4283 rx_ring_size = MIN_RX_SIZE_NONTPA; 4284 bp->rx_ring_size = rx_ring_size; 4285 } else if (!bp->rx_ring_size) { 4286 rx_ring_size = MAX_RX_AVAIL/BNX2X_NUM_RX_QUEUES(bp); 4287 4288 if (CHIP_IS_E3(bp)) { 4289 u32 cfg = SHMEM_RD(bp, 4290 dev_info.port_hw_config[BP_PORT(bp)]. 4291 default_cfg); 4292 4293 /* Decrease ring size for 1G functions */ 4294 if ((cfg & PORT_HW_CFG_NET_SERDES_IF_MASK) == 4295 PORT_HW_CFG_NET_SERDES_IF_SGMII) 4296 rx_ring_size /= 10; 4297 } 4298 4299 /* allocate at least number of buffers required by FW */ 4300 rx_ring_size = max_t(int, bp->disable_tpa ? MIN_RX_SIZE_NONTPA : 4301 MIN_RX_SIZE_TPA, rx_ring_size); 4302 4303 bp->rx_ring_size = rx_ring_size; 4304 } else /* if rx_ring_size specified - use it */ 4305 rx_ring_size = bp->rx_ring_size; 4306 4307 DP(BNX2X_MSG_SP, "calculated rx_ring_size %d\n", rx_ring_size); 4308 4309 /* Common */ 4310 sb = &bnx2x_fp(bp, index, status_blk); 4311 4312 if (!IS_FCOE_IDX(index)) { 4313 /* status blocks */ 4314 if (!CHIP_IS_E1x(bp)) 4315 BNX2X_PCI_ALLOC(sb->e2_sb, 4316 &bnx2x_fp(bp, index, status_blk_mapping), 4317 sizeof(struct host_hc_status_block_e2)); 4318 else 4319 BNX2X_PCI_ALLOC(sb->e1x_sb, 4320 &bnx2x_fp(bp, index, status_blk_mapping), 4321 sizeof(struct host_hc_status_block_e1x)); 4322 } 4323 4324 /* FCoE Queue uses Default SB and doesn't ACK the SB, thus no need to 4325 * set shortcuts for it. 4326 */ 4327 if (!IS_FCOE_IDX(index)) 4328 set_sb_shortcuts(bp, index); 4329 4330 /* Tx */ 4331 if (!skip_tx_queue(bp, index)) { 4332 /* fastpath tx rings: tx_buf tx_desc */ 4333 for_each_cos_in_tx_queue(fp, cos) { 4334 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos]; 4335 4336 DP(NETIF_MSG_IFUP, 4337 "allocating tx memory of fp %d cos %d\n", 4338 index, cos); 4339 4340 BNX2X_ALLOC(txdata->tx_buf_ring, 4341 sizeof(struct sw_tx_bd) * NUM_TX_BD); 4342 BNX2X_PCI_ALLOC(txdata->tx_desc_ring, 4343 &txdata->tx_desc_mapping, 4344 sizeof(union eth_tx_bd_types) * NUM_TX_BD); 4345 } 4346 } 4347 4348 /* Rx */ 4349 if (!skip_rx_queue(bp, index)) { 4350 /* fastpath rx rings: rx_buf rx_desc rx_comp */ 4351 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_buf_ring), 4352 sizeof(struct sw_rx_bd) * NUM_RX_BD); 4353 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_desc_ring), 4354 &bnx2x_fp(bp, index, rx_desc_mapping), 4355 sizeof(struct eth_rx_bd) * NUM_RX_BD); 4356 4357 /* Seed all CQEs by 1s */ 4358 BNX2X_PCI_FALLOC(bnx2x_fp(bp, index, rx_comp_ring), 4359 &bnx2x_fp(bp, index, rx_comp_mapping), 4360 sizeof(struct eth_fast_path_rx_cqe) * 4361 NUM_RCQ_BD); 4362 4363 /* SGE ring */ 4364 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_page_ring), 4365 sizeof(struct sw_rx_page) * NUM_RX_SGE); 4366 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_sge_ring), 4367 &bnx2x_fp(bp, index, rx_sge_mapping), 4368 BCM_PAGE_SIZE * NUM_RX_SGE_PAGES); 4369 /* RX BD ring */ 4370 bnx2x_set_next_page_rx_bd(fp); 4371 4372 /* CQ ring */ 4373 bnx2x_set_next_page_rx_cq(fp); 4374 4375 /* BDs */ 4376 ring_size = bnx2x_alloc_rx_bds(fp, rx_ring_size); 4377 if (ring_size < rx_ring_size) 4378 goto alloc_mem_err; 4379 } 4380 4381 return 0; 4382 4383 /* handles low memory cases */ 4384 alloc_mem_err: 4385 BNX2X_ERR("Unable to allocate full memory for queue %d (size %d)\n", 4386 index, ring_size); 4387 /* FW will drop all packets if queue is not big enough, 4388 * In these cases we disable the queue 4389 * Min size is different for OOO, TPA and non-TPA queues 4390 */ 4391 if (ring_size < (fp->disable_tpa ? 4392 MIN_RX_SIZE_NONTPA : MIN_RX_SIZE_TPA)) { 4393 /* release memory allocated for this queue */ 4394 bnx2x_free_fp_mem_at(bp, index); 4395 return -ENOMEM; 4396 } 4397 return 0; 4398 } 4399 4400 int bnx2x_alloc_fp_mem_cnic(struct bnx2x *bp) 4401 { 4402 if (!NO_FCOE(bp)) 4403 /* FCoE */ 4404 if (bnx2x_alloc_fp_mem_at(bp, FCOE_IDX(bp))) 4405 /* we will fail load process instead of mark 4406 * NO_FCOE_FLAG 4407 */ 4408 return -ENOMEM; 4409 4410 return 0; 4411 } 4412 4413 int bnx2x_alloc_fp_mem(struct bnx2x *bp) 4414 { 4415 int i; 4416 4417 /* 1. Allocate FP for leading - fatal if error 4418 * 2. Allocate RSS - fix number of queues if error 4419 */ 4420 4421 /* leading */ 4422 if (bnx2x_alloc_fp_mem_at(bp, 0)) 4423 return -ENOMEM; 4424 4425 /* RSS */ 4426 for_each_nondefault_eth_queue(bp, i) 4427 if (bnx2x_alloc_fp_mem_at(bp, i)) 4428 break; 4429 4430 /* handle memory failures */ 4431 if (i != BNX2X_NUM_ETH_QUEUES(bp)) { 4432 int delta = BNX2X_NUM_ETH_QUEUES(bp) - i; 4433 4434 WARN_ON(delta < 0); 4435 bnx2x_shrink_eth_fp(bp, delta); 4436 if (CNIC_SUPPORT(bp)) 4437 /* move non eth FPs next to last eth FP 4438 * must be done in that order 4439 * FCOE_IDX < FWD_IDX < OOO_IDX 4440 */ 4441 4442 /* move FCoE fp even NO_FCOE_FLAG is on */ 4443 bnx2x_move_fp(bp, FCOE_IDX(bp), FCOE_IDX(bp) - delta); 4444 bp->num_ethernet_queues -= delta; 4445 bp->num_queues = bp->num_ethernet_queues + 4446 bp->num_cnic_queues; 4447 BNX2X_ERR("Adjusted num of queues from %d to %d\n", 4448 bp->num_queues + delta, bp->num_queues); 4449 } 4450 4451 return 0; 4452 } 4453 4454 void bnx2x_free_mem_bp(struct bnx2x *bp) 4455 { 4456 int i; 4457 4458 for (i = 0; i < bp->fp_array_size; i++) 4459 kfree(bp->fp[i].tpa_info); 4460 kfree(bp->fp); 4461 kfree(bp->sp_objs); 4462 kfree(bp->fp_stats); 4463 kfree(bp->bnx2x_txq); 4464 kfree(bp->msix_table); 4465 kfree(bp->ilt); 4466 } 4467 4468 int bnx2x_alloc_mem_bp(struct bnx2x *bp) 4469 { 4470 struct bnx2x_fastpath *fp; 4471 struct msix_entry *tbl; 4472 struct bnx2x_ilt *ilt; 4473 int msix_table_size = 0; 4474 int fp_array_size, txq_array_size; 4475 int i; 4476 4477 /* 4478 * The biggest MSI-X table we might need is as a maximum number of fast 4479 * path IGU SBs plus default SB (for PF only). 4480 */ 4481 msix_table_size = bp->igu_sb_cnt; 4482 if (IS_PF(bp)) 4483 msix_table_size++; 4484 BNX2X_DEV_INFO("msix_table_size %d\n", msix_table_size); 4485 4486 /* fp array: RSS plus CNIC related L2 queues */ 4487 fp_array_size = BNX2X_MAX_RSS_COUNT(bp) + CNIC_SUPPORT(bp); 4488 bp->fp_array_size = fp_array_size; 4489 BNX2X_DEV_INFO("fp_array_size %d\n", bp->fp_array_size); 4490 4491 fp = kcalloc(bp->fp_array_size, sizeof(*fp), GFP_KERNEL); 4492 if (!fp) 4493 goto alloc_err; 4494 for (i = 0; i < bp->fp_array_size; i++) { 4495 fp[i].tpa_info = 4496 kcalloc(ETH_MAX_AGGREGATION_QUEUES_E1H_E2, 4497 sizeof(struct bnx2x_agg_info), GFP_KERNEL); 4498 if (!(fp[i].tpa_info)) 4499 goto alloc_err; 4500 } 4501 4502 bp->fp = fp; 4503 4504 /* allocate sp objs */ 4505 bp->sp_objs = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_sp_objs), 4506 GFP_KERNEL); 4507 if (!bp->sp_objs) 4508 goto alloc_err; 4509 4510 /* allocate fp_stats */ 4511 bp->fp_stats = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_fp_stats), 4512 GFP_KERNEL); 4513 if (!bp->fp_stats) 4514 goto alloc_err; 4515 4516 /* Allocate memory for the transmission queues array */ 4517 txq_array_size = 4518 BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS + CNIC_SUPPORT(bp); 4519 BNX2X_DEV_INFO("txq_array_size %d", txq_array_size); 4520 4521 bp->bnx2x_txq = kcalloc(txq_array_size, sizeof(struct bnx2x_fp_txdata), 4522 GFP_KERNEL); 4523 if (!bp->bnx2x_txq) 4524 goto alloc_err; 4525 4526 /* msix table */ 4527 tbl = kcalloc(msix_table_size, sizeof(*tbl), GFP_KERNEL); 4528 if (!tbl) 4529 goto alloc_err; 4530 bp->msix_table = tbl; 4531 4532 /* ilt */ 4533 ilt = kzalloc(sizeof(*ilt), GFP_KERNEL); 4534 if (!ilt) 4535 goto alloc_err; 4536 bp->ilt = ilt; 4537 4538 return 0; 4539 alloc_err: 4540 bnx2x_free_mem_bp(bp); 4541 return -ENOMEM; 4542 } 4543 4544 int bnx2x_reload_if_running(struct net_device *dev) 4545 { 4546 struct bnx2x *bp = netdev_priv(dev); 4547 4548 if (unlikely(!netif_running(dev))) 4549 return 0; 4550 4551 bnx2x_nic_unload(bp, UNLOAD_NORMAL, true); 4552 return bnx2x_nic_load(bp, LOAD_NORMAL); 4553 } 4554 4555 int bnx2x_get_cur_phy_idx(struct bnx2x *bp) 4556 { 4557 u32 sel_phy_idx = 0; 4558 if (bp->link_params.num_phys <= 1) 4559 return INT_PHY; 4560 4561 if (bp->link_vars.link_up) { 4562 sel_phy_idx = EXT_PHY1; 4563 /* In case link is SERDES, check if the EXT_PHY2 is the one */ 4564 if ((bp->link_vars.link_status & LINK_STATUS_SERDES_LINK) && 4565 (bp->link_params.phy[EXT_PHY2].supported & SUPPORTED_FIBRE)) 4566 sel_phy_idx = EXT_PHY2; 4567 } else { 4568 4569 switch (bnx2x_phy_selection(&bp->link_params)) { 4570 case PORT_HW_CFG_PHY_SELECTION_HARDWARE_DEFAULT: 4571 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY: 4572 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY_PRIORITY: 4573 sel_phy_idx = EXT_PHY1; 4574 break; 4575 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY: 4576 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY_PRIORITY: 4577 sel_phy_idx = EXT_PHY2; 4578 break; 4579 } 4580 } 4581 4582 return sel_phy_idx; 4583 } 4584 int bnx2x_get_link_cfg_idx(struct bnx2x *bp) 4585 { 4586 u32 sel_phy_idx = bnx2x_get_cur_phy_idx(bp); 4587 /* 4588 * The selected activated PHY is always after swapping (in case PHY 4589 * swapping is enabled). So when swapping is enabled, we need to reverse 4590 * the configuration 4591 */ 4592 4593 if (bp->link_params.multi_phy_config & 4594 PORT_HW_CFG_PHY_SWAPPED_ENABLED) { 4595 if (sel_phy_idx == EXT_PHY1) 4596 sel_phy_idx = EXT_PHY2; 4597 else if (sel_phy_idx == EXT_PHY2) 4598 sel_phy_idx = EXT_PHY1; 4599 } 4600 return LINK_CONFIG_IDX(sel_phy_idx); 4601 } 4602 4603 #ifdef NETDEV_FCOE_WWNN 4604 int bnx2x_fcoe_get_wwn(struct net_device *dev, u64 *wwn, int type) 4605 { 4606 struct bnx2x *bp = netdev_priv(dev); 4607 struct cnic_eth_dev *cp = &bp->cnic_eth_dev; 4608 4609 switch (type) { 4610 case NETDEV_FCOE_WWNN: 4611 *wwn = HILO_U64(cp->fcoe_wwn_node_name_hi, 4612 cp->fcoe_wwn_node_name_lo); 4613 break; 4614 case NETDEV_FCOE_WWPN: 4615 *wwn = HILO_U64(cp->fcoe_wwn_port_name_hi, 4616 cp->fcoe_wwn_port_name_lo); 4617 break; 4618 default: 4619 BNX2X_ERR("Wrong WWN type requested - %d\n", type); 4620 return -EINVAL; 4621 } 4622 4623 return 0; 4624 } 4625 #endif 4626 4627 /* called with rtnl_lock */ 4628 int bnx2x_change_mtu(struct net_device *dev, int new_mtu) 4629 { 4630 struct bnx2x *bp = netdev_priv(dev); 4631 4632 if (bp->recovery_state != BNX2X_RECOVERY_DONE) { 4633 BNX2X_ERR("Can't perform change MTU during parity recovery\n"); 4634 return -EAGAIN; 4635 } 4636 4637 if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) || 4638 ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE)) { 4639 BNX2X_ERR("Can't support requested MTU size\n"); 4640 return -EINVAL; 4641 } 4642 4643 /* This does not race with packet allocation 4644 * because the actual alloc size is 4645 * only updated as part of load 4646 */ 4647 dev->mtu = new_mtu; 4648 4649 return bnx2x_reload_if_running(dev); 4650 } 4651 4652 netdev_features_t bnx2x_fix_features(struct net_device *dev, 4653 netdev_features_t features) 4654 { 4655 struct bnx2x *bp = netdev_priv(dev); 4656 4657 /* TPA requires Rx CSUM offloading */ 4658 if (!(features & NETIF_F_RXCSUM) || bp->disable_tpa) { 4659 features &= ~NETIF_F_LRO; 4660 features &= ~NETIF_F_GRO; 4661 } 4662 4663 return features; 4664 } 4665 4666 int bnx2x_set_features(struct net_device *dev, netdev_features_t features) 4667 { 4668 struct bnx2x *bp = netdev_priv(dev); 4669 u32 flags = bp->flags; 4670 u32 changes; 4671 bool bnx2x_reload = false; 4672 4673 if (features & NETIF_F_LRO) 4674 flags |= TPA_ENABLE_FLAG; 4675 else 4676 flags &= ~TPA_ENABLE_FLAG; 4677 4678 if (features & NETIF_F_GRO) 4679 flags |= GRO_ENABLE_FLAG; 4680 else 4681 flags &= ~GRO_ENABLE_FLAG; 4682 4683 if (features & NETIF_F_LOOPBACK) { 4684 if (bp->link_params.loopback_mode != LOOPBACK_BMAC) { 4685 bp->link_params.loopback_mode = LOOPBACK_BMAC; 4686 bnx2x_reload = true; 4687 } 4688 } else { 4689 if (bp->link_params.loopback_mode != LOOPBACK_NONE) { 4690 bp->link_params.loopback_mode = LOOPBACK_NONE; 4691 bnx2x_reload = true; 4692 } 4693 } 4694 4695 changes = flags ^ bp->flags; 4696 4697 /* if GRO is changed while LRO is enabled, don't force a reload */ 4698 if ((changes & GRO_ENABLE_FLAG) && (flags & TPA_ENABLE_FLAG)) 4699 changes &= ~GRO_ENABLE_FLAG; 4700 4701 if (changes) 4702 bnx2x_reload = true; 4703 4704 bp->flags = flags; 4705 4706 if (bnx2x_reload) { 4707 if (bp->recovery_state == BNX2X_RECOVERY_DONE) 4708 return bnx2x_reload_if_running(dev); 4709 /* else: bnx2x_nic_load() will be called at end of recovery */ 4710 } 4711 4712 return 0; 4713 } 4714 4715 void bnx2x_tx_timeout(struct net_device *dev) 4716 { 4717 struct bnx2x *bp = netdev_priv(dev); 4718 4719 #ifdef BNX2X_STOP_ON_ERROR 4720 if (!bp->panic) 4721 bnx2x_panic(); 4722 #endif 4723 4724 smp_mb__before_clear_bit(); 4725 set_bit(BNX2X_SP_RTNL_TX_TIMEOUT, &bp->sp_rtnl_state); 4726 smp_mb__after_clear_bit(); 4727 4728 /* This allows the netif to be shutdown gracefully before resetting */ 4729 schedule_delayed_work(&bp->sp_rtnl_task, 0); 4730 } 4731 4732 int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state) 4733 { 4734 struct net_device *dev = pci_get_drvdata(pdev); 4735 struct bnx2x *bp; 4736 4737 if (!dev) { 4738 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n"); 4739 return -ENODEV; 4740 } 4741 bp = netdev_priv(dev); 4742 4743 rtnl_lock(); 4744 4745 pci_save_state(pdev); 4746 4747 if (!netif_running(dev)) { 4748 rtnl_unlock(); 4749 return 0; 4750 } 4751 4752 netif_device_detach(dev); 4753 4754 bnx2x_nic_unload(bp, UNLOAD_CLOSE, false); 4755 4756 bnx2x_set_power_state(bp, pci_choose_state(pdev, state)); 4757 4758 rtnl_unlock(); 4759 4760 return 0; 4761 } 4762 4763 int bnx2x_resume(struct pci_dev *pdev) 4764 { 4765 struct net_device *dev = pci_get_drvdata(pdev); 4766 struct bnx2x *bp; 4767 int rc; 4768 4769 if (!dev) { 4770 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n"); 4771 return -ENODEV; 4772 } 4773 bp = netdev_priv(dev); 4774 4775 if (bp->recovery_state != BNX2X_RECOVERY_DONE) { 4776 BNX2X_ERR("Handling parity error recovery. Try again later\n"); 4777 return -EAGAIN; 4778 } 4779 4780 rtnl_lock(); 4781 4782 pci_restore_state(pdev); 4783 4784 if (!netif_running(dev)) { 4785 rtnl_unlock(); 4786 return 0; 4787 } 4788 4789 bnx2x_set_power_state(bp, PCI_D0); 4790 netif_device_attach(dev); 4791 4792 rc = bnx2x_nic_load(bp, LOAD_OPEN); 4793 4794 rtnl_unlock(); 4795 4796 return rc; 4797 } 4798 4799 void bnx2x_set_ctx_validation(struct bnx2x *bp, struct eth_context *cxt, 4800 u32 cid) 4801 { 4802 if (!cxt) { 4803 BNX2X_ERR("bad context pointer %p\n", cxt); 4804 return; 4805 } 4806 4807 /* ustorm cxt validation */ 4808 cxt->ustorm_ag_context.cdu_usage = 4809 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid), 4810 CDU_REGION_NUMBER_UCM_AG, ETH_CONNECTION_TYPE); 4811 /* xcontext validation */ 4812 cxt->xstorm_ag_context.cdu_reserved = 4813 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid), 4814 CDU_REGION_NUMBER_XCM_AG, ETH_CONNECTION_TYPE); 4815 } 4816 4817 static void storm_memset_hc_timeout(struct bnx2x *bp, u8 port, 4818 u8 fw_sb_id, u8 sb_index, 4819 u8 ticks) 4820 { 4821 u32 addr = BAR_CSTRORM_INTMEM + 4822 CSTORM_STATUS_BLOCK_DATA_TIMEOUT_OFFSET(fw_sb_id, sb_index); 4823 REG_WR8(bp, addr, ticks); 4824 DP(NETIF_MSG_IFUP, 4825 "port %x fw_sb_id %d sb_index %d ticks %d\n", 4826 port, fw_sb_id, sb_index, ticks); 4827 } 4828 4829 static void storm_memset_hc_disable(struct bnx2x *bp, u8 port, 4830 u16 fw_sb_id, u8 sb_index, 4831 u8 disable) 4832 { 4833 u32 enable_flag = disable ? 0 : (1 << HC_INDEX_DATA_HC_ENABLED_SHIFT); 4834 u32 addr = BAR_CSTRORM_INTMEM + 4835 CSTORM_STATUS_BLOCK_DATA_FLAGS_OFFSET(fw_sb_id, sb_index); 4836 u8 flags = REG_RD8(bp, addr); 4837 /* clear and set */ 4838 flags &= ~HC_INDEX_DATA_HC_ENABLED; 4839 flags |= enable_flag; 4840 REG_WR8(bp, addr, flags); 4841 DP(NETIF_MSG_IFUP, 4842 "port %x fw_sb_id %d sb_index %d disable %d\n", 4843 port, fw_sb_id, sb_index, disable); 4844 } 4845 4846 void bnx2x_update_coalesce_sb_index(struct bnx2x *bp, u8 fw_sb_id, 4847 u8 sb_index, u8 disable, u16 usec) 4848 { 4849 int port = BP_PORT(bp); 4850 u8 ticks = usec / BNX2X_BTR; 4851 4852 storm_memset_hc_timeout(bp, port, fw_sb_id, sb_index, ticks); 4853 4854 disable = disable ? 1 : (usec ? 0 : 1); 4855 storm_memset_hc_disable(bp, port, fw_sb_id, sb_index, disable); 4856 } 4857