1 /* bnx2x_cmn.c: Broadcom Everest network driver. 2 * 3 * Copyright (c) 2007-2013 Broadcom Corporation 4 * 5 * This program is free software; you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License as published by 7 * the Free Software Foundation. 8 * 9 * Maintained by: Eilon Greenstein <eilong@broadcom.com> 10 * Written by: Eliezer Tamir 11 * Based on code from Michael Chan's bnx2 driver 12 * UDP CSUM errata workaround by Arik Gendelman 13 * Slowpath and fastpath rework by Vladislav Zolotarov 14 * Statistics and Link management by Yitchak Gertner 15 * 16 */ 17 18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 19 20 #include <linux/etherdevice.h> 21 #include <linux/if_vlan.h> 22 #include <linux/interrupt.h> 23 #include <linux/ip.h> 24 #include <net/tcp.h> 25 #include <net/ipv6.h> 26 #include <net/ip6_checksum.h> 27 #include <net/busy_poll.h> 28 #include <linux/prefetch.h> 29 #include "bnx2x_cmn.h" 30 #include "bnx2x_init.h" 31 #include "bnx2x_sp.h" 32 33 /** 34 * bnx2x_move_fp - move content of the fastpath structure. 35 * 36 * @bp: driver handle 37 * @from: source FP index 38 * @to: destination FP index 39 * 40 * Makes sure the contents of the bp->fp[to].napi is kept 41 * intact. This is done by first copying the napi struct from 42 * the target to the source, and then mem copying the entire 43 * source onto the target. Update txdata pointers and related 44 * content. 45 */ 46 static inline void bnx2x_move_fp(struct bnx2x *bp, int from, int to) 47 { 48 struct bnx2x_fastpath *from_fp = &bp->fp[from]; 49 struct bnx2x_fastpath *to_fp = &bp->fp[to]; 50 struct bnx2x_sp_objs *from_sp_objs = &bp->sp_objs[from]; 51 struct bnx2x_sp_objs *to_sp_objs = &bp->sp_objs[to]; 52 struct bnx2x_fp_stats *from_fp_stats = &bp->fp_stats[from]; 53 struct bnx2x_fp_stats *to_fp_stats = &bp->fp_stats[to]; 54 int old_max_eth_txqs, new_max_eth_txqs; 55 int old_txdata_index = 0, new_txdata_index = 0; 56 struct bnx2x_agg_info *old_tpa_info = to_fp->tpa_info; 57 58 /* Copy the NAPI object as it has been already initialized */ 59 from_fp->napi = to_fp->napi; 60 61 /* Move bnx2x_fastpath contents */ 62 memcpy(to_fp, from_fp, sizeof(*to_fp)); 63 to_fp->index = to; 64 65 /* Retain the tpa_info of the original `to' version as we don't want 66 * 2 FPs to contain the same tpa_info pointer. 67 */ 68 to_fp->tpa_info = old_tpa_info; 69 70 /* move sp_objs contents as well, as their indices match fp ones */ 71 memcpy(to_sp_objs, from_sp_objs, sizeof(*to_sp_objs)); 72 73 /* move fp_stats contents as well, as their indices match fp ones */ 74 memcpy(to_fp_stats, from_fp_stats, sizeof(*to_fp_stats)); 75 76 /* Update txdata pointers in fp and move txdata content accordingly: 77 * Each fp consumes 'max_cos' txdata structures, so the index should be 78 * decremented by max_cos x delta. 79 */ 80 81 old_max_eth_txqs = BNX2X_NUM_ETH_QUEUES(bp) * (bp)->max_cos; 82 new_max_eth_txqs = (BNX2X_NUM_ETH_QUEUES(bp) - from + to) * 83 (bp)->max_cos; 84 if (from == FCOE_IDX(bp)) { 85 old_txdata_index = old_max_eth_txqs + FCOE_TXQ_IDX_OFFSET; 86 new_txdata_index = new_max_eth_txqs + FCOE_TXQ_IDX_OFFSET; 87 } 88 89 memcpy(&bp->bnx2x_txq[new_txdata_index], 90 &bp->bnx2x_txq[old_txdata_index], 91 sizeof(struct bnx2x_fp_txdata)); 92 to_fp->txdata_ptr[0] = &bp->bnx2x_txq[new_txdata_index]; 93 } 94 95 /** 96 * bnx2x_fill_fw_str - Fill buffer with FW version string. 97 * 98 * @bp: driver handle 99 * @buf: character buffer to fill with the fw name 100 * @buf_len: length of the above buffer 101 * 102 */ 103 void bnx2x_fill_fw_str(struct bnx2x *bp, char *buf, size_t buf_len) 104 { 105 if (IS_PF(bp)) { 106 u8 phy_fw_ver[PHY_FW_VER_LEN]; 107 108 phy_fw_ver[0] = '\0'; 109 bnx2x_get_ext_phy_fw_version(&bp->link_params, 110 phy_fw_ver, PHY_FW_VER_LEN); 111 strlcpy(buf, bp->fw_ver, buf_len); 112 snprintf(buf + strlen(bp->fw_ver), 32 - strlen(bp->fw_ver), 113 "bc %d.%d.%d%s%s", 114 (bp->common.bc_ver & 0xff0000) >> 16, 115 (bp->common.bc_ver & 0xff00) >> 8, 116 (bp->common.bc_ver & 0xff), 117 ((phy_fw_ver[0] != '\0') ? " phy " : ""), phy_fw_ver); 118 } else { 119 bnx2x_vf_fill_fw_str(bp, buf, buf_len); 120 } 121 } 122 123 /** 124 * bnx2x_shrink_eth_fp - guarantees fastpath structures stay intact 125 * 126 * @bp: driver handle 127 * @delta: number of eth queues which were not allocated 128 */ 129 static void bnx2x_shrink_eth_fp(struct bnx2x *bp, int delta) 130 { 131 int i, cos, old_eth_num = BNX2X_NUM_ETH_QUEUES(bp); 132 133 /* Queue pointer cannot be re-set on an fp-basis, as moving pointer 134 * backward along the array could cause memory to be overridden 135 */ 136 for (cos = 1; cos < bp->max_cos; cos++) { 137 for (i = 0; i < old_eth_num - delta; i++) { 138 struct bnx2x_fastpath *fp = &bp->fp[i]; 139 int new_idx = cos * (old_eth_num - delta) + i; 140 141 memcpy(&bp->bnx2x_txq[new_idx], fp->txdata_ptr[cos], 142 sizeof(struct bnx2x_fp_txdata)); 143 fp->txdata_ptr[cos] = &bp->bnx2x_txq[new_idx]; 144 } 145 } 146 } 147 148 int load_count[2][3] = { {0} }; /* per-path: 0-common, 1-port0, 2-port1 */ 149 150 /* free skb in the packet ring at pos idx 151 * return idx of last bd freed 152 */ 153 static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata, 154 u16 idx, unsigned int *pkts_compl, 155 unsigned int *bytes_compl) 156 { 157 struct sw_tx_bd *tx_buf = &txdata->tx_buf_ring[idx]; 158 struct eth_tx_start_bd *tx_start_bd; 159 struct eth_tx_bd *tx_data_bd; 160 struct sk_buff *skb = tx_buf->skb; 161 u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons; 162 int nbd; 163 164 /* prefetch skb end pointer to speedup dev_kfree_skb() */ 165 prefetch(&skb->end); 166 167 DP(NETIF_MSG_TX_DONE, "fp[%d]: pkt_idx %d buff @(%p)->skb %p\n", 168 txdata->txq_index, idx, tx_buf, skb); 169 170 /* unmap first bd */ 171 tx_start_bd = &txdata->tx_desc_ring[bd_idx].start_bd; 172 dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd), 173 BD_UNMAP_LEN(tx_start_bd), DMA_TO_DEVICE); 174 175 nbd = le16_to_cpu(tx_start_bd->nbd) - 1; 176 #ifdef BNX2X_STOP_ON_ERROR 177 if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) { 178 BNX2X_ERR("BAD nbd!\n"); 179 bnx2x_panic(); 180 } 181 #endif 182 new_cons = nbd + tx_buf->first_bd; 183 184 /* Get the next bd */ 185 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx)); 186 187 /* Skip a parse bd... */ 188 --nbd; 189 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx)); 190 191 /* ...and the TSO split header bd since they have no mapping */ 192 if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) { 193 --nbd; 194 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx)); 195 } 196 197 /* now free frags */ 198 while (nbd > 0) { 199 200 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd; 201 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd), 202 BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE); 203 if (--nbd) 204 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx)); 205 } 206 207 /* release skb */ 208 WARN_ON(!skb); 209 if (likely(skb)) { 210 (*pkts_compl)++; 211 (*bytes_compl) += skb->len; 212 } 213 214 dev_kfree_skb_any(skb); 215 tx_buf->first_bd = 0; 216 tx_buf->skb = NULL; 217 218 return new_cons; 219 } 220 221 int bnx2x_tx_int(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata) 222 { 223 struct netdev_queue *txq; 224 u16 hw_cons, sw_cons, bd_cons = txdata->tx_bd_cons; 225 unsigned int pkts_compl = 0, bytes_compl = 0; 226 227 #ifdef BNX2X_STOP_ON_ERROR 228 if (unlikely(bp->panic)) 229 return -1; 230 #endif 231 232 txq = netdev_get_tx_queue(bp->dev, txdata->txq_index); 233 hw_cons = le16_to_cpu(*txdata->tx_cons_sb); 234 sw_cons = txdata->tx_pkt_cons; 235 236 while (sw_cons != hw_cons) { 237 u16 pkt_cons; 238 239 pkt_cons = TX_BD(sw_cons); 240 241 DP(NETIF_MSG_TX_DONE, 242 "queue[%d]: hw_cons %u sw_cons %u pkt_cons %u\n", 243 txdata->txq_index, hw_cons, sw_cons, pkt_cons); 244 245 bd_cons = bnx2x_free_tx_pkt(bp, txdata, pkt_cons, 246 &pkts_compl, &bytes_compl); 247 248 sw_cons++; 249 } 250 251 netdev_tx_completed_queue(txq, pkts_compl, bytes_compl); 252 253 txdata->tx_pkt_cons = sw_cons; 254 txdata->tx_bd_cons = bd_cons; 255 256 /* Need to make the tx_bd_cons update visible to start_xmit() 257 * before checking for netif_tx_queue_stopped(). Without the 258 * memory barrier, there is a small possibility that 259 * start_xmit() will miss it and cause the queue to be stopped 260 * forever. 261 * On the other hand we need an rmb() here to ensure the proper 262 * ordering of bit testing in the following 263 * netif_tx_queue_stopped(txq) call. 264 */ 265 smp_mb(); 266 267 if (unlikely(netif_tx_queue_stopped(txq))) { 268 /* Taking tx_lock() is needed to prevent re-enabling the queue 269 * while it's empty. This could have happen if rx_action() gets 270 * suspended in bnx2x_tx_int() after the condition before 271 * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()): 272 * 273 * stops the queue->sees fresh tx_bd_cons->releases the queue-> 274 * sends some packets consuming the whole queue again-> 275 * stops the queue 276 */ 277 278 __netif_tx_lock(txq, smp_processor_id()); 279 280 if ((netif_tx_queue_stopped(txq)) && 281 (bp->state == BNX2X_STATE_OPEN) && 282 (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT)) 283 netif_tx_wake_queue(txq); 284 285 __netif_tx_unlock(txq); 286 } 287 return 0; 288 } 289 290 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp, 291 u16 idx) 292 { 293 u16 last_max = fp->last_max_sge; 294 295 if (SUB_S16(idx, last_max) > 0) 296 fp->last_max_sge = idx; 297 } 298 299 static inline void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp, 300 u16 sge_len, 301 struct eth_end_agg_rx_cqe *cqe) 302 { 303 struct bnx2x *bp = fp->bp; 304 u16 last_max, last_elem, first_elem; 305 u16 delta = 0; 306 u16 i; 307 308 if (!sge_len) 309 return; 310 311 /* First mark all used pages */ 312 for (i = 0; i < sge_len; i++) 313 BIT_VEC64_CLEAR_BIT(fp->sge_mask, 314 RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[i]))); 315 316 DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n", 317 sge_len - 1, le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1])); 318 319 /* Here we assume that the last SGE index is the biggest */ 320 prefetch((void *)(fp->sge_mask)); 321 bnx2x_update_last_max_sge(fp, 322 le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1])); 323 324 last_max = RX_SGE(fp->last_max_sge); 325 last_elem = last_max >> BIT_VEC64_ELEM_SHIFT; 326 first_elem = RX_SGE(fp->rx_sge_prod) >> BIT_VEC64_ELEM_SHIFT; 327 328 /* If ring is not full */ 329 if (last_elem + 1 != first_elem) 330 last_elem++; 331 332 /* Now update the prod */ 333 for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) { 334 if (likely(fp->sge_mask[i])) 335 break; 336 337 fp->sge_mask[i] = BIT_VEC64_ELEM_ONE_MASK; 338 delta += BIT_VEC64_ELEM_SZ; 339 } 340 341 if (delta > 0) { 342 fp->rx_sge_prod += delta; 343 /* clear page-end entries */ 344 bnx2x_clear_sge_mask_next_elems(fp); 345 } 346 347 DP(NETIF_MSG_RX_STATUS, 348 "fp->last_max_sge = %d fp->rx_sge_prod = %d\n", 349 fp->last_max_sge, fp->rx_sge_prod); 350 } 351 352 /* Get Toeplitz hash value in the skb using the value from the 353 * CQE (calculated by HW). 354 */ 355 static u32 bnx2x_get_rxhash(const struct bnx2x *bp, 356 const struct eth_fast_path_rx_cqe *cqe, 357 bool *l4_rxhash) 358 { 359 /* Get Toeplitz hash from CQE */ 360 if ((bp->dev->features & NETIF_F_RXHASH) && 361 (cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG)) { 362 enum eth_rss_hash_type htype; 363 364 htype = cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_TYPE; 365 *l4_rxhash = (htype == TCP_IPV4_HASH_TYPE) || 366 (htype == TCP_IPV6_HASH_TYPE); 367 return le32_to_cpu(cqe->rss_hash_result); 368 } 369 *l4_rxhash = false; 370 return 0; 371 } 372 373 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue, 374 u16 cons, u16 prod, 375 struct eth_fast_path_rx_cqe *cqe) 376 { 377 struct bnx2x *bp = fp->bp; 378 struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons]; 379 struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod]; 380 struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod]; 381 dma_addr_t mapping; 382 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue]; 383 struct sw_rx_bd *first_buf = &tpa_info->first_buf; 384 385 /* print error if current state != stop */ 386 if (tpa_info->tpa_state != BNX2X_TPA_STOP) 387 BNX2X_ERR("start of bin not in stop [%d]\n", queue); 388 389 /* Try to map an empty data buffer from the aggregation info */ 390 mapping = dma_map_single(&bp->pdev->dev, 391 first_buf->data + NET_SKB_PAD, 392 fp->rx_buf_size, DMA_FROM_DEVICE); 393 /* 394 * ...if it fails - move the skb from the consumer to the producer 395 * and set the current aggregation state as ERROR to drop it 396 * when TPA_STOP arrives. 397 */ 398 399 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) { 400 /* Move the BD from the consumer to the producer */ 401 bnx2x_reuse_rx_data(fp, cons, prod); 402 tpa_info->tpa_state = BNX2X_TPA_ERROR; 403 return; 404 } 405 406 /* move empty data from pool to prod */ 407 prod_rx_buf->data = first_buf->data; 408 dma_unmap_addr_set(prod_rx_buf, mapping, mapping); 409 /* point prod_bd to new data */ 410 prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping)); 411 prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping)); 412 413 /* move partial skb from cons to pool (don't unmap yet) */ 414 *first_buf = *cons_rx_buf; 415 416 /* mark bin state as START */ 417 tpa_info->parsing_flags = 418 le16_to_cpu(cqe->pars_flags.flags); 419 tpa_info->vlan_tag = le16_to_cpu(cqe->vlan_tag); 420 tpa_info->tpa_state = BNX2X_TPA_START; 421 tpa_info->len_on_bd = le16_to_cpu(cqe->len_on_bd); 422 tpa_info->placement_offset = cqe->placement_offset; 423 tpa_info->rxhash = bnx2x_get_rxhash(bp, cqe, &tpa_info->l4_rxhash); 424 if (fp->mode == TPA_MODE_GRO) { 425 u16 gro_size = le16_to_cpu(cqe->pkt_len_or_gro_seg_len); 426 tpa_info->full_page = SGE_PAGES / gro_size * gro_size; 427 tpa_info->gro_size = gro_size; 428 } 429 430 #ifdef BNX2X_STOP_ON_ERROR 431 fp->tpa_queue_used |= (1 << queue); 432 #ifdef _ASM_GENERIC_INT_L64_H 433 DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n", 434 #else 435 DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n", 436 #endif 437 fp->tpa_queue_used); 438 #endif 439 } 440 441 /* Timestamp option length allowed for TPA aggregation: 442 * 443 * nop nop kind length echo val 444 */ 445 #define TPA_TSTAMP_OPT_LEN 12 446 /** 447 * bnx2x_set_gro_params - compute GRO values 448 * 449 * @skb: packet skb 450 * @parsing_flags: parsing flags from the START CQE 451 * @len_on_bd: total length of the first packet for the 452 * aggregation. 453 * @pkt_len: length of all segments 454 * 455 * Approximate value of the MSS for this aggregation calculated using 456 * the first packet of it. 457 * Compute number of aggregated segments, and gso_type. 458 */ 459 static void bnx2x_set_gro_params(struct sk_buff *skb, u16 parsing_flags, 460 u16 len_on_bd, unsigned int pkt_len, 461 u16 num_of_coalesced_segs) 462 { 463 /* TPA aggregation won't have either IP options or TCP options 464 * other than timestamp or IPv6 extension headers. 465 */ 466 u16 hdrs_len = ETH_HLEN + sizeof(struct tcphdr); 467 468 if (GET_FLAG(parsing_flags, PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) == 469 PRS_FLAG_OVERETH_IPV6) { 470 hdrs_len += sizeof(struct ipv6hdr); 471 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6; 472 } else { 473 hdrs_len += sizeof(struct iphdr); 474 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4; 475 } 476 477 /* Check if there was a TCP timestamp, if there is it's will 478 * always be 12 bytes length: nop nop kind length echo val. 479 * 480 * Otherwise FW would close the aggregation. 481 */ 482 if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG) 483 hdrs_len += TPA_TSTAMP_OPT_LEN; 484 485 skb_shinfo(skb)->gso_size = len_on_bd - hdrs_len; 486 487 /* tcp_gro_complete() will copy NAPI_GRO_CB(skb)->count 488 * to skb_shinfo(skb)->gso_segs 489 */ 490 NAPI_GRO_CB(skb)->count = num_of_coalesced_segs; 491 } 492 493 static int bnx2x_alloc_rx_sge(struct bnx2x *bp, struct bnx2x_fastpath *fp, 494 u16 index, gfp_t gfp_mask) 495 { 496 struct page *page = alloc_pages(gfp_mask, PAGES_PER_SGE_SHIFT); 497 struct sw_rx_page *sw_buf = &fp->rx_page_ring[index]; 498 struct eth_rx_sge *sge = &fp->rx_sge_ring[index]; 499 dma_addr_t mapping; 500 501 if (unlikely(page == NULL)) { 502 BNX2X_ERR("Can't alloc sge\n"); 503 return -ENOMEM; 504 } 505 506 mapping = dma_map_page(&bp->pdev->dev, page, 0, 507 SGE_PAGES, DMA_FROM_DEVICE); 508 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) { 509 __free_pages(page, PAGES_PER_SGE_SHIFT); 510 BNX2X_ERR("Can't map sge\n"); 511 return -ENOMEM; 512 } 513 514 sw_buf->page = page; 515 dma_unmap_addr_set(sw_buf, mapping, mapping); 516 517 sge->addr_hi = cpu_to_le32(U64_HI(mapping)); 518 sge->addr_lo = cpu_to_le32(U64_LO(mapping)); 519 520 return 0; 521 } 522 523 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp, 524 struct bnx2x_agg_info *tpa_info, 525 u16 pages, 526 struct sk_buff *skb, 527 struct eth_end_agg_rx_cqe *cqe, 528 u16 cqe_idx) 529 { 530 struct sw_rx_page *rx_pg, old_rx_pg; 531 u32 i, frag_len, frag_size; 532 int err, j, frag_id = 0; 533 u16 len_on_bd = tpa_info->len_on_bd; 534 u16 full_page = 0, gro_size = 0; 535 536 frag_size = le16_to_cpu(cqe->pkt_len) - len_on_bd; 537 538 if (fp->mode == TPA_MODE_GRO) { 539 gro_size = tpa_info->gro_size; 540 full_page = tpa_info->full_page; 541 } 542 543 /* This is needed in order to enable forwarding support */ 544 if (frag_size) 545 bnx2x_set_gro_params(skb, tpa_info->parsing_flags, len_on_bd, 546 le16_to_cpu(cqe->pkt_len), 547 le16_to_cpu(cqe->num_of_coalesced_segs)); 548 549 #ifdef BNX2X_STOP_ON_ERROR 550 if (pages > min_t(u32, 8, MAX_SKB_FRAGS) * SGE_PAGES) { 551 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n", 552 pages, cqe_idx); 553 BNX2X_ERR("cqe->pkt_len = %d\n", cqe->pkt_len); 554 bnx2x_panic(); 555 return -EINVAL; 556 } 557 #endif 558 559 /* Run through the SGL and compose the fragmented skb */ 560 for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) { 561 u16 sge_idx = RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[j])); 562 563 /* FW gives the indices of the SGE as if the ring is an array 564 (meaning that "next" element will consume 2 indices) */ 565 if (fp->mode == TPA_MODE_GRO) 566 frag_len = min_t(u32, frag_size, (u32)full_page); 567 else /* LRO */ 568 frag_len = min_t(u32, frag_size, (u32)SGE_PAGES); 569 570 rx_pg = &fp->rx_page_ring[sge_idx]; 571 old_rx_pg = *rx_pg; 572 573 /* If we fail to allocate a substitute page, we simply stop 574 where we are and drop the whole packet */ 575 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx, GFP_ATOMIC); 576 if (unlikely(err)) { 577 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++; 578 return err; 579 } 580 581 /* Unmap the page as we're going to pass it to the stack */ 582 dma_unmap_page(&bp->pdev->dev, 583 dma_unmap_addr(&old_rx_pg, mapping), 584 SGE_PAGES, DMA_FROM_DEVICE); 585 /* Add one frag and update the appropriate fields in the skb */ 586 if (fp->mode == TPA_MODE_LRO) 587 skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len); 588 else { /* GRO */ 589 int rem; 590 int offset = 0; 591 for (rem = frag_len; rem > 0; rem -= gro_size) { 592 int len = rem > gro_size ? gro_size : rem; 593 skb_fill_page_desc(skb, frag_id++, 594 old_rx_pg.page, offset, len); 595 if (offset) 596 get_page(old_rx_pg.page); 597 offset += len; 598 } 599 } 600 601 skb->data_len += frag_len; 602 skb->truesize += SGE_PAGES; 603 skb->len += frag_len; 604 605 frag_size -= frag_len; 606 } 607 608 return 0; 609 } 610 611 static void bnx2x_frag_free(const struct bnx2x_fastpath *fp, void *data) 612 { 613 if (fp->rx_frag_size) 614 put_page(virt_to_head_page(data)); 615 else 616 kfree(data); 617 } 618 619 static void *bnx2x_frag_alloc(const struct bnx2x_fastpath *fp, gfp_t gfp_mask) 620 { 621 if (fp->rx_frag_size) { 622 /* GFP_KERNEL allocations are used only during initialization */ 623 if (unlikely(gfp_mask & __GFP_WAIT)) 624 return (void *)__get_free_page(gfp_mask); 625 626 return netdev_alloc_frag(fp->rx_frag_size); 627 } 628 629 return kmalloc(fp->rx_buf_size + NET_SKB_PAD, gfp_mask); 630 } 631 632 #ifdef CONFIG_INET 633 static void bnx2x_gro_ip_csum(struct bnx2x *bp, struct sk_buff *skb) 634 { 635 const struct iphdr *iph = ip_hdr(skb); 636 struct tcphdr *th; 637 638 skb_set_transport_header(skb, sizeof(struct iphdr)); 639 th = tcp_hdr(skb); 640 641 th->check = ~tcp_v4_check(skb->len - skb_transport_offset(skb), 642 iph->saddr, iph->daddr, 0); 643 } 644 645 static void bnx2x_gro_ipv6_csum(struct bnx2x *bp, struct sk_buff *skb) 646 { 647 struct ipv6hdr *iph = ipv6_hdr(skb); 648 struct tcphdr *th; 649 650 skb_set_transport_header(skb, sizeof(struct ipv6hdr)); 651 th = tcp_hdr(skb); 652 653 th->check = ~tcp_v6_check(skb->len - skb_transport_offset(skb), 654 &iph->saddr, &iph->daddr, 0); 655 } 656 657 static void bnx2x_gro_csum(struct bnx2x *bp, struct sk_buff *skb, 658 void (*gro_func)(struct bnx2x*, struct sk_buff*)) 659 { 660 skb_set_network_header(skb, 0); 661 gro_func(bp, skb); 662 tcp_gro_complete(skb); 663 } 664 #endif 665 666 static void bnx2x_gro_receive(struct bnx2x *bp, struct bnx2x_fastpath *fp, 667 struct sk_buff *skb) 668 { 669 #ifdef CONFIG_INET 670 if (skb_shinfo(skb)->gso_size) { 671 switch (be16_to_cpu(skb->protocol)) { 672 case ETH_P_IP: 673 bnx2x_gro_csum(bp, skb, bnx2x_gro_ip_csum); 674 break; 675 case ETH_P_IPV6: 676 bnx2x_gro_csum(bp, skb, bnx2x_gro_ipv6_csum); 677 break; 678 default: 679 BNX2X_ERR("Error: FW GRO supports only IPv4/IPv6, not 0x%04x\n", 680 be16_to_cpu(skb->protocol)); 681 } 682 } 683 #endif 684 skb_record_rx_queue(skb, fp->rx_queue); 685 napi_gro_receive(&fp->napi, skb); 686 } 687 688 static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp, 689 struct bnx2x_agg_info *tpa_info, 690 u16 pages, 691 struct eth_end_agg_rx_cqe *cqe, 692 u16 cqe_idx) 693 { 694 struct sw_rx_bd *rx_buf = &tpa_info->first_buf; 695 u8 pad = tpa_info->placement_offset; 696 u16 len = tpa_info->len_on_bd; 697 struct sk_buff *skb = NULL; 698 u8 *new_data, *data = rx_buf->data; 699 u8 old_tpa_state = tpa_info->tpa_state; 700 701 tpa_info->tpa_state = BNX2X_TPA_STOP; 702 703 /* If we there was an error during the handling of the TPA_START - 704 * drop this aggregation. 705 */ 706 if (old_tpa_state == BNX2X_TPA_ERROR) 707 goto drop; 708 709 /* Try to allocate the new data */ 710 new_data = bnx2x_frag_alloc(fp, GFP_ATOMIC); 711 /* Unmap skb in the pool anyway, as we are going to change 712 pool entry status to BNX2X_TPA_STOP even if new skb allocation 713 fails. */ 714 dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping), 715 fp->rx_buf_size, DMA_FROM_DEVICE); 716 if (likely(new_data)) 717 skb = build_skb(data, fp->rx_frag_size); 718 719 if (likely(skb)) { 720 #ifdef BNX2X_STOP_ON_ERROR 721 if (pad + len > fp->rx_buf_size) { 722 BNX2X_ERR("skb_put is about to fail... pad %d len %d rx_buf_size %d\n", 723 pad, len, fp->rx_buf_size); 724 bnx2x_panic(); 725 return; 726 } 727 #endif 728 729 skb_reserve(skb, pad + NET_SKB_PAD); 730 skb_put(skb, len); 731 skb->rxhash = tpa_info->rxhash; 732 skb->l4_rxhash = tpa_info->l4_rxhash; 733 734 skb->protocol = eth_type_trans(skb, bp->dev); 735 skb->ip_summed = CHECKSUM_UNNECESSARY; 736 737 if (!bnx2x_fill_frag_skb(bp, fp, tpa_info, pages, 738 skb, cqe, cqe_idx)) { 739 if (tpa_info->parsing_flags & PARSING_FLAGS_VLAN) 740 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), tpa_info->vlan_tag); 741 bnx2x_gro_receive(bp, fp, skb); 742 } else { 743 DP(NETIF_MSG_RX_STATUS, 744 "Failed to allocate new pages - dropping packet!\n"); 745 dev_kfree_skb_any(skb); 746 } 747 748 /* put new data in bin */ 749 rx_buf->data = new_data; 750 751 return; 752 } 753 bnx2x_frag_free(fp, new_data); 754 drop: 755 /* drop the packet and keep the buffer in the bin */ 756 DP(NETIF_MSG_RX_STATUS, 757 "Failed to allocate or map a new skb - dropping packet!\n"); 758 bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed++; 759 } 760 761 static int bnx2x_alloc_rx_data(struct bnx2x *bp, struct bnx2x_fastpath *fp, 762 u16 index, gfp_t gfp_mask) 763 { 764 u8 *data; 765 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[index]; 766 struct eth_rx_bd *rx_bd = &fp->rx_desc_ring[index]; 767 dma_addr_t mapping; 768 769 data = bnx2x_frag_alloc(fp, gfp_mask); 770 if (unlikely(data == NULL)) 771 return -ENOMEM; 772 773 mapping = dma_map_single(&bp->pdev->dev, data + NET_SKB_PAD, 774 fp->rx_buf_size, 775 DMA_FROM_DEVICE); 776 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) { 777 bnx2x_frag_free(fp, data); 778 BNX2X_ERR("Can't map rx data\n"); 779 return -ENOMEM; 780 } 781 782 rx_buf->data = data; 783 dma_unmap_addr_set(rx_buf, mapping, mapping); 784 785 rx_bd->addr_hi = cpu_to_le32(U64_HI(mapping)); 786 rx_bd->addr_lo = cpu_to_le32(U64_LO(mapping)); 787 788 return 0; 789 } 790 791 static 792 void bnx2x_csum_validate(struct sk_buff *skb, union eth_rx_cqe *cqe, 793 struct bnx2x_fastpath *fp, 794 struct bnx2x_eth_q_stats *qstats) 795 { 796 /* Do nothing if no L4 csum validation was done. 797 * We do not check whether IP csum was validated. For IPv4 we assume 798 * that if the card got as far as validating the L4 csum, it also 799 * validated the IP csum. IPv6 has no IP csum. 800 */ 801 if (cqe->fast_path_cqe.status_flags & 802 ETH_FAST_PATH_RX_CQE_L4_XSUM_NO_VALIDATION_FLG) 803 return; 804 805 /* If L4 validation was done, check if an error was found. */ 806 807 if (cqe->fast_path_cqe.type_error_flags & 808 (ETH_FAST_PATH_RX_CQE_IP_BAD_XSUM_FLG | 809 ETH_FAST_PATH_RX_CQE_L4_BAD_XSUM_FLG)) 810 qstats->hw_csum_err++; 811 else 812 skb->ip_summed = CHECKSUM_UNNECESSARY; 813 } 814 815 int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget) 816 { 817 struct bnx2x *bp = fp->bp; 818 u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons; 819 u16 sw_comp_cons, sw_comp_prod; 820 int rx_pkt = 0; 821 union eth_rx_cqe *cqe; 822 struct eth_fast_path_rx_cqe *cqe_fp; 823 824 #ifdef BNX2X_STOP_ON_ERROR 825 if (unlikely(bp->panic)) 826 return 0; 827 #endif 828 829 bd_cons = fp->rx_bd_cons; 830 bd_prod = fp->rx_bd_prod; 831 bd_prod_fw = bd_prod; 832 sw_comp_cons = fp->rx_comp_cons; 833 sw_comp_prod = fp->rx_comp_prod; 834 835 comp_ring_cons = RCQ_BD(sw_comp_cons); 836 cqe = &fp->rx_comp_ring[comp_ring_cons]; 837 cqe_fp = &cqe->fast_path_cqe; 838 839 DP(NETIF_MSG_RX_STATUS, 840 "queue[%d]: sw_comp_cons %u\n", fp->index, sw_comp_cons); 841 842 while (BNX2X_IS_CQE_COMPLETED(cqe_fp)) { 843 struct sw_rx_bd *rx_buf = NULL; 844 struct sk_buff *skb; 845 u8 cqe_fp_flags; 846 enum eth_rx_cqe_type cqe_fp_type; 847 u16 len, pad, queue; 848 u8 *data; 849 bool l4_rxhash; 850 851 #ifdef BNX2X_STOP_ON_ERROR 852 if (unlikely(bp->panic)) 853 return 0; 854 #endif 855 856 bd_prod = RX_BD(bd_prod); 857 bd_cons = RX_BD(bd_cons); 858 859 cqe_fp_flags = cqe_fp->type_error_flags; 860 cqe_fp_type = cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE; 861 862 DP(NETIF_MSG_RX_STATUS, 863 "CQE type %x err %x status %x queue %x vlan %x len %u\n", 864 CQE_TYPE(cqe_fp_flags), 865 cqe_fp_flags, cqe_fp->status_flags, 866 le32_to_cpu(cqe_fp->rss_hash_result), 867 le16_to_cpu(cqe_fp->vlan_tag), 868 le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len)); 869 870 /* is this a slowpath msg? */ 871 if (unlikely(CQE_TYPE_SLOW(cqe_fp_type))) { 872 bnx2x_sp_event(fp, cqe); 873 goto next_cqe; 874 } 875 876 rx_buf = &fp->rx_buf_ring[bd_cons]; 877 data = rx_buf->data; 878 879 if (!CQE_TYPE_FAST(cqe_fp_type)) { 880 struct bnx2x_agg_info *tpa_info; 881 u16 frag_size, pages; 882 #ifdef BNX2X_STOP_ON_ERROR 883 /* sanity check */ 884 if (fp->disable_tpa && 885 (CQE_TYPE_START(cqe_fp_type) || 886 CQE_TYPE_STOP(cqe_fp_type))) 887 BNX2X_ERR("START/STOP packet while disable_tpa type %x\n", 888 CQE_TYPE(cqe_fp_type)); 889 #endif 890 891 if (CQE_TYPE_START(cqe_fp_type)) { 892 u16 queue = cqe_fp->queue_index; 893 DP(NETIF_MSG_RX_STATUS, 894 "calling tpa_start on queue %d\n", 895 queue); 896 897 bnx2x_tpa_start(fp, queue, 898 bd_cons, bd_prod, 899 cqe_fp); 900 901 goto next_rx; 902 } 903 queue = cqe->end_agg_cqe.queue_index; 904 tpa_info = &fp->tpa_info[queue]; 905 DP(NETIF_MSG_RX_STATUS, 906 "calling tpa_stop on queue %d\n", 907 queue); 908 909 frag_size = le16_to_cpu(cqe->end_agg_cqe.pkt_len) - 910 tpa_info->len_on_bd; 911 912 if (fp->mode == TPA_MODE_GRO) 913 pages = (frag_size + tpa_info->full_page - 1) / 914 tpa_info->full_page; 915 else 916 pages = SGE_PAGE_ALIGN(frag_size) >> 917 SGE_PAGE_SHIFT; 918 919 bnx2x_tpa_stop(bp, fp, tpa_info, pages, 920 &cqe->end_agg_cqe, comp_ring_cons); 921 #ifdef BNX2X_STOP_ON_ERROR 922 if (bp->panic) 923 return 0; 924 #endif 925 926 bnx2x_update_sge_prod(fp, pages, &cqe->end_agg_cqe); 927 goto next_cqe; 928 } 929 /* non TPA */ 930 len = le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len); 931 pad = cqe_fp->placement_offset; 932 dma_sync_single_for_cpu(&bp->pdev->dev, 933 dma_unmap_addr(rx_buf, mapping), 934 pad + RX_COPY_THRESH, 935 DMA_FROM_DEVICE); 936 pad += NET_SKB_PAD; 937 prefetch(data + pad); /* speedup eth_type_trans() */ 938 /* is this an error packet? */ 939 if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) { 940 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS, 941 "ERROR flags %x rx packet %u\n", 942 cqe_fp_flags, sw_comp_cons); 943 bnx2x_fp_qstats(bp, fp)->rx_err_discard_pkt++; 944 goto reuse_rx; 945 } 946 947 /* Since we don't have a jumbo ring 948 * copy small packets if mtu > 1500 949 */ 950 if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) && 951 (len <= RX_COPY_THRESH)) { 952 skb = netdev_alloc_skb_ip_align(bp->dev, len); 953 if (skb == NULL) { 954 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS, 955 "ERROR packet dropped because of alloc failure\n"); 956 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++; 957 goto reuse_rx; 958 } 959 memcpy(skb->data, data + pad, len); 960 bnx2x_reuse_rx_data(fp, bd_cons, bd_prod); 961 } else { 962 if (likely(bnx2x_alloc_rx_data(bp, fp, bd_prod, 963 GFP_ATOMIC) == 0)) { 964 dma_unmap_single(&bp->pdev->dev, 965 dma_unmap_addr(rx_buf, mapping), 966 fp->rx_buf_size, 967 DMA_FROM_DEVICE); 968 skb = build_skb(data, fp->rx_frag_size); 969 if (unlikely(!skb)) { 970 bnx2x_frag_free(fp, data); 971 bnx2x_fp_qstats(bp, fp)-> 972 rx_skb_alloc_failed++; 973 goto next_rx; 974 } 975 skb_reserve(skb, pad); 976 } else { 977 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS, 978 "ERROR packet dropped because of alloc failure\n"); 979 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++; 980 reuse_rx: 981 bnx2x_reuse_rx_data(fp, bd_cons, bd_prod); 982 goto next_rx; 983 } 984 } 985 986 skb_put(skb, len); 987 skb->protocol = eth_type_trans(skb, bp->dev); 988 989 /* Set Toeplitz hash for a none-LRO skb */ 990 skb->rxhash = bnx2x_get_rxhash(bp, cqe_fp, &l4_rxhash); 991 skb->l4_rxhash = l4_rxhash; 992 993 skb_checksum_none_assert(skb); 994 995 if (bp->dev->features & NETIF_F_RXCSUM) 996 bnx2x_csum_validate(skb, cqe, fp, 997 bnx2x_fp_qstats(bp, fp)); 998 999 skb_record_rx_queue(skb, fp->rx_queue); 1000 1001 if (le16_to_cpu(cqe_fp->pars_flags.flags) & 1002 PARSING_FLAGS_VLAN) 1003 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), 1004 le16_to_cpu(cqe_fp->vlan_tag)); 1005 1006 skb_mark_napi_id(skb, &fp->napi); 1007 1008 if (bnx2x_fp_ll_polling(fp)) 1009 netif_receive_skb(skb); 1010 else 1011 napi_gro_receive(&fp->napi, skb); 1012 next_rx: 1013 rx_buf->data = NULL; 1014 1015 bd_cons = NEXT_RX_IDX(bd_cons); 1016 bd_prod = NEXT_RX_IDX(bd_prod); 1017 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw); 1018 rx_pkt++; 1019 next_cqe: 1020 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod); 1021 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons); 1022 1023 /* mark CQE as free */ 1024 BNX2X_SEED_CQE(cqe_fp); 1025 1026 if (rx_pkt == budget) 1027 break; 1028 1029 comp_ring_cons = RCQ_BD(sw_comp_cons); 1030 cqe = &fp->rx_comp_ring[comp_ring_cons]; 1031 cqe_fp = &cqe->fast_path_cqe; 1032 } /* while */ 1033 1034 fp->rx_bd_cons = bd_cons; 1035 fp->rx_bd_prod = bd_prod_fw; 1036 fp->rx_comp_cons = sw_comp_cons; 1037 fp->rx_comp_prod = sw_comp_prod; 1038 1039 /* Update producers */ 1040 bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod, 1041 fp->rx_sge_prod); 1042 1043 fp->rx_pkt += rx_pkt; 1044 fp->rx_calls++; 1045 1046 return rx_pkt; 1047 } 1048 1049 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie) 1050 { 1051 struct bnx2x_fastpath *fp = fp_cookie; 1052 struct bnx2x *bp = fp->bp; 1053 u8 cos; 1054 1055 DP(NETIF_MSG_INTR, 1056 "got an MSI-X interrupt on IDX:SB [fp %d fw_sd %d igusb %d]\n", 1057 fp->index, fp->fw_sb_id, fp->igu_sb_id); 1058 1059 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0); 1060 1061 #ifdef BNX2X_STOP_ON_ERROR 1062 if (unlikely(bp->panic)) 1063 return IRQ_HANDLED; 1064 #endif 1065 1066 /* Handle Rx and Tx according to MSI-X vector */ 1067 for_each_cos_in_tx_queue(fp, cos) 1068 prefetch(fp->txdata_ptr[cos]->tx_cons_sb); 1069 1070 prefetch(&fp->sb_running_index[SM_RX_ID]); 1071 napi_schedule(&bnx2x_fp(bp, fp->index, napi)); 1072 1073 return IRQ_HANDLED; 1074 } 1075 1076 /* HW Lock for shared dual port PHYs */ 1077 void bnx2x_acquire_phy_lock(struct bnx2x *bp) 1078 { 1079 mutex_lock(&bp->port.phy_mutex); 1080 1081 bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO); 1082 } 1083 1084 void bnx2x_release_phy_lock(struct bnx2x *bp) 1085 { 1086 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO); 1087 1088 mutex_unlock(&bp->port.phy_mutex); 1089 } 1090 1091 /* calculates MF speed according to current linespeed and MF configuration */ 1092 u16 bnx2x_get_mf_speed(struct bnx2x *bp) 1093 { 1094 u16 line_speed = bp->link_vars.line_speed; 1095 if (IS_MF(bp)) { 1096 u16 maxCfg = bnx2x_extract_max_cfg(bp, 1097 bp->mf_config[BP_VN(bp)]); 1098 1099 /* Calculate the current MAX line speed limit for the MF 1100 * devices 1101 */ 1102 if (IS_MF_SI(bp)) 1103 line_speed = (line_speed * maxCfg) / 100; 1104 else { /* SD mode */ 1105 u16 vn_max_rate = maxCfg * 100; 1106 1107 if (vn_max_rate < line_speed) 1108 line_speed = vn_max_rate; 1109 } 1110 } 1111 1112 return line_speed; 1113 } 1114 1115 /** 1116 * bnx2x_fill_report_data - fill link report data to report 1117 * 1118 * @bp: driver handle 1119 * @data: link state to update 1120 * 1121 * It uses a none-atomic bit operations because is called under the mutex. 1122 */ 1123 static void bnx2x_fill_report_data(struct bnx2x *bp, 1124 struct bnx2x_link_report_data *data) 1125 { 1126 u16 line_speed = bnx2x_get_mf_speed(bp); 1127 1128 memset(data, 0, sizeof(*data)); 1129 1130 /* Fill the report data: effective line speed */ 1131 data->line_speed = line_speed; 1132 1133 /* Link is down */ 1134 if (!bp->link_vars.link_up || (bp->flags & MF_FUNC_DIS)) 1135 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN, 1136 &data->link_report_flags); 1137 1138 /* Full DUPLEX */ 1139 if (bp->link_vars.duplex == DUPLEX_FULL) 1140 __set_bit(BNX2X_LINK_REPORT_FD, &data->link_report_flags); 1141 1142 /* Rx Flow Control is ON */ 1143 if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX) 1144 __set_bit(BNX2X_LINK_REPORT_RX_FC_ON, &data->link_report_flags); 1145 1146 /* Tx Flow Control is ON */ 1147 if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX) 1148 __set_bit(BNX2X_LINK_REPORT_TX_FC_ON, &data->link_report_flags); 1149 } 1150 1151 /** 1152 * bnx2x_link_report - report link status to OS. 1153 * 1154 * @bp: driver handle 1155 * 1156 * Calls the __bnx2x_link_report() under the same locking scheme 1157 * as a link/PHY state managing code to ensure a consistent link 1158 * reporting. 1159 */ 1160 1161 void bnx2x_link_report(struct bnx2x *bp) 1162 { 1163 bnx2x_acquire_phy_lock(bp); 1164 __bnx2x_link_report(bp); 1165 bnx2x_release_phy_lock(bp); 1166 } 1167 1168 /** 1169 * __bnx2x_link_report - report link status to OS. 1170 * 1171 * @bp: driver handle 1172 * 1173 * None atomic implementation. 1174 * Should be called under the phy_lock. 1175 */ 1176 void __bnx2x_link_report(struct bnx2x *bp) 1177 { 1178 struct bnx2x_link_report_data cur_data; 1179 1180 /* reread mf_cfg */ 1181 if (IS_PF(bp) && !CHIP_IS_E1(bp)) 1182 bnx2x_read_mf_cfg(bp); 1183 1184 /* Read the current link report info */ 1185 bnx2x_fill_report_data(bp, &cur_data); 1186 1187 /* Don't report link down or exactly the same link status twice */ 1188 if (!memcmp(&cur_data, &bp->last_reported_link, sizeof(cur_data)) || 1189 (test_bit(BNX2X_LINK_REPORT_LINK_DOWN, 1190 &bp->last_reported_link.link_report_flags) && 1191 test_bit(BNX2X_LINK_REPORT_LINK_DOWN, 1192 &cur_data.link_report_flags))) 1193 return; 1194 1195 bp->link_cnt++; 1196 1197 /* We are going to report a new link parameters now - 1198 * remember the current data for the next time. 1199 */ 1200 memcpy(&bp->last_reported_link, &cur_data, sizeof(cur_data)); 1201 1202 if (test_bit(BNX2X_LINK_REPORT_LINK_DOWN, 1203 &cur_data.link_report_flags)) { 1204 netif_carrier_off(bp->dev); 1205 netdev_err(bp->dev, "NIC Link is Down\n"); 1206 return; 1207 } else { 1208 const char *duplex; 1209 const char *flow; 1210 1211 netif_carrier_on(bp->dev); 1212 1213 if (test_and_clear_bit(BNX2X_LINK_REPORT_FD, 1214 &cur_data.link_report_flags)) 1215 duplex = "full"; 1216 else 1217 duplex = "half"; 1218 1219 /* Handle the FC at the end so that only these flags would be 1220 * possibly set. This way we may easily check if there is no FC 1221 * enabled. 1222 */ 1223 if (cur_data.link_report_flags) { 1224 if (test_bit(BNX2X_LINK_REPORT_RX_FC_ON, 1225 &cur_data.link_report_flags)) { 1226 if (test_bit(BNX2X_LINK_REPORT_TX_FC_ON, 1227 &cur_data.link_report_flags)) 1228 flow = "ON - receive & transmit"; 1229 else 1230 flow = "ON - receive"; 1231 } else { 1232 flow = "ON - transmit"; 1233 } 1234 } else { 1235 flow = "none"; 1236 } 1237 netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n", 1238 cur_data.line_speed, duplex, flow); 1239 } 1240 } 1241 1242 static void bnx2x_set_next_page_sgl(struct bnx2x_fastpath *fp) 1243 { 1244 int i; 1245 1246 for (i = 1; i <= NUM_RX_SGE_PAGES; i++) { 1247 struct eth_rx_sge *sge; 1248 1249 sge = &fp->rx_sge_ring[RX_SGE_CNT * i - 2]; 1250 sge->addr_hi = 1251 cpu_to_le32(U64_HI(fp->rx_sge_mapping + 1252 BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES))); 1253 1254 sge->addr_lo = 1255 cpu_to_le32(U64_LO(fp->rx_sge_mapping + 1256 BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES))); 1257 } 1258 } 1259 1260 static void bnx2x_free_tpa_pool(struct bnx2x *bp, 1261 struct bnx2x_fastpath *fp, int last) 1262 { 1263 int i; 1264 1265 for (i = 0; i < last; i++) { 1266 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[i]; 1267 struct sw_rx_bd *first_buf = &tpa_info->first_buf; 1268 u8 *data = first_buf->data; 1269 1270 if (data == NULL) { 1271 DP(NETIF_MSG_IFDOWN, "tpa bin %d empty on free\n", i); 1272 continue; 1273 } 1274 if (tpa_info->tpa_state == BNX2X_TPA_START) 1275 dma_unmap_single(&bp->pdev->dev, 1276 dma_unmap_addr(first_buf, mapping), 1277 fp->rx_buf_size, DMA_FROM_DEVICE); 1278 bnx2x_frag_free(fp, data); 1279 first_buf->data = NULL; 1280 } 1281 } 1282 1283 void bnx2x_init_rx_rings_cnic(struct bnx2x *bp) 1284 { 1285 int j; 1286 1287 for_each_rx_queue_cnic(bp, j) { 1288 struct bnx2x_fastpath *fp = &bp->fp[j]; 1289 1290 fp->rx_bd_cons = 0; 1291 1292 /* Activate BD ring */ 1293 /* Warning! 1294 * this will generate an interrupt (to the TSTORM) 1295 * must only be done after chip is initialized 1296 */ 1297 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod, 1298 fp->rx_sge_prod); 1299 } 1300 } 1301 1302 void bnx2x_init_rx_rings(struct bnx2x *bp) 1303 { 1304 int func = BP_FUNC(bp); 1305 u16 ring_prod; 1306 int i, j; 1307 1308 /* Allocate TPA resources */ 1309 for_each_eth_queue(bp, j) { 1310 struct bnx2x_fastpath *fp = &bp->fp[j]; 1311 1312 DP(NETIF_MSG_IFUP, 1313 "mtu %d rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size); 1314 1315 if (!fp->disable_tpa) { 1316 /* Fill the per-aggregation pool */ 1317 for (i = 0; i < MAX_AGG_QS(bp); i++) { 1318 struct bnx2x_agg_info *tpa_info = 1319 &fp->tpa_info[i]; 1320 struct sw_rx_bd *first_buf = 1321 &tpa_info->first_buf; 1322 1323 first_buf->data = 1324 bnx2x_frag_alloc(fp, GFP_KERNEL); 1325 if (!first_buf->data) { 1326 BNX2X_ERR("Failed to allocate TPA skb pool for queue[%d] - disabling TPA on this queue!\n", 1327 j); 1328 bnx2x_free_tpa_pool(bp, fp, i); 1329 fp->disable_tpa = 1; 1330 break; 1331 } 1332 dma_unmap_addr_set(first_buf, mapping, 0); 1333 tpa_info->tpa_state = BNX2X_TPA_STOP; 1334 } 1335 1336 /* "next page" elements initialization */ 1337 bnx2x_set_next_page_sgl(fp); 1338 1339 /* set SGEs bit mask */ 1340 bnx2x_init_sge_ring_bit_mask(fp); 1341 1342 /* Allocate SGEs and initialize the ring elements */ 1343 for (i = 0, ring_prod = 0; 1344 i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) { 1345 1346 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod, 1347 GFP_KERNEL) < 0) { 1348 BNX2X_ERR("was only able to allocate %d rx sges\n", 1349 i); 1350 BNX2X_ERR("disabling TPA for queue[%d]\n", 1351 j); 1352 /* Cleanup already allocated elements */ 1353 bnx2x_free_rx_sge_range(bp, fp, 1354 ring_prod); 1355 bnx2x_free_tpa_pool(bp, fp, 1356 MAX_AGG_QS(bp)); 1357 fp->disable_tpa = 1; 1358 ring_prod = 0; 1359 break; 1360 } 1361 ring_prod = NEXT_SGE_IDX(ring_prod); 1362 } 1363 1364 fp->rx_sge_prod = ring_prod; 1365 } 1366 } 1367 1368 for_each_eth_queue(bp, j) { 1369 struct bnx2x_fastpath *fp = &bp->fp[j]; 1370 1371 fp->rx_bd_cons = 0; 1372 1373 /* Activate BD ring */ 1374 /* Warning! 1375 * this will generate an interrupt (to the TSTORM) 1376 * must only be done after chip is initialized 1377 */ 1378 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod, 1379 fp->rx_sge_prod); 1380 1381 if (j != 0) 1382 continue; 1383 1384 if (CHIP_IS_E1(bp)) { 1385 REG_WR(bp, BAR_USTRORM_INTMEM + 1386 USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func), 1387 U64_LO(fp->rx_comp_mapping)); 1388 REG_WR(bp, BAR_USTRORM_INTMEM + 1389 USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4, 1390 U64_HI(fp->rx_comp_mapping)); 1391 } 1392 } 1393 } 1394 1395 static void bnx2x_free_tx_skbs_queue(struct bnx2x_fastpath *fp) 1396 { 1397 u8 cos; 1398 struct bnx2x *bp = fp->bp; 1399 1400 for_each_cos_in_tx_queue(fp, cos) { 1401 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos]; 1402 unsigned pkts_compl = 0, bytes_compl = 0; 1403 1404 u16 sw_prod = txdata->tx_pkt_prod; 1405 u16 sw_cons = txdata->tx_pkt_cons; 1406 1407 while (sw_cons != sw_prod) { 1408 bnx2x_free_tx_pkt(bp, txdata, TX_BD(sw_cons), 1409 &pkts_compl, &bytes_compl); 1410 sw_cons++; 1411 } 1412 1413 netdev_tx_reset_queue( 1414 netdev_get_tx_queue(bp->dev, 1415 txdata->txq_index)); 1416 } 1417 } 1418 1419 static void bnx2x_free_tx_skbs_cnic(struct bnx2x *bp) 1420 { 1421 int i; 1422 1423 for_each_tx_queue_cnic(bp, i) { 1424 bnx2x_free_tx_skbs_queue(&bp->fp[i]); 1425 } 1426 } 1427 1428 static void bnx2x_free_tx_skbs(struct bnx2x *bp) 1429 { 1430 int i; 1431 1432 for_each_eth_queue(bp, i) { 1433 bnx2x_free_tx_skbs_queue(&bp->fp[i]); 1434 } 1435 } 1436 1437 static void bnx2x_free_rx_bds(struct bnx2x_fastpath *fp) 1438 { 1439 struct bnx2x *bp = fp->bp; 1440 int i; 1441 1442 /* ring wasn't allocated */ 1443 if (fp->rx_buf_ring == NULL) 1444 return; 1445 1446 for (i = 0; i < NUM_RX_BD; i++) { 1447 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i]; 1448 u8 *data = rx_buf->data; 1449 1450 if (data == NULL) 1451 continue; 1452 dma_unmap_single(&bp->pdev->dev, 1453 dma_unmap_addr(rx_buf, mapping), 1454 fp->rx_buf_size, DMA_FROM_DEVICE); 1455 1456 rx_buf->data = NULL; 1457 bnx2x_frag_free(fp, data); 1458 } 1459 } 1460 1461 static void bnx2x_free_rx_skbs_cnic(struct bnx2x *bp) 1462 { 1463 int j; 1464 1465 for_each_rx_queue_cnic(bp, j) { 1466 bnx2x_free_rx_bds(&bp->fp[j]); 1467 } 1468 } 1469 1470 static void bnx2x_free_rx_skbs(struct bnx2x *bp) 1471 { 1472 int j; 1473 1474 for_each_eth_queue(bp, j) { 1475 struct bnx2x_fastpath *fp = &bp->fp[j]; 1476 1477 bnx2x_free_rx_bds(fp); 1478 1479 if (!fp->disable_tpa) 1480 bnx2x_free_tpa_pool(bp, fp, MAX_AGG_QS(bp)); 1481 } 1482 } 1483 1484 void bnx2x_free_skbs_cnic(struct bnx2x *bp) 1485 { 1486 bnx2x_free_tx_skbs_cnic(bp); 1487 bnx2x_free_rx_skbs_cnic(bp); 1488 } 1489 1490 void bnx2x_free_skbs(struct bnx2x *bp) 1491 { 1492 bnx2x_free_tx_skbs(bp); 1493 bnx2x_free_rx_skbs(bp); 1494 } 1495 1496 void bnx2x_update_max_mf_config(struct bnx2x *bp, u32 value) 1497 { 1498 /* load old values */ 1499 u32 mf_cfg = bp->mf_config[BP_VN(bp)]; 1500 1501 if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) { 1502 /* leave all but MAX value */ 1503 mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK; 1504 1505 /* set new MAX value */ 1506 mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT) 1507 & FUNC_MF_CFG_MAX_BW_MASK; 1508 1509 bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg); 1510 } 1511 } 1512 1513 /** 1514 * bnx2x_free_msix_irqs - free previously requested MSI-X IRQ vectors 1515 * 1516 * @bp: driver handle 1517 * @nvecs: number of vectors to be released 1518 */ 1519 static void bnx2x_free_msix_irqs(struct bnx2x *bp, int nvecs) 1520 { 1521 int i, offset = 0; 1522 1523 if (nvecs == offset) 1524 return; 1525 1526 /* VFs don't have a default SB */ 1527 if (IS_PF(bp)) { 1528 free_irq(bp->msix_table[offset].vector, bp->dev); 1529 DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n", 1530 bp->msix_table[offset].vector); 1531 offset++; 1532 } 1533 1534 if (CNIC_SUPPORT(bp)) { 1535 if (nvecs == offset) 1536 return; 1537 offset++; 1538 } 1539 1540 for_each_eth_queue(bp, i) { 1541 if (nvecs == offset) 1542 return; 1543 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq\n", 1544 i, bp->msix_table[offset].vector); 1545 1546 free_irq(bp->msix_table[offset++].vector, &bp->fp[i]); 1547 } 1548 } 1549 1550 void bnx2x_free_irq(struct bnx2x *bp) 1551 { 1552 if (bp->flags & USING_MSIX_FLAG && 1553 !(bp->flags & USING_SINGLE_MSIX_FLAG)) { 1554 int nvecs = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_SUPPORT(bp); 1555 1556 /* vfs don't have a default status block */ 1557 if (IS_PF(bp)) 1558 nvecs++; 1559 1560 bnx2x_free_msix_irqs(bp, nvecs); 1561 } else { 1562 free_irq(bp->dev->irq, bp->dev); 1563 } 1564 } 1565 1566 int bnx2x_enable_msix(struct bnx2x *bp) 1567 { 1568 int msix_vec = 0, i, rc; 1569 1570 /* VFs don't have a default status block */ 1571 if (IS_PF(bp)) { 1572 bp->msix_table[msix_vec].entry = msix_vec; 1573 BNX2X_DEV_INFO("msix_table[0].entry = %d (slowpath)\n", 1574 bp->msix_table[0].entry); 1575 msix_vec++; 1576 } 1577 1578 /* Cnic requires an msix vector for itself */ 1579 if (CNIC_SUPPORT(bp)) { 1580 bp->msix_table[msix_vec].entry = msix_vec; 1581 BNX2X_DEV_INFO("msix_table[%d].entry = %d (CNIC)\n", 1582 msix_vec, bp->msix_table[msix_vec].entry); 1583 msix_vec++; 1584 } 1585 1586 /* We need separate vectors for ETH queues only (not FCoE) */ 1587 for_each_eth_queue(bp, i) { 1588 bp->msix_table[msix_vec].entry = msix_vec; 1589 BNX2X_DEV_INFO("msix_table[%d].entry = %d (fastpath #%u)\n", 1590 msix_vec, msix_vec, i); 1591 msix_vec++; 1592 } 1593 1594 DP(BNX2X_MSG_SP, "about to request enable msix with %d vectors\n", 1595 msix_vec); 1596 1597 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], msix_vec); 1598 1599 /* 1600 * reconfigure number of tx/rx queues according to available 1601 * MSI-X vectors 1602 */ 1603 if (rc >= BNX2X_MIN_MSIX_VEC_CNT(bp)) { 1604 /* how less vectors we will have? */ 1605 int diff = msix_vec - rc; 1606 1607 BNX2X_DEV_INFO("Trying to use less MSI-X vectors: %d\n", rc); 1608 1609 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], rc); 1610 1611 if (rc) { 1612 BNX2X_DEV_INFO("MSI-X is not attainable rc %d\n", rc); 1613 goto no_msix; 1614 } 1615 /* 1616 * decrease number of queues by number of unallocated entries 1617 */ 1618 bp->num_ethernet_queues -= diff; 1619 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues; 1620 1621 BNX2X_DEV_INFO("New queue configuration set: %d\n", 1622 bp->num_queues); 1623 } else if (rc > 0) { 1624 /* Get by with single vector */ 1625 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], 1); 1626 if (rc) { 1627 BNX2X_DEV_INFO("Single MSI-X is not attainable rc %d\n", 1628 rc); 1629 goto no_msix; 1630 } 1631 1632 BNX2X_DEV_INFO("Using single MSI-X vector\n"); 1633 bp->flags |= USING_SINGLE_MSIX_FLAG; 1634 1635 BNX2X_DEV_INFO("set number of queues to 1\n"); 1636 bp->num_ethernet_queues = 1; 1637 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues; 1638 } else if (rc < 0) { 1639 BNX2X_DEV_INFO("MSI-X is not attainable rc %d\n", rc); 1640 goto no_msix; 1641 } 1642 1643 bp->flags |= USING_MSIX_FLAG; 1644 1645 return 0; 1646 1647 no_msix: 1648 /* fall to INTx if not enough memory */ 1649 if (rc == -ENOMEM) 1650 bp->flags |= DISABLE_MSI_FLAG; 1651 1652 return rc; 1653 } 1654 1655 static int bnx2x_req_msix_irqs(struct bnx2x *bp) 1656 { 1657 int i, rc, offset = 0; 1658 1659 /* no default status block for vf */ 1660 if (IS_PF(bp)) { 1661 rc = request_irq(bp->msix_table[offset++].vector, 1662 bnx2x_msix_sp_int, 0, 1663 bp->dev->name, bp->dev); 1664 if (rc) { 1665 BNX2X_ERR("request sp irq failed\n"); 1666 return -EBUSY; 1667 } 1668 } 1669 1670 if (CNIC_SUPPORT(bp)) 1671 offset++; 1672 1673 for_each_eth_queue(bp, i) { 1674 struct bnx2x_fastpath *fp = &bp->fp[i]; 1675 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d", 1676 bp->dev->name, i); 1677 1678 rc = request_irq(bp->msix_table[offset].vector, 1679 bnx2x_msix_fp_int, 0, fp->name, fp); 1680 if (rc) { 1681 BNX2X_ERR("request fp #%d irq (%d) failed rc %d\n", i, 1682 bp->msix_table[offset].vector, rc); 1683 bnx2x_free_msix_irqs(bp, offset); 1684 return -EBUSY; 1685 } 1686 1687 offset++; 1688 } 1689 1690 i = BNX2X_NUM_ETH_QUEUES(bp); 1691 if (IS_PF(bp)) { 1692 offset = 1 + CNIC_SUPPORT(bp); 1693 netdev_info(bp->dev, 1694 "using MSI-X IRQs: sp %d fp[%d] %d ... fp[%d] %d\n", 1695 bp->msix_table[0].vector, 1696 0, bp->msix_table[offset].vector, 1697 i - 1, bp->msix_table[offset + i - 1].vector); 1698 } else { 1699 offset = CNIC_SUPPORT(bp); 1700 netdev_info(bp->dev, 1701 "using MSI-X IRQs: fp[%d] %d ... fp[%d] %d\n", 1702 0, bp->msix_table[offset].vector, 1703 i - 1, bp->msix_table[offset + i - 1].vector); 1704 } 1705 return 0; 1706 } 1707 1708 int bnx2x_enable_msi(struct bnx2x *bp) 1709 { 1710 int rc; 1711 1712 rc = pci_enable_msi(bp->pdev); 1713 if (rc) { 1714 BNX2X_DEV_INFO("MSI is not attainable\n"); 1715 return -1; 1716 } 1717 bp->flags |= USING_MSI_FLAG; 1718 1719 return 0; 1720 } 1721 1722 static int bnx2x_req_irq(struct bnx2x *bp) 1723 { 1724 unsigned long flags; 1725 unsigned int irq; 1726 1727 if (bp->flags & (USING_MSI_FLAG | USING_MSIX_FLAG)) 1728 flags = 0; 1729 else 1730 flags = IRQF_SHARED; 1731 1732 if (bp->flags & USING_MSIX_FLAG) 1733 irq = bp->msix_table[0].vector; 1734 else 1735 irq = bp->pdev->irq; 1736 1737 return request_irq(irq, bnx2x_interrupt, flags, bp->dev->name, bp->dev); 1738 } 1739 1740 static int bnx2x_setup_irqs(struct bnx2x *bp) 1741 { 1742 int rc = 0; 1743 if (bp->flags & USING_MSIX_FLAG && 1744 !(bp->flags & USING_SINGLE_MSIX_FLAG)) { 1745 rc = bnx2x_req_msix_irqs(bp); 1746 if (rc) 1747 return rc; 1748 } else { 1749 rc = bnx2x_req_irq(bp); 1750 if (rc) { 1751 BNX2X_ERR("IRQ request failed rc %d, aborting\n", rc); 1752 return rc; 1753 } 1754 if (bp->flags & USING_MSI_FLAG) { 1755 bp->dev->irq = bp->pdev->irq; 1756 netdev_info(bp->dev, "using MSI IRQ %d\n", 1757 bp->dev->irq); 1758 } 1759 if (bp->flags & USING_MSIX_FLAG) { 1760 bp->dev->irq = bp->msix_table[0].vector; 1761 netdev_info(bp->dev, "using MSIX IRQ %d\n", 1762 bp->dev->irq); 1763 } 1764 } 1765 1766 return 0; 1767 } 1768 1769 static void bnx2x_napi_enable_cnic(struct bnx2x *bp) 1770 { 1771 int i; 1772 1773 for_each_rx_queue_cnic(bp, i) { 1774 bnx2x_fp_init_lock(&bp->fp[i]); 1775 napi_enable(&bnx2x_fp(bp, i, napi)); 1776 } 1777 } 1778 1779 static void bnx2x_napi_enable(struct bnx2x *bp) 1780 { 1781 int i; 1782 1783 for_each_eth_queue(bp, i) { 1784 bnx2x_fp_init_lock(&bp->fp[i]); 1785 napi_enable(&bnx2x_fp(bp, i, napi)); 1786 } 1787 } 1788 1789 static void bnx2x_napi_disable_cnic(struct bnx2x *bp) 1790 { 1791 int i; 1792 1793 local_bh_disable(); 1794 for_each_rx_queue_cnic(bp, i) { 1795 napi_disable(&bnx2x_fp(bp, i, napi)); 1796 while (!bnx2x_fp_lock_napi(&bp->fp[i])) 1797 mdelay(1); 1798 } 1799 local_bh_enable(); 1800 } 1801 1802 static void bnx2x_napi_disable(struct bnx2x *bp) 1803 { 1804 int i; 1805 1806 local_bh_disable(); 1807 for_each_eth_queue(bp, i) { 1808 napi_disable(&bnx2x_fp(bp, i, napi)); 1809 while (!bnx2x_fp_lock_napi(&bp->fp[i])) 1810 mdelay(1); 1811 } 1812 local_bh_enable(); 1813 } 1814 1815 void bnx2x_netif_start(struct bnx2x *bp) 1816 { 1817 if (netif_running(bp->dev)) { 1818 bnx2x_napi_enable(bp); 1819 if (CNIC_LOADED(bp)) 1820 bnx2x_napi_enable_cnic(bp); 1821 bnx2x_int_enable(bp); 1822 if (bp->state == BNX2X_STATE_OPEN) 1823 netif_tx_wake_all_queues(bp->dev); 1824 } 1825 } 1826 1827 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw) 1828 { 1829 bnx2x_int_disable_sync(bp, disable_hw); 1830 bnx2x_napi_disable(bp); 1831 if (CNIC_LOADED(bp)) 1832 bnx2x_napi_disable_cnic(bp); 1833 } 1834 1835 u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb) 1836 { 1837 struct bnx2x *bp = netdev_priv(dev); 1838 1839 if (CNIC_LOADED(bp) && !NO_FCOE(bp)) { 1840 struct ethhdr *hdr = (struct ethhdr *)skb->data; 1841 u16 ether_type = ntohs(hdr->h_proto); 1842 1843 /* Skip VLAN tag if present */ 1844 if (ether_type == ETH_P_8021Q) { 1845 struct vlan_ethhdr *vhdr = 1846 (struct vlan_ethhdr *)skb->data; 1847 1848 ether_type = ntohs(vhdr->h_vlan_encapsulated_proto); 1849 } 1850 1851 /* If ethertype is FCoE or FIP - use FCoE ring */ 1852 if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP)) 1853 return bnx2x_fcoe_tx(bp, txq_index); 1854 } 1855 1856 /* select a non-FCoE queue */ 1857 return __netdev_pick_tx(dev, skb) % BNX2X_NUM_ETH_QUEUES(bp); 1858 } 1859 1860 void bnx2x_set_num_queues(struct bnx2x *bp) 1861 { 1862 /* RSS queues */ 1863 bp->num_ethernet_queues = bnx2x_calc_num_queues(bp); 1864 1865 /* override in STORAGE SD modes */ 1866 if (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp)) 1867 bp->num_ethernet_queues = 1; 1868 1869 /* Add special queues */ 1870 bp->num_cnic_queues = CNIC_SUPPORT(bp); /* For FCOE */ 1871 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues; 1872 1873 BNX2X_DEV_INFO("set number of queues to %d\n", bp->num_queues); 1874 } 1875 1876 /** 1877 * bnx2x_set_real_num_queues - configure netdev->real_num_[tx,rx]_queues 1878 * 1879 * @bp: Driver handle 1880 * 1881 * We currently support for at most 16 Tx queues for each CoS thus we will 1882 * allocate a multiple of 16 for ETH L2 rings according to the value of the 1883 * bp->max_cos. 1884 * 1885 * If there is an FCoE L2 queue the appropriate Tx queue will have the next 1886 * index after all ETH L2 indices. 1887 * 1888 * If the actual number of Tx queues (for each CoS) is less than 16 then there 1889 * will be the holes at the end of each group of 16 ETh L2 indices (0..15, 1890 * 16..31,...) with indices that are not coupled with any real Tx queue. 1891 * 1892 * The proper configuration of skb->queue_mapping is handled by 1893 * bnx2x_select_queue() and __skb_tx_hash(). 1894 * 1895 * bnx2x_setup_tc() takes care of the proper TC mappings so that __skb_tx_hash() 1896 * will return a proper Tx index if TC is enabled (netdev->num_tc > 0). 1897 */ 1898 static int bnx2x_set_real_num_queues(struct bnx2x *bp, int include_cnic) 1899 { 1900 int rc, tx, rx; 1901 1902 tx = BNX2X_NUM_ETH_QUEUES(bp) * bp->max_cos; 1903 rx = BNX2X_NUM_ETH_QUEUES(bp); 1904 1905 /* account for fcoe queue */ 1906 if (include_cnic && !NO_FCOE(bp)) { 1907 rx++; 1908 tx++; 1909 } 1910 1911 rc = netif_set_real_num_tx_queues(bp->dev, tx); 1912 if (rc) { 1913 BNX2X_ERR("Failed to set real number of Tx queues: %d\n", rc); 1914 return rc; 1915 } 1916 rc = netif_set_real_num_rx_queues(bp->dev, rx); 1917 if (rc) { 1918 BNX2X_ERR("Failed to set real number of Rx queues: %d\n", rc); 1919 return rc; 1920 } 1921 1922 DP(NETIF_MSG_IFUP, "Setting real num queues to (tx, rx) (%d, %d)\n", 1923 tx, rx); 1924 1925 return rc; 1926 } 1927 1928 static void bnx2x_set_rx_buf_size(struct bnx2x *bp) 1929 { 1930 int i; 1931 1932 for_each_queue(bp, i) { 1933 struct bnx2x_fastpath *fp = &bp->fp[i]; 1934 u32 mtu; 1935 1936 /* Always use a mini-jumbo MTU for the FCoE L2 ring */ 1937 if (IS_FCOE_IDX(i)) 1938 /* 1939 * Although there are no IP frames expected to arrive to 1940 * this ring we still want to add an 1941 * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer 1942 * overrun attack. 1943 */ 1944 mtu = BNX2X_FCOE_MINI_JUMBO_MTU; 1945 else 1946 mtu = bp->dev->mtu; 1947 fp->rx_buf_size = BNX2X_FW_RX_ALIGN_START + 1948 IP_HEADER_ALIGNMENT_PADDING + 1949 ETH_OVREHEAD + 1950 mtu + 1951 BNX2X_FW_RX_ALIGN_END; 1952 /* Note : rx_buf_size doesn't take into account NET_SKB_PAD */ 1953 if (fp->rx_buf_size + NET_SKB_PAD <= PAGE_SIZE) 1954 fp->rx_frag_size = fp->rx_buf_size + NET_SKB_PAD; 1955 else 1956 fp->rx_frag_size = 0; 1957 } 1958 } 1959 1960 static int bnx2x_init_rss(struct bnx2x *bp) 1961 { 1962 int i; 1963 u8 num_eth_queues = BNX2X_NUM_ETH_QUEUES(bp); 1964 1965 /* Prepare the initial contents for the indirection table if RSS is 1966 * enabled 1967 */ 1968 for (i = 0; i < sizeof(bp->rss_conf_obj.ind_table); i++) 1969 bp->rss_conf_obj.ind_table[i] = 1970 bp->fp->cl_id + 1971 ethtool_rxfh_indir_default(i, num_eth_queues); 1972 1973 /* 1974 * For 57710 and 57711 SEARCHER configuration (rss_keys) is 1975 * per-port, so if explicit configuration is needed , do it only 1976 * for a PMF. 1977 * 1978 * For 57712 and newer on the other hand it's a per-function 1979 * configuration. 1980 */ 1981 return bnx2x_config_rss_eth(bp, bp->port.pmf || !CHIP_IS_E1x(bp)); 1982 } 1983 1984 int bnx2x_rss(struct bnx2x *bp, struct bnx2x_rss_config_obj *rss_obj, 1985 bool config_hash, bool enable) 1986 { 1987 struct bnx2x_config_rss_params params = {NULL}; 1988 1989 /* Although RSS is meaningless when there is a single HW queue we 1990 * still need it enabled in order to have HW Rx hash generated. 1991 * 1992 * if (!is_eth_multi(bp)) 1993 * bp->multi_mode = ETH_RSS_MODE_DISABLED; 1994 */ 1995 1996 params.rss_obj = rss_obj; 1997 1998 __set_bit(RAMROD_COMP_WAIT, ¶ms.ramrod_flags); 1999 2000 if (enable) { 2001 __set_bit(BNX2X_RSS_MODE_REGULAR, ¶ms.rss_flags); 2002 2003 /* RSS configuration */ 2004 __set_bit(BNX2X_RSS_IPV4, ¶ms.rss_flags); 2005 __set_bit(BNX2X_RSS_IPV4_TCP, ¶ms.rss_flags); 2006 __set_bit(BNX2X_RSS_IPV6, ¶ms.rss_flags); 2007 __set_bit(BNX2X_RSS_IPV6_TCP, ¶ms.rss_flags); 2008 if (rss_obj->udp_rss_v4) 2009 __set_bit(BNX2X_RSS_IPV4_UDP, ¶ms.rss_flags); 2010 if (rss_obj->udp_rss_v6) 2011 __set_bit(BNX2X_RSS_IPV6_UDP, ¶ms.rss_flags); 2012 } else { 2013 __set_bit(BNX2X_RSS_MODE_DISABLED, ¶ms.rss_flags); 2014 } 2015 2016 /* Hash bits */ 2017 params.rss_result_mask = MULTI_MASK; 2018 2019 memcpy(params.ind_table, rss_obj->ind_table, sizeof(params.ind_table)); 2020 2021 if (config_hash) { 2022 /* RSS keys */ 2023 prandom_bytes(params.rss_key, T_ETH_RSS_KEY * 4); 2024 __set_bit(BNX2X_RSS_SET_SRCH, ¶ms.rss_flags); 2025 } 2026 2027 if (IS_PF(bp)) 2028 return bnx2x_config_rss(bp, ¶ms); 2029 else 2030 return bnx2x_vfpf_config_rss(bp, ¶ms); 2031 } 2032 2033 static int bnx2x_init_hw(struct bnx2x *bp, u32 load_code) 2034 { 2035 struct bnx2x_func_state_params func_params = {NULL}; 2036 2037 /* Prepare parameters for function state transitions */ 2038 __set_bit(RAMROD_COMP_WAIT, &func_params.ramrod_flags); 2039 2040 func_params.f_obj = &bp->func_obj; 2041 func_params.cmd = BNX2X_F_CMD_HW_INIT; 2042 2043 func_params.params.hw_init.load_phase = load_code; 2044 2045 return bnx2x_func_state_change(bp, &func_params); 2046 } 2047 2048 /* 2049 * Cleans the object that have internal lists without sending 2050 * ramrods. Should be run when interrupts are disabled. 2051 */ 2052 void bnx2x_squeeze_objects(struct bnx2x *bp) 2053 { 2054 int rc; 2055 unsigned long ramrod_flags = 0, vlan_mac_flags = 0; 2056 struct bnx2x_mcast_ramrod_params rparam = {NULL}; 2057 struct bnx2x_vlan_mac_obj *mac_obj = &bp->sp_objs->mac_obj; 2058 2059 /***************** Cleanup MACs' object first *************************/ 2060 2061 /* Wait for completion of requested */ 2062 __set_bit(RAMROD_COMP_WAIT, &ramrod_flags); 2063 /* Perform a dry cleanup */ 2064 __set_bit(RAMROD_DRV_CLR_ONLY, &ramrod_flags); 2065 2066 /* Clean ETH primary MAC */ 2067 __set_bit(BNX2X_ETH_MAC, &vlan_mac_flags); 2068 rc = mac_obj->delete_all(bp, &bp->sp_objs->mac_obj, &vlan_mac_flags, 2069 &ramrod_flags); 2070 if (rc != 0) 2071 BNX2X_ERR("Failed to clean ETH MACs: %d\n", rc); 2072 2073 /* Cleanup UC list */ 2074 vlan_mac_flags = 0; 2075 __set_bit(BNX2X_UC_LIST_MAC, &vlan_mac_flags); 2076 rc = mac_obj->delete_all(bp, mac_obj, &vlan_mac_flags, 2077 &ramrod_flags); 2078 if (rc != 0) 2079 BNX2X_ERR("Failed to clean UC list MACs: %d\n", rc); 2080 2081 /***************** Now clean mcast object *****************************/ 2082 rparam.mcast_obj = &bp->mcast_obj; 2083 __set_bit(RAMROD_DRV_CLR_ONLY, &rparam.ramrod_flags); 2084 2085 /* Add a DEL command... - Since we're doing a driver cleanup only, 2086 * we take a lock surrounding both the initial send and the CONTs, 2087 * as we don't want a true completion to disrupt us in the middle. 2088 */ 2089 netif_addr_lock_bh(bp->dev); 2090 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_DEL); 2091 if (rc < 0) 2092 BNX2X_ERR("Failed to add a new DEL command to a multi-cast object: %d\n", 2093 rc); 2094 2095 /* ...and wait until all pending commands are cleared */ 2096 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT); 2097 while (rc != 0) { 2098 if (rc < 0) { 2099 BNX2X_ERR("Failed to clean multi-cast object: %d\n", 2100 rc); 2101 netif_addr_unlock_bh(bp->dev); 2102 return; 2103 } 2104 2105 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT); 2106 } 2107 netif_addr_unlock_bh(bp->dev); 2108 } 2109 2110 #ifndef BNX2X_STOP_ON_ERROR 2111 #define LOAD_ERROR_EXIT(bp, label) \ 2112 do { \ 2113 (bp)->state = BNX2X_STATE_ERROR; \ 2114 goto label; \ 2115 } while (0) 2116 2117 #define LOAD_ERROR_EXIT_CNIC(bp, label) \ 2118 do { \ 2119 bp->cnic_loaded = false; \ 2120 goto label; \ 2121 } while (0) 2122 #else /*BNX2X_STOP_ON_ERROR*/ 2123 #define LOAD_ERROR_EXIT(bp, label) \ 2124 do { \ 2125 (bp)->state = BNX2X_STATE_ERROR; \ 2126 (bp)->panic = 1; \ 2127 return -EBUSY; \ 2128 } while (0) 2129 #define LOAD_ERROR_EXIT_CNIC(bp, label) \ 2130 do { \ 2131 bp->cnic_loaded = false; \ 2132 (bp)->panic = 1; \ 2133 return -EBUSY; \ 2134 } while (0) 2135 #endif /*BNX2X_STOP_ON_ERROR*/ 2136 2137 static void bnx2x_free_fw_stats_mem(struct bnx2x *bp) 2138 { 2139 BNX2X_PCI_FREE(bp->fw_stats, bp->fw_stats_mapping, 2140 bp->fw_stats_data_sz + bp->fw_stats_req_sz); 2141 return; 2142 } 2143 2144 static int bnx2x_alloc_fw_stats_mem(struct bnx2x *bp) 2145 { 2146 int num_groups, vf_headroom = 0; 2147 int is_fcoe_stats = NO_FCOE(bp) ? 0 : 1; 2148 2149 /* number of queues for statistics is number of eth queues + FCoE */ 2150 u8 num_queue_stats = BNX2X_NUM_ETH_QUEUES(bp) + is_fcoe_stats; 2151 2152 /* Total number of FW statistics requests = 2153 * 1 for port stats + 1 for PF stats + potential 2 for FCoE (fcoe proper 2154 * and fcoe l2 queue) stats + num of queues (which includes another 1 2155 * for fcoe l2 queue if applicable) 2156 */ 2157 bp->fw_stats_num = 2 + is_fcoe_stats + num_queue_stats; 2158 2159 /* vf stats appear in the request list, but their data is allocated by 2160 * the VFs themselves. We don't include them in the bp->fw_stats_num as 2161 * it is used to determine where to place the vf stats queries in the 2162 * request struct 2163 */ 2164 if (IS_SRIOV(bp)) 2165 vf_headroom = bnx2x_vf_headroom(bp); 2166 2167 /* Request is built from stats_query_header and an array of 2168 * stats_query_cmd_group each of which contains 2169 * STATS_QUERY_CMD_COUNT rules. The real number or requests is 2170 * configured in the stats_query_header. 2171 */ 2172 num_groups = 2173 (((bp->fw_stats_num + vf_headroom) / STATS_QUERY_CMD_COUNT) + 2174 (((bp->fw_stats_num + vf_headroom) % STATS_QUERY_CMD_COUNT) ? 2175 1 : 0)); 2176 2177 DP(BNX2X_MSG_SP, "stats fw_stats_num %d, vf headroom %d, num_groups %d\n", 2178 bp->fw_stats_num, vf_headroom, num_groups); 2179 bp->fw_stats_req_sz = sizeof(struct stats_query_header) + 2180 num_groups * sizeof(struct stats_query_cmd_group); 2181 2182 /* Data for statistics requests + stats_counter 2183 * stats_counter holds per-STORM counters that are incremented 2184 * when STORM has finished with the current request. 2185 * memory for FCoE offloaded statistics are counted anyway, 2186 * even if they will not be sent. 2187 * VF stats are not accounted for here as the data of VF stats is stored 2188 * in memory allocated by the VF, not here. 2189 */ 2190 bp->fw_stats_data_sz = sizeof(struct per_port_stats) + 2191 sizeof(struct per_pf_stats) + 2192 sizeof(struct fcoe_statistics_params) + 2193 sizeof(struct per_queue_stats) * num_queue_stats + 2194 sizeof(struct stats_counter); 2195 2196 BNX2X_PCI_ALLOC(bp->fw_stats, &bp->fw_stats_mapping, 2197 bp->fw_stats_data_sz + bp->fw_stats_req_sz); 2198 2199 /* Set shortcuts */ 2200 bp->fw_stats_req = (struct bnx2x_fw_stats_req *)bp->fw_stats; 2201 bp->fw_stats_req_mapping = bp->fw_stats_mapping; 2202 bp->fw_stats_data = (struct bnx2x_fw_stats_data *) 2203 ((u8 *)bp->fw_stats + bp->fw_stats_req_sz); 2204 bp->fw_stats_data_mapping = bp->fw_stats_mapping + 2205 bp->fw_stats_req_sz; 2206 2207 DP(BNX2X_MSG_SP, "statistics request base address set to %x %x\n", 2208 U64_HI(bp->fw_stats_req_mapping), 2209 U64_LO(bp->fw_stats_req_mapping)); 2210 DP(BNX2X_MSG_SP, "statistics data base address set to %x %x\n", 2211 U64_HI(bp->fw_stats_data_mapping), 2212 U64_LO(bp->fw_stats_data_mapping)); 2213 return 0; 2214 2215 alloc_mem_err: 2216 bnx2x_free_fw_stats_mem(bp); 2217 BNX2X_ERR("Can't allocate FW stats memory\n"); 2218 return -ENOMEM; 2219 } 2220 2221 /* send load request to mcp and analyze response */ 2222 static int bnx2x_nic_load_request(struct bnx2x *bp, u32 *load_code) 2223 { 2224 u32 param; 2225 2226 /* init fw_seq */ 2227 bp->fw_seq = 2228 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_mb_header) & 2229 DRV_MSG_SEQ_NUMBER_MASK); 2230 BNX2X_DEV_INFO("fw_seq 0x%08x\n", bp->fw_seq); 2231 2232 /* Get current FW pulse sequence */ 2233 bp->fw_drv_pulse_wr_seq = 2234 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_pulse_mb) & 2235 DRV_PULSE_SEQ_MASK); 2236 BNX2X_DEV_INFO("drv_pulse 0x%x\n", bp->fw_drv_pulse_wr_seq); 2237 2238 param = DRV_MSG_CODE_LOAD_REQ_WITH_LFA; 2239 2240 if (IS_MF_SD(bp) && bnx2x_port_after_undi(bp)) 2241 param |= DRV_MSG_CODE_LOAD_REQ_FORCE_LFA; 2242 2243 /* load request */ 2244 (*load_code) = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ, param); 2245 2246 /* if mcp fails to respond we must abort */ 2247 if (!(*load_code)) { 2248 BNX2X_ERR("MCP response failure, aborting\n"); 2249 return -EBUSY; 2250 } 2251 2252 /* If mcp refused (e.g. other port is in diagnostic mode) we 2253 * must abort 2254 */ 2255 if ((*load_code) == FW_MSG_CODE_DRV_LOAD_REFUSED) { 2256 BNX2X_ERR("MCP refused load request, aborting\n"); 2257 return -EBUSY; 2258 } 2259 return 0; 2260 } 2261 2262 /* check whether another PF has already loaded FW to chip. In 2263 * virtualized environments a pf from another VM may have already 2264 * initialized the device including loading FW 2265 */ 2266 int bnx2x_nic_load_analyze_req(struct bnx2x *bp, u32 load_code) 2267 { 2268 /* is another pf loaded on this engine? */ 2269 if (load_code != FW_MSG_CODE_DRV_LOAD_COMMON_CHIP && 2270 load_code != FW_MSG_CODE_DRV_LOAD_COMMON) { 2271 /* build my FW version dword */ 2272 u32 my_fw = (BCM_5710_FW_MAJOR_VERSION) + 2273 (BCM_5710_FW_MINOR_VERSION << 8) + 2274 (BCM_5710_FW_REVISION_VERSION << 16) + 2275 (BCM_5710_FW_ENGINEERING_VERSION << 24); 2276 2277 /* read loaded FW from chip */ 2278 u32 loaded_fw = REG_RD(bp, XSEM_REG_PRAM); 2279 2280 DP(BNX2X_MSG_SP, "loaded fw %x, my fw %x\n", 2281 loaded_fw, my_fw); 2282 2283 /* abort nic load if version mismatch */ 2284 if (my_fw != loaded_fw) { 2285 BNX2X_ERR("bnx2x with FW %x was already loaded which mismatches my %x FW. Aborting\n", 2286 loaded_fw, my_fw); 2287 return -EBUSY; 2288 } 2289 } 2290 return 0; 2291 } 2292 2293 /* returns the "mcp load_code" according to global load_count array */ 2294 static int bnx2x_nic_load_no_mcp(struct bnx2x *bp, int port) 2295 { 2296 int path = BP_PATH(bp); 2297 2298 DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d] %d, %d, %d\n", 2299 path, load_count[path][0], load_count[path][1], 2300 load_count[path][2]); 2301 load_count[path][0]++; 2302 load_count[path][1 + port]++; 2303 DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d] %d, %d, %d\n", 2304 path, load_count[path][0], load_count[path][1], 2305 load_count[path][2]); 2306 if (load_count[path][0] == 1) 2307 return FW_MSG_CODE_DRV_LOAD_COMMON; 2308 else if (load_count[path][1 + port] == 1) 2309 return FW_MSG_CODE_DRV_LOAD_PORT; 2310 else 2311 return FW_MSG_CODE_DRV_LOAD_FUNCTION; 2312 } 2313 2314 /* mark PMF if applicable */ 2315 static void bnx2x_nic_load_pmf(struct bnx2x *bp, u32 load_code) 2316 { 2317 if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) || 2318 (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) || 2319 (load_code == FW_MSG_CODE_DRV_LOAD_PORT)) { 2320 bp->port.pmf = 1; 2321 /* We need the barrier to ensure the ordering between the 2322 * writing to bp->port.pmf here and reading it from the 2323 * bnx2x_periodic_task(). 2324 */ 2325 smp_mb(); 2326 } else { 2327 bp->port.pmf = 0; 2328 } 2329 2330 DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf); 2331 } 2332 2333 static void bnx2x_nic_load_afex_dcc(struct bnx2x *bp, int load_code) 2334 { 2335 if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) || 2336 (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) && 2337 (bp->common.shmem2_base)) { 2338 if (SHMEM2_HAS(bp, dcc_support)) 2339 SHMEM2_WR(bp, dcc_support, 2340 (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV | 2341 SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV)); 2342 if (SHMEM2_HAS(bp, afex_driver_support)) 2343 SHMEM2_WR(bp, afex_driver_support, 2344 SHMEM_AFEX_SUPPORTED_VERSION_ONE); 2345 } 2346 2347 /* Set AFEX default VLAN tag to an invalid value */ 2348 bp->afex_def_vlan_tag = -1; 2349 } 2350 2351 /** 2352 * bnx2x_bz_fp - zero content of the fastpath structure. 2353 * 2354 * @bp: driver handle 2355 * @index: fastpath index to be zeroed 2356 * 2357 * Makes sure the contents of the bp->fp[index].napi is kept 2358 * intact. 2359 */ 2360 static void bnx2x_bz_fp(struct bnx2x *bp, int index) 2361 { 2362 struct bnx2x_fastpath *fp = &bp->fp[index]; 2363 int cos; 2364 struct napi_struct orig_napi = fp->napi; 2365 struct bnx2x_agg_info *orig_tpa_info = fp->tpa_info; 2366 2367 /* bzero bnx2x_fastpath contents */ 2368 if (fp->tpa_info) 2369 memset(fp->tpa_info, 0, ETH_MAX_AGGREGATION_QUEUES_E1H_E2 * 2370 sizeof(struct bnx2x_agg_info)); 2371 memset(fp, 0, sizeof(*fp)); 2372 2373 /* Restore the NAPI object as it has been already initialized */ 2374 fp->napi = orig_napi; 2375 fp->tpa_info = orig_tpa_info; 2376 fp->bp = bp; 2377 fp->index = index; 2378 if (IS_ETH_FP(fp)) 2379 fp->max_cos = bp->max_cos; 2380 else 2381 /* Special queues support only one CoS */ 2382 fp->max_cos = 1; 2383 2384 /* Init txdata pointers */ 2385 if (IS_FCOE_FP(fp)) 2386 fp->txdata_ptr[0] = &bp->bnx2x_txq[FCOE_TXQ_IDX(bp)]; 2387 if (IS_ETH_FP(fp)) 2388 for_each_cos_in_tx_queue(fp, cos) 2389 fp->txdata_ptr[cos] = &bp->bnx2x_txq[cos * 2390 BNX2X_NUM_ETH_QUEUES(bp) + index]; 2391 2392 /* set the tpa flag for each queue. The tpa flag determines the queue 2393 * minimal size so it must be set prior to queue memory allocation 2394 */ 2395 fp->disable_tpa = !(bp->flags & TPA_ENABLE_FLAG || 2396 (bp->flags & GRO_ENABLE_FLAG && 2397 bnx2x_mtu_allows_gro(bp->dev->mtu))); 2398 if (bp->flags & TPA_ENABLE_FLAG) 2399 fp->mode = TPA_MODE_LRO; 2400 else if (bp->flags & GRO_ENABLE_FLAG) 2401 fp->mode = TPA_MODE_GRO; 2402 2403 /* We don't want TPA on an FCoE L2 ring */ 2404 if (IS_FCOE_FP(fp)) 2405 fp->disable_tpa = 1; 2406 } 2407 2408 int bnx2x_load_cnic(struct bnx2x *bp) 2409 { 2410 int i, rc, port = BP_PORT(bp); 2411 2412 DP(NETIF_MSG_IFUP, "Starting CNIC-related load\n"); 2413 2414 mutex_init(&bp->cnic_mutex); 2415 2416 if (IS_PF(bp)) { 2417 rc = bnx2x_alloc_mem_cnic(bp); 2418 if (rc) { 2419 BNX2X_ERR("Unable to allocate bp memory for cnic\n"); 2420 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0); 2421 } 2422 } 2423 2424 rc = bnx2x_alloc_fp_mem_cnic(bp); 2425 if (rc) { 2426 BNX2X_ERR("Unable to allocate memory for cnic fps\n"); 2427 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0); 2428 } 2429 2430 /* Update the number of queues with the cnic queues */ 2431 rc = bnx2x_set_real_num_queues(bp, 1); 2432 if (rc) { 2433 BNX2X_ERR("Unable to set real_num_queues including cnic\n"); 2434 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0); 2435 } 2436 2437 /* Add all CNIC NAPI objects */ 2438 bnx2x_add_all_napi_cnic(bp); 2439 DP(NETIF_MSG_IFUP, "cnic napi added\n"); 2440 bnx2x_napi_enable_cnic(bp); 2441 2442 rc = bnx2x_init_hw_func_cnic(bp); 2443 if (rc) 2444 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic1); 2445 2446 bnx2x_nic_init_cnic(bp); 2447 2448 if (IS_PF(bp)) { 2449 /* Enable Timer scan */ 2450 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 1); 2451 2452 /* setup cnic queues */ 2453 for_each_cnic_queue(bp, i) { 2454 rc = bnx2x_setup_queue(bp, &bp->fp[i], 0); 2455 if (rc) { 2456 BNX2X_ERR("Queue setup failed\n"); 2457 LOAD_ERROR_EXIT(bp, load_error_cnic2); 2458 } 2459 } 2460 } 2461 2462 /* Initialize Rx filter. */ 2463 bnx2x_set_rx_mode_inner(bp); 2464 2465 /* re-read iscsi info */ 2466 bnx2x_get_iscsi_info(bp); 2467 bnx2x_setup_cnic_irq_info(bp); 2468 bnx2x_setup_cnic_info(bp); 2469 bp->cnic_loaded = true; 2470 if (bp->state == BNX2X_STATE_OPEN) 2471 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD); 2472 2473 DP(NETIF_MSG_IFUP, "Ending successfully CNIC-related load\n"); 2474 2475 return 0; 2476 2477 #ifndef BNX2X_STOP_ON_ERROR 2478 load_error_cnic2: 2479 /* Disable Timer scan */ 2480 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 0); 2481 2482 load_error_cnic1: 2483 bnx2x_napi_disable_cnic(bp); 2484 /* Update the number of queues without the cnic queues */ 2485 if (bnx2x_set_real_num_queues(bp, 0)) 2486 BNX2X_ERR("Unable to set real_num_queues not including cnic\n"); 2487 load_error_cnic0: 2488 BNX2X_ERR("CNIC-related load failed\n"); 2489 bnx2x_free_fp_mem_cnic(bp); 2490 bnx2x_free_mem_cnic(bp); 2491 return rc; 2492 #endif /* ! BNX2X_STOP_ON_ERROR */ 2493 } 2494 2495 /* must be called with rtnl_lock */ 2496 int bnx2x_nic_load(struct bnx2x *bp, int load_mode) 2497 { 2498 int port = BP_PORT(bp); 2499 int i, rc = 0, load_code = 0; 2500 2501 DP(NETIF_MSG_IFUP, "Starting NIC load\n"); 2502 DP(NETIF_MSG_IFUP, 2503 "CNIC is %s\n", CNIC_ENABLED(bp) ? "enabled" : "disabled"); 2504 2505 #ifdef BNX2X_STOP_ON_ERROR 2506 if (unlikely(bp->panic)) { 2507 BNX2X_ERR("Can't load NIC when there is panic\n"); 2508 return -EPERM; 2509 } 2510 #endif 2511 2512 bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD; 2513 2514 /* zero the structure w/o any lock, before SP handler is initialized */ 2515 memset(&bp->last_reported_link, 0, sizeof(bp->last_reported_link)); 2516 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN, 2517 &bp->last_reported_link.link_report_flags); 2518 2519 if (IS_PF(bp)) 2520 /* must be called before memory allocation and HW init */ 2521 bnx2x_ilt_set_info(bp); 2522 2523 /* 2524 * Zero fastpath structures preserving invariants like napi, which are 2525 * allocated only once, fp index, max_cos, bp pointer. 2526 * Also set fp->disable_tpa and txdata_ptr. 2527 */ 2528 DP(NETIF_MSG_IFUP, "num queues: %d", bp->num_queues); 2529 for_each_queue(bp, i) 2530 bnx2x_bz_fp(bp, i); 2531 memset(bp->bnx2x_txq, 0, (BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS + 2532 bp->num_cnic_queues) * 2533 sizeof(struct bnx2x_fp_txdata)); 2534 2535 bp->fcoe_init = false; 2536 2537 /* Set the receive queues buffer size */ 2538 bnx2x_set_rx_buf_size(bp); 2539 2540 if (IS_PF(bp)) { 2541 rc = bnx2x_alloc_mem(bp); 2542 if (rc) { 2543 BNX2X_ERR("Unable to allocate bp memory\n"); 2544 return rc; 2545 } 2546 } 2547 2548 /* Allocated memory for FW statistics */ 2549 if (bnx2x_alloc_fw_stats_mem(bp)) 2550 LOAD_ERROR_EXIT(bp, load_error0); 2551 2552 /* need to be done after alloc mem, since it's self adjusting to amount 2553 * of memory available for RSS queues 2554 */ 2555 rc = bnx2x_alloc_fp_mem(bp); 2556 if (rc) { 2557 BNX2X_ERR("Unable to allocate memory for fps\n"); 2558 LOAD_ERROR_EXIT(bp, load_error0); 2559 } 2560 2561 /* request pf to initialize status blocks */ 2562 if (IS_VF(bp)) { 2563 rc = bnx2x_vfpf_init(bp); 2564 if (rc) 2565 LOAD_ERROR_EXIT(bp, load_error0); 2566 } 2567 2568 /* As long as bnx2x_alloc_mem() may possibly update 2569 * bp->num_queues, bnx2x_set_real_num_queues() should always 2570 * come after it. At this stage cnic queues are not counted. 2571 */ 2572 rc = bnx2x_set_real_num_queues(bp, 0); 2573 if (rc) { 2574 BNX2X_ERR("Unable to set real_num_queues\n"); 2575 LOAD_ERROR_EXIT(bp, load_error0); 2576 } 2577 2578 /* configure multi cos mappings in kernel. 2579 * this configuration may be overridden by a multi class queue 2580 * discipline or by a dcbx negotiation result. 2581 */ 2582 bnx2x_setup_tc(bp->dev, bp->max_cos); 2583 2584 /* Add all NAPI objects */ 2585 bnx2x_add_all_napi(bp); 2586 DP(NETIF_MSG_IFUP, "napi added\n"); 2587 bnx2x_napi_enable(bp); 2588 2589 if (IS_PF(bp)) { 2590 /* set pf load just before approaching the MCP */ 2591 bnx2x_set_pf_load(bp); 2592 2593 /* if mcp exists send load request and analyze response */ 2594 if (!BP_NOMCP(bp)) { 2595 /* attempt to load pf */ 2596 rc = bnx2x_nic_load_request(bp, &load_code); 2597 if (rc) 2598 LOAD_ERROR_EXIT(bp, load_error1); 2599 2600 /* what did mcp say? */ 2601 rc = bnx2x_nic_load_analyze_req(bp, load_code); 2602 if (rc) { 2603 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0); 2604 LOAD_ERROR_EXIT(bp, load_error2); 2605 } 2606 } else { 2607 load_code = bnx2x_nic_load_no_mcp(bp, port); 2608 } 2609 2610 /* mark pmf if applicable */ 2611 bnx2x_nic_load_pmf(bp, load_code); 2612 2613 /* Init Function state controlling object */ 2614 bnx2x__init_func_obj(bp); 2615 2616 /* Initialize HW */ 2617 rc = bnx2x_init_hw(bp, load_code); 2618 if (rc) { 2619 BNX2X_ERR("HW init failed, aborting\n"); 2620 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0); 2621 LOAD_ERROR_EXIT(bp, load_error2); 2622 } 2623 } 2624 2625 bnx2x_pre_irq_nic_init(bp); 2626 2627 /* Connect to IRQs */ 2628 rc = bnx2x_setup_irqs(bp); 2629 if (rc) { 2630 BNX2X_ERR("setup irqs failed\n"); 2631 if (IS_PF(bp)) 2632 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0); 2633 LOAD_ERROR_EXIT(bp, load_error2); 2634 } 2635 2636 /* Init per-function objects */ 2637 if (IS_PF(bp)) { 2638 /* Setup NIC internals and enable interrupts */ 2639 bnx2x_post_irq_nic_init(bp, load_code); 2640 2641 bnx2x_init_bp_objs(bp); 2642 bnx2x_iov_nic_init(bp); 2643 2644 /* Set AFEX default VLAN tag to an invalid value */ 2645 bp->afex_def_vlan_tag = -1; 2646 bnx2x_nic_load_afex_dcc(bp, load_code); 2647 bp->state = BNX2X_STATE_OPENING_WAIT4_PORT; 2648 rc = bnx2x_func_start(bp); 2649 if (rc) { 2650 BNX2X_ERR("Function start failed!\n"); 2651 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0); 2652 2653 LOAD_ERROR_EXIT(bp, load_error3); 2654 } 2655 2656 /* Send LOAD_DONE command to MCP */ 2657 if (!BP_NOMCP(bp)) { 2658 load_code = bnx2x_fw_command(bp, 2659 DRV_MSG_CODE_LOAD_DONE, 0); 2660 if (!load_code) { 2661 BNX2X_ERR("MCP response failure, aborting\n"); 2662 rc = -EBUSY; 2663 LOAD_ERROR_EXIT(bp, load_error3); 2664 } 2665 } 2666 2667 /* initialize FW coalescing state machines in RAM */ 2668 bnx2x_update_coalesce(bp); 2669 } 2670 2671 /* setup the leading queue */ 2672 rc = bnx2x_setup_leading(bp); 2673 if (rc) { 2674 BNX2X_ERR("Setup leading failed!\n"); 2675 LOAD_ERROR_EXIT(bp, load_error3); 2676 } 2677 2678 /* set up the rest of the queues */ 2679 for_each_nondefault_eth_queue(bp, i) { 2680 if (IS_PF(bp)) 2681 rc = bnx2x_setup_queue(bp, &bp->fp[i], false); 2682 else /* VF */ 2683 rc = bnx2x_vfpf_setup_q(bp, &bp->fp[i], false); 2684 if (rc) { 2685 BNX2X_ERR("Queue %d setup failed\n", i); 2686 LOAD_ERROR_EXIT(bp, load_error3); 2687 } 2688 } 2689 2690 /* setup rss */ 2691 rc = bnx2x_init_rss(bp); 2692 if (rc) { 2693 BNX2X_ERR("PF RSS init failed\n"); 2694 LOAD_ERROR_EXIT(bp, load_error3); 2695 } 2696 2697 /* Now when Clients are configured we are ready to work */ 2698 bp->state = BNX2X_STATE_OPEN; 2699 2700 /* Configure a ucast MAC */ 2701 if (IS_PF(bp)) 2702 rc = bnx2x_set_eth_mac(bp, true); 2703 else /* vf */ 2704 rc = bnx2x_vfpf_config_mac(bp, bp->dev->dev_addr, bp->fp->index, 2705 true); 2706 if (rc) { 2707 BNX2X_ERR("Setting Ethernet MAC failed\n"); 2708 LOAD_ERROR_EXIT(bp, load_error3); 2709 } 2710 2711 if (IS_PF(bp) && bp->pending_max) { 2712 bnx2x_update_max_mf_config(bp, bp->pending_max); 2713 bp->pending_max = 0; 2714 } 2715 2716 if (bp->port.pmf) { 2717 rc = bnx2x_initial_phy_init(bp, load_mode); 2718 if (rc) 2719 LOAD_ERROR_EXIT(bp, load_error3); 2720 } 2721 bp->link_params.feature_config_flags &= ~FEATURE_CONFIG_BOOT_FROM_SAN; 2722 2723 /* Start fast path */ 2724 2725 /* Initialize Rx filter. */ 2726 bnx2x_set_rx_mode_inner(bp); 2727 2728 /* Start the Tx */ 2729 switch (load_mode) { 2730 case LOAD_NORMAL: 2731 /* Tx queue should be only re-enabled */ 2732 netif_tx_wake_all_queues(bp->dev); 2733 break; 2734 2735 case LOAD_OPEN: 2736 netif_tx_start_all_queues(bp->dev); 2737 smp_mb__after_clear_bit(); 2738 break; 2739 2740 case LOAD_DIAG: 2741 case LOAD_LOOPBACK_EXT: 2742 bp->state = BNX2X_STATE_DIAG; 2743 break; 2744 2745 default: 2746 break; 2747 } 2748 2749 if (bp->port.pmf) 2750 bnx2x_update_drv_flags(bp, 1 << DRV_FLAGS_PORT_MASK, 0); 2751 else 2752 bnx2x__link_status_update(bp); 2753 2754 /* start the timer */ 2755 mod_timer(&bp->timer, jiffies + bp->current_interval); 2756 2757 if (CNIC_ENABLED(bp)) 2758 bnx2x_load_cnic(bp); 2759 2760 if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) { 2761 /* mark driver is loaded in shmem2 */ 2762 u32 val; 2763 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]); 2764 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)], 2765 val | DRV_FLAGS_CAPABILITIES_LOADED_SUPPORTED | 2766 DRV_FLAGS_CAPABILITIES_LOADED_L2); 2767 } 2768 2769 /* Wait for all pending SP commands to complete */ 2770 if (IS_PF(bp) && !bnx2x_wait_sp_comp(bp, ~0x0UL)) { 2771 BNX2X_ERR("Timeout waiting for SP elements to complete\n"); 2772 bnx2x_nic_unload(bp, UNLOAD_CLOSE, false); 2773 return -EBUSY; 2774 } 2775 2776 /* If PMF - send ADMIN DCBX msg to MFW to initiate DCBX FSM */ 2777 if (bp->port.pmf && (bp->state != BNX2X_STATE_DIAG)) 2778 bnx2x_dcbx_init(bp, false); 2779 2780 DP(NETIF_MSG_IFUP, "Ending successfully NIC load\n"); 2781 2782 return 0; 2783 2784 #ifndef BNX2X_STOP_ON_ERROR 2785 load_error3: 2786 if (IS_PF(bp)) { 2787 bnx2x_int_disable_sync(bp, 1); 2788 2789 /* Clean queueable objects */ 2790 bnx2x_squeeze_objects(bp); 2791 } 2792 2793 /* Free SKBs, SGEs, TPA pool and driver internals */ 2794 bnx2x_free_skbs(bp); 2795 for_each_rx_queue(bp, i) 2796 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE); 2797 2798 /* Release IRQs */ 2799 bnx2x_free_irq(bp); 2800 load_error2: 2801 if (IS_PF(bp) && !BP_NOMCP(bp)) { 2802 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0); 2803 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0); 2804 } 2805 2806 bp->port.pmf = 0; 2807 load_error1: 2808 bnx2x_napi_disable(bp); 2809 bnx2x_del_all_napi(bp); 2810 2811 /* clear pf_load status, as it was already set */ 2812 if (IS_PF(bp)) 2813 bnx2x_clear_pf_load(bp); 2814 load_error0: 2815 bnx2x_free_fp_mem(bp); 2816 bnx2x_free_fw_stats_mem(bp); 2817 bnx2x_free_mem(bp); 2818 2819 return rc; 2820 #endif /* ! BNX2X_STOP_ON_ERROR */ 2821 } 2822 2823 int bnx2x_drain_tx_queues(struct bnx2x *bp) 2824 { 2825 u8 rc = 0, cos, i; 2826 2827 /* Wait until tx fastpath tasks complete */ 2828 for_each_tx_queue(bp, i) { 2829 struct bnx2x_fastpath *fp = &bp->fp[i]; 2830 2831 for_each_cos_in_tx_queue(fp, cos) 2832 rc = bnx2x_clean_tx_queue(bp, fp->txdata_ptr[cos]); 2833 if (rc) 2834 return rc; 2835 } 2836 return 0; 2837 } 2838 2839 /* must be called with rtnl_lock */ 2840 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode, bool keep_link) 2841 { 2842 int i; 2843 bool global = false; 2844 2845 DP(NETIF_MSG_IFUP, "Starting NIC unload\n"); 2846 2847 /* mark driver is unloaded in shmem2 */ 2848 if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) { 2849 u32 val; 2850 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]); 2851 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)], 2852 val & ~DRV_FLAGS_CAPABILITIES_LOADED_L2); 2853 } 2854 2855 if (IS_PF(bp) && bp->recovery_state != BNX2X_RECOVERY_DONE && 2856 (bp->state == BNX2X_STATE_CLOSED || 2857 bp->state == BNX2X_STATE_ERROR)) { 2858 /* We can get here if the driver has been unloaded 2859 * during parity error recovery and is either waiting for a 2860 * leader to complete or for other functions to unload and 2861 * then ifdown has been issued. In this case we want to 2862 * unload and let other functions to complete a recovery 2863 * process. 2864 */ 2865 bp->recovery_state = BNX2X_RECOVERY_DONE; 2866 bp->is_leader = 0; 2867 bnx2x_release_leader_lock(bp); 2868 smp_mb(); 2869 2870 DP(NETIF_MSG_IFDOWN, "Releasing a leadership...\n"); 2871 BNX2X_ERR("Can't unload in closed or error state\n"); 2872 return -EINVAL; 2873 } 2874 2875 /* Nothing to do during unload if previous bnx2x_nic_load() 2876 * have not completed successfully - all resources are released. 2877 * 2878 * we can get here only after unsuccessful ndo_* callback, during which 2879 * dev->IFF_UP flag is still on. 2880 */ 2881 if (bp->state == BNX2X_STATE_CLOSED || bp->state == BNX2X_STATE_ERROR) 2882 return 0; 2883 2884 /* It's important to set the bp->state to the value different from 2885 * BNX2X_STATE_OPEN and only then stop the Tx. Otherwise bnx2x_tx_int() 2886 * may restart the Tx from the NAPI context (see bnx2x_tx_int()). 2887 */ 2888 bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT; 2889 smp_mb(); 2890 2891 /* indicate to VFs that the PF is going down */ 2892 bnx2x_iov_channel_down(bp); 2893 2894 if (CNIC_LOADED(bp)) 2895 bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD); 2896 2897 /* Stop Tx */ 2898 bnx2x_tx_disable(bp); 2899 netdev_reset_tc(bp->dev); 2900 2901 bp->rx_mode = BNX2X_RX_MODE_NONE; 2902 2903 del_timer_sync(&bp->timer); 2904 2905 if (IS_PF(bp)) { 2906 /* Set ALWAYS_ALIVE bit in shmem */ 2907 bp->fw_drv_pulse_wr_seq |= DRV_PULSE_ALWAYS_ALIVE; 2908 bnx2x_drv_pulse(bp); 2909 bnx2x_stats_handle(bp, STATS_EVENT_STOP); 2910 bnx2x_save_statistics(bp); 2911 } 2912 2913 /* wait till consumers catch up with producers in all queues */ 2914 bnx2x_drain_tx_queues(bp); 2915 2916 /* if VF indicate to PF this function is going down (PF will delete sp 2917 * elements and clear initializations 2918 */ 2919 if (IS_VF(bp)) 2920 bnx2x_vfpf_close_vf(bp); 2921 else if (unload_mode != UNLOAD_RECOVERY) 2922 /* if this is a normal/close unload need to clean up chip*/ 2923 bnx2x_chip_cleanup(bp, unload_mode, keep_link); 2924 else { 2925 /* Send the UNLOAD_REQUEST to the MCP */ 2926 bnx2x_send_unload_req(bp, unload_mode); 2927 2928 /* Prevent transactions to host from the functions on the 2929 * engine that doesn't reset global blocks in case of global 2930 * attention once global blocks are reset and gates are opened 2931 * (the engine which leader will perform the recovery 2932 * last). 2933 */ 2934 if (!CHIP_IS_E1x(bp)) 2935 bnx2x_pf_disable(bp); 2936 2937 /* Disable HW interrupts, NAPI */ 2938 bnx2x_netif_stop(bp, 1); 2939 /* Delete all NAPI objects */ 2940 bnx2x_del_all_napi(bp); 2941 if (CNIC_LOADED(bp)) 2942 bnx2x_del_all_napi_cnic(bp); 2943 /* Release IRQs */ 2944 bnx2x_free_irq(bp); 2945 2946 /* Report UNLOAD_DONE to MCP */ 2947 bnx2x_send_unload_done(bp, false); 2948 } 2949 2950 /* 2951 * At this stage no more interrupts will arrive so we may safely clean 2952 * the queueable objects here in case they failed to get cleaned so far. 2953 */ 2954 if (IS_PF(bp)) 2955 bnx2x_squeeze_objects(bp); 2956 2957 /* There should be no more pending SP commands at this stage */ 2958 bp->sp_state = 0; 2959 2960 bp->port.pmf = 0; 2961 2962 /* Free SKBs, SGEs, TPA pool and driver internals */ 2963 bnx2x_free_skbs(bp); 2964 if (CNIC_LOADED(bp)) 2965 bnx2x_free_skbs_cnic(bp); 2966 for_each_rx_queue(bp, i) 2967 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE); 2968 2969 bnx2x_free_fp_mem(bp); 2970 if (CNIC_LOADED(bp)) 2971 bnx2x_free_fp_mem_cnic(bp); 2972 2973 if (IS_PF(bp)) { 2974 if (CNIC_LOADED(bp)) 2975 bnx2x_free_mem_cnic(bp); 2976 } 2977 bnx2x_free_mem(bp); 2978 2979 bp->state = BNX2X_STATE_CLOSED; 2980 bp->cnic_loaded = false; 2981 2982 /* Check if there are pending parity attentions. If there are - set 2983 * RECOVERY_IN_PROGRESS. 2984 */ 2985 if (IS_PF(bp) && bnx2x_chk_parity_attn(bp, &global, false)) { 2986 bnx2x_set_reset_in_progress(bp); 2987 2988 /* Set RESET_IS_GLOBAL if needed */ 2989 if (global) 2990 bnx2x_set_reset_global(bp); 2991 } 2992 2993 /* The last driver must disable a "close the gate" if there is no 2994 * parity attention or "process kill" pending. 2995 */ 2996 if (IS_PF(bp) && 2997 !bnx2x_clear_pf_load(bp) && 2998 bnx2x_reset_is_done(bp, BP_PATH(bp))) 2999 bnx2x_disable_close_the_gate(bp); 3000 3001 DP(NETIF_MSG_IFUP, "Ending NIC unload\n"); 3002 3003 return 0; 3004 } 3005 3006 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state) 3007 { 3008 u16 pmcsr; 3009 3010 /* If there is no power capability, silently succeed */ 3011 if (!bp->pdev->pm_cap) { 3012 BNX2X_DEV_INFO("No power capability. Breaking.\n"); 3013 return 0; 3014 } 3015 3016 pci_read_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL, &pmcsr); 3017 3018 switch (state) { 3019 case PCI_D0: 3020 pci_write_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL, 3021 ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) | 3022 PCI_PM_CTRL_PME_STATUS)); 3023 3024 if (pmcsr & PCI_PM_CTRL_STATE_MASK) 3025 /* delay required during transition out of D3hot */ 3026 msleep(20); 3027 break; 3028 3029 case PCI_D3hot: 3030 /* If there are other clients above don't 3031 shut down the power */ 3032 if (atomic_read(&bp->pdev->enable_cnt) != 1) 3033 return 0; 3034 /* Don't shut down the power for emulation and FPGA */ 3035 if (CHIP_REV_IS_SLOW(bp)) 3036 return 0; 3037 3038 pmcsr &= ~PCI_PM_CTRL_STATE_MASK; 3039 pmcsr |= 3; 3040 3041 if (bp->wol) 3042 pmcsr |= PCI_PM_CTRL_PME_ENABLE; 3043 3044 pci_write_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL, 3045 pmcsr); 3046 3047 /* No more memory access after this point until 3048 * device is brought back to D0. 3049 */ 3050 break; 3051 3052 default: 3053 dev_err(&bp->pdev->dev, "Can't support state = %d\n", state); 3054 return -EINVAL; 3055 } 3056 return 0; 3057 } 3058 3059 /* 3060 * net_device service functions 3061 */ 3062 int bnx2x_poll(struct napi_struct *napi, int budget) 3063 { 3064 int work_done = 0; 3065 u8 cos; 3066 struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath, 3067 napi); 3068 struct bnx2x *bp = fp->bp; 3069 3070 while (1) { 3071 #ifdef BNX2X_STOP_ON_ERROR 3072 if (unlikely(bp->panic)) { 3073 napi_complete(napi); 3074 return 0; 3075 } 3076 #endif 3077 if (!bnx2x_fp_lock_napi(fp)) 3078 return work_done; 3079 3080 for_each_cos_in_tx_queue(fp, cos) 3081 if (bnx2x_tx_queue_has_work(fp->txdata_ptr[cos])) 3082 bnx2x_tx_int(bp, fp->txdata_ptr[cos]); 3083 3084 if (bnx2x_has_rx_work(fp)) { 3085 work_done += bnx2x_rx_int(fp, budget - work_done); 3086 3087 /* must not complete if we consumed full budget */ 3088 if (work_done >= budget) { 3089 bnx2x_fp_unlock_napi(fp); 3090 break; 3091 } 3092 } 3093 3094 /* Fall out from the NAPI loop if needed */ 3095 if (!bnx2x_fp_unlock_napi(fp) && 3096 !(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) { 3097 3098 /* No need to update SB for FCoE L2 ring as long as 3099 * it's connected to the default SB and the SB 3100 * has been updated when NAPI was scheduled. 3101 */ 3102 if (IS_FCOE_FP(fp)) { 3103 napi_complete(napi); 3104 break; 3105 } 3106 bnx2x_update_fpsb_idx(fp); 3107 /* bnx2x_has_rx_work() reads the status block, 3108 * thus we need to ensure that status block indices 3109 * have been actually read (bnx2x_update_fpsb_idx) 3110 * prior to this check (bnx2x_has_rx_work) so that 3111 * we won't write the "newer" value of the status block 3112 * to IGU (if there was a DMA right after 3113 * bnx2x_has_rx_work and if there is no rmb, the memory 3114 * reading (bnx2x_update_fpsb_idx) may be postponed 3115 * to right before bnx2x_ack_sb). In this case there 3116 * will never be another interrupt until there is 3117 * another update of the status block, while there 3118 * is still unhandled work. 3119 */ 3120 rmb(); 3121 3122 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) { 3123 napi_complete(napi); 3124 /* Re-enable interrupts */ 3125 DP(NETIF_MSG_RX_STATUS, 3126 "Update index to %d\n", fp->fp_hc_idx); 3127 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 3128 le16_to_cpu(fp->fp_hc_idx), 3129 IGU_INT_ENABLE, 1); 3130 break; 3131 } 3132 } 3133 } 3134 3135 return work_done; 3136 } 3137 3138 #ifdef CONFIG_NET_RX_BUSY_POLL 3139 /* must be called with local_bh_disable()d */ 3140 int bnx2x_low_latency_recv(struct napi_struct *napi) 3141 { 3142 struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath, 3143 napi); 3144 struct bnx2x *bp = fp->bp; 3145 int found = 0; 3146 3147 if ((bp->state == BNX2X_STATE_CLOSED) || 3148 (bp->state == BNX2X_STATE_ERROR) || 3149 (bp->flags & (TPA_ENABLE_FLAG | GRO_ENABLE_FLAG))) 3150 return LL_FLUSH_FAILED; 3151 3152 if (!bnx2x_fp_lock_poll(fp)) 3153 return LL_FLUSH_BUSY; 3154 3155 if (bnx2x_has_rx_work(fp)) 3156 found = bnx2x_rx_int(fp, 4); 3157 3158 bnx2x_fp_unlock_poll(fp); 3159 3160 return found; 3161 } 3162 #endif 3163 3164 /* we split the first BD into headers and data BDs 3165 * to ease the pain of our fellow microcode engineers 3166 * we use one mapping for both BDs 3167 */ 3168 static u16 bnx2x_tx_split(struct bnx2x *bp, 3169 struct bnx2x_fp_txdata *txdata, 3170 struct sw_tx_bd *tx_buf, 3171 struct eth_tx_start_bd **tx_bd, u16 hlen, 3172 u16 bd_prod) 3173 { 3174 struct eth_tx_start_bd *h_tx_bd = *tx_bd; 3175 struct eth_tx_bd *d_tx_bd; 3176 dma_addr_t mapping; 3177 int old_len = le16_to_cpu(h_tx_bd->nbytes); 3178 3179 /* first fix first BD */ 3180 h_tx_bd->nbytes = cpu_to_le16(hlen); 3181 3182 DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d (%x:%x)\n", 3183 h_tx_bd->nbytes, h_tx_bd->addr_hi, h_tx_bd->addr_lo); 3184 3185 /* now get a new data BD 3186 * (after the pbd) and fill it */ 3187 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod)); 3188 d_tx_bd = &txdata->tx_desc_ring[bd_prod].reg_bd; 3189 3190 mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi), 3191 le32_to_cpu(h_tx_bd->addr_lo)) + hlen; 3192 3193 d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping)); 3194 d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping)); 3195 d_tx_bd->nbytes = cpu_to_le16(old_len - hlen); 3196 3197 /* this marks the BD as one that has no individual mapping */ 3198 tx_buf->flags |= BNX2X_TSO_SPLIT_BD; 3199 3200 DP(NETIF_MSG_TX_QUEUED, 3201 "TSO split data size is %d (%x:%x)\n", 3202 d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo); 3203 3204 /* update tx_bd */ 3205 *tx_bd = (struct eth_tx_start_bd *)d_tx_bd; 3206 3207 return bd_prod; 3208 } 3209 3210 #define bswab32(b32) ((__force __le32) swab32((__force __u32) (b32))) 3211 #define bswab16(b16) ((__force __le16) swab16((__force __u16) (b16))) 3212 static __le16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix) 3213 { 3214 __sum16 tsum = (__force __sum16) csum; 3215 3216 if (fix > 0) 3217 tsum = ~csum_fold(csum_sub((__force __wsum) csum, 3218 csum_partial(t_header - fix, fix, 0))); 3219 3220 else if (fix < 0) 3221 tsum = ~csum_fold(csum_add((__force __wsum) csum, 3222 csum_partial(t_header, -fix, 0))); 3223 3224 return bswab16(tsum); 3225 } 3226 3227 static u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb) 3228 { 3229 u32 rc; 3230 __u8 prot = 0; 3231 __be16 protocol; 3232 3233 if (skb->ip_summed != CHECKSUM_PARTIAL) 3234 return XMIT_PLAIN; 3235 3236 protocol = vlan_get_protocol(skb); 3237 if (protocol == htons(ETH_P_IPV6)) { 3238 rc = XMIT_CSUM_V6; 3239 prot = ipv6_hdr(skb)->nexthdr; 3240 } else { 3241 rc = XMIT_CSUM_V4; 3242 prot = ip_hdr(skb)->protocol; 3243 } 3244 3245 if (!CHIP_IS_E1x(bp) && skb->encapsulation) { 3246 if (inner_ip_hdr(skb)->version == 6) { 3247 rc |= XMIT_CSUM_ENC_V6; 3248 if (inner_ipv6_hdr(skb)->nexthdr == IPPROTO_TCP) 3249 rc |= XMIT_CSUM_TCP; 3250 } else { 3251 rc |= XMIT_CSUM_ENC_V4; 3252 if (inner_ip_hdr(skb)->protocol == IPPROTO_TCP) 3253 rc |= XMIT_CSUM_TCP; 3254 } 3255 } 3256 if (prot == IPPROTO_TCP) 3257 rc |= XMIT_CSUM_TCP; 3258 3259 if (skb_is_gso_v6(skb)) { 3260 rc |= (XMIT_GSO_V6 | XMIT_CSUM_TCP); 3261 if (rc & XMIT_CSUM_ENC) 3262 rc |= XMIT_GSO_ENC_V6; 3263 } else if (skb_is_gso(skb)) { 3264 rc |= (XMIT_GSO_V4 | XMIT_CSUM_TCP); 3265 if (rc & XMIT_CSUM_ENC) 3266 rc |= XMIT_GSO_ENC_V4; 3267 } 3268 3269 return rc; 3270 } 3271 3272 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3) 3273 /* check if packet requires linearization (packet is too fragmented) 3274 no need to check fragmentation if page size > 8K (there will be no 3275 violation to FW restrictions) */ 3276 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb, 3277 u32 xmit_type) 3278 { 3279 int to_copy = 0; 3280 int hlen = 0; 3281 int first_bd_sz = 0; 3282 3283 /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */ 3284 if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) { 3285 3286 if (xmit_type & XMIT_GSO) { 3287 unsigned short lso_mss = skb_shinfo(skb)->gso_size; 3288 /* Check if LSO packet needs to be copied: 3289 3 = 1 (for headers BD) + 2 (for PBD and last BD) */ 3290 int wnd_size = MAX_FETCH_BD - 3; 3291 /* Number of windows to check */ 3292 int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size; 3293 int wnd_idx = 0; 3294 int frag_idx = 0; 3295 u32 wnd_sum = 0; 3296 3297 /* Headers length */ 3298 hlen = (int)(skb_transport_header(skb) - skb->data) + 3299 tcp_hdrlen(skb); 3300 3301 /* Amount of data (w/o headers) on linear part of SKB*/ 3302 first_bd_sz = skb_headlen(skb) - hlen; 3303 3304 wnd_sum = first_bd_sz; 3305 3306 /* Calculate the first sum - it's special */ 3307 for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++) 3308 wnd_sum += 3309 skb_frag_size(&skb_shinfo(skb)->frags[frag_idx]); 3310 3311 /* If there was data on linear skb data - check it */ 3312 if (first_bd_sz > 0) { 3313 if (unlikely(wnd_sum < lso_mss)) { 3314 to_copy = 1; 3315 goto exit_lbl; 3316 } 3317 3318 wnd_sum -= first_bd_sz; 3319 } 3320 3321 /* Others are easier: run through the frag list and 3322 check all windows */ 3323 for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) { 3324 wnd_sum += 3325 skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1]); 3326 3327 if (unlikely(wnd_sum < lso_mss)) { 3328 to_copy = 1; 3329 break; 3330 } 3331 wnd_sum -= 3332 skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx]); 3333 } 3334 } else { 3335 /* in non-LSO too fragmented packet should always 3336 be linearized */ 3337 to_copy = 1; 3338 } 3339 } 3340 3341 exit_lbl: 3342 if (unlikely(to_copy)) 3343 DP(NETIF_MSG_TX_QUEUED, 3344 "Linearization IS REQUIRED for %s packet. num_frags %d hlen %d first_bd_sz %d\n", 3345 (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO", 3346 skb_shinfo(skb)->nr_frags, hlen, first_bd_sz); 3347 3348 return to_copy; 3349 } 3350 #endif 3351 3352 static void bnx2x_set_pbd_gso_e2(struct sk_buff *skb, u32 *parsing_data, 3353 u32 xmit_type) 3354 { 3355 struct ipv6hdr *ipv6; 3356 3357 *parsing_data |= (skb_shinfo(skb)->gso_size << 3358 ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT) & 3359 ETH_TX_PARSE_BD_E2_LSO_MSS; 3360 3361 if (xmit_type & XMIT_GSO_ENC_V6) 3362 ipv6 = inner_ipv6_hdr(skb); 3363 else if (xmit_type & XMIT_GSO_V6) 3364 ipv6 = ipv6_hdr(skb); 3365 else 3366 ipv6 = NULL; 3367 3368 if (ipv6 && ipv6->nexthdr == NEXTHDR_IPV6) 3369 *parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR; 3370 } 3371 3372 /** 3373 * bnx2x_set_pbd_gso - update PBD in GSO case. 3374 * 3375 * @skb: packet skb 3376 * @pbd: parse BD 3377 * @xmit_type: xmit flags 3378 */ 3379 static void bnx2x_set_pbd_gso(struct sk_buff *skb, 3380 struct eth_tx_parse_bd_e1x *pbd, 3381 struct eth_tx_start_bd *tx_start_bd, 3382 u32 xmit_type) 3383 { 3384 pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size); 3385 pbd->tcp_send_seq = bswab32(tcp_hdr(skb)->seq); 3386 pbd->tcp_flags = pbd_tcp_flags(tcp_hdr(skb)); 3387 3388 if (xmit_type & XMIT_GSO_V4) { 3389 pbd->ip_id = bswab16(ip_hdr(skb)->id); 3390 pbd->tcp_pseudo_csum = 3391 bswab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr, 3392 ip_hdr(skb)->daddr, 3393 0, IPPROTO_TCP, 0)); 3394 3395 /* GSO on 57710/57711 needs FW to calculate IP checksum */ 3396 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IP_CSUM; 3397 } else { 3398 pbd->tcp_pseudo_csum = 3399 bswab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 3400 &ipv6_hdr(skb)->daddr, 3401 0, IPPROTO_TCP, 0)); 3402 } 3403 3404 pbd->global_data |= 3405 cpu_to_le16(ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN); 3406 } 3407 3408 /** 3409 * bnx2x_set_pbd_csum_enc - update PBD with checksum and return header length 3410 * 3411 * @bp: driver handle 3412 * @skb: packet skb 3413 * @parsing_data: data to be updated 3414 * @xmit_type: xmit flags 3415 * 3416 * 57712/578xx related, when skb has encapsulation 3417 */ 3418 static u8 bnx2x_set_pbd_csum_enc(struct bnx2x *bp, struct sk_buff *skb, 3419 u32 *parsing_data, u32 xmit_type) 3420 { 3421 *parsing_data |= 3422 ((((u8 *)skb_inner_transport_header(skb) - skb->data) >> 1) << 3423 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) & 3424 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W; 3425 3426 if (xmit_type & XMIT_CSUM_TCP) { 3427 *parsing_data |= ((inner_tcp_hdrlen(skb) / 4) << 3428 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) & 3429 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW; 3430 3431 return skb_inner_transport_header(skb) + 3432 inner_tcp_hdrlen(skb) - skb->data; 3433 } 3434 3435 /* We support checksum offload for TCP and UDP only. 3436 * No need to pass the UDP header length - it's a constant. 3437 */ 3438 return skb_inner_transport_header(skb) + 3439 sizeof(struct udphdr) - skb->data; 3440 } 3441 3442 /** 3443 * bnx2x_set_pbd_csum_e2 - update PBD with checksum and return header length 3444 * 3445 * @bp: driver handle 3446 * @skb: packet skb 3447 * @parsing_data: data to be updated 3448 * @xmit_type: xmit flags 3449 * 3450 * 57712/578xx related 3451 */ 3452 static u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb, 3453 u32 *parsing_data, u32 xmit_type) 3454 { 3455 *parsing_data |= 3456 ((((u8 *)skb_transport_header(skb) - skb->data) >> 1) << 3457 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) & 3458 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W; 3459 3460 if (xmit_type & XMIT_CSUM_TCP) { 3461 *parsing_data |= ((tcp_hdrlen(skb) / 4) << 3462 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) & 3463 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW; 3464 3465 return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data; 3466 } 3467 /* We support checksum offload for TCP and UDP only. 3468 * No need to pass the UDP header length - it's a constant. 3469 */ 3470 return skb_transport_header(skb) + sizeof(struct udphdr) - skb->data; 3471 } 3472 3473 /* set FW indication according to inner or outer protocols if tunneled */ 3474 static void bnx2x_set_sbd_csum(struct bnx2x *bp, struct sk_buff *skb, 3475 struct eth_tx_start_bd *tx_start_bd, 3476 u32 xmit_type) 3477 { 3478 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM; 3479 3480 if (xmit_type & (XMIT_CSUM_ENC_V6 | XMIT_CSUM_V6)) 3481 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IPV6; 3482 3483 if (!(xmit_type & XMIT_CSUM_TCP)) 3484 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IS_UDP; 3485 } 3486 3487 /** 3488 * bnx2x_set_pbd_csum - update PBD with checksum and return header length 3489 * 3490 * @bp: driver handle 3491 * @skb: packet skb 3492 * @pbd: parse BD to be updated 3493 * @xmit_type: xmit flags 3494 */ 3495 static u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb, 3496 struct eth_tx_parse_bd_e1x *pbd, 3497 u32 xmit_type) 3498 { 3499 u8 hlen = (skb_network_header(skb) - skb->data) >> 1; 3500 3501 /* for now NS flag is not used in Linux */ 3502 pbd->global_data = 3503 cpu_to_le16(hlen | 3504 ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) << 3505 ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT)); 3506 3507 pbd->ip_hlen_w = (skb_transport_header(skb) - 3508 skb_network_header(skb)) >> 1; 3509 3510 hlen += pbd->ip_hlen_w; 3511 3512 /* We support checksum offload for TCP and UDP only */ 3513 if (xmit_type & XMIT_CSUM_TCP) 3514 hlen += tcp_hdrlen(skb) / 2; 3515 else 3516 hlen += sizeof(struct udphdr) / 2; 3517 3518 pbd->total_hlen_w = cpu_to_le16(hlen); 3519 hlen = hlen*2; 3520 3521 if (xmit_type & XMIT_CSUM_TCP) { 3522 pbd->tcp_pseudo_csum = bswab16(tcp_hdr(skb)->check); 3523 3524 } else { 3525 s8 fix = SKB_CS_OFF(skb); /* signed! */ 3526 3527 DP(NETIF_MSG_TX_QUEUED, 3528 "hlen %d fix %d csum before fix %x\n", 3529 le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb)); 3530 3531 /* HW bug: fixup the CSUM */ 3532 pbd->tcp_pseudo_csum = 3533 bnx2x_csum_fix(skb_transport_header(skb), 3534 SKB_CS(skb), fix); 3535 3536 DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n", 3537 pbd->tcp_pseudo_csum); 3538 } 3539 3540 return hlen; 3541 } 3542 3543 static void bnx2x_update_pbds_gso_enc(struct sk_buff *skb, 3544 struct eth_tx_parse_bd_e2 *pbd_e2, 3545 struct eth_tx_parse_2nd_bd *pbd2, 3546 u16 *global_data, 3547 u32 xmit_type) 3548 { 3549 u16 hlen_w = 0; 3550 u8 outerip_off, outerip_len = 0; 3551 3552 /* from outer IP to transport */ 3553 hlen_w = (skb_inner_transport_header(skb) - 3554 skb_network_header(skb)) >> 1; 3555 3556 /* transport len */ 3557 hlen_w += inner_tcp_hdrlen(skb) >> 1; 3558 3559 pbd2->fw_ip_hdr_to_payload_w = hlen_w; 3560 3561 /* outer IP header info */ 3562 if (xmit_type & XMIT_CSUM_V4) { 3563 struct iphdr *iph = ip_hdr(skb); 3564 u32 csum = (__force u32)(~iph->check) - 3565 (__force u32)iph->tot_len - 3566 (__force u32)iph->frag_off; 3567 3568 pbd2->fw_ip_csum_wo_len_flags_frag = 3569 bswab16(csum_fold((__force __wsum)csum)); 3570 } else { 3571 pbd2->fw_ip_hdr_to_payload_w = 3572 hlen_w - ((sizeof(struct ipv6hdr)) >> 1); 3573 } 3574 3575 pbd2->tcp_send_seq = bswab32(inner_tcp_hdr(skb)->seq); 3576 3577 pbd2->tcp_flags = pbd_tcp_flags(inner_tcp_hdr(skb)); 3578 3579 if (xmit_type & XMIT_GSO_V4) { 3580 pbd2->hw_ip_id = bswab16(inner_ip_hdr(skb)->id); 3581 3582 pbd_e2->data.tunnel_data.pseudo_csum = 3583 bswab16(~csum_tcpudp_magic( 3584 inner_ip_hdr(skb)->saddr, 3585 inner_ip_hdr(skb)->daddr, 3586 0, IPPROTO_TCP, 0)); 3587 3588 outerip_len = ip_hdr(skb)->ihl << 1; 3589 } else { 3590 pbd_e2->data.tunnel_data.pseudo_csum = 3591 bswab16(~csum_ipv6_magic( 3592 &inner_ipv6_hdr(skb)->saddr, 3593 &inner_ipv6_hdr(skb)->daddr, 3594 0, IPPROTO_TCP, 0)); 3595 } 3596 3597 outerip_off = (skb_network_header(skb) - skb->data) >> 1; 3598 3599 *global_data |= 3600 outerip_off | 3601 (!!(xmit_type & XMIT_CSUM_V6) << 3602 ETH_TX_PARSE_2ND_BD_IP_HDR_TYPE_OUTER_SHIFT) | 3603 (outerip_len << 3604 ETH_TX_PARSE_2ND_BD_IP_HDR_LEN_OUTER_W_SHIFT) | 3605 ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) << 3606 ETH_TX_PARSE_2ND_BD_LLC_SNAP_EN_SHIFT); 3607 3608 if (ip_hdr(skb)->protocol == IPPROTO_UDP) { 3609 SET_FLAG(*global_data, ETH_TX_PARSE_2ND_BD_TUNNEL_UDP_EXIST, 1); 3610 pbd2->tunnel_udp_hdr_start_w = skb_transport_offset(skb) >> 1; 3611 } 3612 } 3613 3614 /* called with netif_tx_lock 3615 * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call 3616 * netif_wake_queue() 3617 */ 3618 netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev) 3619 { 3620 struct bnx2x *bp = netdev_priv(dev); 3621 3622 struct netdev_queue *txq; 3623 struct bnx2x_fp_txdata *txdata; 3624 struct sw_tx_bd *tx_buf; 3625 struct eth_tx_start_bd *tx_start_bd, *first_bd; 3626 struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL; 3627 struct eth_tx_parse_bd_e1x *pbd_e1x = NULL; 3628 struct eth_tx_parse_bd_e2 *pbd_e2 = NULL; 3629 struct eth_tx_parse_2nd_bd *pbd2 = NULL; 3630 u32 pbd_e2_parsing_data = 0; 3631 u16 pkt_prod, bd_prod; 3632 int nbd, txq_index; 3633 dma_addr_t mapping; 3634 u32 xmit_type = bnx2x_xmit_type(bp, skb); 3635 int i; 3636 u8 hlen = 0; 3637 __le16 pkt_size = 0; 3638 struct ethhdr *eth; 3639 u8 mac_type = UNICAST_ADDRESS; 3640 3641 #ifdef BNX2X_STOP_ON_ERROR 3642 if (unlikely(bp->panic)) 3643 return NETDEV_TX_BUSY; 3644 #endif 3645 3646 txq_index = skb_get_queue_mapping(skb); 3647 txq = netdev_get_tx_queue(dev, txq_index); 3648 3649 BUG_ON(txq_index >= MAX_ETH_TXQ_IDX(bp) + (CNIC_LOADED(bp) ? 1 : 0)); 3650 3651 txdata = &bp->bnx2x_txq[txq_index]; 3652 3653 /* enable this debug print to view the transmission queue being used 3654 DP(NETIF_MSG_TX_QUEUED, "indices: txq %d, fp %d, txdata %d\n", 3655 txq_index, fp_index, txdata_index); */ 3656 3657 /* enable this debug print to view the transmission details 3658 DP(NETIF_MSG_TX_QUEUED, 3659 "transmitting packet cid %d fp index %d txdata_index %d tx_data ptr %p fp pointer %p\n", 3660 txdata->cid, fp_index, txdata_index, txdata, fp); */ 3661 3662 if (unlikely(bnx2x_tx_avail(bp, txdata) < 3663 skb_shinfo(skb)->nr_frags + 3664 BDS_PER_TX_PKT + 3665 NEXT_CNT_PER_TX_PKT(MAX_BDS_PER_TX_PKT))) { 3666 /* Handle special storage cases separately */ 3667 if (txdata->tx_ring_size == 0) { 3668 struct bnx2x_eth_q_stats *q_stats = 3669 bnx2x_fp_qstats(bp, txdata->parent_fp); 3670 q_stats->driver_filtered_tx_pkt++; 3671 dev_kfree_skb(skb); 3672 return NETDEV_TX_OK; 3673 } 3674 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++; 3675 netif_tx_stop_queue(txq); 3676 BNX2X_ERR("BUG! Tx ring full when queue awake!\n"); 3677 3678 return NETDEV_TX_BUSY; 3679 } 3680 3681 DP(NETIF_MSG_TX_QUEUED, 3682 "queue[%d]: SKB: summed %x protocol %x protocol(%x,%x) gso type %x xmit_type %x len %d\n", 3683 txq_index, skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr, 3684 ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type, 3685 skb->len); 3686 3687 eth = (struct ethhdr *)skb->data; 3688 3689 /* set flag according to packet type (UNICAST_ADDRESS is default)*/ 3690 if (unlikely(is_multicast_ether_addr(eth->h_dest))) { 3691 if (is_broadcast_ether_addr(eth->h_dest)) 3692 mac_type = BROADCAST_ADDRESS; 3693 else 3694 mac_type = MULTICAST_ADDRESS; 3695 } 3696 3697 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - BDS_PER_TX_PKT) 3698 /* First, check if we need to linearize the skb (due to FW 3699 restrictions). No need to check fragmentation if page size > 8K 3700 (there will be no violation to FW restrictions) */ 3701 if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) { 3702 /* Statistics of linearization */ 3703 bp->lin_cnt++; 3704 if (skb_linearize(skb) != 0) { 3705 DP(NETIF_MSG_TX_QUEUED, 3706 "SKB linearization failed - silently dropping this SKB\n"); 3707 dev_kfree_skb_any(skb); 3708 return NETDEV_TX_OK; 3709 } 3710 } 3711 #endif 3712 /* Map skb linear data for DMA */ 3713 mapping = dma_map_single(&bp->pdev->dev, skb->data, 3714 skb_headlen(skb), DMA_TO_DEVICE); 3715 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) { 3716 DP(NETIF_MSG_TX_QUEUED, 3717 "SKB mapping failed - silently dropping this SKB\n"); 3718 dev_kfree_skb_any(skb); 3719 return NETDEV_TX_OK; 3720 } 3721 /* 3722 Please read carefully. First we use one BD which we mark as start, 3723 then we have a parsing info BD (used for TSO or xsum), 3724 and only then we have the rest of the TSO BDs. 3725 (don't forget to mark the last one as last, 3726 and to unmap only AFTER you write to the BD ...) 3727 And above all, all pdb sizes are in words - NOT DWORDS! 3728 */ 3729 3730 /* get current pkt produced now - advance it just before sending packet 3731 * since mapping of pages may fail and cause packet to be dropped 3732 */ 3733 pkt_prod = txdata->tx_pkt_prod; 3734 bd_prod = TX_BD(txdata->tx_bd_prod); 3735 3736 /* get a tx_buf and first BD 3737 * tx_start_bd may be changed during SPLIT, 3738 * but first_bd will always stay first 3739 */ 3740 tx_buf = &txdata->tx_buf_ring[TX_BD(pkt_prod)]; 3741 tx_start_bd = &txdata->tx_desc_ring[bd_prod].start_bd; 3742 first_bd = tx_start_bd; 3743 3744 tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD; 3745 3746 /* header nbd: indirectly zero other flags! */ 3747 tx_start_bd->general_data = 1 << ETH_TX_START_BD_HDR_NBDS_SHIFT; 3748 3749 /* remember the first BD of the packet */ 3750 tx_buf->first_bd = txdata->tx_bd_prod; 3751 tx_buf->skb = skb; 3752 tx_buf->flags = 0; 3753 3754 DP(NETIF_MSG_TX_QUEUED, 3755 "sending pkt %u @%p next_idx %u bd %u @%p\n", 3756 pkt_prod, tx_buf, txdata->tx_pkt_prod, bd_prod, tx_start_bd); 3757 3758 if (vlan_tx_tag_present(skb)) { 3759 tx_start_bd->vlan_or_ethertype = 3760 cpu_to_le16(vlan_tx_tag_get(skb)); 3761 tx_start_bd->bd_flags.as_bitfield |= 3762 (X_ETH_OUTBAND_VLAN << ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT); 3763 } else { 3764 /* when transmitting in a vf, start bd must hold the ethertype 3765 * for fw to enforce it 3766 */ 3767 if (IS_VF(bp)) 3768 tx_start_bd->vlan_or_ethertype = 3769 cpu_to_le16(ntohs(eth->h_proto)); 3770 else 3771 /* used by FW for packet accounting */ 3772 tx_start_bd->vlan_or_ethertype = cpu_to_le16(pkt_prod); 3773 } 3774 3775 nbd = 2; /* start_bd + pbd + frags (updated when pages are mapped) */ 3776 3777 /* turn on parsing and get a BD */ 3778 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod)); 3779 3780 if (xmit_type & XMIT_CSUM) 3781 bnx2x_set_sbd_csum(bp, skb, tx_start_bd, xmit_type); 3782 3783 if (!CHIP_IS_E1x(bp)) { 3784 pbd_e2 = &txdata->tx_desc_ring[bd_prod].parse_bd_e2; 3785 memset(pbd_e2, 0, sizeof(struct eth_tx_parse_bd_e2)); 3786 3787 if (xmit_type & XMIT_CSUM_ENC) { 3788 u16 global_data = 0; 3789 3790 /* Set PBD in enc checksum offload case */ 3791 hlen = bnx2x_set_pbd_csum_enc(bp, skb, 3792 &pbd_e2_parsing_data, 3793 xmit_type); 3794 3795 /* turn on 2nd parsing and get a BD */ 3796 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod)); 3797 3798 pbd2 = &txdata->tx_desc_ring[bd_prod].parse_2nd_bd; 3799 3800 memset(pbd2, 0, sizeof(*pbd2)); 3801 3802 pbd_e2->data.tunnel_data.ip_hdr_start_inner_w = 3803 (skb_inner_network_header(skb) - 3804 skb->data) >> 1; 3805 3806 if (xmit_type & XMIT_GSO_ENC) 3807 bnx2x_update_pbds_gso_enc(skb, pbd_e2, pbd2, 3808 &global_data, 3809 xmit_type); 3810 3811 pbd2->global_data = cpu_to_le16(global_data); 3812 3813 /* add addition parse BD indication to start BD */ 3814 SET_FLAG(tx_start_bd->general_data, 3815 ETH_TX_START_BD_PARSE_NBDS, 1); 3816 /* set encapsulation flag in start BD */ 3817 SET_FLAG(tx_start_bd->general_data, 3818 ETH_TX_START_BD_TUNNEL_EXIST, 1); 3819 nbd++; 3820 } else if (xmit_type & XMIT_CSUM) { 3821 /* Set PBD in checksum offload case w/o encapsulation */ 3822 hlen = bnx2x_set_pbd_csum_e2(bp, skb, 3823 &pbd_e2_parsing_data, 3824 xmit_type); 3825 } 3826 3827 /* Add the macs to the parsing BD this is a vf */ 3828 if (IS_VF(bp)) { 3829 /* override GRE parameters in BD */ 3830 bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.src_hi, 3831 &pbd_e2->data.mac_addr.src_mid, 3832 &pbd_e2->data.mac_addr.src_lo, 3833 eth->h_source); 3834 3835 bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.dst_hi, 3836 &pbd_e2->data.mac_addr.dst_mid, 3837 &pbd_e2->data.mac_addr.dst_lo, 3838 eth->h_dest); 3839 } 3840 3841 SET_FLAG(pbd_e2_parsing_data, 3842 ETH_TX_PARSE_BD_E2_ETH_ADDR_TYPE, mac_type); 3843 } else { 3844 u16 global_data = 0; 3845 pbd_e1x = &txdata->tx_desc_ring[bd_prod].parse_bd_e1x; 3846 memset(pbd_e1x, 0, sizeof(struct eth_tx_parse_bd_e1x)); 3847 /* Set PBD in checksum offload case */ 3848 if (xmit_type & XMIT_CSUM) 3849 hlen = bnx2x_set_pbd_csum(bp, skb, pbd_e1x, xmit_type); 3850 3851 SET_FLAG(global_data, 3852 ETH_TX_PARSE_BD_E1X_ETH_ADDR_TYPE, mac_type); 3853 pbd_e1x->global_data |= cpu_to_le16(global_data); 3854 } 3855 3856 /* Setup the data pointer of the first BD of the packet */ 3857 tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping)); 3858 tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping)); 3859 tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb)); 3860 pkt_size = tx_start_bd->nbytes; 3861 3862 DP(NETIF_MSG_TX_QUEUED, 3863 "first bd @%p addr (%x:%x) nbytes %d flags %x vlan %x\n", 3864 tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo, 3865 le16_to_cpu(tx_start_bd->nbytes), 3866 tx_start_bd->bd_flags.as_bitfield, 3867 le16_to_cpu(tx_start_bd->vlan_or_ethertype)); 3868 3869 if (xmit_type & XMIT_GSO) { 3870 3871 DP(NETIF_MSG_TX_QUEUED, 3872 "TSO packet len %d hlen %d total len %d tso size %d\n", 3873 skb->len, hlen, skb_headlen(skb), 3874 skb_shinfo(skb)->gso_size); 3875 3876 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO; 3877 3878 if (unlikely(skb_headlen(skb) > hlen)) { 3879 nbd++; 3880 bd_prod = bnx2x_tx_split(bp, txdata, tx_buf, 3881 &tx_start_bd, hlen, 3882 bd_prod); 3883 } 3884 if (!CHIP_IS_E1x(bp)) 3885 bnx2x_set_pbd_gso_e2(skb, &pbd_e2_parsing_data, 3886 xmit_type); 3887 else 3888 bnx2x_set_pbd_gso(skb, pbd_e1x, first_bd, xmit_type); 3889 } 3890 3891 /* Set the PBD's parsing_data field if not zero 3892 * (for the chips newer than 57711). 3893 */ 3894 if (pbd_e2_parsing_data) 3895 pbd_e2->parsing_data = cpu_to_le32(pbd_e2_parsing_data); 3896 3897 tx_data_bd = (struct eth_tx_bd *)tx_start_bd; 3898 3899 /* Handle fragmented skb */ 3900 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 3901 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 3902 3903 mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 0, 3904 skb_frag_size(frag), DMA_TO_DEVICE); 3905 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) { 3906 unsigned int pkts_compl = 0, bytes_compl = 0; 3907 3908 DP(NETIF_MSG_TX_QUEUED, 3909 "Unable to map page - dropping packet...\n"); 3910 3911 /* we need unmap all buffers already mapped 3912 * for this SKB; 3913 * first_bd->nbd need to be properly updated 3914 * before call to bnx2x_free_tx_pkt 3915 */ 3916 first_bd->nbd = cpu_to_le16(nbd); 3917 bnx2x_free_tx_pkt(bp, txdata, 3918 TX_BD(txdata->tx_pkt_prod), 3919 &pkts_compl, &bytes_compl); 3920 return NETDEV_TX_OK; 3921 } 3922 3923 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod)); 3924 tx_data_bd = &txdata->tx_desc_ring[bd_prod].reg_bd; 3925 if (total_pkt_bd == NULL) 3926 total_pkt_bd = &txdata->tx_desc_ring[bd_prod].reg_bd; 3927 3928 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping)); 3929 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping)); 3930 tx_data_bd->nbytes = cpu_to_le16(skb_frag_size(frag)); 3931 le16_add_cpu(&pkt_size, skb_frag_size(frag)); 3932 nbd++; 3933 3934 DP(NETIF_MSG_TX_QUEUED, 3935 "frag %d bd @%p addr (%x:%x) nbytes %d\n", 3936 i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo, 3937 le16_to_cpu(tx_data_bd->nbytes)); 3938 } 3939 3940 DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd); 3941 3942 /* update with actual num BDs */ 3943 first_bd->nbd = cpu_to_le16(nbd); 3944 3945 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod)); 3946 3947 /* now send a tx doorbell, counting the next BD 3948 * if the packet contains or ends with it 3949 */ 3950 if (TX_BD_POFF(bd_prod) < nbd) 3951 nbd++; 3952 3953 /* total_pkt_bytes should be set on the first data BD if 3954 * it's not an LSO packet and there is more than one 3955 * data BD. In this case pkt_size is limited by an MTU value. 3956 * However we prefer to set it for an LSO packet (while we don't 3957 * have to) in order to save some CPU cycles in a none-LSO 3958 * case, when we much more care about them. 3959 */ 3960 if (total_pkt_bd != NULL) 3961 total_pkt_bd->total_pkt_bytes = pkt_size; 3962 3963 if (pbd_e1x) 3964 DP(NETIF_MSG_TX_QUEUED, 3965 "PBD (E1X) @%p ip_data %x ip_hlen %u ip_id %u lso_mss %u tcp_flags %x xsum %x seq %u hlen %u\n", 3966 pbd_e1x, pbd_e1x->global_data, pbd_e1x->ip_hlen_w, 3967 pbd_e1x->ip_id, pbd_e1x->lso_mss, pbd_e1x->tcp_flags, 3968 pbd_e1x->tcp_pseudo_csum, pbd_e1x->tcp_send_seq, 3969 le16_to_cpu(pbd_e1x->total_hlen_w)); 3970 if (pbd_e2) 3971 DP(NETIF_MSG_TX_QUEUED, 3972 "PBD (E2) @%p dst %x %x %x src %x %x %x parsing_data %x\n", 3973 pbd_e2, 3974 pbd_e2->data.mac_addr.dst_hi, 3975 pbd_e2->data.mac_addr.dst_mid, 3976 pbd_e2->data.mac_addr.dst_lo, 3977 pbd_e2->data.mac_addr.src_hi, 3978 pbd_e2->data.mac_addr.src_mid, 3979 pbd_e2->data.mac_addr.src_lo, 3980 pbd_e2->parsing_data); 3981 DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d bd %u\n", nbd, bd_prod); 3982 3983 netdev_tx_sent_queue(txq, skb->len); 3984 3985 skb_tx_timestamp(skb); 3986 3987 txdata->tx_pkt_prod++; 3988 /* 3989 * Make sure that the BD data is updated before updating the producer 3990 * since FW might read the BD right after the producer is updated. 3991 * This is only applicable for weak-ordered memory model archs such 3992 * as IA-64. The following barrier is also mandatory since FW will 3993 * assumes packets must have BDs. 3994 */ 3995 wmb(); 3996 3997 txdata->tx_db.data.prod += nbd; 3998 barrier(); 3999 4000 DOORBELL(bp, txdata->cid, txdata->tx_db.raw); 4001 4002 mmiowb(); 4003 4004 txdata->tx_bd_prod += nbd; 4005 4006 if (unlikely(bnx2x_tx_avail(bp, txdata) < MAX_DESC_PER_TX_PKT)) { 4007 netif_tx_stop_queue(txq); 4008 4009 /* paired memory barrier is in bnx2x_tx_int(), we have to keep 4010 * ordering of set_bit() in netif_tx_stop_queue() and read of 4011 * fp->bd_tx_cons */ 4012 smp_mb(); 4013 4014 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++; 4015 if (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT) 4016 netif_tx_wake_queue(txq); 4017 } 4018 txdata->tx_pkt++; 4019 4020 return NETDEV_TX_OK; 4021 } 4022 4023 /** 4024 * bnx2x_setup_tc - routine to configure net_device for multi tc 4025 * 4026 * @netdev: net device to configure 4027 * @tc: number of traffic classes to enable 4028 * 4029 * callback connected to the ndo_setup_tc function pointer 4030 */ 4031 int bnx2x_setup_tc(struct net_device *dev, u8 num_tc) 4032 { 4033 int cos, prio, count, offset; 4034 struct bnx2x *bp = netdev_priv(dev); 4035 4036 /* setup tc must be called under rtnl lock */ 4037 ASSERT_RTNL(); 4038 4039 /* no traffic classes requested. Aborting */ 4040 if (!num_tc) { 4041 netdev_reset_tc(dev); 4042 return 0; 4043 } 4044 4045 /* requested to support too many traffic classes */ 4046 if (num_tc > bp->max_cos) { 4047 BNX2X_ERR("support for too many traffic classes requested: %d. Max supported is %d\n", 4048 num_tc, bp->max_cos); 4049 return -EINVAL; 4050 } 4051 4052 /* declare amount of supported traffic classes */ 4053 if (netdev_set_num_tc(dev, num_tc)) { 4054 BNX2X_ERR("failed to declare %d traffic classes\n", num_tc); 4055 return -EINVAL; 4056 } 4057 4058 /* configure priority to traffic class mapping */ 4059 for (prio = 0; prio < BNX2X_MAX_PRIORITY; prio++) { 4060 netdev_set_prio_tc_map(dev, prio, bp->prio_to_cos[prio]); 4061 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP, 4062 "mapping priority %d to tc %d\n", 4063 prio, bp->prio_to_cos[prio]); 4064 } 4065 4066 /* Use this configuration to differentiate tc0 from other COSes 4067 This can be used for ets or pfc, and save the effort of setting 4068 up a multio class queue disc or negotiating DCBX with a switch 4069 netdev_set_prio_tc_map(dev, 0, 0); 4070 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", 0, 0); 4071 for (prio = 1; prio < 16; prio++) { 4072 netdev_set_prio_tc_map(dev, prio, 1); 4073 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", prio, 1); 4074 } */ 4075 4076 /* configure traffic class to transmission queue mapping */ 4077 for (cos = 0; cos < bp->max_cos; cos++) { 4078 count = BNX2X_NUM_ETH_QUEUES(bp); 4079 offset = cos * BNX2X_NUM_NON_CNIC_QUEUES(bp); 4080 netdev_set_tc_queue(dev, cos, count, offset); 4081 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP, 4082 "mapping tc %d to offset %d count %d\n", 4083 cos, offset, count); 4084 } 4085 4086 return 0; 4087 } 4088 4089 /* called with rtnl_lock */ 4090 int bnx2x_change_mac_addr(struct net_device *dev, void *p) 4091 { 4092 struct sockaddr *addr = p; 4093 struct bnx2x *bp = netdev_priv(dev); 4094 int rc = 0; 4095 4096 if (!bnx2x_is_valid_ether_addr(bp, addr->sa_data)) { 4097 BNX2X_ERR("Requested MAC address is not valid\n"); 4098 return -EINVAL; 4099 } 4100 4101 if ((IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp)) && 4102 !is_zero_ether_addr(addr->sa_data)) { 4103 BNX2X_ERR("Can't configure non-zero address on iSCSI or FCoE functions in MF-SD mode\n"); 4104 return -EINVAL; 4105 } 4106 4107 if (netif_running(dev)) { 4108 rc = bnx2x_set_eth_mac(bp, false); 4109 if (rc) 4110 return rc; 4111 } 4112 4113 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len); 4114 4115 if (netif_running(dev)) 4116 rc = bnx2x_set_eth_mac(bp, true); 4117 4118 return rc; 4119 } 4120 4121 static void bnx2x_free_fp_mem_at(struct bnx2x *bp, int fp_index) 4122 { 4123 union host_hc_status_block *sb = &bnx2x_fp(bp, fp_index, status_blk); 4124 struct bnx2x_fastpath *fp = &bp->fp[fp_index]; 4125 u8 cos; 4126 4127 /* Common */ 4128 4129 if (IS_FCOE_IDX(fp_index)) { 4130 memset(sb, 0, sizeof(union host_hc_status_block)); 4131 fp->status_blk_mapping = 0; 4132 } else { 4133 /* status blocks */ 4134 if (!CHIP_IS_E1x(bp)) 4135 BNX2X_PCI_FREE(sb->e2_sb, 4136 bnx2x_fp(bp, fp_index, 4137 status_blk_mapping), 4138 sizeof(struct host_hc_status_block_e2)); 4139 else 4140 BNX2X_PCI_FREE(sb->e1x_sb, 4141 bnx2x_fp(bp, fp_index, 4142 status_blk_mapping), 4143 sizeof(struct host_hc_status_block_e1x)); 4144 } 4145 4146 /* Rx */ 4147 if (!skip_rx_queue(bp, fp_index)) { 4148 bnx2x_free_rx_bds(fp); 4149 4150 /* fastpath rx rings: rx_buf rx_desc rx_comp */ 4151 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_buf_ring)); 4152 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_desc_ring), 4153 bnx2x_fp(bp, fp_index, rx_desc_mapping), 4154 sizeof(struct eth_rx_bd) * NUM_RX_BD); 4155 4156 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_comp_ring), 4157 bnx2x_fp(bp, fp_index, rx_comp_mapping), 4158 sizeof(struct eth_fast_path_rx_cqe) * 4159 NUM_RCQ_BD); 4160 4161 /* SGE ring */ 4162 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_page_ring)); 4163 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_sge_ring), 4164 bnx2x_fp(bp, fp_index, rx_sge_mapping), 4165 BCM_PAGE_SIZE * NUM_RX_SGE_PAGES); 4166 } 4167 4168 /* Tx */ 4169 if (!skip_tx_queue(bp, fp_index)) { 4170 /* fastpath tx rings: tx_buf tx_desc */ 4171 for_each_cos_in_tx_queue(fp, cos) { 4172 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos]; 4173 4174 DP(NETIF_MSG_IFDOWN, 4175 "freeing tx memory of fp %d cos %d cid %d\n", 4176 fp_index, cos, txdata->cid); 4177 4178 BNX2X_FREE(txdata->tx_buf_ring); 4179 BNX2X_PCI_FREE(txdata->tx_desc_ring, 4180 txdata->tx_desc_mapping, 4181 sizeof(union eth_tx_bd_types) * NUM_TX_BD); 4182 } 4183 } 4184 /* end of fastpath */ 4185 } 4186 4187 void bnx2x_free_fp_mem_cnic(struct bnx2x *bp) 4188 { 4189 int i; 4190 for_each_cnic_queue(bp, i) 4191 bnx2x_free_fp_mem_at(bp, i); 4192 } 4193 4194 void bnx2x_free_fp_mem(struct bnx2x *bp) 4195 { 4196 int i; 4197 for_each_eth_queue(bp, i) 4198 bnx2x_free_fp_mem_at(bp, i); 4199 } 4200 4201 static void set_sb_shortcuts(struct bnx2x *bp, int index) 4202 { 4203 union host_hc_status_block status_blk = bnx2x_fp(bp, index, status_blk); 4204 if (!CHIP_IS_E1x(bp)) { 4205 bnx2x_fp(bp, index, sb_index_values) = 4206 (__le16 *)status_blk.e2_sb->sb.index_values; 4207 bnx2x_fp(bp, index, sb_running_index) = 4208 (__le16 *)status_blk.e2_sb->sb.running_index; 4209 } else { 4210 bnx2x_fp(bp, index, sb_index_values) = 4211 (__le16 *)status_blk.e1x_sb->sb.index_values; 4212 bnx2x_fp(bp, index, sb_running_index) = 4213 (__le16 *)status_blk.e1x_sb->sb.running_index; 4214 } 4215 } 4216 4217 /* Returns the number of actually allocated BDs */ 4218 static int bnx2x_alloc_rx_bds(struct bnx2x_fastpath *fp, 4219 int rx_ring_size) 4220 { 4221 struct bnx2x *bp = fp->bp; 4222 u16 ring_prod, cqe_ring_prod; 4223 int i, failure_cnt = 0; 4224 4225 fp->rx_comp_cons = 0; 4226 cqe_ring_prod = ring_prod = 0; 4227 4228 /* This routine is called only during fo init so 4229 * fp->eth_q_stats.rx_skb_alloc_failed = 0 4230 */ 4231 for (i = 0; i < rx_ring_size; i++) { 4232 if (bnx2x_alloc_rx_data(bp, fp, ring_prod, GFP_KERNEL) < 0) { 4233 failure_cnt++; 4234 continue; 4235 } 4236 ring_prod = NEXT_RX_IDX(ring_prod); 4237 cqe_ring_prod = NEXT_RCQ_IDX(cqe_ring_prod); 4238 WARN_ON(ring_prod <= (i - failure_cnt)); 4239 } 4240 4241 if (failure_cnt) 4242 BNX2X_ERR("was only able to allocate %d rx skbs on queue[%d]\n", 4243 i - failure_cnt, fp->index); 4244 4245 fp->rx_bd_prod = ring_prod; 4246 /* Limit the CQE producer by the CQE ring size */ 4247 fp->rx_comp_prod = min_t(u16, NUM_RCQ_RINGS*RCQ_DESC_CNT, 4248 cqe_ring_prod); 4249 fp->rx_pkt = fp->rx_calls = 0; 4250 4251 bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed += failure_cnt; 4252 4253 return i - failure_cnt; 4254 } 4255 4256 static void bnx2x_set_next_page_rx_cq(struct bnx2x_fastpath *fp) 4257 { 4258 int i; 4259 4260 for (i = 1; i <= NUM_RCQ_RINGS; i++) { 4261 struct eth_rx_cqe_next_page *nextpg; 4262 4263 nextpg = (struct eth_rx_cqe_next_page *) 4264 &fp->rx_comp_ring[RCQ_DESC_CNT * i - 1]; 4265 nextpg->addr_hi = 4266 cpu_to_le32(U64_HI(fp->rx_comp_mapping + 4267 BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS))); 4268 nextpg->addr_lo = 4269 cpu_to_le32(U64_LO(fp->rx_comp_mapping + 4270 BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS))); 4271 } 4272 } 4273 4274 static int bnx2x_alloc_fp_mem_at(struct bnx2x *bp, int index) 4275 { 4276 union host_hc_status_block *sb; 4277 struct bnx2x_fastpath *fp = &bp->fp[index]; 4278 int ring_size = 0; 4279 u8 cos; 4280 int rx_ring_size = 0; 4281 4282 if (!bp->rx_ring_size && 4283 (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))) { 4284 rx_ring_size = MIN_RX_SIZE_NONTPA; 4285 bp->rx_ring_size = rx_ring_size; 4286 } else if (!bp->rx_ring_size) { 4287 rx_ring_size = MAX_RX_AVAIL/BNX2X_NUM_RX_QUEUES(bp); 4288 4289 if (CHIP_IS_E3(bp)) { 4290 u32 cfg = SHMEM_RD(bp, 4291 dev_info.port_hw_config[BP_PORT(bp)]. 4292 default_cfg); 4293 4294 /* Decrease ring size for 1G functions */ 4295 if ((cfg & PORT_HW_CFG_NET_SERDES_IF_MASK) == 4296 PORT_HW_CFG_NET_SERDES_IF_SGMII) 4297 rx_ring_size /= 10; 4298 } 4299 4300 /* allocate at least number of buffers required by FW */ 4301 rx_ring_size = max_t(int, bp->disable_tpa ? MIN_RX_SIZE_NONTPA : 4302 MIN_RX_SIZE_TPA, rx_ring_size); 4303 4304 bp->rx_ring_size = rx_ring_size; 4305 } else /* if rx_ring_size specified - use it */ 4306 rx_ring_size = bp->rx_ring_size; 4307 4308 DP(BNX2X_MSG_SP, "calculated rx_ring_size %d\n", rx_ring_size); 4309 4310 /* Common */ 4311 sb = &bnx2x_fp(bp, index, status_blk); 4312 4313 if (!IS_FCOE_IDX(index)) { 4314 /* status blocks */ 4315 if (!CHIP_IS_E1x(bp)) 4316 BNX2X_PCI_ALLOC(sb->e2_sb, 4317 &bnx2x_fp(bp, index, status_blk_mapping), 4318 sizeof(struct host_hc_status_block_e2)); 4319 else 4320 BNX2X_PCI_ALLOC(sb->e1x_sb, 4321 &bnx2x_fp(bp, index, status_blk_mapping), 4322 sizeof(struct host_hc_status_block_e1x)); 4323 } 4324 4325 /* FCoE Queue uses Default SB and doesn't ACK the SB, thus no need to 4326 * set shortcuts for it. 4327 */ 4328 if (!IS_FCOE_IDX(index)) 4329 set_sb_shortcuts(bp, index); 4330 4331 /* Tx */ 4332 if (!skip_tx_queue(bp, index)) { 4333 /* fastpath tx rings: tx_buf tx_desc */ 4334 for_each_cos_in_tx_queue(fp, cos) { 4335 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos]; 4336 4337 DP(NETIF_MSG_IFUP, 4338 "allocating tx memory of fp %d cos %d\n", 4339 index, cos); 4340 4341 BNX2X_ALLOC(txdata->tx_buf_ring, 4342 sizeof(struct sw_tx_bd) * NUM_TX_BD); 4343 BNX2X_PCI_ALLOC(txdata->tx_desc_ring, 4344 &txdata->tx_desc_mapping, 4345 sizeof(union eth_tx_bd_types) * NUM_TX_BD); 4346 } 4347 } 4348 4349 /* Rx */ 4350 if (!skip_rx_queue(bp, index)) { 4351 /* fastpath rx rings: rx_buf rx_desc rx_comp */ 4352 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_buf_ring), 4353 sizeof(struct sw_rx_bd) * NUM_RX_BD); 4354 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_desc_ring), 4355 &bnx2x_fp(bp, index, rx_desc_mapping), 4356 sizeof(struct eth_rx_bd) * NUM_RX_BD); 4357 4358 /* Seed all CQEs by 1s */ 4359 BNX2X_PCI_FALLOC(bnx2x_fp(bp, index, rx_comp_ring), 4360 &bnx2x_fp(bp, index, rx_comp_mapping), 4361 sizeof(struct eth_fast_path_rx_cqe) * 4362 NUM_RCQ_BD); 4363 4364 /* SGE ring */ 4365 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_page_ring), 4366 sizeof(struct sw_rx_page) * NUM_RX_SGE); 4367 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_sge_ring), 4368 &bnx2x_fp(bp, index, rx_sge_mapping), 4369 BCM_PAGE_SIZE * NUM_RX_SGE_PAGES); 4370 /* RX BD ring */ 4371 bnx2x_set_next_page_rx_bd(fp); 4372 4373 /* CQ ring */ 4374 bnx2x_set_next_page_rx_cq(fp); 4375 4376 /* BDs */ 4377 ring_size = bnx2x_alloc_rx_bds(fp, rx_ring_size); 4378 if (ring_size < rx_ring_size) 4379 goto alloc_mem_err; 4380 } 4381 4382 return 0; 4383 4384 /* handles low memory cases */ 4385 alloc_mem_err: 4386 BNX2X_ERR("Unable to allocate full memory for queue %d (size %d)\n", 4387 index, ring_size); 4388 /* FW will drop all packets if queue is not big enough, 4389 * In these cases we disable the queue 4390 * Min size is different for OOO, TPA and non-TPA queues 4391 */ 4392 if (ring_size < (fp->disable_tpa ? 4393 MIN_RX_SIZE_NONTPA : MIN_RX_SIZE_TPA)) { 4394 /* release memory allocated for this queue */ 4395 bnx2x_free_fp_mem_at(bp, index); 4396 return -ENOMEM; 4397 } 4398 return 0; 4399 } 4400 4401 int bnx2x_alloc_fp_mem_cnic(struct bnx2x *bp) 4402 { 4403 if (!NO_FCOE(bp)) 4404 /* FCoE */ 4405 if (bnx2x_alloc_fp_mem_at(bp, FCOE_IDX(bp))) 4406 /* we will fail load process instead of mark 4407 * NO_FCOE_FLAG 4408 */ 4409 return -ENOMEM; 4410 4411 return 0; 4412 } 4413 4414 int bnx2x_alloc_fp_mem(struct bnx2x *bp) 4415 { 4416 int i; 4417 4418 /* 1. Allocate FP for leading - fatal if error 4419 * 2. Allocate RSS - fix number of queues if error 4420 */ 4421 4422 /* leading */ 4423 if (bnx2x_alloc_fp_mem_at(bp, 0)) 4424 return -ENOMEM; 4425 4426 /* RSS */ 4427 for_each_nondefault_eth_queue(bp, i) 4428 if (bnx2x_alloc_fp_mem_at(bp, i)) 4429 break; 4430 4431 /* handle memory failures */ 4432 if (i != BNX2X_NUM_ETH_QUEUES(bp)) { 4433 int delta = BNX2X_NUM_ETH_QUEUES(bp) - i; 4434 4435 WARN_ON(delta < 0); 4436 bnx2x_shrink_eth_fp(bp, delta); 4437 if (CNIC_SUPPORT(bp)) 4438 /* move non eth FPs next to last eth FP 4439 * must be done in that order 4440 * FCOE_IDX < FWD_IDX < OOO_IDX 4441 */ 4442 4443 /* move FCoE fp even NO_FCOE_FLAG is on */ 4444 bnx2x_move_fp(bp, FCOE_IDX(bp), FCOE_IDX(bp) - delta); 4445 bp->num_ethernet_queues -= delta; 4446 bp->num_queues = bp->num_ethernet_queues + 4447 bp->num_cnic_queues; 4448 BNX2X_ERR("Adjusted num of queues from %d to %d\n", 4449 bp->num_queues + delta, bp->num_queues); 4450 } 4451 4452 return 0; 4453 } 4454 4455 void bnx2x_free_mem_bp(struct bnx2x *bp) 4456 { 4457 int i; 4458 4459 for (i = 0; i < bp->fp_array_size; i++) 4460 kfree(bp->fp[i].tpa_info); 4461 kfree(bp->fp); 4462 kfree(bp->sp_objs); 4463 kfree(bp->fp_stats); 4464 kfree(bp->bnx2x_txq); 4465 kfree(bp->msix_table); 4466 kfree(bp->ilt); 4467 } 4468 4469 int bnx2x_alloc_mem_bp(struct bnx2x *bp) 4470 { 4471 struct bnx2x_fastpath *fp; 4472 struct msix_entry *tbl; 4473 struct bnx2x_ilt *ilt; 4474 int msix_table_size = 0; 4475 int fp_array_size, txq_array_size; 4476 int i; 4477 4478 /* 4479 * The biggest MSI-X table we might need is as a maximum number of fast 4480 * path IGU SBs plus default SB (for PF only). 4481 */ 4482 msix_table_size = bp->igu_sb_cnt; 4483 if (IS_PF(bp)) 4484 msix_table_size++; 4485 BNX2X_DEV_INFO("msix_table_size %d\n", msix_table_size); 4486 4487 /* fp array: RSS plus CNIC related L2 queues */ 4488 fp_array_size = BNX2X_MAX_RSS_COUNT(bp) + CNIC_SUPPORT(bp); 4489 bp->fp_array_size = fp_array_size; 4490 BNX2X_DEV_INFO("fp_array_size %d\n", bp->fp_array_size); 4491 4492 fp = kcalloc(bp->fp_array_size, sizeof(*fp), GFP_KERNEL); 4493 if (!fp) 4494 goto alloc_err; 4495 for (i = 0; i < bp->fp_array_size; i++) { 4496 fp[i].tpa_info = 4497 kcalloc(ETH_MAX_AGGREGATION_QUEUES_E1H_E2, 4498 sizeof(struct bnx2x_agg_info), GFP_KERNEL); 4499 if (!(fp[i].tpa_info)) 4500 goto alloc_err; 4501 } 4502 4503 bp->fp = fp; 4504 4505 /* allocate sp objs */ 4506 bp->sp_objs = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_sp_objs), 4507 GFP_KERNEL); 4508 if (!bp->sp_objs) 4509 goto alloc_err; 4510 4511 /* allocate fp_stats */ 4512 bp->fp_stats = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_fp_stats), 4513 GFP_KERNEL); 4514 if (!bp->fp_stats) 4515 goto alloc_err; 4516 4517 /* Allocate memory for the transmission queues array */ 4518 txq_array_size = 4519 BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS + CNIC_SUPPORT(bp); 4520 BNX2X_DEV_INFO("txq_array_size %d", txq_array_size); 4521 4522 bp->bnx2x_txq = kcalloc(txq_array_size, sizeof(struct bnx2x_fp_txdata), 4523 GFP_KERNEL); 4524 if (!bp->bnx2x_txq) 4525 goto alloc_err; 4526 4527 /* msix table */ 4528 tbl = kcalloc(msix_table_size, sizeof(*tbl), GFP_KERNEL); 4529 if (!tbl) 4530 goto alloc_err; 4531 bp->msix_table = tbl; 4532 4533 /* ilt */ 4534 ilt = kzalloc(sizeof(*ilt), GFP_KERNEL); 4535 if (!ilt) 4536 goto alloc_err; 4537 bp->ilt = ilt; 4538 4539 return 0; 4540 alloc_err: 4541 bnx2x_free_mem_bp(bp); 4542 return -ENOMEM; 4543 } 4544 4545 int bnx2x_reload_if_running(struct net_device *dev) 4546 { 4547 struct bnx2x *bp = netdev_priv(dev); 4548 4549 if (unlikely(!netif_running(dev))) 4550 return 0; 4551 4552 bnx2x_nic_unload(bp, UNLOAD_NORMAL, true); 4553 return bnx2x_nic_load(bp, LOAD_NORMAL); 4554 } 4555 4556 int bnx2x_get_cur_phy_idx(struct bnx2x *bp) 4557 { 4558 u32 sel_phy_idx = 0; 4559 if (bp->link_params.num_phys <= 1) 4560 return INT_PHY; 4561 4562 if (bp->link_vars.link_up) { 4563 sel_phy_idx = EXT_PHY1; 4564 /* In case link is SERDES, check if the EXT_PHY2 is the one */ 4565 if ((bp->link_vars.link_status & LINK_STATUS_SERDES_LINK) && 4566 (bp->link_params.phy[EXT_PHY2].supported & SUPPORTED_FIBRE)) 4567 sel_phy_idx = EXT_PHY2; 4568 } else { 4569 4570 switch (bnx2x_phy_selection(&bp->link_params)) { 4571 case PORT_HW_CFG_PHY_SELECTION_HARDWARE_DEFAULT: 4572 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY: 4573 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY_PRIORITY: 4574 sel_phy_idx = EXT_PHY1; 4575 break; 4576 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY: 4577 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY_PRIORITY: 4578 sel_phy_idx = EXT_PHY2; 4579 break; 4580 } 4581 } 4582 4583 return sel_phy_idx; 4584 } 4585 int bnx2x_get_link_cfg_idx(struct bnx2x *bp) 4586 { 4587 u32 sel_phy_idx = bnx2x_get_cur_phy_idx(bp); 4588 /* 4589 * The selected activated PHY is always after swapping (in case PHY 4590 * swapping is enabled). So when swapping is enabled, we need to reverse 4591 * the configuration 4592 */ 4593 4594 if (bp->link_params.multi_phy_config & 4595 PORT_HW_CFG_PHY_SWAPPED_ENABLED) { 4596 if (sel_phy_idx == EXT_PHY1) 4597 sel_phy_idx = EXT_PHY2; 4598 else if (sel_phy_idx == EXT_PHY2) 4599 sel_phy_idx = EXT_PHY1; 4600 } 4601 return LINK_CONFIG_IDX(sel_phy_idx); 4602 } 4603 4604 #ifdef NETDEV_FCOE_WWNN 4605 int bnx2x_fcoe_get_wwn(struct net_device *dev, u64 *wwn, int type) 4606 { 4607 struct bnx2x *bp = netdev_priv(dev); 4608 struct cnic_eth_dev *cp = &bp->cnic_eth_dev; 4609 4610 switch (type) { 4611 case NETDEV_FCOE_WWNN: 4612 *wwn = HILO_U64(cp->fcoe_wwn_node_name_hi, 4613 cp->fcoe_wwn_node_name_lo); 4614 break; 4615 case NETDEV_FCOE_WWPN: 4616 *wwn = HILO_U64(cp->fcoe_wwn_port_name_hi, 4617 cp->fcoe_wwn_port_name_lo); 4618 break; 4619 default: 4620 BNX2X_ERR("Wrong WWN type requested - %d\n", type); 4621 return -EINVAL; 4622 } 4623 4624 return 0; 4625 } 4626 #endif 4627 4628 /* called with rtnl_lock */ 4629 int bnx2x_change_mtu(struct net_device *dev, int new_mtu) 4630 { 4631 struct bnx2x *bp = netdev_priv(dev); 4632 4633 if (bp->recovery_state != BNX2X_RECOVERY_DONE) { 4634 BNX2X_ERR("Can't perform change MTU during parity recovery\n"); 4635 return -EAGAIN; 4636 } 4637 4638 if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) || 4639 ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE)) { 4640 BNX2X_ERR("Can't support requested MTU size\n"); 4641 return -EINVAL; 4642 } 4643 4644 /* This does not race with packet allocation 4645 * because the actual alloc size is 4646 * only updated as part of load 4647 */ 4648 dev->mtu = new_mtu; 4649 4650 return bnx2x_reload_if_running(dev); 4651 } 4652 4653 netdev_features_t bnx2x_fix_features(struct net_device *dev, 4654 netdev_features_t features) 4655 { 4656 struct bnx2x *bp = netdev_priv(dev); 4657 4658 /* TPA requires Rx CSUM offloading */ 4659 if (!(features & NETIF_F_RXCSUM) || bp->disable_tpa) { 4660 features &= ~NETIF_F_LRO; 4661 features &= ~NETIF_F_GRO; 4662 } 4663 4664 return features; 4665 } 4666 4667 int bnx2x_set_features(struct net_device *dev, netdev_features_t features) 4668 { 4669 struct bnx2x *bp = netdev_priv(dev); 4670 u32 flags = bp->flags; 4671 u32 changes; 4672 bool bnx2x_reload = false; 4673 4674 if (features & NETIF_F_LRO) 4675 flags |= TPA_ENABLE_FLAG; 4676 else 4677 flags &= ~TPA_ENABLE_FLAG; 4678 4679 if (features & NETIF_F_GRO) 4680 flags |= GRO_ENABLE_FLAG; 4681 else 4682 flags &= ~GRO_ENABLE_FLAG; 4683 4684 if (features & NETIF_F_LOOPBACK) { 4685 if (bp->link_params.loopback_mode != LOOPBACK_BMAC) { 4686 bp->link_params.loopback_mode = LOOPBACK_BMAC; 4687 bnx2x_reload = true; 4688 } 4689 } else { 4690 if (bp->link_params.loopback_mode != LOOPBACK_NONE) { 4691 bp->link_params.loopback_mode = LOOPBACK_NONE; 4692 bnx2x_reload = true; 4693 } 4694 } 4695 4696 changes = flags ^ bp->flags; 4697 4698 /* if GRO is changed while LRO is enabled, don't force a reload */ 4699 if ((changes & GRO_ENABLE_FLAG) && (flags & TPA_ENABLE_FLAG)) 4700 changes &= ~GRO_ENABLE_FLAG; 4701 4702 if (changes) 4703 bnx2x_reload = true; 4704 4705 bp->flags = flags; 4706 4707 if (bnx2x_reload) { 4708 if (bp->recovery_state == BNX2X_RECOVERY_DONE) 4709 return bnx2x_reload_if_running(dev); 4710 /* else: bnx2x_nic_load() will be called at end of recovery */ 4711 } 4712 4713 return 0; 4714 } 4715 4716 void bnx2x_tx_timeout(struct net_device *dev) 4717 { 4718 struct bnx2x *bp = netdev_priv(dev); 4719 4720 #ifdef BNX2X_STOP_ON_ERROR 4721 if (!bp->panic) 4722 bnx2x_panic(); 4723 #endif 4724 4725 smp_mb__before_clear_bit(); 4726 set_bit(BNX2X_SP_RTNL_TX_TIMEOUT, &bp->sp_rtnl_state); 4727 smp_mb__after_clear_bit(); 4728 4729 /* This allows the netif to be shutdown gracefully before resetting */ 4730 schedule_delayed_work(&bp->sp_rtnl_task, 0); 4731 } 4732 4733 int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state) 4734 { 4735 struct net_device *dev = pci_get_drvdata(pdev); 4736 struct bnx2x *bp; 4737 4738 if (!dev) { 4739 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n"); 4740 return -ENODEV; 4741 } 4742 bp = netdev_priv(dev); 4743 4744 rtnl_lock(); 4745 4746 pci_save_state(pdev); 4747 4748 if (!netif_running(dev)) { 4749 rtnl_unlock(); 4750 return 0; 4751 } 4752 4753 netif_device_detach(dev); 4754 4755 bnx2x_nic_unload(bp, UNLOAD_CLOSE, false); 4756 4757 bnx2x_set_power_state(bp, pci_choose_state(pdev, state)); 4758 4759 rtnl_unlock(); 4760 4761 return 0; 4762 } 4763 4764 int bnx2x_resume(struct pci_dev *pdev) 4765 { 4766 struct net_device *dev = pci_get_drvdata(pdev); 4767 struct bnx2x *bp; 4768 int rc; 4769 4770 if (!dev) { 4771 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n"); 4772 return -ENODEV; 4773 } 4774 bp = netdev_priv(dev); 4775 4776 if (bp->recovery_state != BNX2X_RECOVERY_DONE) { 4777 BNX2X_ERR("Handling parity error recovery. Try again later\n"); 4778 return -EAGAIN; 4779 } 4780 4781 rtnl_lock(); 4782 4783 pci_restore_state(pdev); 4784 4785 if (!netif_running(dev)) { 4786 rtnl_unlock(); 4787 return 0; 4788 } 4789 4790 bnx2x_set_power_state(bp, PCI_D0); 4791 netif_device_attach(dev); 4792 4793 rc = bnx2x_nic_load(bp, LOAD_OPEN); 4794 4795 rtnl_unlock(); 4796 4797 return rc; 4798 } 4799 4800 void bnx2x_set_ctx_validation(struct bnx2x *bp, struct eth_context *cxt, 4801 u32 cid) 4802 { 4803 if (!cxt) { 4804 BNX2X_ERR("bad context pointer %p\n", cxt); 4805 return; 4806 } 4807 4808 /* ustorm cxt validation */ 4809 cxt->ustorm_ag_context.cdu_usage = 4810 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid), 4811 CDU_REGION_NUMBER_UCM_AG, ETH_CONNECTION_TYPE); 4812 /* xcontext validation */ 4813 cxt->xstorm_ag_context.cdu_reserved = 4814 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid), 4815 CDU_REGION_NUMBER_XCM_AG, ETH_CONNECTION_TYPE); 4816 } 4817 4818 static void storm_memset_hc_timeout(struct bnx2x *bp, u8 port, 4819 u8 fw_sb_id, u8 sb_index, 4820 u8 ticks) 4821 { 4822 u32 addr = BAR_CSTRORM_INTMEM + 4823 CSTORM_STATUS_BLOCK_DATA_TIMEOUT_OFFSET(fw_sb_id, sb_index); 4824 REG_WR8(bp, addr, ticks); 4825 DP(NETIF_MSG_IFUP, 4826 "port %x fw_sb_id %d sb_index %d ticks %d\n", 4827 port, fw_sb_id, sb_index, ticks); 4828 } 4829 4830 static void storm_memset_hc_disable(struct bnx2x *bp, u8 port, 4831 u16 fw_sb_id, u8 sb_index, 4832 u8 disable) 4833 { 4834 u32 enable_flag = disable ? 0 : (1 << HC_INDEX_DATA_HC_ENABLED_SHIFT); 4835 u32 addr = BAR_CSTRORM_INTMEM + 4836 CSTORM_STATUS_BLOCK_DATA_FLAGS_OFFSET(fw_sb_id, sb_index); 4837 u8 flags = REG_RD8(bp, addr); 4838 /* clear and set */ 4839 flags &= ~HC_INDEX_DATA_HC_ENABLED; 4840 flags |= enable_flag; 4841 REG_WR8(bp, addr, flags); 4842 DP(NETIF_MSG_IFUP, 4843 "port %x fw_sb_id %d sb_index %d disable %d\n", 4844 port, fw_sb_id, sb_index, disable); 4845 } 4846 4847 void bnx2x_update_coalesce_sb_index(struct bnx2x *bp, u8 fw_sb_id, 4848 u8 sb_index, u8 disable, u16 usec) 4849 { 4850 int port = BP_PORT(bp); 4851 u8 ticks = usec / BNX2X_BTR; 4852 4853 storm_memset_hc_timeout(bp, port, fw_sb_id, sb_index, ticks); 4854 4855 disable = disable ? 1 : (usec ? 0 : 1); 4856 storm_memset_hc_disable(bp, port, fw_sb_id, sb_index, disable); 4857 } 4858