1 /* Broadcom NetXtreme-C/E network driver. 2 * 3 * Copyright (c) 2014-2016 Broadcom Corporation 4 * Copyright (c) 2016-2018 Broadcom Limited 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License as published by 8 * the Free Software Foundation. 9 */ 10 11 #include <linux/module.h> 12 13 #include <linux/stringify.h> 14 #include <linux/kernel.h> 15 #include <linux/timer.h> 16 #include <linux/errno.h> 17 #include <linux/ioport.h> 18 #include <linux/slab.h> 19 #include <linux/vmalloc.h> 20 #include <linux/interrupt.h> 21 #include <linux/pci.h> 22 #include <linux/netdevice.h> 23 #include <linux/etherdevice.h> 24 #include <linux/skbuff.h> 25 #include <linux/dma-mapping.h> 26 #include <linux/bitops.h> 27 #include <linux/io.h> 28 #include <linux/irq.h> 29 #include <linux/delay.h> 30 #include <asm/byteorder.h> 31 #include <asm/page.h> 32 #include <linux/time.h> 33 #include <linux/mii.h> 34 #include <linux/if.h> 35 #include <linux/if_vlan.h> 36 #include <linux/if_bridge.h> 37 #include <linux/rtc.h> 38 #include <linux/bpf.h> 39 #include <net/ip.h> 40 #include <net/tcp.h> 41 #include <net/udp.h> 42 #include <net/checksum.h> 43 #include <net/ip6_checksum.h> 44 #include <net/udp_tunnel.h> 45 #include <linux/workqueue.h> 46 #include <linux/prefetch.h> 47 #include <linux/cache.h> 48 #include <linux/log2.h> 49 #include <linux/aer.h> 50 #include <linux/bitmap.h> 51 #include <linux/cpu_rmap.h> 52 #include <linux/cpumask.h> 53 #include <net/pkt_cls.h> 54 55 #include "bnxt_hsi.h" 56 #include "bnxt.h" 57 #include "bnxt_ulp.h" 58 #include "bnxt_sriov.h" 59 #include "bnxt_ethtool.h" 60 #include "bnxt_dcb.h" 61 #include "bnxt_xdp.h" 62 #include "bnxt_vfr.h" 63 #include "bnxt_tc.h" 64 #include "bnxt_devlink.h" 65 #include "bnxt_debugfs.h" 66 67 #define BNXT_TX_TIMEOUT (5 * HZ) 68 69 static const char version[] = 70 "Broadcom NetXtreme-C/E driver " DRV_MODULE_NAME " v" DRV_MODULE_VERSION "\n"; 71 72 MODULE_LICENSE("GPL"); 73 MODULE_DESCRIPTION("Broadcom BCM573xx network driver"); 74 MODULE_VERSION(DRV_MODULE_VERSION); 75 76 #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN) 77 #define BNXT_RX_DMA_OFFSET NET_SKB_PAD 78 #define BNXT_RX_COPY_THRESH 256 79 80 #define BNXT_TX_PUSH_THRESH 164 81 82 enum board_idx { 83 BCM57301, 84 BCM57302, 85 BCM57304, 86 BCM57417_NPAR, 87 BCM58700, 88 BCM57311, 89 BCM57312, 90 BCM57402, 91 BCM57404, 92 BCM57406, 93 BCM57402_NPAR, 94 BCM57407, 95 BCM57412, 96 BCM57414, 97 BCM57416, 98 BCM57417, 99 BCM57412_NPAR, 100 BCM57314, 101 BCM57417_SFP, 102 BCM57416_SFP, 103 BCM57404_NPAR, 104 BCM57406_NPAR, 105 BCM57407_SFP, 106 BCM57407_NPAR, 107 BCM57414_NPAR, 108 BCM57416_NPAR, 109 BCM57452, 110 BCM57454, 111 BCM5745x_NPAR, 112 BCM58802, 113 BCM58804, 114 BCM58808, 115 NETXTREME_E_VF, 116 NETXTREME_C_VF, 117 NETXTREME_S_VF, 118 }; 119 120 /* indexed by enum above */ 121 static const struct { 122 char *name; 123 } board_info[] = { 124 [BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" }, 125 [BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" }, 126 [BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" }, 127 [BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" }, 128 [BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" }, 129 [BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" }, 130 [BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" }, 131 [BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" }, 132 [BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" }, 133 [BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" }, 134 [BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" }, 135 [BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" }, 136 [BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" }, 137 [BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" }, 138 [BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" }, 139 [BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" }, 140 [BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" }, 141 [BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" }, 142 [BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" }, 143 [BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" }, 144 [BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" }, 145 [BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" }, 146 [BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" }, 147 [BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" }, 148 [BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" }, 149 [BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" }, 150 [BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" }, 151 [BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" }, 152 [BCM5745x_NPAR] = { "Broadcom BCM5745x NetXtreme-E Ethernet Partition" }, 153 [BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" }, 154 [BCM58804] = { "Broadcom BCM58804 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" }, 155 [BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" }, 156 [NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" }, 157 [NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" }, 158 [NETXTREME_S_VF] = { "Broadcom NetXtreme-S Ethernet Virtual Function" }, 159 }; 160 161 static const struct pci_device_id bnxt_pci_tbl[] = { 162 { PCI_VDEVICE(BROADCOM, 0x1604), .driver_data = BCM5745x_NPAR }, 163 { PCI_VDEVICE(BROADCOM, 0x1605), .driver_data = BCM5745x_NPAR }, 164 { PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 }, 165 { PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR }, 166 { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 }, 167 { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 }, 168 { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 }, 169 { PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR }, 170 { PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 }, 171 { PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 }, 172 { PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 }, 173 { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 }, 174 { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 }, 175 { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 }, 176 { PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR }, 177 { PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 }, 178 { PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 }, 179 { PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 }, 180 { PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 }, 181 { PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 }, 182 { PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR }, 183 { PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 }, 184 { PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP }, 185 { PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP }, 186 { PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR }, 187 { PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR }, 188 { PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP }, 189 { PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR }, 190 { PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR }, 191 { PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR }, 192 { PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR }, 193 { PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR }, 194 { PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR }, 195 { PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 }, 196 { PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 }, 197 { PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 }, 198 { PCI_VDEVICE(BROADCOM, 0xd804), .driver_data = BCM58804 }, 199 #ifdef CONFIG_BNXT_SRIOV 200 { PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF }, 201 { PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF }, 202 { PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF }, 203 { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF }, 204 { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF }, 205 { PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF }, 206 { PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF }, 207 { PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF }, 208 { PCI_VDEVICE(BROADCOM, 0xd800), .driver_data = NETXTREME_S_VF }, 209 #endif 210 { 0 } 211 }; 212 213 MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl); 214 215 static const u16 bnxt_vf_req_snif[] = { 216 HWRM_FUNC_CFG, 217 HWRM_FUNC_VF_CFG, 218 HWRM_PORT_PHY_QCFG, 219 HWRM_CFA_L2_FILTER_ALLOC, 220 }; 221 222 static const u16 bnxt_async_events_arr[] = { 223 ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE, 224 ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD, 225 ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED, 226 ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE, 227 ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE, 228 }; 229 230 static struct workqueue_struct *bnxt_pf_wq; 231 232 static bool bnxt_vf_pciid(enum board_idx idx) 233 { 234 return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF || 235 idx == NETXTREME_S_VF); 236 } 237 238 #define DB_CP_REARM_FLAGS (DB_KEY_CP | DB_IDX_VALID) 239 #define DB_CP_FLAGS (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS) 240 #define DB_CP_IRQ_DIS_FLAGS (DB_KEY_CP | DB_IRQ_DIS) 241 242 #define BNXT_CP_DB_REARM(db, raw_cons) \ 243 writel(DB_CP_REARM_FLAGS | RING_CMP(raw_cons), db) 244 245 #define BNXT_CP_DB(db, raw_cons) \ 246 writel(DB_CP_FLAGS | RING_CMP(raw_cons), db) 247 248 #define BNXT_CP_DB_IRQ_DIS(db) \ 249 writel(DB_CP_IRQ_DIS_FLAGS, db) 250 251 const u16 bnxt_lhint_arr[] = { 252 TX_BD_FLAGS_LHINT_512_AND_SMALLER, 253 TX_BD_FLAGS_LHINT_512_TO_1023, 254 TX_BD_FLAGS_LHINT_1024_TO_2047, 255 TX_BD_FLAGS_LHINT_1024_TO_2047, 256 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 257 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 258 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 259 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 260 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 261 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 262 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 263 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 264 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 265 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 266 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 267 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 268 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 269 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 270 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 271 }; 272 273 static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb) 274 { 275 struct metadata_dst *md_dst = skb_metadata_dst(skb); 276 277 if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX) 278 return 0; 279 280 return md_dst->u.port_info.port_id; 281 } 282 283 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev) 284 { 285 struct bnxt *bp = netdev_priv(dev); 286 struct tx_bd *txbd; 287 struct tx_bd_ext *txbd1; 288 struct netdev_queue *txq; 289 int i; 290 dma_addr_t mapping; 291 unsigned int length, pad = 0; 292 u32 len, free_size, vlan_tag_flags, cfa_action, flags; 293 u16 prod, last_frag; 294 struct pci_dev *pdev = bp->pdev; 295 struct bnxt_tx_ring_info *txr; 296 struct bnxt_sw_tx_bd *tx_buf; 297 298 i = skb_get_queue_mapping(skb); 299 if (unlikely(i >= bp->tx_nr_rings)) { 300 dev_kfree_skb_any(skb); 301 return NETDEV_TX_OK; 302 } 303 304 txq = netdev_get_tx_queue(dev, i); 305 txr = &bp->tx_ring[bp->tx_ring_map[i]]; 306 prod = txr->tx_prod; 307 308 free_size = bnxt_tx_avail(bp, txr); 309 if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) { 310 netif_tx_stop_queue(txq); 311 return NETDEV_TX_BUSY; 312 } 313 314 length = skb->len; 315 len = skb_headlen(skb); 316 last_frag = skb_shinfo(skb)->nr_frags; 317 318 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 319 320 txbd->tx_bd_opaque = prod; 321 322 tx_buf = &txr->tx_buf_ring[prod]; 323 tx_buf->skb = skb; 324 tx_buf->nr_frags = last_frag; 325 326 vlan_tag_flags = 0; 327 cfa_action = bnxt_xmit_get_cfa_action(skb); 328 if (skb_vlan_tag_present(skb)) { 329 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN | 330 skb_vlan_tag_get(skb); 331 /* Currently supports 8021Q, 8021AD vlan offloads 332 * QINQ1, QINQ2, QINQ3 vlan headers are deprecated 333 */ 334 if (skb->vlan_proto == htons(ETH_P_8021Q)) 335 vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT; 336 } 337 338 if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh) { 339 struct tx_push_buffer *tx_push_buf = txr->tx_push; 340 struct tx_push_bd *tx_push = &tx_push_buf->push_bd; 341 struct tx_bd_ext *tx_push1 = &tx_push->txbd2; 342 void *pdata = tx_push_buf->data; 343 u64 *end; 344 int j, push_len; 345 346 /* Set COAL_NOW to be ready quickly for the next push */ 347 tx_push->tx_bd_len_flags_type = 348 cpu_to_le32((length << TX_BD_LEN_SHIFT) | 349 TX_BD_TYPE_LONG_TX_BD | 350 TX_BD_FLAGS_LHINT_512_AND_SMALLER | 351 TX_BD_FLAGS_COAL_NOW | 352 TX_BD_FLAGS_PACKET_END | 353 (2 << TX_BD_FLAGS_BD_CNT_SHIFT)); 354 355 if (skb->ip_summed == CHECKSUM_PARTIAL) 356 tx_push1->tx_bd_hsize_lflags = 357 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM); 358 else 359 tx_push1->tx_bd_hsize_lflags = 0; 360 361 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags); 362 tx_push1->tx_bd_cfa_action = 363 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT); 364 365 end = pdata + length; 366 end = PTR_ALIGN(end, 8) - 1; 367 *end = 0; 368 369 skb_copy_from_linear_data(skb, pdata, len); 370 pdata += len; 371 for (j = 0; j < last_frag; j++) { 372 skb_frag_t *frag = &skb_shinfo(skb)->frags[j]; 373 void *fptr; 374 375 fptr = skb_frag_address_safe(frag); 376 if (!fptr) 377 goto normal_tx; 378 379 memcpy(pdata, fptr, skb_frag_size(frag)); 380 pdata += skb_frag_size(frag); 381 } 382 383 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type; 384 txbd->tx_bd_haddr = txr->data_mapping; 385 prod = NEXT_TX(prod); 386 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 387 memcpy(txbd, tx_push1, sizeof(*txbd)); 388 prod = NEXT_TX(prod); 389 tx_push->doorbell = 390 cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | prod); 391 txr->tx_prod = prod; 392 393 tx_buf->is_push = 1; 394 netdev_tx_sent_queue(txq, skb->len); 395 wmb(); /* Sync is_push and byte queue before pushing data */ 396 397 push_len = (length + sizeof(*tx_push) + 7) / 8; 398 if (push_len > 16) { 399 __iowrite64_copy(txr->tx_doorbell, tx_push_buf, 16); 400 __iowrite32_copy(txr->tx_doorbell + 4, tx_push_buf + 1, 401 (push_len - 16) << 1); 402 } else { 403 __iowrite64_copy(txr->tx_doorbell, tx_push_buf, 404 push_len); 405 } 406 407 goto tx_done; 408 } 409 410 normal_tx: 411 if (length < BNXT_MIN_PKT_SIZE) { 412 pad = BNXT_MIN_PKT_SIZE - length; 413 if (skb_pad(skb, pad)) { 414 /* SKB already freed. */ 415 tx_buf->skb = NULL; 416 return NETDEV_TX_OK; 417 } 418 length = BNXT_MIN_PKT_SIZE; 419 } 420 421 mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE); 422 423 if (unlikely(dma_mapping_error(&pdev->dev, mapping))) { 424 dev_kfree_skb_any(skb); 425 tx_buf->skb = NULL; 426 return NETDEV_TX_OK; 427 } 428 429 dma_unmap_addr_set(tx_buf, mapping, mapping); 430 flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD | 431 ((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT); 432 433 txbd->tx_bd_haddr = cpu_to_le64(mapping); 434 435 prod = NEXT_TX(prod); 436 txbd1 = (struct tx_bd_ext *) 437 &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 438 439 txbd1->tx_bd_hsize_lflags = 0; 440 if (skb_is_gso(skb)) { 441 u32 hdr_len; 442 443 if (skb->encapsulation) 444 hdr_len = skb_inner_network_offset(skb) + 445 skb_inner_network_header_len(skb) + 446 inner_tcp_hdrlen(skb); 447 else 448 hdr_len = skb_transport_offset(skb) + 449 tcp_hdrlen(skb); 450 451 txbd1->tx_bd_hsize_lflags = cpu_to_le32(TX_BD_FLAGS_LSO | 452 TX_BD_FLAGS_T_IPID | 453 (hdr_len << (TX_BD_HSIZE_SHIFT - 1))); 454 length = skb_shinfo(skb)->gso_size; 455 txbd1->tx_bd_mss = cpu_to_le32(length); 456 length += hdr_len; 457 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { 458 txbd1->tx_bd_hsize_lflags = 459 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM); 460 txbd1->tx_bd_mss = 0; 461 } 462 463 length >>= 9; 464 flags |= bnxt_lhint_arr[length]; 465 txbd->tx_bd_len_flags_type = cpu_to_le32(flags); 466 467 txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags); 468 txbd1->tx_bd_cfa_action = 469 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT); 470 for (i = 0; i < last_frag; i++) { 471 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 472 473 prod = NEXT_TX(prod); 474 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 475 476 len = skb_frag_size(frag); 477 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len, 478 DMA_TO_DEVICE); 479 480 if (unlikely(dma_mapping_error(&pdev->dev, mapping))) 481 goto tx_dma_error; 482 483 tx_buf = &txr->tx_buf_ring[prod]; 484 dma_unmap_addr_set(tx_buf, mapping, mapping); 485 486 txbd->tx_bd_haddr = cpu_to_le64(mapping); 487 488 flags = len << TX_BD_LEN_SHIFT; 489 txbd->tx_bd_len_flags_type = cpu_to_le32(flags); 490 } 491 492 flags &= ~TX_BD_LEN; 493 txbd->tx_bd_len_flags_type = 494 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags | 495 TX_BD_FLAGS_PACKET_END); 496 497 netdev_tx_sent_queue(txq, skb->len); 498 499 /* Sync BD data before updating doorbell */ 500 wmb(); 501 502 prod = NEXT_TX(prod); 503 txr->tx_prod = prod; 504 505 if (!skb->xmit_more || netif_xmit_stopped(txq)) 506 bnxt_db_write(bp, txr->tx_doorbell, DB_KEY_TX | prod); 507 508 tx_done: 509 510 mmiowb(); 511 512 if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) { 513 if (skb->xmit_more && !tx_buf->is_push) 514 bnxt_db_write(bp, txr->tx_doorbell, DB_KEY_TX | prod); 515 516 netif_tx_stop_queue(txq); 517 518 /* netif_tx_stop_queue() must be done before checking 519 * tx index in bnxt_tx_avail() below, because in 520 * bnxt_tx_int(), we update tx index before checking for 521 * netif_tx_queue_stopped(). 522 */ 523 smp_mb(); 524 if (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh) 525 netif_tx_wake_queue(txq); 526 } 527 return NETDEV_TX_OK; 528 529 tx_dma_error: 530 last_frag = i; 531 532 /* start back at beginning and unmap skb */ 533 prod = txr->tx_prod; 534 tx_buf = &txr->tx_buf_ring[prod]; 535 tx_buf->skb = NULL; 536 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping), 537 skb_headlen(skb), PCI_DMA_TODEVICE); 538 prod = NEXT_TX(prod); 539 540 /* unmap remaining mapped pages */ 541 for (i = 0; i < last_frag; i++) { 542 prod = NEXT_TX(prod); 543 tx_buf = &txr->tx_buf_ring[prod]; 544 dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping), 545 skb_frag_size(&skb_shinfo(skb)->frags[i]), 546 PCI_DMA_TODEVICE); 547 } 548 549 dev_kfree_skb_any(skb); 550 return NETDEV_TX_OK; 551 } 552 553 static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts) 554 { 555 struct bnxt_tx_ring_info *txr = bnapi->tx_ring; 556 struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index); 557 u16 cons = txr->tx_cons; 558 struct pci_dev *pdev = bp->pdev; 559 int i; 560 unsigned int tx_bytes = 0; 561 562 for (i = 0; i < nr_pkts; i++) { 563 struct bnxt_sw_tx_bd *tx_buf; 564 struct sk_buff *skb; 565 int j, last; 566 567 tx_buf = &txr->tx_buf_ring[cons]; 568 cons = NEXT_TX(cons); 569 skb = tx_buf->skb; 570 tx_buf->skb = NULL; 571 572 if (tx_buf->is_push) { 573 tx_buf->is_push = 0; 574 goto next_tx_int; 575 } 576 577 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping), 578 skb_headlen(skb), PCI_DMA_TODEVICE); 579 last = tx_buf->nr_frags; 580 581 for (j = 0; j < last; j++) { 582 cons = NEXT_TX(cons); 583 tx_buf = &txr->tx_buf_ring[cons]; 584 dma_unmap_page( 585 &pdev->dev, 586 dma_unmap_addr(tx_buf, mapping), 587 skb_frag_size(&skb_shinfo(skb)->frags[j]), 588 PCI_DMA_TODEVICE); 589 } 590 591 next_tx_int: 592 cons = NEXT_TX(cons); 593 594 tx_bytes += skb->len; 595 dev_kfree_skb_any(skb); 596 } 597 598 netdev_tx_completed_queue(txq, nr_pkts, tx_bytes); 599 txr->tx_cons = cons; 600 601 /* Need to make the tx_cons update visible to bnxt_start_xmit() 602 * before checking for netif_tx_queue_stopped(). Without the 603 * memory barrier, there is a small possibility that bnxt_start_xmit() 604 * will miss it and cause the queue to be stopped forever. 605 */ 606 smp_mb(); 607 608 if (unlikely(netif_tx_queue_stopped(txq)) && 609 (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)) { 610 __netif_tx_lock(txq, smp_processor_id()); 611 if (netif_tx_queue_stopped(txq) && 612 bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh && 613 txr->dev_state != BNXT_DEV_STATE_CLOSING) 614 netif_tx_wake_queue(txq); 615 __netif_tx_unlock(txq); 616 } 617 } 618 619 static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping, 620 gfp_t gfp) 621 { 622 struct device *dev = &bp->pdev->dev; 623 struct page *page; 624 625 page = alloc_page(gfp); 626 if (!page) 627 return NULL; 628 629 *mapping = dma_map_page_attrs(dev, page, 0, PAGE_SIZE, bp->rx_dir, 630 DMA_ATTR_WEAK_ORDERING); 631 if (dma_mapping_error(dev, *mapping)) { 632 __free_page(page); 633 return NULL; 634 } 635 *mapping += bp->rx_dma_offset; 636 return page; 637 } 638 639 static inline u8 *__bnxt_alloc_rx_data(struct bnxt *bp, dma_addr_t *mapping, 640 gfp_t gfp) 641 { 642 u8 *data; 643 struct pci_dev *pdev = bp->pdev; 644 645 data = kmalloc(bp->rx_buf_size, gfp); 646 if (!data) 647 return NULL; 648 649 *mapping = dma_map_single_attrs(&pdev->dev, data + bp->rx_dma_offset, 650 bp->rx_buf_use_size, bp->rx_dir, 651 DMA_ATTR_WEAK_ORDERING); 652 653 if (dma_mapping_error(&pdev->dev, *mapping)) { 654 kfree(data); 655 data = NULL; 656 } 657 return data; 658 } 659 660 int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, 661 u16 prod, gfp_t gfp) 662 { 663 struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 664 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod]; 665 dma_addr_t mapping; 666 667 if (BNXT_RX_PAGE_MODE(bp)) { 668 struct page *page = __bnxt_alloc_rx_page(bp, &mapping, gfp); 669 670 if (!page) 671 return -ENOMEM; 672 673 rx_buf->data = page; 674 rx_buf->data_ptr = page_address(page) + bp->rx_offset; 675 } else { 676 u8 *data = __bnxt_alloc_rx_data(bp, &mapping, gfp); 677 678 if (!data) 679 return -ENOMEM; 680 681 rx_buf->data = data; 682 rx_buf->data_ptr = data + bp->rx_offset; 683 } 684 rx_buf->mapping = mapping; 685 686 rxbd->rx_bd_haddr = cpu_to_le64(mapping); 687 return 0; 688 } 689 690 void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data) 691 { 692 u16 prod = rxr->rx_prod; 693 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf; 694 struct rx_bd *cons_bd, *prod_bd; 695 696 prod_rx_buf = &rxr->rx_buf_ring[prod]; 697 cons_rx_buf = &rxr->rx_buf_ring[cons]; 698 699 prod_rx_buf->data = data; 700 prod_rx_buf->data_ptr = cons_rx_buf->data_ptr; 701 702 prod_rx_buf->mapping = cons_rx_buf->mapping; 703 704 prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 705 cons_bd = &rxr->rx_desc_ring[RX_RING(cons)][RX_IDX(cons)]; 706 707 prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr; 708 } 709 710 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx) 711 { 712 u16 next, max = rxr->rx_agg_bmap_size; 713 714 next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx); 715 if (next >= max) 716 next = find_first_zero_bit(rxr->rx_agg_bmap, max); 717 return next; 718 } 719 720 static inline int bnxt_alloc_rx_page(struct bnxt *bp, 721 struct bnxt_rx_ring_info *rxr, 722 u16 prod, gfp_t gfp) 723 { 724 struct rx_bd *rxbd = 725 &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 726 struct bnxt_sw_rx_agg_bd *rx_agg_buf; 727 struct pci_dev *pdev = bp->pdev; 728 struct page *page; 729 dma_addr_t mapping; 730 u16 sw_prod = rxr->rx_sw_agg_prod; 731 unsigned int offset = 0; 732 733 if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) { 734 page = rxr->rx_page; 735 if (!page) { 736 page = alloc_page(gfp); 737 if (!page) 738 return -ENOMEM; 739 rxr->rx_page = page; 740 rxr->rx_page_offset = 0; 741 } 742 offset = rxr->rx_page_offset; 743 rxr->rx_page_offset += BNXT_RX_PAGE_SIZE; 744 if (rxr->rx_page_offset == PAGE_SIZE) 745 rxr->rx_page = NULL; 746 else 747 get_page(page); 748 } else { 749 page = alloc_page(gfp); 750 if (!page) 751 return -ENOMEM; 752 } 753 754 mapping = dma_map_page_attrs(&pdev->dev, page, offset, 755 BNXT_RX_PAGE_SIZE, PCI_DMA_FROMDEVICE, 756 DMA_ATTR_WEAK_ORDERING); 757 if (dma_mapping_error(&pdev->dev, mapping)) { 758 __free_page(page); 759 return -EIO; 760 } 761 762 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap))) 763 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod); 764 765 __set_bit(sw_prod, rxr->rx_agg_bmap); 766 rx_agg_buf = &rxr->rx_agg_ring[sw_prod]; 767 rxr->rx_sw_agg_prod = NEXT_RX_AGG(sw_prod); 768 769 rx_agg_buf->page = page; 770 rx_agg_buf->offset = offset; 771 rx_agg_buf->mapping = mapping; 772 rxbd->rx_bd_haddr = cpu_to_le64(mapping); 773 rxbd->rx_bd_opaque = sw_prod; 774 return 0; 775 } 776 777 static void bnxt_reuse_rx_agg_bufs(struct bnxt_napi *bnapi, u16 cp_cons, 778 u32 agg_bufs) 779 { 780 struct bnxt *bp = bnapi->bp; 781 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 782 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 783 u16 prod = rxr->rx_agg_prod; 784 u16 sw_prod = rxr->rx_sw_agg_prod; 785 u32 i; 786 787 for (i = 0; i < agg_bufs; i++) { 788 u16 cons; 789 struct rx_agg_cmp *agg; 790 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf; 791 struct rx_bd *prod_bd; 792 struct page *page; 793 794 agg = (struct rx_agg_cmp *) 795 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 796 cons = agg->rx_agg_cmp_opaque; 797 __clear_bit(cons, rxr->rx_agg_bmap); 798 799 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap))) 800 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod); 801 802 __set_bit(sw_prod, rxr->rx_agg_bmap); 803 prod_rx_buf = &rxr->rx_agg_ring[sw_prod]; 804 cons_rx_buf = &rxr->rx_agg_ring[cons]; 805 806 /* It is possible for sw_prod to be equal to cons, so 807 * set cons_rx_buf->page to NULL first. 808 */ 809 page = cons_rx_buf->page; 810 cons_rx_buf->page = NULL; 811 prod_rx_buf->page = page; 812 prod_rx_buf->offset = cons_rx_buf->offset; 813 814 prod_rx_buf->mapping = cons_rx_buf->mapping; 815 816 prod_bd = &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 817 818 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping); 819 prod_bd->rx_bd_opaque = sw_prod; 820 821 prod = NEXT_RX_AGG(prod); 822 sw_prod = NEXT_RX_AGG(sw_prod); 823 cp_cons = NEXT_CMP(cp_cons); 824 } 825 rxr->rx_agg_prod = prod; 826 rxr->rx_sw_agg_prod = sw_prod; 827 } 828 829 static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp, 830 struct bnxt_rx_ring_info *rxr, 831 u16 cons, void *data, u8 *data_ptr, 832 dma_addr_t dma_addr, 833 unsigned int offset_and_len) 834 { 835 unsigned int payload = offset_and_len >> 16; 836 unsigned int len = offset_and_len & 0xffff; 837 struct skb_frag_struct *frag; 838 struct page *page = data; 839 u16 prod = rxr->rx_prod; 840 struct sk_buff *skb; 841 int off, err; 842 843 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC); 844 if (unlikely(err)) { 845 bnxt_reuse_rx_data(rxr, cons, data); 846 return NULL; 847 } 848 dma_addr -= bp->rx_dma_offset; 849 dma_unmap_page_attrs(&bp->pdev->dev, dma_addr, PAGE_SIZE, bp->rx_dir, 850 DMA_ATTR_WEAK_ORDERING); 851 852 if (unlikely(!payload)) 853 payload = eth_get_headlen(data_ptr, len); 854 855 skb = napi_alloc_skb(&rxr->bnapi->napi, payload); 856 if (!skb) { 857 __free_page(page); 858 return NULL; 859 } 860 861 off = (void *)data_ptr - page_address(page); 862 skb_add_rx_frag(skb, 0, page, off, len, PAGE_SIZE); 863 memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN, 864 payload + NET_IP_ALIGN); 865 866 frag = &skb_shinfo(skb)->frags[0]; 867 skb_frag_size_sub(frag, payload); 868 frag->page_offset += payload; 869 skb->data_len -= payload; 870 skb->tail += payload; 871 872 return skb; 873 } 874 875 static struct sk_buff *bnxt_rx_skb(struct bnxt *bp, 876 struct bnxt_rx_ring_info *rxr, u16 cons, 877 void *data, u8 *data_ptr, 878 dma_addr_t dma_addr, 879 unsigned int offset_and_len) 880 { 881 u16 prod = rxr->rx_prod; 882 struct sk_buff *skb; 883 int err; 884 885 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC); 886 if (unlikely(err)) { 887 bnxt_reuse_rx_data(rxr, cons, data); 888 return NULL; 889 } 890 891 skb = build_skb(data, 0); 892 dma_unmap_single_attrs(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size, 893 bp->rx_dir, DMA_ATTR_WEAK_ORDERING); 894 if (!skb) { 895 kfree(data); 896 return NULL; 897 } 898 899 skb_reserve(skb, bp->rx_offset); 900 skb_put(skb, offset_and_len & 0xffff); 901 return skb; 902 } 903 904 static struct sk_buff *bnxt_rx_pages(struct bnxt *bp, struct bnxt_napi *bnapi, 905 struct sk_buff *skb, u16 cp_cons, 906 u32 agg_bufs) 907 { 908 struct pci_dev *pdev = bp->pdev; 909 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 910 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 911 u16 prod = rxr->rx_agg_prod; 912 u32 i; 913 914 for (i = 0; i < agg_bufs; i++) { 915 u16 cons, frag_len; 916 struct rx_agg_cmp *agg; 917 struct bnxt_sw_rx_agg_bd *cons_rx_buf; 918 struct page *page; 919 dma_addr_t mapping; 920 921 agg = (struct rx_agg_cmp *) 922 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 923 cons = agg->rx_agg_cmp_opaque; 924 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) & 925 RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT; 926 927 cons_rx_buf = &rxr->rx_agg_ring[cons]; 928 skb_fill_page_desc(skb, i, cons_rx_buf->page, 929 cons_rx_buf->offset, frag_len); 930 __clear_bit(cons, rxr->rx_agg_bmap); 931 932 /* It is possible for bnxt_alloc_rx_page() to allocate 933 * a sw_prod index that equals the cons index, so we 934 * need to clear the cons entry now. 935 */ 936 mapping = cons_rx_buf->mapping; 937 page = cons_rx_buf->page; 938 cons_rx_buf->page = NULL; 939 940 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) { 941 struct skb_shared_info *shinfo; 942 unsigned int nr_frags; 943 944 shinfo = skb_shinfo(skb); 945 nr_frags = --shinfo->nr_frags; 946 __skb_frag_set_page(&shinfo->frags[nr_frags], NULL); 947 948 dev_kfree_skb(skb); 949 950 cons_rx_buf->page = page; 951 952 /* Update prod since possibly some pages have been 953 * allocated already. 954 */ 955 rxr->rx_agg_prod = prod; 956 bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs - i); 957 return NULL; 958 } 959 960 dma_unmap_page_attrs(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE, 961 PCI_DMA_FROMDEVICE, 962 DMA_ATTR_WEAK_ORDERING); 963 964 skb->data_len += frag_len; 965 skb->len += frag_len; 966 skb->truesize += PAGE_SIZE; 967 968 prod = NEXT_RX_AGG(prod); 969 cp_cons = NEXT_CMP(cp_cons); 970 } 971 rxr->rx_agg_prod = prod; 972 return skb; 973 } 974 975 static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 976 u8 agg_bufs, u32 *raw_cons) 977 { 978 u16 last; 979 struct rx_agg_cmp *agg; 980 981 *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs); 982 last = RING_CMP(*raw_cons); 983 agg = (struct rx_agg_cmp *) 984 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)]; 985 return RX_AGG_CMP_VALID(agg, *raw_cons); 986 } 987 988 static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data, 989 unsigned int len, 990 dma_addr_t mapping) 991 { 992 struct bnxt *bp = bnapi->bp; 993 struct pci_dev *pdev = bp->pdev; 994 struct sk_buff *skb; 995 996 skb = napi_alloc_skb(&bnapi->napi, len); 997 if (!skb) 998 return NULL; 999 1000 dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copy_thresh, 1001 bp->rx_dir); 1002 1003 memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN, 1004 len + NET_IP_ALIGN); 1005 1006 dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copy_thresh, 1007 bp->rx_dir); 1008 1009 skb_put(skb, len); 1010 return skb; 1011 } 1012 1013 static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_napi *bnapi, 1014 u32 *raw_cons, void *cmp) 1015 { 1016 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 1017 struct rx_cmp *rxcmp = cmp; 1018 u32 tmp_raw_cons = *raw_cons; 1019 u8 cmp_type, agg_bufs = 0; 1020 1021 cmp_type = RX_CMP_TYPE(rxcmp); 1022 1023 if (cmp_type == CMP_TYPE_RX_L2_CMP) { 1024 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) & 1025 RX_CMP_AGG_BUFS) >> 1026 RX_CMP_AGG_BUFS_SHIFT; 1027 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) { 1028 struct rx_tpa_end_cmp *tpa_end = cmp; 1029 1030 agg_bufs = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) & 1031 RX_TPA_END_CMP_AGG_BUFS) >> 1032 RX_TPA_END_CMP_AGG_BUFS_SHIFT; 1033 } 1034 1035 if (agg_bufs) { 1036 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons)) 1037 return -EBUSY; 1038 } 1039 *raw_cons = tmp_raw_cons; 1040 return 0; 1041 } 1042 1043 static void bnxt_queue_sp_work(struct bnxt *bp) 1044 { 1045 if (BNXT_PF(bp)) 1046 queue_work(bnxt_pf_wq, &bp->sp_task); 1047 else 1048 schedule_work(&bp->sp_task); 1049 } 1050 1051 static void bnxt_cancel_sp_work(struct bnxt *bp) 1052 { 1053 if (BNXT_PF(bp)) 1054 flush_workqueue(bnxt_pf_wq); 1055 else 1056 cancel_work_sync(&bp->sp_task); 1057 } 1058 1059 static void bnxt_sched_reset(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) 1060 { 1061 if (!rxr->bnapi->in_reset) { 1062 rxr->bnapi->in_reset = true; 1063 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event); 1064 bnxt_queue_sp_work(bp); 1065 } 1066 rxr->rx_next_cons = 0xffff; 1067 } 1068 1069 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, 1070 struct rx_tpa_start_cmp *tpa_start, 1071 struct rx_tpa_start_cmp_ext *tpa_start1) 1072 { 1073 u8 agg_id = TPA_START_AGG_ID(tpa_start); 1074 u16 cons, prod; 1075 struct bnxt_tpa_info *tpa_info; 1076 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf; 1077 struct rx_bd *prod_bd; 1078 dma_addr_t mapping; 1079 1080 cons = tpa_start->rx_tpa_start_cmp_opaque; 1081 prod = rxr->rx_prod; 1082 cons_rx_buf = &rxr->rx_buf_ring[cons]; 1083 prod_rx_buf = &rxr->rx_buf_ring[prod]; 1084 tpa_info = &rxr->rx_tpa[agg_id]; 1085 1086 if (unlikely(cons != rxr->rx_next_cons)) { 1087 bnxt_sched_reset(bp, rxr); 1088 return; 1089 } 1090 /* Store cfa_code in tpa_info to use in tpa_end 1091 * completion processing. 1092 */ 1093 tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1); 1094 prod_rx_buf->data = tpa_info->data; 1095 prod_rx_buf->data_ptr = tpa_info->data_ptr; 1096 1097 mapping = tpa_info->mapping; 1098 prod_rx_buf->mapping = mapping; 1099 1100 prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 1101 1102 prod_bd->rx_bd_haddr = cpu_to_le64(mapping); 1103 1104 tpa_info->data = cons_rx_buf->data; 1105 tpa_info->data_ptr = cons_rx_buf->data_ptr; 1106 cons_rx_buf->data = NULL; 1107 tpa_info->mapping = cons_rx_buf->mapping; 1108 1109 tpa_info->len = 1110 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >> 1111 RX_TPA_START_CMP_LEN_SHIFT; 1112 if (likely(TPA_START_HASH_VALID(tpa_start))) { 1113 u32 hash_type = TPA_START_HASH_TYPE(tpa_start); 1114 1115 tpa_info->hash_type = PKT_HASH_TYPE_L4; 1116 tpa_info->gso_type = SKB_GSO_TCPV4; 1117 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */ 1118 if (hash_type == 3) 1119 tpa_info->gso_type = SKB_GSO_TCPV6; 1120 tpa_info->rss_hash = 1121 le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash); 1122 } else { 1123 tpa_info->hash_type = PKT_HASH_TYPE_NONE; 1124 tpa_info->gso_type = 0; 1125 if (netif_msg_rx_err(bp)) 1126 netdev_warn(bp->dev, "TPA packet without valid hash\n"); 1127 } 1128 tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2); 1129 tpa_info->metadata = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata); 1130 tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info); 1131 1132 rxr->rx_prod = NEXT_RX(prod); 1133 cons = NEXT_RX(cons); 1134 rxr->rx_next_cons = NEXT_RX(cons); 1135 cons_rx_buf = &rxr->rx_buf_ring[cons]; 1136 1137 bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data); 1138 rxr->rx_prod = NEXT_RX(rxr->rx_prod); 1139 cons_rx_buf->data = NULL; 1140 } 1141 1142 static void bnxt_abort_tpa(struct bnxt *bp, struct bnxt_napi *bnapi, 1143 u16 cp_cons, u32 agg_bufs) 1144 { 1145 if (agg_bufs) 1146 bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs); 1147 } 1148 1149 static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info, 1150 int payload_off, int tcp_ts, 1151 struct sk_buff *skb) 1152 { 1153 #ifdef CONFIG_INET 1154 struct tcphdr *th; 1155 int len, nw_off; 1156 u16 outer_ip_off, inner_ip_off, inner_mac_off; 1157 u32 hdr_info = tpa_info->hdr_info; 1158 bool loopback = false; 1159 1160 inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info); 1161 inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info); 1162 outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info); 1163 1164 /* If the packet is an internal loopback packet, the offsets will 1165 * have an extra 4 bytes. 1166 */ 1167 if (inner_mac_off == 4) { 1168 loopback = true; 1169 } else if (inner_mac_off > 4) { 1170 __be16 proto = *((__be16 *)(skb->data + inner_ip_off - 1171 ETH_HLEN - 2)); 1172 1173 /* We only support inner iPv4/ipv6. If we don't see the 1174 * correct protocol ID, it must be a loopback packet where 1175 * the offsets are off by 4. 1176 */ 1177 if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6)) 1178 loopback = true; 1179 } 1180 if (loopback) { 1181 /* internal loopback packet, subtract all offsets by 4 */ 1182 inner_ip_off -= 4; 1183 inner_mac_off -= 4; 1184 outer_ip_off -= 4; 1185 } 1186 1187 nw_off = inner_ip_off - ETH_HLEN; 1188 skb_set_network_header(skb, nw_off); 1189 if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) { 1190 struct ipv6hdr *iph = ipv6_hdr(skb); 1191 1192 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr)); 1193 len = skb->len - skb_transport_offset(skb); 1194 th = tcp_hdr(skb); 1195 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0); 1196 } else { 1197 struct iphdr *iph = ip_hdr(skb); 1198 1199 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr)); 1200 len = skb->len - skb_transport_offset(skb); 1201 th = tcp_hdr(skb); 1202 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0); 1203 } 1204 1205 if (inner_mac_off) { /* tunnel */ 1206 struct udphdr *uh = NULL; 1207 __be16 proto = *((__be16 *)(skb->data + outer_ip_off - 1208 ETH_HLEN - 2)); 1209 1210 if (proto == htons(ETH_P_IP)) { 1211 struct iphdr *iph = (struct iphdr *)skb->data; 1212 1213 if (iph->protocol == IPPROTO_UDP) 1214 uh = (struct udphdr *)(iph + 1); 1215 } else { 1216 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data; 1217 1218 if (iph->nexthdr == IPPROTO_UDP) 1219 uh = (struct udphdr *)(iph + 1); 1220 } 1221 if (uh) { 1222 if (uh->check) 1223 skb_shinfo(skb)->gso_type |= 1224 SKB_GSO_UDP_TUNNEL_CSUM; 1225 else 1226 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL; 1227 } 1228 } 1229 #endif 1230 return skb; 1231 } 1232 1233 #define BNXT_IPV4_HDR_SIZE (sizeof(struct iphdr) + sizeof(struct tcphdr)) 1234 #define BNXT_IPV6_HDR_SIZE (sizeof(struct ipv6hdr) + sizeof(struct tcphdr)) 1235 1236 static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info, 1237 int payload_off, int tcp_ts, 1238 struct sk_buff *skb) 1239 { 1240 #ifdef CONFIG_INET 1241 struct tcphdr *th; 1242 int len, nw_off, tcp_opt_len = 0; 1243 1244 if (tcp_ts) 1245 tcp_opt_len = 12; 1246 1247 if (tpa_info->gso_type == SKB_GSO_TCPV4) { 1248 struct iphdr *iph; 1249 1250 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len - 1251 ETH_HLEN; 1252 skb_set_network_header(skb, nw_off); 1253 iph = ip_hdr(skb); 1254 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr)); 1255 len = skb->len - skb_transport_offset(skb); 1256 th = tcp_hdr(skb); 1257 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0); 1258 } else if (tpa_info->gso_type == SKB_GSO_TCPV6) { 1259 struct ipv6hdr *iph; 1260 1261 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len - 1262 ETH_HLEN; 1263 skb_set_network_header(skb, nw_off); 1264 iph = ipv6_hdr(skb); 1265 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr)); 1266 len = skb->len - skb_transport_offset(skb); 1267 th = tcp_hdr(skb); 1268 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0); 1269 } else { 1270 dev_kfree_skb_any(skb); 1271 return NULL; 1272 } 1273 1274 if (nw_off) { /* tunnel */ 1275 struct udphdr *uh = NULL; 1276 1277 if (skb->protocol == htons(ETH_P_IP)) { 1278 struct iphdr *iph = (struct iphdr *)skb->data; 1279 1280 if (iph->protocol == IPPROTO_UDP) 1281 uh = (struct udphdr *)(iph + 1); 1282 } else { 1283 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data; 1284 1285 if (iph->nexthdr == IPPROTO_UDP) 1286 uh = (struct udphdr *)(iph + 1); 1287 } 1288 if (uh) { 1289 if (uh->check) 1290 skb_shinfo(skb)->gso_type |= 1291 SKB_GSO_UDP_TUNNEL_CSUM; 1292 else 1293 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL; 1294 } 1295 } 1296 #endif 1297 return skb; 1298 } 1299 1300 static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp, 1301 struct bnxt_tpa_info *tpa_info, 1302 struct rx_tpa_end_cmp *tpa_end, 1303 struct rx_tpa_end_cmp_ext *tpa_end1, 1304 struct sk_buff *skb) 1305 { 1306 #ifdef CONFIG_INET 1307 int payload_off; 1308 u16 segs; 1309 1310 segs = TPA_END_TPA_SEGS(tpa_end); 1311 if (segs == 1) 1312 return skb; 1313 1314 NAPI_GRO_CB(skb)->count = segs; 1315 skb_shinfo(skb)->gso_size = 1316 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len); 1317 skb_shinfo(skb)->gso_type = tpa_info->gso_type; 1318 payload_off = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) & 1319 RX_TPA_END_CMP_PAYLOAD_OFFSET) >> 1320 RX_TPA_END_CMP_PAYLOAD_OFFSET_SHIFT; 1321 skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb); 1322 if (likely(skb)) 1323 tcp_gro_complete(skb); 1324 #endif 1325 return skb; 1326 } 1327 1328 /* Given the cfa_code of a received packet determine which 1329 * netdev (vf-rep or PF) the packet is destined to. 1330 */ 1331 static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code) 1332 { 1333 struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code); 1334 1335 /* if vf-rep dev is NULL, the must belongs to the PF */ 1336 return dev ? dev : bp->dev; 1337 } 1338 1339 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp, 1340 struct bnxt_napi *bnapi, 1341 u32 *raw_cons, 1342 struct rx_tpa_end_cmp *tpa_end, 1343 struct rx_tpa_end_cmp_ext *tpa_end1, 1344 u8 *event) 1345 { 1346 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 1347 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1348 u8 agg_id = TPA_END_AGG_ID(tpa_end); 1349 u8 *data_ptr, agg_bufs; 1350 u16 cp_cons = RING_CMP(*raw_cons); 1351 unsigned int len; 1352 struct bnxt_tpa_info *tpa_info; 1353 dma_addr_t mapping; 1354 struct sk_buff *skb; 1355 void *data; 1356 1357 if (unlikely(bnapi->in_reset)) { 1358 int rc = bnxt_discard_rx(bp, bnapi, raw_cons, tpa_end); 1359 1360 if (rc < 0) 1361 return ERR_PTR(-EBUSY); 1362 return NULL; 1363 } 1364 1365 tpa_info = &rxr->rx_tpa[agg_id]; 1366 data = tpa_info->data; 1367 data_ptr = tpa_info->data_ptr; 1368 prefetch(data_ptr); 1369 len = tpa_info->len; 1370 mapping = tpa_info->mapping; 1371 1372 agg_bufs = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) & 1373 RX_TPA_END_CMP_AGG_BUFS) >> RX_TPA_END_CMP_AGG_BUFS_SHIFT; 1374 1375 if (agg_bufs) { 1376 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons)) 1377 return ERR_PTR(-EBUSY); 1378 1379 *event |= BNXT_AGG_EVENT; 1380 cp_cons = NEXT_CMP(cp_cons); 1381 } 1382 1383 if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) { 1384 bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs); 1385 if (agg_bufs > MAX_SKB_FRAGS) 1386 netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n", 1387 agg_bufs, (int)MAX_SKB_FRAGS); 1388 return NULL; 1389 } 1390 1391 if (len <= bp->rx_copy_thresh) { 1392 skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping); 1393 if (!skb) { 1394 bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs); 1395 return NULL; 1396 } 1397 } else { 1398 u8 *new_data; 1399 dma_addr_t new_mapping; 1400 1401 new_data = __bnxt_alloc_rx_data(bp, &new_mapping, GFP_ATOMIC); 1402 if (!new_data) { 1403 bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs); 1404 return NULL; 1405 } 1406 1407 tpa_info->data = new_data; 1408 tpa_info->data_ptr = new_data + bp->rx_offset; 1409 tpa_info->mapping = new_mapping; 1410 1411 skb = build_skb(data, 0); 1412 dma_unmap_single_attrs(&bp->pdev->dev, mapping, 1413 bp->rx_buf_use_size, bp->rx_dir, 1414 DMA_ATTR_WEAK_ORDERING); 1415 1416 if (!skb) { 1417 kfree(data); 1418 bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs); 1419 return NULL; 1420 } 1421 skb_reserve(skb, bp->rx_offset); 1422 skb_put(skb, len); 1423 } 1424 1425 if (agg_bufs) { 1426 skb = bnxt_rx_pages(bp, bnapi, skb, cp_cons, agg_bufs); 1427 if (!skb) { 1428 /* Page reuse already handled by bnxt_rx_pages(). */ 1429 return NULL; 1430 } 1431 } 1432 1433 skb->protocol = 1434 eth_type_trans(skb, bnxt_get_pkt_dev(bp, tpa_info->cfa_code)); 1435 1436 if (tpa_info->hash_type != PKT_HASH_TYPE_NONE) 1437 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type); 1438 1439 if ((tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) && 1440 (skb->dev->features & NETIF_F_HW_VLAN_CTAG_RX)) { 1441 u16 vlan_proto = tpa_info->metadata >> 1442 RX_CMP_FLAGS2_METADATA_TPID_SFT; 1443 u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_TCI_MASK; 1444 1445 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag); 1446 } 1447 1448 skb_checksum_none_assert(skb); 1449 if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) { 1450 skb->ip_summed = CHECKSUM_UNNECESSARY; 1451 skb->csum_level = 1452 (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3; 1453 } 1454 1455 if (TPA_END_GRO(tpa_end)) 1456 skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb); 1457 1458 return skb; 1459 } 1460 1461 static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi, 1462 struct sk_buff *skb) 1463 { 1464 if (skb->dev != bp->dev) { 1465 /* this packet belongs to a vf-rep */ 1466 bnxt_vf_rep_rx(bp, skb); 1467 return; 1468 } 1469 skb_record_rx_queue(skb, bnapi->index); 1470 napi_gro_receive(&bnapi->napi, skb); 1471 } 1472 1473 /* returns the following: 1474 * 1 - 1 packet successfully received 1475 * 0 - successful TPA_START, packet not completed yet 1476 * -EBUSY - completion ring does not have all the agg buffers yet 1477 * -ENOMEM - packet aborted due to out of memory 1478 * -EIO - packet aborted due to hw error indicated in BD 1479 */ 1480 static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_napi *bnapi, u32 *raw_cons, 1481 u8 *event) 1482 { 1483 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 1484 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1485 struct net_device *dev = bp->dev; 1486 struct rx_cmp *rxcmp; 1487 struct rx_cmp_ext *rxcmp1; 1488 u32 tmp_raw_cons = *raw_cons; 1489 u16 cfa_code, cons, prod, cp_cons = RING_CMP(tmp_raw_cons); 1490 struct bnxt_sw_rx_bd *rx_buf; 1491 unsigned int len; 1492 u8 *data_ptr, agg_bufs, cmp_type; 1493 dma_addr_t dma_addr; 1494 struct sk_buff *skb; 1495 void *data; 1496 int rc = 0; 1497 u32 misc; 1498 1499 rxcmp = (struct rx_cmp *) 1500 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1501 1502 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons); 1503 cp_cons = RING_CMP(tmp_raw_cons); 1504 rxcmp1 = (struct rx_cmp_ext *) 1505 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1506 1507 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons)) 1508 return -EBUSY; 1509 1510 cmp_type = RX_CMP_TYPE(rxcmp); 1511 1512 prod = rxr->rx_prod; 1513 1514 if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) { 1515 bnxt_tpa_start(bp, rxr, (struct rx_tpa_start_cmp *)rxcmp, 1516 (struct rx_tpa_start_cmp_ext *)rxcmp1); 1517 1518 *event |= BNXT_RX_EVENT; 1519 goto next_rx_no_prod_no_len; 1520 1521 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) { 1522 skb = bnxt_tpa_end(bp, bnapi, &tmp_raw_cons, 1523 (struct rx_tpa_end_cmp *)rxcmp, 1524 (struct rx_tpa_end_cmp_ext *)rxcmp1, event); 1525 1526 if (IS_ERR(skb)) 1527 return -EBUSY; 1528 1529 rc = -ENOMEM; 1530 if (likely(skb)) { 1531 bnxt_deliver_skb(bp, bnapi, skb); 1532 rc = 1; 1533 } 1534 *event |= BNXT_RX_EVENT; 1535 goto next_rx_no_prod_no_len; 1536 } 1537 1538 cons = rxcmp->rx_cmp_opaque; 1539 rx_buf = &rxr->rx_buf_ring[cons]; 1540 data = rx_buf->data; 1541 data_ptr = rx_buf->data_ptr; 1542 if (unlikely(cons != rxr->rx_next_cons)) { 1543 int rc1 = bnxt_discard_rx(bp, bnapi, raw_cons, rxcmp); 1544 1545 bnxt_sched_reset(bp, rxr); 1546 return rc1; 1547 } 1548 prefetch(data_ptr); 1549 1550 misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1); 1551 agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT; 1552 1553 if (agg_bufs) { 1554 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons)) 1555 return -EBUSY; 1556 1557 cp_cons = NEXT_CMP(cp_cons); 1558 *event |= BNXT_AGG_EVENT; 1559 } 1560 *event |= BNXT_RX_EVENT; 1561 1562 rx_buf->data = NULL; 1563 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) { 1564 bnxt_reuse_rx_data(rxr, cons, data); 1565 if (agg_bufs) 1566 bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs); 1567 1568 rc = -EIO; 1569 goto next_rx; 1570 } 1571 1572 len = le32_to_cpu(rxcmp->rx_cmp_len_flags_type) >> RX_CMP_LEN_SHIFT; 1573 dma_addr = rx_buf->mapping; 1574 1575 if (bnxt_rx_xdp(bp, rxr, cons, data, &data_ptr, &len, event)) { 1576 rc = 1; 1577 goto next_rx; 1578 } 1579 1580 if (len <= bp->rx_copy_thresh) { 1581 skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr); 1582 bnxt_reuse_rx_data(rxr, cons, data); 1583 if (!skb) { 1584 rc = -ENOMEM; 1585 goto next_rx; 1586 } 1587 } else { 1588 u32 payload; 1589 1590 if (rx_buf->data_ptr == data_ptr) 1591 payload = misc & RX_CMP_PAYLOAD_OFFSET; 1592 else 1593 payload = 0; 1594 skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr, 1595 payload | len); 1596 if (!skb) { 1597 rc = -ENOMEM; 1598 goto next_rx; 1599 } 1600 } 1601 1602 if (agg_bufs) { 1603 skb = bnxt_rx_pages(bp, bnapi, skb, cp_cons, agg_bufs); 1604 if (!skb) { 1605 rc = -ENOMEM; 1606 goto next_rx; 1607 } 1608 } 1609 1610 if (RX_CMP_HASH_VALID(rxcmp)) { 1611 u32 hash_type = RX_CMP_HASH_TYPE(rxcmp); 1612 enum pkt_hash_types type = PKT_HASH_TYPE_L4; 1613 1614 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */ 1615 if (hash_type != 1 && hash_type != 3) 1616 type = PKT_HASH_TYPE_L3; 1617 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type); 1618 } 1619 1620 cfa_code = RX_CMP_CFA_CODE(rxcmp1); 1621 skb->protocol = eth_type_trans(skb, bnxt_get_pkt_dev(bp, cfa_code)); 1622 1623 if ((rxcmp1->rx_cmp_flags2 & 1624 cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) && 1625 (skb->dev->features & NETIF_F_HW_VLAN_CTAG_RX)) { 1626 u32 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data); 1627 u16 vtag = meta_data & RX_CMP_FLAGS2_METADATA_TCI_MASK; 1628 u16 vlan_proto = meta_data >> RX_CMP_FLAGS2_METADATA_TPID_SFT; 1629 1630 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag); 1631 } 1632 1633 skb_checksum_none_assert(skb); 1634 if (RX_CMP_L4_CS_OK(rxcmp1)) { 1635 if (dev->features & NETIF_F_RXCSUM) { 1636 skb->ip_summed = CHECKSUM_UNNECESSARY; 1637 skb->csum_level = RX_CMP_ENCAP(rxcmp1); 1638 } 1639 } else { 1640 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) { 1641 if (dev->features & NETIF_F_RXCSUM) 1642 cpr->rx_l4_csum_errors++; 1643 } 1644 } 1645 1646 bnxt_deliver_skb(bp, bnapi, skb); 1647 rc = 1; 1648 1649 next_rx: 1650 rxr->rx_prod = NEXT_RX(prod); 1651 rxr->rx_next_cons = NEXT_RX(cons); 1652 1653 cpr->rx_packets += 1; 1654 cpr->rx_bytes += len; 1655 1656 next_rx_no_prod_no_len: 1657 *raw_cons = tmp_raw_cons; 1658 1659 return rc; 1660 } 1661 1662 /* In netpoll mode, if we are using a combined completion ring, we need to 1663 * discard the rx packets and recycle the buffers. 1664 */ 1665 static int bnxt_force_rx_discard(struct bnxt *bp, struct bnxt_napi *bnapi, 1666 u32 *raw_cons, u8 *event) 1667 { 1668 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 1669 u32 tmp_raw_cons = *raw_cons; 1670 struct rx_cmp_ext *rxcmp1; 1671 struct rx_cmp *rxcmp; 1672 u16 cp_cons; 1673 u8 cmp_type; 1674 1675 cp_cons = RING_CMP(tmp_raw_cons); 1676 rxcmp = (struct rx_cmp *) 1677 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1678 1679 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons); 1680 cp_cons = RING_CMP(tmp_raw_cons); 1681 rxcmp1 = (struct rx_cmp_ext *) 1682 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1683 1684 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons)) 1685 return -EBUSY; 1686 1687 cmp_type = RX_CMP_TYPE(rxcmp); 1688 if (cmp_type == CMP_TYPE_RX_L2_CMP) { 1689 rxcmp1->rx_cmp_cfa_code_errors_v2 |= 1690 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR); 1691 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) { 1692 struct rx_tpa_end_cmp_ext *tpa_end1; 1693 1694 tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1; 1695 tpa_end1->rx_tpa_end_cmp_errors_v2 |= 1696 cpu_to_le32(RX_TPA_END_CMP_ERRORS); 1697 } 1698 return bnxt_rx_pkt(bp, bnapi, raw_cons, event); 1699 } 1700 1701 #define BNXT_GET_EVENT_PORT(data) \ 1702 ((data) & \ 1703 ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK) 1704 1705 static int bnxt_async_event_process(struct bnxt *bp, 1706 struct hwrm_async_event_cmpl *cmpl) 1707 { 1708 u16 event_id = le16_to_cpu(cmpl->event_id); 1709 1710 /* TODO CHIMP_FW: Define event id's for link change, error etc */ 1711 switch (event_id) { 1712 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: { 1713 u32 data1 = le32_to_cpu(cmpl->event_data1); 1714 struct bnxt_link_info *link_info = &bp->link_info; 1715 1716 if (BNXT_VF(bp)) 1717 goto async_event_process_exit; 1718 1719 /* print unsupported speed warning in forced speed mode only */ 1720 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED) && 1721 (data1 & 0x20000)) { 1722 u16 fw_speed = link_info->force_link_speed; 1723 u32 speed = bnxt_fw_to_ethtool_speed(fw_speed); 1724 1725 if (speed != SPEED_UNKNOWN) 1726 netdev_warn(bp->dev, "Link speed %d no longer supported\n", 1727 speed); 1728 } 1729 set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event); 1730 /* fall thru */ 1731 } 1732 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE: 1733 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event); 1734 break; 1735 case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD: 1736 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event); 1737 break; 1738 case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: { 1739 u32 data1 = le32_to_cpu(cmpl->event_data1); 1740 u16 port_id = BNXT_GET_EVENT_PORT(data1); 1741 1742 if (BNXT_VF(bp)) 1743 break; 1744 1745 if (bp->pf.port_id != port_id) 1746 break; 1747 1748 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event); 1749 break; 1750 } 1751 case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE: 1752 if (BNXT_PF(bp)) 1753 goto async_event_process_exit; 1754 set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event); 1755 break; 1756 default: 1757 goto async_event_process_exit; 1758 } 1759 bnxt_queue_sp_work(bp); 1760 async_event_process_exit: 1761 bnxt_ulp_async_events(bp, cmpl); 1762 return 0; 1763 } 1764 1765 static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp) 1766 { 1767 u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id; 1768 struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp; 1769 struct hwrm_fwd_req_cmpl *fwd_req_cmpl = 1770 (struct hwrm_fwd_req_cmpl *)txcmp; 1771 1772 switch (cmpl_type) { 1773 case CMPL_BASE_TYPE_HWRM_DONE: 1774 seq_id = le16_to_cpu(h_cmpl->sequence_id); 1775 if (seq_id == bp->hwrm_intr_seq_id) 1776 bp->hwrm_intr_seq_id = HWRM_SEQ_ID_INVALID; 1777 else 1778 netdev_err(bp->dev, "Invalid hwrm seq id %d\n", seq_id); 1779 break; 1780 1781 case CMPL_BASE_TYPE_HWRM_FWD_REQ: 1782 vf_id = le16_to_cpu(fwd_req_cmpl->source_id); 1783 1784 if ((vf_id < bp->pf.first_vf_id) || 1785 (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) { 1786 netdev_err(bp->dev, "Msg contains invalid VF id %x\n", 1787 vf_id); 1788 return -EINVAL; 1789 } 1790 1791 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap); 1792 set_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event); 1793 bnxt_queue_sp_work(bp); 1794 break; 1795 1796 case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT: 1797 bnxt_async_event_process(bp, 1798 (struct hwrm_async_event_cmpl *)txcmp); 1799 1800 default: 1801 break; 1802 } 1803 1804 return 0; 1805 } 1806 1807 static irqreturn_t bnxt_msix(int irq, void *dev_instance) 1808 { 1809 struct bnxt_napi *bnapi = dev_instance; 1810 struct bnxt *bp = bnapi->bp; 1811 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 1812 u32 cons = RING_CMP(cpr->cp_raw_cons); 1813 1814 cpr->event_ctr++; 1815 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]); 1816 napi_schedule(&bnapi->napi); 1817 return IRQ_HANDLED; 1818 } 1819 1820 static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr) 1821 { 1822 u32 raw_cons = cpr->cp_raw_cons; 1823 u16 cons = RING_CMP(raw_cons); 1824 struct tx_cmp *txcmp; 1825 1826 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]; 1827 1828 return TX_CMP_VALID(txcmp, raw_cons); 1829 } 1830 1831 static irqreturn_t bnxt_inta(int irq, void *dev_instance) 1832 { 1833 struct bnxt_napi *bnapi = dev_instance; 1834 struct bnxt *bp = bnapi->bp; 1835 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 1836 u32 cons = RING_CMP(cpr->cp_raw_cons); 1837 u32 int_status; 1838 1839 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]); 1840 1841 if (!bnxt_has_work(bp, cpr)) { 1842 int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS); 1843 /* return if erroneous interrupt */ 1844 if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id))) 1845 return IRQ_NONE; 1846 } 1847 1848 /* disable ring IRQ */ 1849 BNXT_CP_DB_IRQ_DIS(cpr->cp_doorbell); 1850 1851 /* Return here if interrupt is shared and is disabled. */ 1852 if (unlikely(atomic_read(&bp->intr_sem) != 0)) 1853 return IRQ_HANDLED; 1854 1855 napi_schedule(&bnapi->napi); 1856 return IRQ_HANDLED; 1857 } 1858 1859 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_napi *bnapi, int budget) 1860 { 1861 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 1862 u32 raw_cons = cpr->cp_raw_cons; 1863 u32 cons; 1864 int tx_pkts = 0; 1865 int rx_pkts = 0; 1866 u8 event = 0; 1867 struct tx_cmp *txcmp; 1868 1869 while (1) { 1870 int rc; 1871 1872 cons = RING_CMP(raw_cons); 1873 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]; 1874 1875 if (!TX_CMP_VALID(txcmp, raw_cons)) 1876 break; 1877 1878 /* The valid test of the entry must be done first before 1879 * reading any further. 1880 */ 1881 dma_rmb(); 1882 if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) { 1883 tx_pkts++; 1884 /* return full budget so NAPI will complete. */ 1885 if (unlikely(tx_pkts > bp->tx_wake_thresh)) 1886 rx_pkts = budget; 1887 } else if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) { 1888 if (likely(budget)) 1889 rc = bnxt_rx_pkt(bp, bnapi, &raw_cons, &event); 1890 else 1891 rc = bnxt_force_rx_discard(bp, bnapi, &raw_cons, 1892 &event); 1893 if (likely(rc >= 0)) 1894 rx_pkts += rc; 1895 /* Increment rx_pkts when rc is -ENOMEM to count towards 1896 * the NAPI budget. Otherwise, we may potentially loop 1897 * here forever if we consistently cannot allocate 1898 * buffers. 1899 */ 1900 else if (rc == -ENOMEM && budget) 1901 rx_pkts++; 1902 else if (rc == -EBUSY) /* partial completion */ 1903 break; 1904 } else if (unlikely((TX_CMP_TYPE(txcmp) == 1905 CMPL_BASE_TYPE_HWRM_DONE) || 1906 (TX_CMP_TYPE(txcmp) == 1907 CMPL_BASE_TYPE_HWRM_FWD_REQ) || 1908 (TX_CMP_TYPE(txcmp) == 1909 CMPL_BASE_TYPE_HWRM_ASYNC_EVENT))) { 1910 bnxt_hwrm_handler(bp, txcmp); 1911 } 1912 raw_cons = NEXT_RAW_CMP(raw_cons); 1913 1914 if (rx_pkts == budget) 1915 break; 1916 } 1917 1918 if (event & BNXT_TX_EVENT) { 1919 struct bnxt_tx_ring_info *txr = bnapi->tx_ring; 1920 void __iomem *db = txr->tx_doorbell; 1921 u16 prod = txr->tx_prod; 1922 1923 /* Sync BD data before updating doorbell */ 1924 wmb(); 1925 1926 bnxt_db_write_relaxed(bp, db, DB_KEY_TX | prod); 1927 } 1928 1929 cpr->cp_raw_cons = raw_cons; 1930 /* ACK completion ring before freeing tx ring and producing new 1931 * buffers in rx/agg rings to prevent overflowing the completion 1932 * ring. 1933 */ 1934 BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons); 1935 1936 if (tx_pkts) 1937 bnapi->tx_int(bp, bnapi, tx_pkts); 1938 1939 if (event & BNXT_RX_EVENT) { 1940 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1941 1942 bnxt_db_write(bp, rxr->rx_doorbell, DB_KEY_RX | rxr->rx_prod); 1943 if (event & BNXT_AGG_EVENT) 1944 bnxt_db_write(bp, rxr->rx_agg_doorbell, 1945 DB_KEY_RX | rxr->rx_agg_prod); 1946 } 1947 return rx_pkts; 1948 } 1949 1950 static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget) 1951 { 1952 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi); 1953 struct bnxt *bp = bnapi->bp; 1954 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 1955 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1956 struct tx_cmp *txcmp; 1957 struct rx_cmp_ext *rxcmp1; 1958 u32 cp_cons, tmp_raw_cons; 1959 u32 raw_cons = cpr->cp_raw_cons; 1960 u32 rx_pkts = 0; 1961 u8 event = 0; 1962 1963 while (1) { 1964 int rc; 1965 1966 cp_cons = RING_CMP(raw_cons); 1967 txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1968 1969 if (!TX_CMP_VALID(txcmp, raw_cons)) 1970 break; 1971 1972 if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) { 1973 tmp_raw_cons = NEXT_RAW_CMP(raw_cons); 1974 cp_cons = RING_CMP(tmp_raw_cons); 1975 rxcmp1 = (struct rx_cmp_ext *) 1976 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1977 1978 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons)) 1979 break; 1980 1981 /* force an error to recycle the buffer */ 1982 rxcmp1->rx_cmp_cfa_code_errors_v2 |= 1983 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR); 1984 1985 rc = bnxt_rx_pkt(bp, bnapi, &raw_cons, &event); 1986 if (likely(rc == -EIO) && budget) 1987 rx_pkts++; 1988 else if (rc == -EBUSY) /* partial completion */ 1989 break; 1990 } else if (unlikely(TX_CMP_TYPE(txcmp) == 1991 CMPL_BASE_TYPE_HWRM_DONE)) { 1992 bnxt_hwrm_handler(bp, txcmp); 1993 } else { 1994 netdev_err(bp->dev, 1995 "Invalid completion received on special ring\n"); 1996 } 1997 raw_cons = NEXT_RAW_CMP(raw_cons); 1998 1999 if (rx_pkts == budget) 2000 break; 2001 } 2002 2003 cpr->cp_raw_cons = raw_cons; 2004 BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons); 2005 bnxt_db_write(bp, rxr->rx_doorbell, DB_KEY_RX | rxr->rx_prod); 2006 2007 if (event & BNXT_AGG_EVENT) 2008 bnxt_db_write(bp, rxr->rx_agg_doorbell, 2009 DB_KEY_RX | rxr->rx_agg_prod); 2010 2011 if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) { 2012 napi_complete_done(napi, rx_pkts); 2013 BNXT_CP_DB_REARM(cpr->cp_doorbell, cpr->cp_raw_cons); 2014 } 2015 return rx_pkts; 2016 } 2017 2018 static int bnxt_poll(struct napi_struct *napi, int budget) 2019 { 2020 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi); 2021 struct bnxt *bp = bnapi->bp; 2022 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2023 int work_done = 0; 2024 2025 while (1) { 2026 work_done += bnxt_poll_work(bp, bnapi, budget - work_done); 2027 2028 if (work_done >= budget) 2029 break; 2030 2031 if (!bnxt_has_work(bp, cpr)) { 2032 if (napi_complete_done(napi, work_done)) 2033 BNXT_CP_DB_REARM(cpr->cp_doorbell, 2034 cpr->cp_raw_cons); 2035 break; 2036 } 2037 } 2038 if (bp->flags & BNXT_FLAG_DIM) { 2039 struct net_dim_sample dim_sample; 2040 2041 net_dim_sample(cpr->event_ctr, 2042 cpr->rx_packets, 2043 cpr->rx_bytes, 2044 &dim_sample); 2045 net_dim(&cpr->dim, dim_sample); 2046 } 2047 mmiowb(); 2048 return work_done; 2049 } 2050 2051 static void bnxt_free_tx_skbs(struct bnxt *bp) 2052 { 2053 int i, max_idx; 2054 struct pci_dev *pdev = bp->pdev; 2055 2056 if (!bp->tx_ring) 2057 return; 2058 2059 max_idx = bp->tx_nr_pages * TX_DESC_CNT; 2060 for (i = 0; i < bp->tx_nr_rings; i++) { 2061 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 2062 int j; 2063 2064 for (j = 0; j < max_idx;) { 2065 struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j]; 2066 struct sk_buff *skb = tx_buf->skb; 2067 int k, last; 2068 2069 if (!skb) { 2070 j++; 2071 continue; 2072 } 2073 2074 tx_buf->skb = NULL; 2075 2076 if (tx_buf->is_push) { 2077 dev_kfree_skb(skb); 2078 j += 2; 2079 continue; 2080 } 2081 2082 dma_unmap_single(&pdev->dev, 2083 dma_unmap_addr(tx_buf, mapping), 2084 skb_headlen(skb), 2085 PCI_DMA_TODEVICE); 2086 2087 last = tx_buf->nr_frags; 2088 j += 2; 2089 for (k = 0; k < last; k++, j++) { 2090 int ring_idx = j & bp->tx_ring_mask; 2091 skb_frag_t *frag = &skb_shinfo(skb)->frags[k]; 2092 2093 tx_buf = &txr->tx_buf_ring[ring_idx]; 2094 dma_unmap_page( 2095 &pdev->dev, 2096 dma_unmap_addr(tx_buf, mapping), 2097 skb_frag_size(frag), PCI_DMA_TODEVICE); 2098 } 2099 dev_kfree_skb(skb); 2100 } 2101 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i)); 2102 } 2103 } 2104 2105 static void bnxt_free_rx_skbs(struct bnxt *bp) 2106 { 2107 int i, max_idx, max_agg_idx; 2108 struct pci_dev *pdev = bp->pdev; 2109 2110 if (!bp->rx_ring) 2111 return; 2112 2113 max_idx = bp->rx_nr_pages * RX_DESC_CNT; 2114 max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT; 2115 for (i = 0; i < bp->rx_nr_rings; i++) { 2116 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 2117 int j; 2118 2119 if (rxr->rx_tpa) { 2120 for (j = 0; j < MAX_TPA; j++) { 2121 struct bnxt_tpa_info *tpa_info = 2122 &rxr->rx_tpa[j]; 2123 u8 *data = tpa_info->data; 2124 2125 if (!data) 2126 continue; 2127 2128 dma_unmap_single_attrs(&pdev->dev, 2129 tpa_info->mapping, 2130 bp->rx_buf_use_size, 2131 bp->rx_dir, 2132 DMA_ATTR_WEAK_ORDERING); 2133 2134 tpa_info->data = NULL; 2135 2136 kfree(data); 2137 } 2138 } 2139 2140 for (j = 0; j < max_idx; j++) { 2141 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[j]; 2142 dma_addr_t mapping = rx_buf->mapping; 2143 void *data = rx_buf->data; 2144 2145 if (!data) 2146 continue; 2147 2148 rx_buf->data = NULL; 2149 2150 if (BNXT_RX_PAGE_MODE(bp)) { 2151 mapping -= bp->rx_dma_offset; 2152 dma_unmap_page_attrs(&pdev->dev, mapping, 2153 PAGE_SIZE, bp->rx_dir, 2154 DMA_ATTR_WEAK_ORDERING); 2155 __free_page(data); 2156 } else { 2157 dma_unmap_single_attrs(&pdev->dev, mapping, 2158 bp->rx_buf_use_size, 2159 bp->rx_dir, 2160 DMA_ATTR_WEAK_ORDERING); 2161 kfree(data); 2162 } 2163 } 2164 2165 for (j = 0; j < max_agg_idx; j++) { 2166 struct bnxt_sw_rx_agg_bd *rx_agg_buf = 2167 &rxr->rx_agg_ring[j]; 2168 struct page *page = rx_agg_buf->page; 2169 2170 if (!page) 2171 continue; 2172 2173 dma_unmap_page_attrs(&pdev->dev, rx_agg_buf->mapping, 2174 BNXT_RX_PAGE_SIZE, 2175 PCI_DMA_FROMDEVICE, 2176 DMA_ATTR_WEAK_ORDERING); 2177 2178 rx_agg_buf->page = NULL; 2179 __clear_bit(j, rxr->rx_agg_bmap); 2180 2181 __free_page(page); 2182 } 2183 if (rxr->rx_page) { 2184 __free_page(rxr->rx_page); 2185 rxr->rx_page = NULL; 2186 } 2187 } 2188 } 2189 2190 static void bnxt_free_skbs(struct bnxt *bp) 2191 { 2192 bnxt_free_tx_skbs(bp); 2193 bnxt_free_rx_skbs(bp); 2194 } 2195 2196 static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_struct *ring) 2197 { 2198 struct pci_dev *pdev = bp->pdev; 2199 int i; 2200 2201 for (i = 0; i < ring->nr_pages; i++) { 2202 if (!ring->pg_arr[i]) 2203 continue; 2204 2205 dma_free_coherent(&pdev->dev, ring->page_size, 2206 ring->pg_arr[i], ring->dma_arr[i]); 2207 2208 ring->pg_arr[i] = NULL; 2209 } 2210 if (ring->pg_tbl) { 2211 dma_free_coherent(&pdev->dev, ring->nr_pages * 8, 2212 ring->pg_tbl, ring->pg_tbl_map); 2213 ring->pg_tbl = NULL; 2214 } 2215 if (ring->vmem_size && *ring->vmem) { 2216 vfree(*ring->vmem); 2217 *ring->vmem = NULL; 2218 } 2219 } 2220 2221 static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_struct *ring) 2222 { 2223 int i; 2224 struct pci_dev *pdev = bp->pdev; 2225 2226 if (ring->nr_pages > 1) { 2227 ring->pg_tbl = dma_alloc_coherent(&pdev->dev, 2228 ring->nr_pages * 8, 2229 &ring->pg_tbl_map, 2230 GFP_KERNEL); 2231 if (!ring->pg_tbl) 2232 return -ENOMEM; 2233 } 2234 2235 for (i = 0; i < ring->nr_pages; i++) { 2236 ring->pg_arr[i] = dma_alloc_coherent(&pdev->dev, 2237 ring->page_size, 2238 &ring->dma_arr[i], 2239 GFP_KERNEL); 2240 if (!ring->pg_arr[i]) 2241 return -ENOMEM; 2242 2243 if (ring->nr_pages > 1) 2244 ring->pg_tbl[i] = cpu_to_le64(ring->dma_arr[i]); 2245 } 2246 2247 if (ring->vmem_size) { 2248 *ring->vmem = vzalloc(ring->vmem_size); 2249 if (!(*ring->vmem)) 2250 return -ENOMEM; 2251 } 2252 return 0; 2253 } 2254 2255 static void bnxt_free_rx_rings(struct bnxt *bp) 2256 { 2257 int i; 2258 2259 if (!bp->rx_ring) 2260 return; 2261 2262 for (i = 0; i < bp->rx_nr_rings; i++) { 2263 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 2264 struct bnxt_ring_struct *ring; 2265 2266 if (rxr->xdp_prog) 2267 bpf_prog_put(rxr->xdp_prog); 2268 2269 if (xdp_rxq_info_is_reg(&rxr->xdp_rxq)) 2270 xdp_rxq_info_unreg(&rxr->xdp_rxq); 2271 2272 kfree(rxr->rx_tpa); 2273 rxr->rx_tpa = NULL; 2274 2275 kfree(rxr->rx_agg_bmap); 2276 rxr->rx_agg_bmap = NULL; 2277 2278 ring = &rxr->rx_ring_struct; 2279 bnxt_free_ring(bp, ring); 2280 2281 ring = &rxr->rx_agg_ring_struct; 2282 bnxt_free_ring(bp, ring); 2283 } 2284 } 2285 2286 static int bnxt_alloc_rx_rings(struct bnxt *bp) 2287 { 2288 int i, rc, agg_rings = 0, tpa_rings = 0; 2289 2290 if (!bp->rx_ring) 2291 return -ENOMEM; 2292 2293 if (bp->flags & BNXT_FLAG_AGG_RINGS) 2294 agg_rings = 1; 2295 2296 if (bp->flags & BNXT_FLAG_TPA) 2297 tpa_rings = 1; 2298 2299 for (i = 0; i < bp->rx_nr_rings; i++) { 2300 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 2301 struct bnxt_ring_struct *ring; 2302 2303 ring = &rxr->rx_ring_struct; 2304 2305 rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i); 2306 if (rc < 0) 2307 return rc; 2308 2309 rc = bnxt_alloc_ring(bp, ring); 2310 if (rc) 2311 return rc; 2312 2313 if (agg_rings) { 2314 u16 mem_size; 2315 2316 ring = &rxr->rx_agg_ring_struct; 2317 rc = bnxt_alloc_ring(bp, ring); 2318 if (rc) 2319 return rc; 2320 2321 ring->grp_idx = i; 2322 rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1; 2323 mem_size = rxr->rx_agg_bmap_size / 8; 2324 rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL); 2325 if (!rxr->rx_agg_bmap) 2326 return -ENOMEM; 2327 2328 if (tpa_rings) { 2329 rxr->rx_tpa = kcalloc(MAX_TPA, 2330 sizeof(struct bnxt_tpa_info), 2331 GFP_KERNEL); 2332 if (!rxr->rx_tpa) 2333 return -ENOMEM; 2334 } 2335 } 2336 } 2337 return 0; 2338 } 2339 2340 static void bnxt_free_tx_rings(struct bnxt *bp) 2341 { 2342 int i; 2343 struct pci_dev *pdev = bp->pdev; 2344 2345 if (!bp->tx_ring) 2346 return; 2347 2348 for (i = 0; i < bp->tx_nr_rings; i++) { 2349 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 2350 struct bnxt_ring_struct *ring; 2351 2352 if (txr->tx_push) { 2353 dma_free_coherent(&pdev->dev, bp->tx_push_size, 2354 txr->tx_push, txr->tx_push_mapping); 2355 txr->tx_push = NULL; 2356 } 2357 2358 ring = &txr->tx_ring_struct; 2359 2360 bnxt_free_ring(bp, ring); 2361 } 2362 } 2363 2364 static int bnxt_alloc_tx_rings(struct bnxt *bp) 2365 { 2366 int i, j, rc; 2367 struct pci_dev *pdev = bp->pdev; 2368 2369 bp->tx_push_size = 0; 2370 if (bp->tx_push_thresh) { 2371 int push_size; 2372 2373 push_size = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) + 2374 bp->tx_push_thresh); 2375 2376 if (push_size > 256) { 2377 push_size = 0; 2378 bp->tx_push_thresh = 0; 2379 } 2380 2381 bp->tx_push_size = push_size; 2382 } 2383 2384 for (i = 0, j = 0; i < bp->tx_nr_rings; i++) { 2385 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 2386 struct bnxt_ring_struct *ring; 2387 u8 qidx; 2388 2389 ring = &txr->tx_ring_struct; 2390 2391 rc = bnxt_alloc_ring(bp, ring); 2392 if (rc) 2393 return rc; 2394 2395 ring->grp_idx = txr->bnapi->index; 2396 if (bp->tx_push_size) { 2397 dma_addr_t mapping; 2398 2399 /* One pre-allocated DMA buffer to backup 2400 * TX push operation 2401 */ 2402 txr->tx_push = dma_alloc_coherent(&pdev->dev, 2403 bp->tx_push_size, 2404 &txr->tx_push_mapping, 2405 GFP_KERNEL); 2406 2407 if (!txr->tx_push) 2408 return -ENOMEM; 2409 2410 mapping = txr->tx_push_mapping + 2411 sizeof(struct tx_push_bd); 2412 txr->data_mapping = cpu_to_le64(mapping); 2413 2414 memset(txr->tx_push, 0, sizeof(struct tx_push_bd)); 2415 } 2416 qidx = bp->tc_to_qidx[j]; 2417 ring->queue_id = bp->q_info[qidx].queue_id; 2418 if (i < bp->tx_nr_rings_xdp) 2419 continue; 2420 if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1)) 2421 j++; 2422 } 2423 return 0; 2424 } 2425 2426 static void bnxt_free_cp_rings(struct bnxt *bp) 2427 { 2428 int i; 2429 2430 if (!bp->bnapi) 2431 return; 2432 2433 for (i = 0; i < bp->cp_nr_rings; i++) { 2434 struct bnxt_napi *bnapi = bp->bnapi[i]; 2435 struct bnxt_cp_ring_info *cpr; 2436 struct bnxt_ring_struct *ring; 2437 2438 if (!bnapi) 2439 continue; 2440 2441 cpr = &bnapi->cp_ring; 2442 ring = &cpr->cp_ring_struct; 2443 2444 bnxt_free_ring(bp, ring); 2445 } 2446 } 2447 2448 static int bnxt_alloc_cp_rings(struct bnxt *bp) 2449 { 2450 int i, rc, ulp_base_vec, ulp_msix; 2451 2452 ulp_msix = bnxt_get_ulp_msix_num(bp); 2453 ulp_base_vec = bnxt_get_ulp_msix_base(bp); 2454 for (i = 0; i < bp->cp_nr_rings; i++) { 2455 struct bnxt_napi *bnapi = bp->bnapi[i]; 2456 struct bnxt_cp_ring_info *cpr; 2457 struct bnxt_ring_struct *ring; 2458 2459 if (!bnapi) 2460 continue; 2461 2462 cpr = &bnapi->cp_ring; 2463 ring = &cpr->cp_ring_struct; 2464 2465 rc = bnxt_alloc_ring(bp, ring); 2466 if (rc) 2467 return rc; 2468 2469 if (ulp_msix && i >= ulp_base_vec) 2470 ring->map_idx = i + ulp_msix; 2471 else 2472 ring->map_idx = i; 2473 } 2474 return 0; 2475 } 2476 2477 static void bnxt_init_ring_struct(struct bnxt *bp) 2478 { 2479 int i; 2480 2481 for (i = 0; i < bp->cp_nr_rings; i++) { 2482 struct bnxt_napi *bnapi = bp->bnapi[i]; 2483 struct bnxt_cp_ring_info *cpr; 2484 struct bnxt_rx_ring_info *rxr; 2485 struct bnxt_tx_ring_info *txr; 2486 struct bnxt_ring_struct *ring; 2487 2488 if (!bnapi) 2489 continue; 2490 2491 cpr = &bnapi->cp_ring; 2492 ring = &cpr->cp_ring_struct; 2493 ring->nr_pages = bp->cp_nr_pages; 2494 ring->page_size = HW_CMPD_RING_SIZE; 2495 ring->pg_arr = (void **)cpr->cp_desc_ring; 2496 ring->dma_arr = cpr->cp_desc_mapping; 2497 ring->vmem_size = 0; 2498 2499 rxr = bnapi->rx_ring; 2500 if (!rxr) 2501 goto skip_rx; 2502 2503 ring = &rxr->rx_ring_struct; 2504 ring->nr_pages = bp->rx_nr_pages; 2505 ring->page_size = HW_RXBD_RING_SIZE; 2506 ring->pg_arr = (void **)rxr->rx_desc_ring; 2507 ring->dma_arr = rxr->rx_desc_mapping; 2508 ring->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages; 2509 ring->vmem = (void **)&rxr->rx_buf_ring; 2510 2511 ring = &rxr->rx_agg_ring_struct; 2512 ring->nr_pages = bp->rx_agg_nr_pages; 2513 ring->page_size = HW_RXBD_RING_SIZE; 2514 ring->pg_arr = (void **)rxr->rx_agg_desc_ring; 2515 ring->dma_arr = rxr->rx_agg_desc_mapping; 2516 ring->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages; 2517 ring->vmem = (void **)&rxr->rx_agg_ring; 2518 2519 skip_rx: 2520 txr = bnapi->tx_ring; 2521 if (!txr) 2522 continue; 2523 2524 ring = &txr->tx_ring_struct; 2525 ring->nr_pages = bp->tx_nr_pages; 2526 ring->page_size = HW_RXBD_RING_SIZE; 2527 ring->pg_arr = (void **)txr->tx_desc_ring; 2528 ring->dma_arr = txr->tx_desc_mapping; 2529 ring->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages; 2530 ring->vmem = (void **)&txr->tx_buf_ring; 2531 } 2532 } 2533 2534 static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type) 2535 { 2536 int i; 2537 u32 prod; 2538 struct rx_bd **rx_buf_ring; 2539 2540 rx_buf_ring = (struct rx_bd **)ring->pg_arr; 2541 for (i = 0, prod = 0; i < ring->nr_pages; i++) { 2542 int j; 2543 struct rx_bd *rxbd; 2544 2545 rxbd = rx_buf_ring[i]; 2546 if (!rxbd) 2547 continue; 2548 2549 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) { 2550 rxbd->rx_bd_len_flags_type = cpu_to_le32(type); 2551 rxbd->rx_bd_opaque = prod; 2552 } 2553 } 2554 } 2555 2556 static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr) 2557 { 2558 struct net_device *dev = bp->dev; 2559 struct bnxt_rx_ring_info *rxr; 2560 struct bnxt_ring_struct *ring; 2561 u32 prod, type; 2562 int i; 2563 2564 type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) | 2565 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP; 2566 2567 if (NET_IP_ALIGN == 2) 2568 type |= RX_BD_FLAGS_SOP; 2569 2570 rxr = &bp->rx_ring[ring_nr]; 2571 ring = &rxr->rx_ring_struct; 2572 bnxt_init_rxbd_pages(ring, type); 2573 2574 if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) { 2575 rxr->xdp_prog = bpf_prog_add(bp->xdp_prog, 1); 2576 if (IS_ERR(rxr->xdp_prog)) { 2577 int rc = PTR_ERR(rxr->xdp_prog); 2578 2579 rxr->xdp_prog = NULL; 2580 return rc; 2581 } 2582 } 2583 prod = rxr->rx_prod; 2584 for (i = 0; i < bp->rx_ring_size; i++) { 2585 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL) != 0) { 2586 netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n", 2587 ring_nr, i, bp->rx_ring_size); 2588 break; 2589 } 2590 prod = NEXT_RX(prod); 2591 } 2592 rxr->rx_prod = prod; 2593 ring->fw_ring_id = INVALID_HW_RING_ID; 2594 2595 ring = &rxr->rx_agg_ring_struct; 2596 ring->fw_ring_id = INVALID_HW_RING_ID; 2597 2598 if (!(bp->flags & BNXT_FLAG_AGG_RINGS)) 2599 return 0; 2600 2601 type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) | 2602 RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP; 2603 2604 bnxt_init_rxbd_pages(ring, type); 2605 2606 prod = rxr->rx_agg_prod; 2607 for (i = 0; i < bp->rx_agg_ring_size; i++) { 2608 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL) != 0) { 2609 netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n", 2610 ring_nr, i, bp->rx_ring_size); 2611 break; 2612 } 2613 prod = NEXT_RX_AGG(prod); 2614 } 2615 rxr->rx_agg_prod = prod; 2616 2617 if (bp->flags & BNXT_FLAG_TPA) { 2618 if (rxr->rx_tpa) { 2619 u8 *data; 2620 dma_addr_t mapping; 2621 2622 for (i = 0; i < MAX_TPA; i++) { 2623 data = __bnxt_alloc_rx_data(bp, &mapping, 2624 GFP_KERNEL); 2625 if (!data) 2626 return -ENOMEM; 2627 2628 rxr->rx_tpa[i].data = data; 2629 rxr->rx_tpa[i].data_ptr = data + bp->rx_offset; 2630 rxr->rx_tpa[i].mapping = mapping; 2631 } 2632 } else { 2633 netdev_err(bp->dev, "No resource allocated for LRO/GRO\n"); 2634 return -ENOMEM; 2635 } 2636 } 2637 2638 return 0; 2639 } 2640 2641 static void bnxt_init_cp_rings(struct bnxt *bp) 2642 { 2643 int i; 2644 2645 for (i = 0; i < bp->cp_nr_rings; i++) { 2646 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring; 2647 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 2648 2649 ring->fw_ring_id = INVALID_HW_RING_ID; 2650 cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks; 2651 cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs; 2652 } 2653 } 2654 2655 static int bnxt_init_rx_rings(struct bnxt *bp) 2656 { 2657 int i, rc = 0; 2658 2659 if (BNXT_RX_PAGE_MODE(bp)) { 2660 bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM; 2661 bp->rx_dma_offset = XDP_PACKET_HEADROOM; 2662 } else { 2663 bp->rx_offset = BNXT_RX_OFFSET; 2664 bp->rx_dma_offset = BNXT_RX_DMA_OFFSET; 2665 } 2666 2667 for (i = 0; i < bp->rx_nr_rings; i++) { 2668 rc = bnxt_init_one_rx_ring(bp, i); 2669 if (rc) 2670 break; 2671 } 2672 2673 return rc; 2674 } 2675 2676 static int bnxt_init_tx_rings(struct bnxt *bp) 2677 { 2678 u16 i; 2679 2680 bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2, 2681 MAX_SKB_FRAGS + 1); 2682 2683 for (i = 0; i < bp->tx_nr_rings; i++) { 2684 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 2685 struct bnxt_ring_struct *ring = &txr->tx_ring_struct; 2686 2687 ring->fw_ring_id = INVALID_HW_RING_ID; 2688 } 2689 2690 return 0; 2691 } 2692 2693 static void bnxt_free_ring_grps(struct bnxt *bp) 2694 { 2695 kfree(bp->grp_info); 2696 bp->grp_info = NULL; 2697 } 2698 2699 static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init) 2700 { 2701 int i; 2702 2703 if (irq_re_init) { 2704 bp->grp_info = kcalloc(bp->cp_nr_rings, 2705 sizeof(struct bnxt_ring_grp_info), 2706 GFP_KERNEL); 2707 if (!bp->grp_info) 2708 return -ENOMEM; 2709 } 2710 for (i = 0; i < bp->cp_nr_rings; i++) { 2711 if (irq_re_init) 2712 bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID; 2713 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID; 2714 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID; 2715 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID; 2716 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID; 2717 } 2718 return 0; 2719 } 2720 2721 static void bnxt_free_vnics(struct bnxt *bp) 2722 { 2723 kfree(bp->vnic_info); 2724 bp->vnic_info = NULL; 2725 bp->nr_vnics = 0; 2726 } 2727 2728 static int bnxt_alloc_vnics(struct bnxt *bp) 2729 { 2730 int num_vnics = 1; 2731 2732 #ifdef CONFIG_RFS_ACCEL 2733 if (bp->flags & BNXT_FLAG_RFS) 2734 num_vnics += bp->rx_nr_rings; 2735 #endif 2736 2737 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 2738 num_vnics++; 2739 2740 bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info), 2741 GFP_KERNEL); 2742 if (!bp->vnic_info) 2743 return -ENOMEM; 2744 2745 bp->nr_vnics = num_vnics; 2746 return 0; 2747 } 2748 2749 static void bnxt_init_vnics(struct bnxt *bp) 2750 { 2751 int i; 2752 2753 for (i = 0; i < bp->nr_vnics; i++) { 2754 struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; 2755 2756 vnic->fw_vnic_id = INVALID_HW_RING_ID; 2757 vnic->fw_rss_cos_lb_ctx[0] = INVALID_HW_RING_ID; 2758 vnic->fw_rss_cos_lb_ctx[1] = INVALID_HW_RING_ID; 2759 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID; 2760 2761 if (bp->vnic_info[i].rss_hash_key) { 2762 if (i == 0) 2763 prandom_bytes(vnic->rss_hash_key, 2764 HW_HASH_KEY_SIZE); 2765 else 2766 memcpy(vnic->rss_hash_key, 2767 bp->vnic_info[0].rss_hash_key, 2768 HW_HASH_KEY_SIZE); 2769 } 2770 } 2771 } 2772 2773 static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg) 2774 { 2775 int pages; 2776 2777 pages = ring_size / desc_per_pg; 2778 2779 if (!pages) 2780 return 1; 2781 2782 pages++; 2783 2784 while (pages & (pages - 1)) 2785 pages++; 2786 2787 return pages; 2788 } 2789 2790 void bnxt_set_tpa_flags(struct bnxt *bp) 2791 { 2792 bp->flags &= ~BNXT_FLAG_TPA; 2793 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS) 2794 return; 2795 if (bp->dev->features & NETIF_F_LRO) 2796 bp->flags |= BNXT_FLAG_LRO; 2797 else if (bp->dev->features & NETIF_F_GRO_HW) 2798 bp->flags |= BNXT_FLAG_GRO; 2799 } 2800 2801 /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must 2802 * be set on entry. 2803 */ 2804 void bnxt_set_ring_params(struct bnxt *bp) 2805 { 2806 u32 ring_size, rx_size, rx_space; 2807 u32 agg_factor = 0, agg_ring_size = 0; 2808 2809 /* 8 for CRC and VLAN */ 2810 rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8); 2811 2812 rx_space = rx_size + NET_SKB_PAD + 2813 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 2814 2815 bp->rx_copy_thresh = BNXT_RX_COPY_THRESH; 2816 ring_size = bp->rx_ring_size; 2817 bp->rx_agg_ring_size = 0; 2818 bp->rx_agg_nr_pages = 0; 2819 2820 if (bp->flags & BNXT_FLAG_TPA) 2821 agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE); 2822 2823 bp->flags &= ~BNXT_FLAG_JUMBO; 2824 if (rx_space > PAGE_SIZE && !(bp->flags & BNXT_FLAG_NO_AGG_RINGS)) { 2825 u32 jumbo_factor; 2826 2827 bp->flags |= BNXT_FLAG_JUMBO; 2828 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT; 2829 if (jumbo_factor > agg_factor) 2830 agg_factor = jumbo_factor; 2831 } 2832 agg_ring_size = ring_size * agg_factor; 2833 2834 if (agg_ring_size) { 2835 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size, 2836 RX_DESC_CNT); 2837 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) { 2838 u32 tmp = agg_ring_size; 2839 2840 bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES; 2841 agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1; 2842 netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n", 2843 tmp, agg_ring_size); 2844 } 2845 bp->rx_agg_ring_size = agg_ring_size; 2846 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1; 2847 rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN); 2848 rx_space = rx_size + NET_SKB_PAD + 2849 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 2850 } 2851 2852 bp->rx_buf_use_size = rx_size; 2853 bp->rx_buf_size = rx_space; 2854 2855 bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT); 2856 bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1; 2857 2858 ring_size = bp->tx_ring_size; 2859 bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT); 2860 bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1; 2861 2862 ring_size = bp->rx_ring_size * (2 + agg_factor) + bp->tx_ring_size; 2863 bp->cp_ring_size = ring_size; 2864 2865 bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT); 2866 if (bp->cp_nr_pages > MAX_CP_PAGES) { 2867 bp->cp_nr_pages = MAX_CP_PAGES; 2868 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1; 2869 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n", 2870 ring_size, bp->cp_ring_size); 2871 } 2872 bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT; 2873 bp->cp_ring_mask = bp->cp_bit - 1; 2874 } 2875 2876 /* Changing allocation mode of RX rings. 2877 * TODO: Update when extending xdp_rxq_info to support allocation modes. 2878 */ 2879 int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode) 2880 { 2881 if (page_mode) { 2882 if (bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU) 2883 return -EOPNOTSUPP; 2884 bp->dev->max_mtu = 2885 min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU); 2886 bp->flags &= ~BNXT_FLAG_AGG_RINGS; 2887 bp->flags |= BNXT_FLAG_NO_AGG_RINGS | BNXT_FLAG_RX_PAGE_MODE; 2888 bp->rx_dir = DMA_BIDIRECTIONAL; 2889 bp->rx_skb_func = bnxt_rx_page_skb; 2890 /* Disable LRO or GRO_HW */ 2891 netdev_update_features(bp->dev); 2892 } else { 2893 bp->dev->max_mtu = bp->max_mtu; 2894 bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE; 2895 bp->rx_dir = DMA_FROM_DEVICE; 2896 bp->rx_skb_func = bnxt_rx_skb; 2897 } 2898 return 0; 2899 } 2900 2901 static void bnxt_free_vnic_attributes(struct bnxt *bp) 2902 { 2903 int i; 2904 struct bnxt_vnic_info *vnic; 2905 struct pci_dev *pdev = bp->pdev; 2906 2907 if (!bp->vnic_info) 2908 return; 2909 2910 for (i = 0; i < bp->nr_vnics; i++) { 2911 vnic = &bp->vnic_info[i]; 2912 2913 kfree(vnic->fw_grp_ids); 2914 vnic->fw_grp_ids = NULL; 2915 2916 kfree(vnic->uc_list); 2917 vnic->uc_list = NULL; 2918 2919 if (vnic->mc_list) { 2920 dma_free_coherent(&pdev->dev, vnic->mc_list_size, 2921 vnic->mc_list, vnic->mc_list_mapping); 2922 vnic->mc_list = NULL; 2923 } 2924 2925 if (vnic->rss_table) { 2926 dma_free_coherent(&pdev->dev, PAGE_SIZE, 2927 vnic->rss_table, 2928 vnic->rss_table_dma_addr); 2929 vnic->rss_table = NULL; 2930 } 2931 2932 vnic->rss_hash_key = NULL; 2933 vnic->flags = 0; 2934 } 2935 } 2936 2937 static int bnxt_alloc_vnic_attributes(struct bnxt *bp) 2938 { 2939 int i, rc = 0, size; 2940 struct bnxt_vnic_info *vnic; 2941 struct pci_dev *pdev = bp->pdev; 2942 int max_rings; 2943 2944 for (i = 0; i < bp->nr_vnics; i++) { 2945 vnic = &bp->vnic_info[i]; 2946 2947 if (vnic->flags & BNXT_VNIC_UCAST_FLAG) { 2948 int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN; 2949 2950 if (mem_size > 0) { 2951 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL); 2952 if (!vnic->uc_list) { 2953 rc = -ENOMEM; 2954 goto out; 2955 } 2956 } 2957 } 2958 2959 if (vnic->flags & BNXT_VNIC_MCAST_FLAG) { 2960 vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN; 2961 vnic->mc_list = 2962 dma_alloc_coherent(&pdev->dev, 2963 vnic->mc_list_size, 2964 &vnic->mc_list_mapping, 2965 GFP_KERNEL); 2966 if (!vnic->mc_list) { 2967 rc = -ENOMEM; 2968 goto out; 2969 } 2970 } 2971 2972 if (vnic->flags & BNXT_VNIC_RSS_FLAG) 2973 max_rings = bp->rx_nr_rings; 2974 else 2975 max_rings = 1; 2976 2977 vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL); 2978 if (!vnic->fw_grp_ids) { 2979 rc = -ENOMEM; 2980 goto out; 2981 } 2982 2983 if ((bp->flags & BNXT_FLAG_NEW_RSS_CAP) && 2984 !(vnic->flags & BNXT_VNIC_RSS_FLAG)) 2985 continue; 2986 2987 /* Allocate rss table and hash key */ 2988 vnic->rss_table = dma_alloc_coherent(&pdev->dev, PAGE_SIZE, 2989 &vnic->rss_table_dma_addr, 2990 GFP_KERNEL); 2991 if (!vnic->rss_table) { 2992 rc = -ENOMEM; 2993 goto out; 2994 } 2995 2996 size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16)); 2997 2998 vnic->rss_hash_key = ((void *)vnic->rss_table) + size; 2999 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size; 3000 } 3001 return 0; 3002 3003 out: 3004 return rc; 3005 } 3006 3007 static void bnxt_free_hwrm_resources(struct bnxt *bp) 3008 { 3009 struct pci_dev *pdev = bp->pdev; 3010 3011 dma_free_coherent(&pdev->dev, PAGE_SIZE, bp->hwrm_cmd_resp_addr, 3012 bp->hwrm_cmd_resp_dma_addr); 3013 3014 bp->hwrm_cmd_resp_addr = NULL; 3015 if (bp->hwrm_dbg_resp_addr) { 3016 dma_free_coherent(&pdev->dev, HWRM_DBG_REG_BUF_SIZE, 3017 bp->hwrm_dbg_resp_addr, 3018 bp->hwrm_dbg_resp_dma_addr); 3019 3020 bp->hwrm_dbg_resp_addr = NULL; 3021 } 3022 } 3023 3024 static int bnxt_alloc_hwrm_resources(struct bnxt *bp) 3025 { 3026 struct pci_dev *pdev = bp->pdev; 3027 3028 bp->hwrm_cmd_resp_addr = dma_alloc_coherent(&pdev->dev, PAGE_SIZE, 3029 &bp->hwrm_cmd_resp_dma_addr, 3030 GFP_KERNEL); 3031 if (!bp->hwrm_cmd_resp_addr) 3032 return -ENOMEM; 3033 bp->hwrm_dbg_resp_addr = dma_alloc_coherent(&pdev->dev, 3034 HWRM_DBG_REG_BUF_SIZE, 3035 &bp->hwrm_dbg_resp_dma_addr, 3036 GFP_KERNEL); 3037 if (!bp->hwrm_dbg_resp_addr) 3038 netdev_warn(bp->dev, "fail to alloc debug register dma mem\n"); 3039 3040 return 0; 3041 } 3042 3043 static void bnxt_free_hwrm_short_cmd_req(struct bnxt *bp) 3044 { 3045 if (bp->hwrm_short_cmd_req_addr) { 3046 struct pci_dev *pdev = bp->pdev; 3047 3048 dma_free_coherent(&pdev->dev, BNXT_HWRM_MAX_REQ_LEN, 3049 bp->hwrm_short_cmd_req_addr, 3050 bp->hwrm_short_cmd_req_dma_addr); 3051 bp->hwrm_short_cmd_req_addr = NULL; 3052 } 3053 } 3054 3055 static int bnxt_alloc_hwrm_short_cmd_req(struct bnxt *bp) 3056 { 3057 struct pci_dev *pdev = bp->pdev; 3058 3059 bp->hwrm_short_cmd_req_addr = 3060 dma_alloc_coherent(&pdev->dev, BNXT_HWRM_MAX_REQ_LEN, 3061 &bp->hwrm_short_cmd_req_dma_addr, 3062 GFP_KERNEL); 3063 if (!bp->hwrm_short_cmd_req_addr) 3064 return -ENOMEM; 3065 3066 return 0; 3067 } 3068 3069 static void bnxt_free_stats(struct bnxt *bp) 3070 { 3071 u32 size, i; 3072 struct pci_dev *pdev = bp->pdev; 3073 3074 bp->flags &= ~BNXT_FLAG_PORT_STATS; 3075 bp->flags &= ~BNXT_FLAG_PORT_STATS_EXT; 3076 3077 if (bp->hw_rx_port_stats) { 3078 dma_free_coherent(&pdev->dev, bp->hw_port_stats_size, 3079 bp->hw_rx_port_stats, 3080 bp->hw_rx_port_stats_map); 3081 bp->hw_rx_port_stats = NULL; 3082 } 3083 3084 if (bp->hw_rx_port_stats_ext) { 3085 dma_free_coherent(&pdev->dev, sizeof(struct rx_port_stats_ext), 3086 bp->hw_rx_port_stats_ext, 3087 bp->hw_rx_port_stats_ext_map); 3088 bp->hw_rx_port_stats_ext = NULL; 3089 } 3090 3091 if (!bp->bnapi) 3092 return; 3093 3094 size = sizeof(struct ctx_hw_stats); 3095 3096 for (i = 0; i < bp->cp_nr_rings; i++) { 3097 struct bnxt_napi *bnapi = bp->bnapi[i]; 3098 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 3099 3100 if (cpr->hw_stats) { 3101 dma_free_coherent(&pdev->dev, size, cpr->hw_stats, 3102 cpr->hw_stats_map); 3103 cpr->hw_stats = NULL; 3104 } 3105 } 3106 } 3107 3108 static int bnxt_alloc_stats(struct bnxt *bp) 3109 { 3110 u32 size, i; 3111 struct pci_dev *pdev = bp->pdev; 3112 3113 size = sizeof(struct ctx_hw_stats); 3114 3115 for (i = 0; i < bp->cp_nr_rings; i++) { 3116 struct bnxt_napi *bnapi = bp->bnapi[i]; 3117 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 3118 3119 cpr->hw_stats = dma_alloc_coherent(&pdev->dev, size, 3120 &cpr->hw_stats_map, 3121 GFP_KERNEL); 3122 if (!cpr->hw_stats) 3123 return -ENOMEM; 3124 3125 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID; 3126 } 3127 3128 if (BNXT_PF(bp) && bp->chip_num != CHIP_NUM_58700) { 3129 bp->hw_port_stats_size = sizeof(struct rx_port_stats) + 3130 sizeof(struct tx_port_stats) + 1024; 3131 3132 bp->hw_rx_port_stats = 3133 dma_alloc_coherent(&pdev->dev, bp->hw_port_stats_size, 3134 &bp->hw_rx_port_stats_map, 3135 GFP_KERNEL); 3136 if (!bp->hw_rx_port_stats) 3137 return -ENOMEM; 3138 3139 bp->hw_tx_port_stats = (void *)(bp->hw_rx_port_stats + 1) + 3140 512; 3141 bp->hw_tx_port_stats_map = bp->hw_rx_port_stats_map + 3142 sizeof(struct rx_port_stats) + 512; 3143 bp->flags |= BNXT_FLAG_PORT_STATS; 3144 3145 /* Display extended statistics only if FW supports it */ 3146 if (bp->hwrm_spec_code < 0x10804 || 3147 bp->hwrm_spec_code == 0x10900) 3148 return 0; 3149 3150 bp->hw_rx_port_stats_ext = 3151 dma_zalloc_coherent(&pdev->dev, 3152 sizeof(struct rx_port_stats_ext), 3153 &bp->hw_rx_port_stats_ext_map, 3154 GFP_KERNEL); 3155 if (!bp->hw_rx_port_stats_ext) 3156 return 0; 3157 3158 bp->flags |= BNXT_FLAG_PORT_STATS_EXT; 3159 } 3160 return 0; 3161 } 3162 3163 static void bnxt_clear_ring_indices(struct bnxt *bp) 3164 { 3165 int i; 3166 3167 if (!bp->bnapi) 3168 return; 3169 3170 for (i = 0; i < bp->cp_nr_rings; i++) { 3171 struct bnxt_napi *bnapi = bp->bnapi[i]; 3172 struct bnxt_cp_ring_info *cpr; 3173 struct bnxt_rx_ring_info *rxr; 3174 struct bnxt_tx_ring_info *txr; 3175 3176 if (!bnapi) 3177 continue; 3178 3179 cpr = &bnapi->cp_ring; 3180 cpr->cp_raw_cons = 0; 3181 3182 txr = bnapi->tx_ring; 3183 if (txr) { 3184 txr->tx_prod = 0; 3185 txr->tx_cons = 0; 3186 } 3187 3188 rxr = bnapi->rx_ring; 3189 if (rxr) { 3190 rxr->rx_prod = 0; 3191 rxr->rx_agg_prod = 0; 3192 rxr->rx_sw_agg_prod = 0; 3193 rxr->rx_next_cons = 0; 3194 } 3195 } 3196 } 3197 3198 static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool irq_reinit) 3199 { 3200 #ifdef CONFIG_RFS_ACCEL 3201 int i; 3202 3203 /* Under rtnl_lock and all our NAPIs have been disabled. It's 3204 * safe to delete the hash table. 3205 */ 3206 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) { 3207 struct hlist_head *head; 3208 struct hlist_node *tmp; 3209 struct bnxt_ntuple_filter *fltr; 3210 3211 head = &bp->ntp_fltr_hash_tbl[i]; 3212 hlist_for_each_entry_safe(fltr, tmp, head, hash) { 3213 hlist_del(&fltr->hash); 3214 kfree(fltr); 3215 } 3216 } 3217 if (irq_reinit) { 3218 kfree(bp->ntp_fltr_bmap); 3219 bp->ntp_fltr_bmap = NULL; 3220 } 3221 bp->ntp_fltr_count = 0; 3222 #endif 3223 } 3224 3225 static int bnxt_alloc_ntp_fltrs(struct bnxt *bp) 3226 { 3227 #ifdef CONFIG_RFS_ACCEL 3228 int i, rc = 0; 3229 3230 if (!(bp->flags & BNXT_FLAG_RFS)) 3231 return 0; 3232 3233 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) 3234 INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]); 3235 3236 bp->ntp_fltr_count = 0; 3237 bp->ntp_fltr_bmap = kcalloc(BITS_TO_LONGS(BNXT_NTP_FLTR_MAX_FLTR), 3238 sizeof(long), 3239 GFP_KERNEL); 3240 3241 if (!bp->ntp_fltr_bmap) 3242 rc = -ENOMEM; 3243 3244 return rc; 3245 #else 3246 return 0; 3247 #endif 3248 } 3249 3250 static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init) 3251 { 3252 bnxt_free_vnic_attributes(bp); 3253 bnxt_free_tx_rings(bp); 3254 bnxt_free_rx_rings(bp); 3255 bnxt_free_cp_rings(bp); 3256 bnxt_free_ntp_fltrs(bp, irq_re_init); 3257 if (irq_re_init) { 3258 bnxt_free_stats(bp); 3259 bnxt_free_ring_grps(bp); 3260 bnxt_free_vnics(bp); 3261 kfree(bp->tx_ring_map); 3262 bp->tx_ring_map = NULL; 3263 kfree(bp->tx_ring); 3264 bp->tx_ring = NULL; 3265 kfree(bp->rx_ring); 3266 bp->rx_ring = NULL; 3267 kfree(bp->bnapi); 3268 bp->bnapi = NULL; 3269 } else { 3270 bnxt_clear_ring_indices(bp); 3271 } 3272 } 3273 3274 static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init) 3275 { 3276 int i, j, rc, size, arr_size; 3277 void *bnapi; 3278 3279 if (irq_re_init) { 3280 /* Allocate bnapi mem pointer array and mem block for 3281 * all queues 3282 */ 3283 arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) * 3284 bp->cp_nr_rings); 3285 size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi)); 3286 bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL); 3287 if (!bnapi) 3288 return -ENOMEM; 3289 3290 bp->bnapi = bnapi; 3291 bnapi += arr_size; 3292 for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) { 3293 bp->bnapi[i] = bnapi; 3294 bp->bnapi[i]->index = i; 3295 bp->bnapi[i]->bp = bp; 3296 } 3297 3298 bp->rx_ring = kcalloc(bp->rx_nr_rings, 3299 sizeof(struct bnxt_rx_ring_info), 3300 GFP_KERNEL); 3301 if (!bp->rx_ring) 3302 return -ENOMEM; 3303 3304 for (i = 0; i < bp->rx_nr_rings; i++) { 3305 bp->rx_ring[i].bnapi = bp->bnapi[i]; 3306 bp->bnapi[i]->rx_ring = &bp->rx_ring[i]; 3307 } 3308 3309 bp->tx_ring = kcalloc(bp->tx_nr_rings, 3310 sizeof(struct bnxt_tx_ring_info), 3311 GFP_KERNEL); 3312 if (!bp->tx_ring) 3313 return -ENOMEM; 3314 3315 bp->tx_ring_map = kcalloc(bp->tx_nr_rings, sizeof(u16), 3316 GFP_KERNEL); 3317 3318 if (!bp->tx_ring_map) 3319 return -ENOMEM; 3320 3321 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 3322 j = 0; 3323 else 3324 j = bp->rx_nr_rings; 3325 3326 for (i = 0; i < bp->tx_nr_rings; i++, j++) { 3327 bp->tx_ring[i].bnapi = bp->bnapi[j]; 3328 bp->bnapi[j]->tx_ring = &bp->tx_ring[i]; 3329 bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i; 3330 if (i >= bp->tx_nr_rings_xdp) { 3331 bp->tx_ring[i].txq_index = i - 3332 bp->tx_nr_rings_xdp; 3333 bp->bnapi[j]->tx_int = bnxt_tx_int; 3334 } else { 3335 bp->bnapi[j]->flags |= BNXT_NAPI_FLAG_XDP; 3336 bp->bnapi[j]->tx_int = bnxt_tx_int_xdp; 3337 } 3338 } 3339 3340 rc = bnxt_alloc_stats(bp); 3341 if (rc) 3342 goto alloc_mem_err; 3343 3344 rc = bnxt_alloc_ntp_fltrs(bp); 3345 if (rc) 3346 goto alloc_mem_err; 3347 3348 rc = bnxt_alloc_vnics(bp); 3349 if (rc) 3350 goto alloc_mem_err; 3351 } 3352 3353 bnxt_init_ring_struct(bp); 3354 3355 rc = bnxt_alloc_rx_rings(bp); 3356 if (rc) 3357 goto alloc_mem_err; 3358 3359 rc = bnxt_alloc_tx_rings(bp); 3360 if (rc) 3361 goto alloc_mem_err; 3362 3363 rc = bnxt_alloc_cp_rings(bp); 3364 if (rc) 3365 goto alloc_mem_err; 3366 3367 bp->vnic_info[0].flags |= BNXT_VNIC_RSS_FLAG | BNXT_VNIC_MCAST_FLAG | 3368 BNXT_VNIC_UCAST_FLAG; 3369 rc = bnxt_alloc_vnic_attributes(bp); 3370 if (rc) 3371 goto alloc_mem_err; 3372 return 0; 3373 3374 alloc_mem_err: 3375 bnxt_free_mem(bp, true); 3376 return rc; 3377 } 3378 3379 static void bnxt_disable_int(struct bnxt *bp) 3380 { 3381 int i; 3382 3383 if (!bp->bnapi) 3384 return; 3385 3386 for (i = 0; i < bp->cp_nr_rings; i++) { 3387 struct bnxt_napi *bnapi = bp->bnapi[i]; 3388 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 3389 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 3390 3391 if (ring->fw_ring_id != INVALID_HW_RING_ID) 3392 BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons); 3393 } 3394 } 3395 3396 static int bnxt_cp_num_to_irq_num(struct bnxt *bp, int n) 3397 { 3398 struct bnxt_napi *bnapi = bp->bnapi[n]; 3399 struct bnxt_cp_ring_info *cpr; 3400 3401 cpr = &bnapi->cp_ring; 3402 return cpr->cp_ring_struct.map_idx; 3403 } 3404 3405 static void bnxt_disable_int_sync(struct bnxt *bp) 3406 { 3407 int i; 3408 3409 atomic_inc(&bp->intr_sem); 3410 3411 bnxt_disable_int(bp); 3412 for (i = 0; i < bp->cp_nr_rings; i++) { 3413 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 3414 3415 synchronize_irq(bp->irq_tbl[map_idx].vector); 3416 } 3417 } 3418 3419 static void bnxt_enable_int(struct bnxt *bp) 3420 { 3421 int i; 3422 3423 atomic_set(&bp->intr_sem, 0); 3424 for (i = 0; i < bp->cp_nr_rings; i++) { 3425 struct bnxt_napi *bnapi = bp->bnapi[i]; 3426 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 3427 3428 BNXT_CP_DB_REARM(cpr->cp_doorbell, cpr->cp_raw_cons); 3429 } 3430 } 3431 3432 void bnxt_hwrm_cmd_hdr_init(struct bnxt *bp, void *request, u16 req_type, 3433 u16 cmpl_ring, u16 target_id) 3434 { 3435 struct input *req = request; 3436 3437 req->req_type = cpu_to_le16(req_type); 3438 req->cmpl_ring = cpu_to_le16(cmpl_ring); 3439 req->target_id = cpu_to_le16(target_id); 3440 req->resp_addr = cpu_to_le64(bp->hwrm_cmd_resp_dma_addr); 3441 } 3442 3443 static int bnxt_hwrm_do_send_msg(struct bnxt *bp, void *msg, u32 msg_len, 3444 int timeout, bool silent) 3445 { 3446 int i, intr_process, rc, tmo_count; 3447 struct input *req = msg; 3448 u32 *data = msg; 3449 __le32 *resp_len; 3450 u8 *valid; 3451 u16 cp_ring_id, len = 0; 3452 struct hwrm_err_output *resp = bp->hwrm_cmd_resp_addr; 3453 u16 max_req_len = BNXT_HWRM_MAX_REQ_LEN; 3454 struct hwrm_short_input short_input = {0}; 3455 3456 req->seq_id = cpu_to_le16(bp->hwrm_cmd_seq++); 3457 memset(resp, 0, PAGE_SIZE); 3458 cp_ring_id = le16_to_cpu(req->cmpl_ring); 3459 intr_process = (cp_ring_id == INVALID_HW_RING_ID) ? 0 : 1; 3460 3461 if (bp->flags & BNXT_FLAG_SHORT_CMD) { 3462 void *short_cmd_req = bp->hwrm_short_cmd_req_addr; 3463 3464 memcpy(short_cmd_req, req, msg_len); 3465 memset(short_cmd_req + msg_len, 0, BNXT_HWRM_MAX_REQ_LEN - 3466 msg_len); 3467 3468 short_input.req_type = req->req_type; 3469 short_input.signature = 3470 cpu_to_le16(SHORT_REQ_SIGNATURE_SHORT_CMD); 3471 short_input.size = cpu_to_le16(msg_len); 3472 short_input.req_addr = 3473 cpu_to_le64(bp->hwrm_short_cmd_req_dma_addr); 3474 3475 data = (u32 *)&short_input; 3476 msg_len = sizeof(short_input); 3477 3478 /* Sync memory write before updating doorbell */ 3479 wmb(); 3480 3481 max_req_len = BNXT_HWRM_SHORT_REQ_LEN; 3482 } 3483 3484 /* Write request msg to hwrm channel */ 3485 __iowrite32_copy(bp->bar0, data, msg_len / 4); 3486 3487 for (i = msg_len; i < max_req_len; i += 4) 3488 writel(0, bp->bar0 + i); 3489 3490 /* currently supports only one outstanding message */ 3491 if (intr_process) 3492 bp->hwrm_intr_seq_id = le16_to_cpu(req->seq_id); 3493 3494 /* Ring channel doorbell */ 3495 writel(1, bp->bar0 + 0x100); 3496 3497 if (!timeout) 3498 timeout = DFLT_HWRM_CMD_TIMEOUT; 3499 /* convert timeout to usec */ 3500 timeout *= 1000; 3501 3502 i = 0; 3503 /* Short timeout for the first few iterations: 3504 * number of loops = number of loops for short timeout + 3505 * number of loops for standard timeout. 3506 */ 3507 tmo_count = HWRM_SHORT_TIMEOUT_COUNTER; 3508 timeout = timeout - HWRM_SHORT_MIN_TIMEOUT * HWRM_SHORT_TIMEOUT_COUNTER; 3509 tmo_count += DIV_ROUND_UP(timeout, HWRM_MIN_TIMEOUT); 3510 resp_len = bp->hwrm_cmd_resp_addr + HWRM_RESP_LEN_OFFSET; 3511 if (intr_process) { 3512 /* Wait until hwrm response cmpl interrupt is processed */ 3513 while (bp->hwrm_intr_seq_id != HWRM_SEQ_ID_INVALID && 3514 i++ < tmo_count) { 3515 /* on first few passes, just barely sleep */ 3516 if (i < HWRM_SHORT_TIMEOUT_COUNTER) 3517 usleep_range(HWRM_SHORT_MIN_TIMEOUT, 3518 HWRM_SHORT_MAX_TIMEOUT); 3519 else 3520 usleep_range(HWRM_MIN_TIMEOUT, 3521 HWRM_MAX_TIMEOUT); 3522 } 3523 3524 if (bp->hwrm_intr_seq_id != HWRM_SEQ_ID_INVALID) { 3525 netdev_err(bp->dev, "Resp cmpl intr err msg: 0x%x\n", 3526 le16_to_cpu(req->req_type)); 3527 return -1; 3528 } 3529 len = (le32_to_cpu(*resp_len) & HWRM_RESP_LEN_MASK) >> 3530 HWRM_RESP_LEN_SFT; 3531 valid = bp->hwrm_cmd_resp_addr + len - 1; 3532 } else { 3533 int j; 3534 3535 /* Check if response len is updated */ 3536 for (i = 0; i < tmo_count; i++) { 3537 len = (le32_to_cpu(*resp_len) & HWRM_RESP_LEN_MASK) >> 3538 HWRM_RESP_LEN_SFT; 3539 if (len) 3540 break; 3541 /* on first few passes, just barely sleep */ 3542 if (i < DFLT_HWRM_CMD_TIMEOUT) 3543 usleep_range(HWRM_SHORT_MIN_TIMEOUT, 3544 HWRM_SHORT_MAX_TIMEOUT); 3545 else 3546 usleep_range(HWRM_MIN_TIMEOUT, 3547 HWRM_MAX_TIMEOUT); 3548 } 3549 3550 if (i >= tmo_count) { 3551 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d\n", 3552 HWRM_TOTAL_TIMEOUT(i), 3553 le16_to_cpu(req->req_type), 3554 le16_to_cpu(req->seq_id), len); 3555 return -1; 3556 } 3557 3558 /* Last byte of resp contains valid bit */ 3559 valid = bp->hwrm_cmd_resp_addr + len - 1; 3560 for (j = 0; j < HWRM_VALID_BIT_DELAY_USEC; j++) { 3561 /* make sure we read from updated DMA memory */ 3562 dma_rmb(); 3563 if (*valid) 3564 break; 3565 udelay(1); 3566 } 3567 3568 if (j >= HWRM_VALID_BIT_DELAY_USEC) { 3569 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d v:%d\n", 3570 HWRM_TOTAL_TIMEOUT(i), 3571 le16_to_cpu(req->req_type), 3572 le16_to_cpu(req->seq_id), len, *valid); 3573 return -1; 3574 } 3575 } 3576 3577 /* Zero valid bit for compatibility. Valid bit in an older spec 3578 * may become a new field in a newer spec. We must make sure that 3579 * a new field not implemented by old spec will read zero. 3580 */ 3581 *valid = 0; 3582 rc = le16_to_cpu(resp->error_code); 3583 if (rc && !silent) 3584 netdev_err(bp->dev, "hwrm req_type 0x%x seq id 0x%x error 0x%x\n", 3585 le16_to_cpu(resp->req_type), 3586 le16_to_cpu(resp->seq_id), rc); 3587 return rc; 3588 } 3589 3590 int _hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout) 3591 { 3592 return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, false); 3593 } 3594 3595 int _hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len, 3596 int timeout) 3597 { 3598 return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true); 3599 } 3600 3601 int hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout) 3602 { 3603 int rc; 3604 3605 mutex_lock(&bp->hwrm_cmd_lock); 3606 rc = _hwrm_send_message(bp, msg, msg_len, timeout); 3607 mutex_unlock(&bp->hwrm_cmd_lock); 3608 return rc; 3609 } 3610 3611 int hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len, 3612 int timeout) 3613 { 3614 int rc; 3615 3616 mutex_lock(&bp->hwrm_cmd_lock); 3617 rc = bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true); 3618 mutex_unlock(&bp->hwrm_cmd_lock); 3619 return rc; 3620 } 3621 3622 int bnxt_hwrm_func_rgtr_async_events(struct bnxt *bp, unsigned long *bmap, 3623 int bmap_size) 3624 { 3625 struct hwrm_func_drv_rgtr_input req = {0}; 3626 DECLARE_BITMAP(async_events_bmap, 256); 3627 u32 *events = (u32 *)async_events_bmap; 3628 int i; 3629 3630 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1); 3631 3632 req.enables = 3633 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD); 3634 3635 memset(async_events_bmap, 0, sizeof(async_events_bmap)); 3636 for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++) 3637 __set_bit(bnxt_async_events_arr[i], async_events_bmap); 3638 3639 if (bmap && bmap_size) { 3640 for (i = 0; i < bmap_size; i++) { 3641 if (test_bit(i, bmap)) 3642 __set_bit(i, async_events_bmap); 3643 } 3644 } 3645 3646 for (i = 0; i < 8; i++) 3647 req.async_event_fwd[i] |= cpu_to_le32(events[i]); 3648 3649 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3650 } 3651 3652 static int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp) 3653 { 3654 struct hwrm_func_drv_rgtr_input req = {0}; 3655 3656 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1); 3657 3658 req.enables = 3659 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE | 3660 FUNC_DRV_RGTR_REQ_ENABLES_VER); 3661 3662 req.os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX); 3663 req.flags = cpu_to_le32(FUNC_DRV_RGTR_REQ_FLAGS_16BIT_VER_MODE); 3664 req.ver_maj_8b = DRV_VER_MAJ; 3665 req.ver_min_8b = DRV_VER_MIN; 3666 req.ver_upd_8b = DRV_VER_UPD; 3667 req.ver_maj = cpu_to_le16(DRV_VER_MAJ); 3668 req.ver_min = cpu_to_le16(DRV_VER_MIN); 3669 req.ver_upd = cpu_to_le16(DRV_VER_UPD); 3670 3671 if (BNXT_PF(bp)) { 3672 u32 data[8]; 3673 int i; 3674 3675 memset(data, 0, sizeof(data)); 3676 for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) { 3677 u16 cmd = bnxt_vf_req_snif[i]; 3678 unsigned int bit, idx; 3679 3680 idx = cmd / 32; 3681 bit = cmd % 32; 3682 data[idx] |= 1 << bit; 3683 } 3684 3685 for (i = 0; i < 8; i++) 3686 req.vf_req_fwd[i] = cpu_to_le32(data[i]); 3687 3688 req.enables |= 3689 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD); 3690 } 3691 3692 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3693 } 3694 3695 static int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp) 3696 { 3697 struct hwrm_func_drv_unrgtr_input req = {0}; 3698 3699 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_UNRGTR, -1, -1); 3700 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3701 } 3702 3703 static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type) 3704 { 3705 u32 rc = 0; 3706 struct hwrm_tunnel_dst_port_free_input req = {0}; 3707 3708 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_FREE, -1, -1); 3709 req.tunnel_type = tunnel_type; 3710 3711 switch (tunnel_type) { 3712 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN: 3713 req.tunnel_dst_port_id = bp->vxlan_fw_dst_port_id; 3714 break; 3715 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE: 3716 req.tunnel_dst_port_id = bp->nge_fw_dst_port_id; 3717 break; 3718 default: 3719 break; 3720 } 3721 3722 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3723 if (rc) 3724 netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n", 3725 rc); 3726 return rc; 3727 } 3728 3729 static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port, 3730 u8 tunnel_type) 3731 { 3732 u32 rc = 0; 3733 struct hwrm_tunnel_dst_port_alloc_input req = {0}; 3734 struct hwrm_tunnel_dst_port_alloc_output *resp = bp->hwrm_cmd_resp_addr; 3735 3736 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_ALLOC, -1, -1); 3737 3738 req.tunnel_type = tunnel_type; 3739 req.tunnel_dst_port_val = port; 3740 3741 mutex_lock(&bp->hwrm_cmd_lock); 3742 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3743 if (rc) { 3744 netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n", 3745 rc); 3746 goto err_out; 3747 } 3748 3749 switch (tunnel_type) { 3750 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN: 3751 bp->vxlan_fw_dst_port_id = resp->tunnel_dst_port_id; 3752 break; 3753 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE: 3754 bp->nge_fw_dst_port_id = resp->tunnel_dst_port_id; 3755 break; 3756 default: 3757 break; 3758 } 3759 3760 err_out: 3761 mutex_unlock(&bp->hwrm_cmd_lock); 3762 return rc; 3763 } 3764 3765 static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id) 3766 { 3767 struct hwrm_cfa_l2_set_rx_mask_input req = {0}; 3768 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 3769 3770 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_SET_RX_MASK, -1, -1); 3771 req.vnic_id = cpu_to_le32(vnic->fw_vnic_id); 3772 3773 req.num_mc_entries = cpu_to_le32(vnic->mc_list_count); 3774 req.mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping); 3775 req.mask = cpu_to_le32(vnic->rx_mask); 3776 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3777 } 3778 3779 #ifdef CONFIG_RFS_ACCEL 3780 static int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp, 3781 struct bnxt_ntuple_filter *fltr) 3782 { 3783 struct hwrm_cfa_ntuple_filter_free_input req = {0}; 3784 3785 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_FREE, -1, -1); 3786 req.ntuple_filter_id = fltr->filter_id; 3787 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3788 } 3789 3790 #define BNXT_NTP_FLTR_FLAGS \ 3791 (CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID | \ 3792 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE | \ 3793 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR | \ 3794 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE | \ 3795 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR | \ 3796 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK | \ 3797 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR | \ 3798 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK | \ 3799 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL | \ 3800 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT | \ 3801 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK | \ 3802 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT | \ 3803 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK | \ 3804 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID) 3805 3806 #define BNXT_NTP_TUNNEL_FLTR_FLAG \ 3807 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE 3808 3809 static int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp, 3810 struct bnxt_ntuple_filter *fltr) 3811 { 3812 int rc = 0; 3813 struct hwrm_cfa_ntuple_filter_alloc_input req = {0}; 3814 struct hwrm_cfa_ntuple_filter_alloc_output *resp = 3815 bp->hwrm_cmd_resp_addr; 3816 struct flow_keys *keys = &fltr->fkeys; 3817 struct bnxt_vnic_info *vnic = &bp->vnic_info[fltr->rxq + 1]; 3818 3819 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_ALLOC, -1, -1); 3820 req.l2_filter_id = bp->vnic_info[0].fw_l2_filter_id[fltr->l2_fltr_idx]; 3821 3822 req.enables = cpu_to_le32(BNXT_NTP_FLTR_FLAGS); 3823 3824 req.ethertype = htons(ETH_P_IP); 3825 memcpy(req.src_macaddr, fltr->src_mac_addr, ETH_ALEN); 3826 req.ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4; 3827 req.ip_protocol = keys->basic.ip_proto; 3828 3829 if (keys->basic.n_proto == htons(ETH_P_IPV6)) { 3830 int i; 3831 3832 req.ethertype = htons(ETH_P_IPV6); 3833 req.ip_addr_type = 3834 CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6; 3835 *(struct in6_addr *)&req.src_ipaddr[0] = 3836 keys->addrs.v6addrs.src; 3837 *(struct in6_addr *)&req.dst_ipaddr[0] = 3838 keys->addrs.v6addrs.dst; 3839 for (i = 0; i < 4; i++) { 3840 req.src_ipaddr_mask[i] = cpu_to_be32(0xffffffff); 3841 req.dst_ipaddr_mask[i] = cpu_to_be32(0xffffffff); 3842 } 3843 } else { 3844 req.src_ipaddr[0] = keys->addrs.v4addrs.src; 3845 req.src_ipaddr_mask[0] = cpu_to_be32(0xffffffff); 3846 req.dst_ipaddr[0] = keys->addrs.v4addrs.dst; 3847 req.dst_ipaddr_mask[0] = cpu_to_be32(0xffffffff); 3848 } 3849 if (keys->control.flags & FLOW_DIS_ENCAPSULATION) { 3850 req.enables |= cpu_to_le32(BNXT_NTP_TUNNEL_FLTR_FLAG); 3851 req.tunnel_type = 3852 CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL; 3853 } 3854 3855 req.src_port = keys->ports.src; 3856 req.src_port_mask = cpu_to_be16(0xffff); 3857 req.dst_port = keys->ports.dst; 3858 req.dst_port_mask = cpu_to_be16(0xffff); 3859 3860 req.dst_id = cpu_to_le16(vnic->fw_vnic_id); 3861 mutex_lock(&bp->hwrm_cmd_lock); 3862 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3863 if (!rc) 3864 fltr->filter_id = resp->ntuple_filter_id; 3865 mutex_unlock(&bp->hwrm_cmd_lock); 3866 return rc; 3867 } 3868 #endif 3869 3870 static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx, 3871 u8 *mac_addr) 3872 { 3873 u32 rc = 0; 3874 struct hwrm_cfa_l2_filter_alloc_input req = {0}; 3875 struct hwrm_cfa_l2_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr; 3876 3877 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_ALLOC, -1, -1); 3878 req.flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX); 3879 if (!BNXT_CHIP_TYPE_NITRO_A0(bp)) 3880 req.flags |= 3881 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST); 3882 req.dst_id = cpu_to_le16(bp->vnic_info[vnic_id].fw_vnic_id); 3883 req.enables = 3884 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR | 3885 CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID | 3886 CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK); 3887 memcpy(req.l2_addr, mac_addr, ETH_ALEN); 3888 req.l2_addr_mask[0] = 0xff; 3889 req.l2_addr_mask[1] = 0xff; 3890 req.l2_addr_mask[2] = 0xff; 3891 req.l2_addr_mask[3] = 0xff; 3892 req.l2_addr_mask[4] = 0xff; 3893 req.l2_addr_mask[5] = 0xff; 3894 3895 mutex_lock(&bp->hwrm_cmd_lock); 3896 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3897 if (!rc) 3898 bp->vnic_info[vnic_id].fw_l2_filter_id[idx] = 3899 resp->l2_filter_id; 3900 mutex_unlock(&bp->hwrm_cmd_lock); 3901 return rc; 3902 } 3903 3904 static int bnxt_hwrm_clear_vnic_filter(struct bnxt *bp) 3905 { 3906 u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */ 3907 int rc = 0; 3908 3909 /* Any associated ntuple filters will also be cleared by firmware. */ 3910 mutex_lock(&bp->hwrm_cmd_lock); 3911 for (i = 0; i < num_of_vnics; i++) { 3912 struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; 3913 3914 for (j = 0; j < vnic->uc_filter_count; j++) { 3915 struct hwrm_cfa_l2_filter_free_input req = {0}; 3916 3917 bnxt_hwrm_cmd_hdr_init(bp, &req, 3918 HWRM_CFA_L2_FILTER_FREE, -1, -1); 3919 3920 req.l2_filter_id = vnic->fw_l2_filter_id[j]; 3921 3922 rc = _hwrm_send_message(bp, &req, sizeof(req), 3923 HWRM_CMD_TIMEOUT); 3924 } 3925 vnic->uc_filter_count = 0; 3926 } 3927 mutex_unlock(&bp->hwrm_cmd_lock); 3928 3929 return rc; 3930 } 3931 3932 static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags) 3933 { 3934 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 3935 struct hwrm_vnic_tpa_cfg_input req = {0}; 3936 3937 if (vnic->fw_vnic_id == INVALID_HW_RING_ID) 3938 return 0; 3939 3940 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_TPA_CFG, -1, -1); 3941 3942 if (tpa_flags) { 3943 u16 mss = bp->dev->mtu - 40; 3944 u32 nsegs, n, segs = 0, flags; 3945 3946 flags = VNIC_TPA_CFG_REQ_FLAGS_TPA | 3947 VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA | 3948 VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE | 3949 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN | 3950 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ; 3951 if (tpa_flags & BNXT_FLAG_GRO) 3952 flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO; 3953 3954 req.flags = cpu_to_le32(flags); 3955 3956 req.enables = 3957 cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS | 3958 VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS | 3959 VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN); 3960 3961 /* Number of segs are log2 units, and first packet is not 3962 * included as part of this units. 3963 */ 3964 if (mss <= BNXT_RX_PAGE_SIZE) { 3965 n = BNXT_RX_PAGE_SIZE / mss; 3966 nsegs = (MAX_SKB_FRAGS - 1) * n; 3967 } else { 3968 n = mss / BNXT_RX_PAGE_SIZE; 3969 if (mss & (BNXT_RX_PAGE_SIZE - 1)) 3970 n++; 3971 nsegs = (MAX_SKB_FRAGS - n) / n; 3972 } 3973 3974 segs = ilog2(nsegs); 3975 req.max_agg_segs = cpu_to_le16(segs); 3976 req.max_aggs = cpu_to_le16(VNIC_TPA_CFG_REQ_MAX_AGGS_MAX); 3977 3978 req.min_agg_len = cpu_to_le32(512); 3979 } 3980 req.vnic_id = cpu_to_le16(vnic->fw_vnic_id); 3981 3982 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3983 } 3984 3985 static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss) 3986 { 3987 u32 i, j, max_rings; 3988 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 3989 struct hwrm_vnic_rss_cfg_input req = {0}; 3990 3991 if (vnic->fw_rss_cos_lb_ctx[0] == INVALID_HW_RING_ID) 3992 return 0; 3993 3994 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1); 3995 if (set_rss) { 3996 req.hash_type = cpu_to_le32(bp->rss_hash_cfg); 3997 if (vnic->flags & BNXT_VNIC_RSS_FLAG) { 3998 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 3999 max_rings = bp->rx_nr_rings - 1; 4000 else 4001 max_rings = bp->rx_nr_rings; 4002 } else { 4003 max_rings = 1; 4004 } 4005 4006 /* Fill the RSS indirection table with ring group ids */ 4007 for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++, j++) { 4008 if (j == max_rings) 4009 j = 0; 4010 vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]); 4011 } 4012 4013 req.ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr); 4014 req.hash_key_tbl_addr = 4015 cpu_to_le64(vnic->rss_hash_key_dma_addr); 4016 } 4017 req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]); 4018 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4019 } 4020 4021 static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id) 4022 { 4023 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 4024 struct hwrm_vnic_plcmodes_cfg_input req = {0}; 4025 4026 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_PLCMODES_CFG, -1, -1); 4027 req.flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT | 4028 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 | 4029 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6); 4030 req.enables = 4031 cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID | 4032 VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID); 4033 /* thresholds not implemented in firmware yet */ 4034 req.jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh); 4035 req.hds_threshold = cpu_to_le16(bp->rx_copy_thresh); 4036 req.vnic_id = cpu_to_le32(vnic->fw_vnic_id); 4037 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4038 } 4039 4040 static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id, 4041 u16 ctx_idx) 4042 { 4043 struct hwrm_vnic_rss_cos_lb_ctx_free_input req = {0}; 4044 4045 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_FREE, -1, -1); 4046 req.rss_cos_lb_ctx_id = 4047 cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx]); 4048 4049 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4050 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID; 4051 } 4052 4053 static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp) 4054 { 4055 int i, j; 4056 4057 for (i = 0; i < bp->nr_vnics; i++) { 4058 struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; 4059 4060 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) { 4061 if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID) 4062 bnxt_hwrm_vnic_ctx_free_one(bp, i, j); 4063 } 4064 } 4065 bp->rsscos_nr_ctxs = 0; 4066 } 4067 4068 static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id, u16 ctx_idx) 4069 { 4070 int rc; 4071 struct hwrm_vnic_rss_cos_lb_ctx_alloc_input req = {0}; 4072 struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp = 4073 bp->hwrm_cmd_resp_addr; 4074 4075 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC, -1, 4076 -1); 4077 4078 mutex_lock(&bp->hwrm_cmd_lock); 4079 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4080 if (!rc) 4081 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = 4082 le16_to_cpu(resp->rss_cos_lb_ctx_id); 4083 mutex_unlock(&bp->hwrm_cmd_lock); 4084 4085 return rc; 4086 } 4087 4088 static u32 bnxt_get_roce_vnic_mode(struct bnxt *bp) 4089 { 4090 if (bp->flags & BNXT_FLAG_ROCE_MIRROR_CAP) 4091 return VNIC_CFG_REQ_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE; 4092 return VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE; 4093 } 4094 4095 int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id) 4096 { 4097 unsigned int ring = 0, grp_idx; 4098 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 4099 struct hwrm_vnic_cfg_input req = {0}; 4100 u16 def_vlan = 0; 4101 4102 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_CFG, -1, -1); 4103 4104 req.enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP); 4105 /* Only RSS support for now TBD: COS & LB */ 4106 if (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID) { 4107 req.rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]); 4108 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE | 4109 VNIC_CFG_REQ_ENABLES_MRU); 4110 } else if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) { 4111 req.rss_rule = 4112 cpu_to_le16(bp->vnic_info[0].fw_rss_cos_lb_ctx[0]); 4113 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE | 4114 VNIC_CFG_REQ_ENABLES_MRU); 4115 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE); 4116 } else { 4117 req.rss_rule = cpu_to_le16(0xffff); 4118 } 4119 4120 if (BNXT_CHIP_TYPE_NITRO_A0(bp) && 4121 (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID)) { 4122 req.cos_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[1]); 4123 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_COS_RULE); 4124 } else { 4125 req.cos_rule = cpu_to_le16(0xffff); 4126 } 4127 4128 if (vnic->flags & BNXT_VNIC_RSS_FLAG) 4129 ring = 0; 4130 else if (vnic->flags & BNXT_VNIC_RFS_FLAG) 4131 ring = vnic_id - 1; 4132 else if ((vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp)) 4133 ring = bp->rx_nr_rings - 1; 4134 4135 grp_idx = bp->rx_ring[ring].bnapi->index; 4136 req.vnic_id = cpu_to_le16(vnic->fw_vnic_id); 4137 req.dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id); 4138 4139 req.lb_rule = cpu_to_le16(0xffff); 4140 req.mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + ETH_FCS_LEN + 4141 VLAN_HLEN); 4142 4143 #ifdef CONFIG_BNXT_SRIOV 4144 if (BNXT_VF(bp)) 4145 def_vlan = bp->vf.vlan; 4146 #endif 4147 if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan) 4148 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE); 4149 if (!vnic_id && bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP)) 4150 req.flags |= cpu_to_le32(bnxt_get_roce_vnic_mode(bp)); 4151 4152 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4153 } 4154 4155 static int bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id) 4156 { 4157 u32 rc = 0; 4158 4159 if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) { 4160 struct hwrm_vnic_free_input req = {0}; 4161 4162 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_FREE, -1, -1); 4163 req.vnic_id = 4164 cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id); 4165 4166 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4167 if (rc) 4168 return rc; 4169 bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID; 4170 } 4171 return rc; 4172 } 4173 4174 static void bnxt_hwrm_vnic_free(struct bnxt *bp) 4175 { 4176 u16 i; 4177 4178 for (i = 0; i < bp->nr_vnics; i++) 4179 bnxt_hwrm_vnic_free_one(bp, i); 4180 } 4181 4182 static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id, 4183 unsigned int start_rx_ring_idx, 4184 unsigned int nr_rings) 4185 { 4186 int rc = 0; 4187 unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings; 4188 struct hwrm_vnic_alloc_input req = {0}; 4189 struct hwrm_vnic_alloc_output *resp = bp->hwrm_cmd_resp_addr; 4190 4191 /* map ring groups to this vnic */ 4192 for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) { 4193 grp_idx = bp->rx_ring[i].bnapi->index; 4194 if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) { 4195 netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n", 4196 j, nr_rings); 4197 break; 4198 } 4199 bp->vnic_info[vnic_id].fw_grp_ids[j] = 4200 bp->grp_info[grp_idx].fw_grp_id; 4201 } 4202 4203 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[0] = INVALID_HW_RING_ID; 4204 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[1] = INVALID_HW_RING_ID; 4205 if (vnic_id == 0) 4206 req.flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT); 4207 4208 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_ALLOC, -1, -1); 4209 4210 mutex_lock(&bp->hwrm_cmd_lock); 4211 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4212 if (!rc) 4213 bp->vnic_info[vnic_id].fw_vnic_id = le32_to_cpu(resp->vnic_id); 4214 mutex_unlock(&bp->hwrm_cmd_lock); 4215 return rc; 4216 } 4217 4218 static int bnxt_hwrm_vnic_qcaps(struct bnxt *bp) 4219 { 4220 struct hwrm_vnic_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 4221 struct hwrm_vnic_qcaps_input req = {0}; 4222 int rc; 4223 4224 if (bp->hwrm_spec_code < 0x10600) 4225 return 0; 4226 4227 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_QCAPS, -1, -1); 4228 mutex_lock(&bp->hwrm_cmd_lock); 4229 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4230 if (!rc) { 4231 u32 flags = le32_to_cpu(resp->flags); 4232 4233 if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP) 4234 bp->flags |= BNXT_FLAG_NEW_RSS_CAP; 4235 if (flags & 4236 VNIC_QCAPS_RESP_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_CAP) 4237 bp->flags |= BNXT_FLAG_ROCE_MIRROR_CAP; 4238 } 4239 mutex_unlock(&bp->hwrm_cmd_lock); 4240 return rc; 4241 } 4242 4243 static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp) 4244 { 4245 u16 i; 4246 u32 rc = 0; 4247 4248 mutex_lock(&bp->hwrm_cmd_lock); 4249 for (i = 0; i < bp->rx_nr_rings; i++) { 4250 struct hwrm_ring_grp_alloc_input req = {0}; 4251 struct hwrm_ring_grp_alloc_output *resp = 4252 bp->hwrm_cmd_resp_addr; 4253 unsigned int grp_idx = bp->rx_ring[i].bnapi->index; 4254 4255 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_ALLOC, -1, -1); 4256 4257 req.cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id); 4258 req.rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id); 4259 req.ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id); 4260 req.sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx); 4261 4262 rc = _hwrm_send_message(bp, &req, sizeof(req), 4263 HWRM_CMD_TIMEOUT); 4264 if (rc) 4265 break; 4266 4267 bp->grp_info[grp_idx].fw_grp_id = 4268 le32_to_cpu(resp->ring_group_id); 4269 } 4270 mutex_unlock(&bp->hwrm_cmd_lock); 4271 return rc; 4272 } 4273 4274 static int bnxt_hwrm_ring_grp_free(struct bnxt *bp) 4275 { 4276 u16 i; 4277 u32 rc = 0; 4278 struct hwrm_ring_grp_free_input req = {0}; 4279 4280 if (!bp->grp_info) 4281 return 0; 4282 4283 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_FREE, -1, -1); 4284 4285 mutex_lock(&bp->hwrm_cmd_lock); 4286 for (i = 0; i < bp->cp_nr_rings; i++) { 4287 if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID) 4288 continue; 4289 req.ring_group_id = 4290 cpu_to_le32(bp->grp_info[i].fw_grp_id); 4291 4292 rc = _hwrm_send_message(bp, &req, sizeof(req), 4293 HWRM_CMD_TIMEOUT); 4294 if (rc) 4295 break; 4296 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID; 4297 } 4298 mutex_unlock(&bp->hwrm_cmd_lock); 4299 return rc; 4300 } 4301 4302 static int hwrm_ring_alloc_send_msg(struct bnxt *bp, 4303 struct bnxt_ring_struct *ring, 4304 u32 ring_type, u32 map_index) 4305 { 4306 int rc = 0, err = 0; 4307 struct hwrm_ring_alloc_input req = {0}; 4308 struct hwrm_ring_alloc_output *resp = bp->hwrm_cmd_resp_addr; 4309 struct bnxt_ring_grp_info *grp_info; 4310 u16 ring_id; 4311 4312 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_ALLOC, -1, -1); 4313 4314 req.enables = 0; 4315 if (ring->nr_pages > 1) { 4316 req.page_tbl_addr = cpu_to_le64(ring->pg_tbl_map); 4317 /* Page size is in log2 units */ 4318 req.page_size = BNXT_PAGE_SHIFT; 4319 req.page_tbl_depth = 1; 4320 } else { 4321 req.page_tbl_addr = cpu_to_le64(ring->dma_arr[0]); 4322 } 4323 req.fbo = 0; 4324 /* Association of ring index with doorbell index and MSIX number */ 4325 req.logical_id = cpu_to_le16(map_index); 4326 4327 switch (ring_type) { 4328 case HWRM_RING_ALLOC_TX: 4329 req.ring_type = RING_ALLOC_REQ_RING_TYPE_TX; 4330 /* Association of transmit ring with completion ring */ 4331 grp_info = &bp->grp_info[ring->grp_idx]; 4332 req.cmpl_ring_id = cpu_to_le16(grp_info->cp_fw_ring_id); 4333 req.length = cpu_to_le32(bp->tx_ring_mask + 1); 4334 req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx); 4335 req.queue_id = cpu_to_le16(ring->queue_id); 4336 break; 4337 case HWRM_RING_ALLOC_RX: 4338 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX; 4339 req.length = cpu_to_le32(bp->rx_ring_mask + 1); 4340 break; 4341 case HWRM_RING_ALLOC_AGG: 4342 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX; 4343 req.length = cpu_to_le32(bp->rx_agg_ring_mask + 1); 4344 break; 4345 case HWRM_RING_ALLOC_CMPL: 4346 req.ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL; 4347 req.length = cpu_to_le32(bp->cp_ring_mask + 1); 4348 if (bp->flags & BNXT_FLAG_USING_MSIX) 4349 req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX; 4350 break; 4351 default: 4352 netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n", 4353 ring_type); 4354 return -1; 4355 } 4356 4357 mutex_lock(&bp->hwrm_cmd_lock); 4358 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4359 err = le16_to_cpu(resp->error_code); 4360 ring_id = le16_to_cpu(resp->ring_id); 4361 mutex_unlock(&bp->hwrm_cmd_lock); 4362 4363 if (rc || err) { 4364 netdev_err(bp->dev, "hwrm_ring_alloc type %d failed. rc:%x err:%x\n", 4365 ring_type, rc, err); 4366 return -EIO; 4367 } 4368 ring->fw_ring_id = ring_id; 4369 return rc; 4370 } 4371 4372 static int bnxt_hwrm_set_async_event_cr(struct bnxt *bp, int idx) 4373 { 4374 int rc; 4375 4376 if (BNXT_PF(bp)) { 4377 struct hwrm_func_cfg_input req = {0}; 4378 4379 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1); 4380 req.fid = cpu_to_le16(0xffff); 4381 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR); 4382 req.async_event_cr = cpu_to_le16(idx); 4383 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4384 } else { 4385 struct hwrm_func_vf_cfg_input req = {0}; 4386 4387 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_VF_CFG, -1, -1); 4388 req.enables = 4389 cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR); 4390 req.async_event_cr = cpu_to_le16(idx); 4391 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4392 } 4393 return rc; 4394 } 4395 4396 static int bnxt_hwrm_ring_alloc(struct bnxt *bp) 4397 { 4398 int i, rc = 0; 4399 4400 for (i = 0; i < bp->cp_nr_rings; i++) { 4401 struct bnxt_napi *bnapi = bp->bnapi[i]; 4402 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 4403 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 4404 u32 map_idx = ring->map_idx; 4405 4406 cpr->cp_doorbell = bp->bar1 + map_idx * 0x80; 4407 rc = hwrm_ring_alloc_send_msg(bp, ring, HWRM_RING_ALLOC_CMPL, 4408 map_idx); 4409 if (rc) 4410 goto err_out; 4411 BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons); 4412 bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id; 4413 4414 if (!i) { 4415 rc = bnxt_hwrm_set_async_event_cr(bp, ring->fw_ring_id); 4416 if (rc) 4417 netdev_warn(bp->dev, "Failed to set async event completion ring.\n"); 4418 } 4419 } 4420 4421 for (i = 0; i < bp->tx_nr_rings; i++) { 4422 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 4423 struct bnxt_ring_struct *ring = &txr->tx_ring_struct; 4424 u32 map_idx = i; 4425 4426 rc = hwrm_ring_alloc_send_msg(bp, ring, HWRM_RING_ALLOC_TX, 4427 map_idx); 4428 if (rc) 4429 goto err_out; 4430 txr->tx_doorbell = bp->bar1 + map_idx * 0x80; 4431 } 4432 4433 for (i = 0; i < bp->rx_nr_rings; i++) { 4434 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 4435 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct; 4436 u32 map_idx = rxr->bnapi->index; 4437 4438 rc = hwrm_ring_alloc_send_msg(bp, ring, HWRM_RING_ALLOC_RX, 4439 map_idx); 4440 if (rc) 4441 goto err_out; 4442 rxr->rx_doorbell = bp->bar1 + map_idx * 0x80; 4443 writel(DB_KEY_RX | rxr->rx_prod, rxr->rx_doorbell); 4444 bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id; 4445 } 4446 4447 if (bp->flags & BNXT_FLAG_AGG_RINGS) { 4448 for (i = 0; i < bp->rx_nr_rings; i++) { 4449 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 4450 struct bnxt_ring_struct *ring = 4451 &rxr->rx_agg_ring_struct; 4452 u32 grp_idx = ring->grp_idx; 4453 u32 map_idx = grp_idx + bp->rx_nr_rings; 4454 4455 rc = hwrm_ring_alloc_send_msg(bp, ring, 4456 HWRM_RING_ALLOC_AGG, 4457 map_idx); 4458 if (rc) 4459 goto err_out; 4460 4461 rxr->rx_agg_doorbell = bp->bar1 + map_idx * 0x80; 4462 writel(DB_KEY_RX | rxr->rx_agg_prod, 4463 rxr->rx_agg_doorbell); 4464 bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id; 4465 } 4466 } 4467 err_out: 4468 return rc; 4469 } 4470 4471 static int hwrm_ring_free_send_msg(struct bnxt *bp, 4472 struct bnxt_ring_struct *ring, 4473 u32 ring_type, int cmpl_ring_id) 4474 { 4475 int rc; 4476 struct hwrm_ring_free_input req = {0}; 4477 struct hwrm_ring_free_output *resp = bp->hwrm_cmd_resp_addr; 4478 u16 error_code; 4479 4480 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_FREE, cmpl_ring_id, -1); 4481 req.ring_type = ring_type; 4482 req.ring_id = cpu_to_le16(ring->fw_ring_id); 4483 4484 mutex_lock(&bp->hwrm_cmd_lock); 4485 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4486 error_code = le16_to_cpu(resp->error_code); 4487 mutex_unlock(&bp->hwrm_cmd_lock); 4488 4489 if (rc || error_code) { 4490 netdev_err(bp->dev, "hwrm_ring_free type %d failed. rc:%x err:%x\n", 4491 ring_type, rc, error_code); 4492 return -EIO; 4493 } 4494 return 0; 4495 } 4496 4497 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path) 4498 { 4499 int i; 4500 4501 if (!bp->bnapi) 4502 return; 4503 4504 for (i = 0; i < bp->tx_nr_rings; i++) { 4505 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 4506 struct bnxt_ring_struct *ring = &txr->tx_ring_struct; 4507 u32 grp_idx = txr->bnapi->index; 4508 u32 cmpl_ring_id = bp->grp_info[grp_idx].cp_fw_ring_id; 4509 4510 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 4511 hwrm_ring_free_send_msg(bp, ring, 4512 RING_FREE_REQ_RING_TYPE_TX, 4513 close_path ? cmpl_ring_id : 4514 INVALID_HW_RING_ID); 4515 ring->fw_ring_id = INVALID_HW_RING_ID; 4516 } 4517 } 4518 4519 for (i = 0; i < bp->rx_nr_rings; i++) { 4520 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 4521 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct; 4522 u32 grp_idx = rxr->bnapi->index; 4523 u32 cmpl_ring_id = bp->grp_info[grp_idx].cp_fw_ring_id; 4524 4525 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 4526 hwrm_ring_free_send_msg(bp, ring, 4527 RING_FREE_REQ_RING_TYPE_RX, 4528 close_path ? cmpl_ring_id : 4529 INVALID_HW_RING_ID); 4530 ring->fw_ring_id = INVALID_HW_RING_ID; 4531 bp->grp_info[grp_idx].rx_fw_ring_id = 4532 INVALID_HW_RING_ID; 4533 } 4534 } 4535 4536 for (i = 0; i < bp->rx_nr_rings; i++) { 4537 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 4538 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct; 4539 u32 grp_idx = rxr->bnapi->index; 4540 u32 cmpl_ring_id = bp->grp_info[grp_idx].cp_fw_ring_id; 4541 4542 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 4543 hwrm_ring_free_send_msg(bp, ring, 4544 RING_FREE_REQ_RING_TYPE_RX, 4545 close_path ? cmpl_ring_id : 4546 INVALID_HW_RING_ID); 4547 ring->fw_ring_id = INVALID_HW_RING_ID; 4548 bp->grp_info[grp_idx].agg_fw_ring_id = 4549 INVALID_HW_RING_ID; 4550 } 4551 } 4552 4553 /* The completion rings are about to be freed. After that the 4554 * IRQ doorbell will not work anymore. So we need to disable 4555 * IRQ here. 4556 */ 4557 bnxt_disable_int_sync(bp); 4558 4559 for (i = 0; i < bp->cp_nr_rings; i++) { 4560 struct bnxt_napi *bnapi = bp->bnapi[i]; 4561 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 4562 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 4563 4564 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 4565 hwrm_ring_free_send_msg(bp, ring, 4566 RING_FREE_REQ_RING_TYPE_L2_CMPL, 4567 INVALID_HW_RING_ID); 4568 ring->fw_ring_id = INVALID_HW_RING_ID; 4569 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID; 4570 } 4571 } 4572 } 4573 4574 static int bnxt_hwrm_get_rings(struct bnxt *bp) 4575 { 4576 struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 4577 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 4578 struct hwrm_func_qcfg_input req = {0}; 4579 int rc; 4580 4581 if (bp->hwrm_spec_code < 0x10601) 4582 return 0; 4583 4584 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1); 4585 req.fid = cpu_to_le16(0xffff); 4586 mutex_lock(&bp->hwrm_cmd_lock); 4587 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4588 if (rc) { 4589 mutex_unlock(&bp->hwrm_cmd_lock); 4590 return -EIO; 4591 } 4592 4593 hw_resc->resv_tx_rings = le16_to_cpu(resp->alloc_tx_rings); 4594 if (bp->flags & BNXT_FLAG_NEW_RM) { 4595 u16 cp, stats; 4596 4597 hw_resc->resv_rx_rings = le16_to_cpu(resp->alloc_rx_rings); 4598 hw_resc->resv_hw_ring_grps = 4599 le32_to_cpu(resp->alloc_hw_ring_grps); 4600 hw_resc->resv_vnics = le16_to_cpu(resp->alloc_vnics); 4601 cp = le16_to_cpu(resp->alloc_cmpl_rings); 4602 stats = le16_to_cpu(resp->alloc_stat_ctx); 4603 cp = min_t(u16, cp, stats); 4604 hw_resc->resv_cp_rings = cp; 4605 } 4606 mutex_unlock(&bp->hwrm_cmd_lock); 4607 return 0; 4608 } 4609 4610 /* Caller must hold bp->hwrm_cmd_lock */ 4611 int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings) 4612 { 4613 struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 4614 struct hwrm_func_qcfg_input req = {0}; 4615 int rc; 4616 4617 if (bp->hwrm_spec_code < 0x10601) 4618 return 0; 4619 4620 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1); 4621 req.fid = cpu_to_le16(fid); 4622 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4623 if (!rc) 4624 *tx_rings = le16_to_cpu(resp->alloc_tx_rings); 4625 4626 return rc; 4627 } 4628 4629 static void 4630 __bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, struct hwrm_func_cfg_input *req, 4631 int tx_rings, int rx_rings, int ring_grps, 4632 int cp_rings, int vnics) 4633 { 4634 u32 enables = 0; 4635 4636 bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_CFG, -1, -1); 4637 req->fid = cpu_to_le16(0xffff); 4638 enables |= tx_rings ? FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS : 0; 4639 req->num_tx_rings = cpu_to_le16(tx_rings); 4640 if (bp->flags & BNXT_FLAG_NEW_RM) { 4641 enables |= rx_rings ? FUNC_CFG_REQ_ENABLES_NUM_RX_RINGS : 0; 4642 enables |= cp_rings ? FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS | 4643 FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0; 4644 enables |= ring_grps ? 4645 FUNC_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0; 4646 enables |= vnics ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0; 4647 4648 req->num_rx_rings = cpu_to_le16(rx_rings); 4649 req->num_hw_ring_grps = cpu_to_le16(ring_grps); 4650 req->num_cmpl_rings = cpu_to_le16(cp_rings); 4651 req->num_stat_ctxs = req->num_cmpl_rings; 4652 req->num_vnics = cpu_to_le16(vnics); 4653 } 4654 req->enables = cpu_to_le32(enables); 4655 } 4656 4657 static void 4658 __bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, 4659 struct hwrm_func_vf_cfg_input *req, int tx_rings, 4660 int rx_rings, int ring_grps, int cp_rings, 4661 int vnics) 4662 { 4663 u32 enables = 0; 4664 4665 bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_VF_CFG, -1, -1); 4666 enables |= tx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_TX_RINGS : 0; 4667 enables |= rx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_RX_RINGS : 0; 4668 enables |= cp_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS | 4669 FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0; 4670 enables |= ring_grps ? FUNC_VF_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0; 4671 enables |= vnics ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0; 4672 4673 req->num_tx_rings = cpu_to_le16(tx_rings); 4674 req->num_rx_rings = cpu_to_le16(rx_rings); 4675 req->num_hw_ring_grps = cpu_to_le16(ring_grps); 4676 req->num_cmpl_rings = cpu_to_le16(cp_rings); 4677 req->num_stat_ctxs = req->num_cmpl_rings; 4678 req->num_vnics = cpu_to_le16(vnics); 4679 4680 req->enables = cpu_to_le32(enables); 4681 } 4682 4683 static int 4684 bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 4685 int ring_grps, int cp_rings, int vnics) 4686 { 4687 struct hwrm_func_cfg_input req = {0}; 4688 int rc; 4689 4690 __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps, 4691 cp_rings, vnics); 4692 if (!req.enables) 4693 return 0; 4694 4695 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4696 if (rc) 4697 return -ENOMEM; 4698 4699 if (bp->hwrm_spec_code < 0x10601) 4700 bp->hw_resc.resv_tx_rings = tx_rings; 4701 4702 rc = bnxt_hwrm_get_rings(bp); 4703 return rc; 4704 } 4705 4706 static int 4707 bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 4708 int ring_grps, int cp_rings, int vnics) 4709 { 4710 struct hwrm_func_vf_cfg_input req = {0}; 4711 int rc; 4712 4713 if (!(bp->flags & BNXT_FLAG_NEW_RM)) { 4714 bp->hw_resc.resv_tx_rings = tx_rings; 4715 return 0; 4716 } 4717 4718 __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps, 4719 cp_rings, vnics); 4720 req.enables |= cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS | 4721 FUNC_VF_CFG_REQ_ENABLES_NUM_L2_CTXS); 4722 req.num_rsscos_ctxs = cpu_to_le16(BNXT_VF_MAX_RSS_CTX); 4723 req.num_l2_ctxs = cpu_to_le16(BNXT_VF_MAX_L2_CTX); 4724 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4725 if (rc) 4726 return -ENOMEM; 4727 4728 rc = bnxt_hwrm_get_rings(bp); 4729 return rc; 4730 } 4731 4732 static int bnxt_hwrm_reserve_rings(struct bnxt *bp, int tx, int rx, int grp, 4733 int cp, int vnic) 4734 { 4735 if (BNXT_PF(bp)) 4736 return bnxt_hwrm_reserve_pf_rings(bp, tx, rx, grp, cp, vnic); 4737 else 4738 return bnxt_hwrm_reserve_vf_rings(bp, tx, rx, grp, cp, vnic); 4739 } 4740 4741 static int bnxt_cp_rings_in_use(struct bnxt *bp) 4742 { 4743 int cp = bp->cp_nr_rings; 4744 int ulp_msix, ulp_base; 4745 4746 ulp_msix = bnxt_get_ulp_msix_num(bp); 4747 if (ulp_msix) { 4748 ulp_base = bnxt_get_ulp_msix_base(bp); 4749 cp += ulp_msix; 4750 if ((ulp_base + ulp_msix) > cp) 4751 cp = ulp_base + ulp_msix; 4752 } 4753 return cp; 4754 } 4755 4756 static bool bnxt_need_reserve_rings(struct bnxt *bp) 4757 { 4758 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 4759 int cp = bnxt_cp_rings_in_use(bp); 4760 int rx = bp->rx_nr_rings; 4761 int vnic = 1, grp = rx; 4762 4763 if (bp->hwrm_spec_code < 0x10601) 4764 return false; 4765 4766 if (hw_resc->resv_tx_rings != bp->tx_nr_rings) 4767 return true; 4768 4769 if (bp->flags & BNXT_FLAG_RFS) 4770 vnic = rx + 1; 4771 if (bp->flags & BNXT_FLAG_AGG_RINGS) 4772 rx <<= 1; 4773 if ((bp->flags & BNXT_FLAG_NEW_RM) && 4774 (hw_resc->resv_rx_rings != rx || hw_resc->resv_cp_rings != cp || 4775 hw_resc->resv_hw_ring_grps != grp || hw_resc->resv_vnics != vnic)) 4776 return true; 4777 return false; 4778 } 4779 4780 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max, 4781 bool shared); 4782 4783 static int __bnxt_reserve_rings(struct bnxt *bp) 4784 { 4785 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 4786 int cp = bnxt_cp_rings_in_use(bp); 4787 int tx = bp->tx_nr_rings; 4788 int rx = bp->rx_nr_rings; 4789 int grp, rx_rings, rc; 4790 bool sh = false; 4791 int vnic = 1; 4792 4793 if (!bnxt_need_reserve_rings(bp)) 4794 return 0; 4795 4796 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 4797 sh = true; 4798 if (bp->flags & BNXT_FLAG_RFS) 4799 vnic = rx + 1; 4800 if (bp->flags & BNXT_FLAG_AGG_RINGS) 4801 rx <<= 1; 4802 grp = bp->rx_nr_rings; 4803 4804 rc = bnxt_hwrm_reserve_rings(bp, tx, rx, grp, cp, vnic); 4805 if (rc) 4806 return rc; 4807 4808 tx = hw_resc->resv_tx_rings; 4809 if (bp->flags & BNXT_FLAG_NEW_RM) { 4810 rx = hw_resc->resv_rx_rings; 4811 cp = hw_resc->resv_cp_rings; 4812 grp = hw_resc->resv_hw_ring_grps; 4813 vnic = hw_resc->resv_vnics; 4814 } 4815 4816 rx_rings = rx; 4817 if (bp->flags & BNXT_FLAG_AGG_RINGS) { 4818 if (rx >= 2) { 4819 rx_rings = rx >> 1; 4820 } else { 4821 if (netif_running(bp->dev)) 4822 return -ENOMEM; 4823 4824 bp->flags &= ~BNXT_FLAG_AGG_RINGS; 4825 bp->flags |= BNXT_FLAG_NO_AGG_RINGS; 4826 bp->dev->hw_features &= ~NETIF_F_LRO; 4827 bp->dev->features &= ~NETIF_F_LRO; 4828 bnxt_set_ring_params(bp); 4829 } 4830 } 4831 rx_rings = min_t(int, rx_rings, grp); 4832 rc = bnxt_trim_rings(bp, &rx_rings, &tx, cp, sh); 4833 if (bp->flags & BNXT_FLAG_AGG_RINGS) 4834 rx = rx_rings << 1; 4835 cp = sh ? max_t(int, tx, rx_rings) : tx + rx_rings; 4836 bp->tx_nr_rings = tx; 4837 bp->rx_nr_rings = rx_rings; 4838 bp->cp_nr_rings = cp; 4839 4840 if (!tx || !rx || !cp || !grp || !vnic) 4841 return -ENOMEM; 4842 4843 return rc; 4844 } 4845 4846 static int bnxt_hwrm_check_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 4847 int ring_grps, int cp_rings, int vnics) 4848 { 4849 struct hwrm_func_vf_cfg_input req = {0}; 4850 u32 flags; 4851 int rc; 4852 4853 if (!(bp->flags & BNXT_FLAG_NEW_RM)) 4854 return 0; 4855 4856 __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps, 4857 cp_rings, vnics); 4858 flags = FUNC_VF_CFG_REQ_FLAGS_TX_ASSETS_TEST | 4859 FUNC_VF_CFG_REQ_FLAGS_RX_ASSETS_TEST | 4860 FUNC_VF_CFG_REQ_FLAGS_CMPL_ASSETS_TEST | 4861 FUNC_VF_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST | 4862 FUNC_VF_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST | 4863 FUNC_VF_CFG_REQ_FLAGS_VNIC_ASSETS_TEST; 4864 4865 req.flags = cpu_to_le32(flags); 4866 rc = hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4867 if (rc) 4868 return -ENOMEM; 4869 return 0; 4870 } 4871 4872 static int bnxt_hwrm_check_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 4873 int ring_grps, int cp_rings, int vnics) 4874 { 4875 struct hwrm_func_cfg_input req = {0}; 4876 u32 flags; 4877 int rc; 4878 4879 __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps, 4880 cp_rings, vnics); 4881 flags = FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST; 4882 if (bp->flags & BNXT_FLAG_NEW_RM) 4883 flags |= FUNC_CFG_REQ_FLAGS_RX_ASSETS_TEST | 4884 FUNC_CFG_REQ_FLAGS_CMPL_ASSETS_TEST | 4885 FUNC_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST | 4886 FUNC_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST | 4887 FUNC_CFG_REQ_FLAGS_VNIC_ASSETS_TEST; 4888 4889 req.flags = cpu_to_le32(flags); 4890 rc = hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4891 if (rc) 4892 return -ENOMEM; 4893 return 0; 4894 } 4895 4896 static int bnxt_hwrm_check_rings(struct bnxt *bp, int tx_rings, int rx_rings, 4897 int ring_grps, int cp_rings, int vnics) 4898 { 4899 if (bp->hwrm_spec_code < 0x10801) 4900 return 0; 4901 4902 if (BNXT_PF(bp)) 4903 return bnxt_hwrm_check_pf_rings(bp, tx_rings, rx_rings, 4904 ring_grps, cp_rings, vnics); 4905 4906 return bnxt_hwrm_check_vf_rings(bp, tx_rings, rx_rings, ring_grps, 4907 cp_rings, vnics); 4908 } 4909 4910 static void bnxt_hwrm_set_coal_params(struct bnxt_coal *hw_coal, 4911 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req) 4912 { 4913 u16 val, tmr, max, flags; 4914 4915 max = hw_coal->bufs_per_record * 128; 4916 if (hw_coal->budget) 4917 max = hw_coal->bufs_per_record * hw_coal->budget; 4918 4919 val = clamp_t(u16, hw_coal->coal_bufs, 1, max); 4920 req->num_cmpl_aggr_int = cpu_to_le16(val); 4921 4922 /* This is a 6-bit value and must not be 0, or we'll get non stop IRQ */ 4923 val = min_t(u16, val, 63); 4924 req->num_cmpl_dma_aggr = cpu_to_le16(val); 4925 4926 /* This is a 6-bit value and must not be 0, or we'll get non stop IRQ */ 4927 val = clamp_t(u16, hw_coal->coal_bufs_irq, 1, 63); 4928 req->num_cmpl_dma_aggr_during_int = cpu_to_le16(val); 4929 4930 tmr = BNXT_USEC_TO_COAL_TIMER(hw_coal->coal_ticks); 4931 tmr = max_t(u16, tmr, 1); 4932 req->int_lat_tmr_max = cpu_to_le16(tmr); 4933 4934 /* min timer set to 1/2 of interrupt timer */ 4935 val = tmr / 2; 4936 req->int_lat_tmr_min = cpu_to_le16(val); 4937 4938 /* buf timer set to 1/4 of interrupt timer */ 4939 val = max_t(u16, tmr / 4, 1); 4940 req->cmpl_aggr_dma_tmr = cpu_to_le16(val); 4941 4942 tmr = BNXT_USEC_TO_COAL_TIMER(hw_coal->coal_ticks_irq); 4943 tmr = max_t(u16, tmr, 1); 4944 req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(tmr); 4945 4946 flags = RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET; 4947 if (hw_coal->idle_thresh && hw_coal->coal_ticks < hw_coal->idle_thresh) 4948 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE; 4949 req->flags = cpu_to_le16(flags); 4950 } 4951 4952 int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi) 4953 { 4954 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0}; 4955 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 4956 struct bnxt_coal coal; 4957 unsigned int grp_idx; 4958 4959 /* Tick values in micro seconds. 4960 * 1 coal_buf x bufs_per_record = 1 completion record. 4961 */ 4962 memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal)); 4963 4964 coal.coal_ticks = cpr->rx_ring_coal.coal_ticks; 4965 coal.coal_bufs = cpr->rx_ring_coal.coal_bufs; 4966 4967 if (!bnapi->rx_ring) 4968 return -ENODEV; 4969 4970 bnxt_hwrm_cmd_hdr_init(bp, &req_rx, 4971 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1); 4972 4973 bnxt_hwrm_set_coal_params(&coal, &req_rx); 4974 4975 grp_idx = bnapi->index; 4976 req_rx.ring_id = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id); 4977 4978 return hwrm_send_message(bp, &req_rx, sizeof(req_rx), 4979 HWRM_CMD_TIMEOUT); 4980 } 4981 4982 int bnxt_hwrm_set_coal(struct bnxt *bp) 4983 { 4984 int i, rc = 0; 4985 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0}, 4986 req_tx = {0}, *req; 4987 4988 bnxt_hwrm_cmd_hdr_init(bp, &req_rx, 4989 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1); 4990 bnxt_hwrm_cmd_hdr_init(bp, &req_tx, 4991 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1); 4992 4993 bnxt_hwrm_set_coal_params(&bp->rx_coal, &req_rx); 4994 bnxt_hwrm_set_coal_params(&bp->tx_coal, &req_tx); 4995 4996 mutex_lock(&bp->hwrm_cmd_lock); 4997 for (i = 0; i < bp->cp_nr_rings; i++) { 4998 struct bnxt_napi *bnapi = bp->bnapi[i]; 4999 5000 req = &req_rx; 5001 if (!bnapi->rx_ring) 5002 req = &req_tx; 5003 req->ring_id = cpu_to_le16(bp->grp_info[i].cp_fw_ring_id); 5004 5005 rc = _hwrm_send_message(bp, req, sizeof(*req), 5006 HWRM_CMD_TIMEOUT); 5007 if (rc) 5008 break; 5009 } 5010 mutex_unlock(&bp->hwrm_cmd_lock); 5011 return rc; 5012 } 5013 5014 static int bnxt_hwrm_stat_ctx_free(struct bnxt *bp) 5015 { 5016 int rc = 0, i; 5017 struct hwrm_stat_ctx_free_input req = {0}; 5018 5019 if (!bp->bnapi) 5020 return 0; 5021 5022 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 5023 return 0; 5024 5025 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_FREE, -1, -1); 5026 5027 mutex_lock(&bp->hwrm_cmd_lock); 5028 for (i = 0; i < bp->cp_nr_rings; i++) { 5029 struct bnxt_napi *bnapi = bp->bnapi[i]; 5030 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 5031 5032 if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) { 5033 req.stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id); 5034 5035 rc = _hwrm_send_message(bp, &req, sizeof(req), 5036 HWRM_CMD_TIMEOUT); 5037 if (rc) 5038 break; 5039 5040 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID; 5041 } 5042 } 5043 mutex_unlock(&bp->hwrm_cmd_lock); 5044 return rc; 5045 } 5046 5047 static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp) 5048 { 5049 int rc = 0, i; 5050 struct hwrm_stat_ctx_alloc_input req = {0}; 5051 struct hwrm_stat_ctx_alloc_output *resp = bp->hwrm_cmd_resp_addr; 5052 5053 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 5054 return 0; 5055 5056 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_ALLOC, -1, -1); 5057 5058 req.update_period_ms = cpu_to_le32(bp->stats_coal_ticks / 1000); 5059 5060 mutex_lock(&bp->hwrm_cmd_lock); 5061 for (i = 0; i < bp->cp_nr_rings; i++) { 5062 struct bnxt_napi *bnapi = bp->bnapi[i]; 5063 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 5064 5065 req.stats_dma_addr = cpu_to_le64(cpr->hw_stats_map); 5066 5067 rc = _hwrm_send_message(bp, &req, sizeof(req), 5068 HWRM_CMD_TIMEOUT); 5069 if (rc) 5070 break; 5071 5072 cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id); 5073 5074 bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id; 5075 } 5076 mutex_unlock(&bp->hwrm_cmd_lock); 5077 return rc; 5078 } 5079 5080 static int bnxt_hwrm_func_qcfg(struct bnxt *bp) 5081 { 5082 struct hwrm_func_qcfg_input req = {0}; 5083 struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 5084 u16 flags; 5085 int rc; 5086 5087 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1); 5088 req.fid = cpu_to_le16(0xffff); 5089 mutex_lock(&bp->hwrm_cmd_lock); 5090 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5091 if (rc) 5092 goto func_qcfg_exit; 5093 5094 #ifdef CONFIG_BNXT_SRIOV 5095 if (BNXT_VF(bp)) { 5096 struct bnxt_vf_info *vf = &bp->vf; 5097 5098 vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK; 5099 } 5100 #endif 5101 flags = le16_to_cpu(resp->flags); 5102 if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED | 5103 FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED)) { 5104 bp->flags |= BNXT_FLAG_FW_LLDP_AGENT; 5105 if (flags & FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED) 5106 bp->flags |= BNXT_FLAG_FW_DCBX_AGENT; 5107 } 5108 if (BNXT_PF(bp) && (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST)) 5109 bp->flags |= BNXT_FLAG_MULTI_HOST; 5110 5111 switch (resp->port_partition_type) { 5112 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0: 5113 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5: 5114 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0: 5115 bp->port_partition_type = resp->port_partition_type; 5116 break; 5117 } 5118 if (bp->hwrm_spec_code < 0x10707 || 5119 resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEB) 5120 bp->br_mode = BRIDGE_MODE_VEB; 5121 else if (resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEPA) 5122 bp->br_mode = BRIDGE_MODE_VEPA; 5123 else 5124 bp->br_mode = BRIDGE_MODE_UNDEF; 5125 5126 bp->max_mtu = le16_to_cpu(resp->max_mtu_configured); 5127 if (!bp->max_mtu) 5128 bp->max_mtu = BNXT_MAX_MTU; 5129 5130 func_qcfg_exit: 5131 mutex_unlock(&bp->hwrm_cmd_lock); 5132 return rc; 5133 } 5134 5135 int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp, bool all) 5136 { 5137 struct hwrm_func_resource_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 5138 struct hwrm_func_resource_qcaps_input req = {0}; 5139 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 5140 int rc; 5141 5142 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESOURCE_QCAPS, -1, -1); 5143 req.fid = cpu_to_le16(0xffff); 5144 5145 mutex_lock(&bp->hwrm_cmd_lock); 5146 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5147 if (rc) { 5148 rc = -EIO; 5149 goto hwrm_func_resc_qcaps_exit; 5150 } 5151 5152 hw_resc->max_tx_sch_inputs = le16_to_cpu(resp->max_tx_scheduler_inputs); 5153 if (!all) 5154 goto hwrm_func_resc_qcaps_exit; 5155 5156 hw_resc->min_rsscos_ctxs = le16_to_cpu(resp->min_rsscos_ctx); 5157 hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx); 5158 hw_resc->min_cp_rings = le16_to_cpu(resp->min_cmpl_rings); 5159 hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings); 5160 hw_resc->min_tx_rings = le16_to_cpu(resp->min_tx_rings); 5161 hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings); 5162 hw_resc->min_rx_rings = le16_to_cpu(resp->min_rx_rings); 5163 hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings); 5164 hw_resc->min_hw_ring_grps = le16_to_cpu(resp->min_hw_ring_grps); 5165 hw_resc->max_hw_ring_grps = le16_to_cpu(resp->max_hw_ring_grps); 5166 hw_resc->min_l2_ctxs = le16_to_cpu(resp->min_l2_ctxs); 5167 hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs); 5168 hw_resc->min_vnics = le16_to_cpu(resp->min_vnics); 5169 hw_resc->max_vnics = le16_to_cpu(resp->max_vnics); 5170 hw_resc->min_stat_ctxs = le16_to_cpu(resp->min_stat_ctx); 5171 hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx); 5172 5173 if (BNXT_PF(bp)) { 5174 struct bnxt_pf_info *pf = &bp->pf; 5175 5176 pf->vf_resv_strategy = 5177 le16_to_cpu(resp->vf_reservation_strategy); 5178 if (pf->vf_resv_strategy > BNXT_VF_RESV_STRATEGY_MINIMAL) 5179 pf->vf_resv_strategy = BNXT_VF_RESV_STRATEGY_MAXIMAL; 5180 } 5181 hwrm_func_resc_qcaps_exit: 5182 mutex_unlock(&bp->hwrm_cmd_lock); 5183 return rc; 5184 } 5185 5186 static int __bnxt_hwrm_func_qcaps(struct bnxt *bp) 5187 { 5188 int rc = 0; 5189 struct hwrm_func_qcaps_input req = {0}; 5190 struct hwrm_func_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 5191 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 5192 u32 flags; 5193 5194 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCAPS, -1, -1); 5195 req.fid = cpu_to_le16(0xffff); 5196 5197 mutex_lock(&bp->hwrm_cmd_lock); 5198 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5199 if (rc) 5200 goto hwrm_func_qcaps_exit; 5201 5202 flags = le32_to_cpu(resp->flags); 5203 if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED) 5204 bp->flags |= BNXT_FLAG_ROCEV1_CAP; 5205 if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED) 5206 bp->flags |= BNXT_FLAG_ROCEV2_CAP; 5207 5208 bp->tx_push_thresh = 0; 5209 if (flags & FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED) 5210 bp->tx_push_thresh = BNXT_TX_PUSH_THRESH; 5211 5212 hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx); 5213 hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings); 5214 hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings); 5215 hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings); 5216 hw_resc->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps); 5217 if (!hw_resc->max_hw_ring_grps) 5218 hw_resc->max_hw_ring_grps = hw_resc->max_tx_rings; 5219 hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs); 5220 hw_resc->max_vnics = le16_to_cpu(resp->max_vnics); 5221 hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx); 5222 5223 if (BNXT_PF(bp)) { 5224 struct bnxt_pf_info *pf = &bp->pf; 5225 5226 pf->fw_fid = le16_to_cpu(resp->fid); 5227 pf->port_id = le16_to_cpu(resp->port_id); 5228 bp->dev->dev_port = pf->port_id; 5229 memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN); 5230 pf->first_vf_id = le16_to_cpu(resp->first_vf_id); 5231 pf->max_vfs = le16_to_cpu(resp->max_vfs); 5232 pf->max_encap_records = le32_to_cpu(resp->max_encap_records); 5233 pf->max_decap_records = le32_to_cpu(resp->max_decap_records); 5234 pf->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows); 5235 pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows); 5236 pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows); 5237 pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows); 5238 if (flags & FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED) 5239 bp->flags |= BNXT_FLAG_WOL_CAP; 5240 } else { 5241 #ifdef CONFIG_BNXT_SRIOV 5242 struct bnxt_vf_info *vf = &bp->vf; 5243 5244 vf->fw_fid = le16_to_cpu(resp->fid); 5245 memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN); 5246 #endif 5247 } 5248 5249 hwrm_func_qcaps_exit: 5250 mutex_unlock(&bp->hwrm_cmd_lock); 5251 return rc; 5252 } 5253 5254 static int bnxt_hwrm_func_qcaps(struct bnxt *bp) 5255 { 5256 int rc; 5257 5258 rc = __bnxt_hwrm_func_qcaps(bp); 5259 if (rc) 5260 return rc; 5261 if (bp->hwrm_spec_code >= 0x10803) { 5262 rc = bnxt_hwrm_func_resc_qcaps(bp, true); 5263 if (!rc) 5264 bp->flags |= BNXT_FLAG_NEW_RM; 5265 } 5266 return 0; 5267 } 5268 5269 static int bnxt_hwrm_func_reset(struct bnxt *bp) 5270 { 5271 struct hwrm_func_reset_input req = {0}; 5272 5273 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESET, -1, -1); 5274 req.enables = 0; 5275 5276 return hwrm_send_message(bp, &req, sizeof(req), HWRM_RESET_TIMEOUT); 5277 } 5278 5279 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp) 5280 { 5281 int rc = 0; 5282 struct hwrm_queue_qportcfg_input req = {0}; 5283 struct hwrm_queue_qportcfg_output *resp = bp->hwrm_cmd_resp_addr; 5284 u8 i, *qptr; 5285 5286 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_QUEUE_QPORTCFG, -1, -1); 5287 5288 mutex_lock(&bp->hwrm_cmd_lock); 5289 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5290 if (rc) 5291 goto qportcfg_exit; 5292 5293 if (!resp->max_configurable_queues) { 5294 rc = -EINVAL; 5295 goto qportcfg_exit; 5296 } 5297 bp->max_tc = resp->max_configurable_queues; 5298 bp->max_lltc = resp->max_configurable_lossless_queues; 5299 if (bp->max_tc > BNXT_MAX_QUEUE) 5300 bp->max_tc = BNXT_MAX_QUEUE; 5301 5302 if (resp->queue_cfg_info & QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG) 5303 bp->max_tc = 1; 5304 5305 if (bp->max_lltc > bp->max_tc) 5306 bp->max_lltc = bp->max_tc; 5307 5308 qptr = &resp->queue_id0; 5309 for (i = 0; i < bp->max_tc; i++) { 5310 bp->q_info[i].queue_id = *qptr++; 5311 bp->q_info[i].queue_profile = *qptr++; 5312 bp->tc_to_qidx[i] = i; 5313 } 5314 5315 qportcfg_exit: 5316 mutex_unlock(&bp->hwrm_cmd_lock); 5317 return rc; 5318 } 5319 5320 static int bnxt_hwrm_ver_get(struct bnxt *bp) 5321 { 5322 int rc; 5323 struct hwrm_ver_get_input req = {0}; 5324 struct hwrm_ver_get_output *resp = bp->hwrm_cmd_resp_addr; 5325 u32 dev_caps_cfg; 5326 5327 bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN; 5328 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VER_GET, -1, -1); 5329 req.hwrm_intf_maj = HWRM_VERSION_MAJOR; 5330 req.hwrm_intf_min = HWRM_VERSION_MINOR; 5331 req.hwrm_intf_upd = HWRM_VERSION_UPDATE; 5332 mutex_lock(&bp->hwrm_cmd_lock); 5333 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5334 if (rc) 5335 goto hwrm_ver_get_exit; 5336 5337 memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output)); 5338 5339 bp->hwrm_spec_code = resp->hwrm_intf_maj_8b << 16 | 5340 resp->hwrm_intf_min_8b << 8 | 5341 resp->hwrm_intf_upd_8b; 5342 if (resp->hwrm_intf_maj_8b < 1) { 5343 netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n", 5344 resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b, 5345 resp->hwrm_intf_upd_8b); 5346 netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n"); 5347 } 5348 snprintf(bp->fw_ver_str, BC_HWRM_STR_LEN, "%d.%d.%d.%d", 5349 resp->hwrm_fw_maj_8b, resp->hwrm_fw_min_8b, 5350 resp->hwrm_fw_bld_8b, resp->hwrm_fw_rsvd_8b); 5351 5352 bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout); 5353 if (!bp->hwrm_cmd_timeout) 5354 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT; 5355 5356 if (resp->hwrm_intf_maj_8b >= 1) 5357 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len); 5358 5359 bp->chip_num = le16_to_cpu(resp->chip_num); 5360 if (bp->chip_num == CHIP_NUM_58700 && !resp->chip_rev && 5361 !resp->chip_metal) 5362 bp->flags |= BNXT_FLAG_CHIP_NITRO_A0; 5363 5364 dev_caps_cfg = le32_to_cpu(resp->dev_caps_cfg); 5365 if ((dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) && 5366 (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED)) 5367 bp->flags |= BNXT_FLAG_SHORT_CMD; 5368 5369 hwrm_ver_get_exit: 5370 mutex_unlock(&bp->hwrm_cmd_lock); 5371 return rc; 5372 } 5373 5374 int bnxt_hwrm_fw_set_time(struct bnxt *bp) 5375 { 5376 struct hwrm_fw_set_time_input req = {0}; 5377 struct tm tm; 5378 time64_t now = ktime_get_real_seconds(); 5379 5380 if ((BNXT_VF(bp) && bp->hwrm_spec_code < 0x10901) || 5381 bp->hwrm_spec_code < 0x10400) 5382 return -EOPNOTSUPP; 5383 5384 time64_to_tm(now, 0, &tm); 5385 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FW_SET_TIME, -1, -1); 5386 req.year = cpu_to_le16(1900 + tm.tm_year); 5387 req.month = 1 + tm.tm_mon; 5388 req.day = tm.tm_mday; 5389 req.hour = tm.tm_hour; 5390 req.minute = tm.tm_min; 5391 req.second = tm.tm_sec; 5392 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5393 } 5394 5395 static int bnxt_hwrm_port_qstats(struct bnxt *bp) 5396 { 5397 int rc; 5398 struct bnxt_pf_info *pf = &bp->pf; 5399 struct hwrm_port_qstats_input req = {0}; 5400 5401 if (!(bp->flags & BNXT_FLAG_PORT_STATS)) 5402 return 0; 5403 5404 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS, -1, -1); 5405 req.port_id = cpu_to_le16(pf->port_id); 5406 req.tx_stat_host_addr = cpu_to_le64(bp->hw_tx_port_stats_map); 5407 req.rx_stat_host_addr = cpu_to_le64(bp->hw_rx_port_stats_map); 5408 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5409 return rc; 5410 } 5411 5412 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp) 5413 { 5414 struct hwrm_port_qstats_ext_input req = {0}; 5415 struct bnxt_pf_info *pf = &bp->pf; 5416 5417 if (!(bp->flags & BNXT_FLAG_PORT_STATS_EXT)) 5418 return 0; 5419 5420 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS_EXT, -1, -1); 5421 req.port_id = cpu_to_le16(pf->port_id); 5422 req.rx_stat_size = cpu_to_le16(sizeof(struct rx_port_stats_ext)); 5423 req.rx_stat_host_addr = cpu_to_le64(bp->hw_rx_port_stats_ext_map); 5424 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5425 } 5426 5427 static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp) 5428 { 5429 if (bp->vxlan_port_cnt) { 5430 bnxt_hwrm_tunnel_dst_port_free( 5431 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN); 5432 } 5433 bp->vxlan_port_cnt = 0; 5434 if (bp->nge_port_cnt) { 5435 bnxt_hwrm_tunnel_dst_port_free( 5436 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE); 5437 } 5438 bp->nge_port_cnt = 0; 5439 } 5440 5441 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa) 5442 { 5443 int rc, i; 5444 u32 tpa_flags = 0; 5445 5446 if (set_tpa) 5447 tpa_flags = bp->flags & BNXT_FLAG_TPA; 5448 for (i = 0; i < bp->nr_vnics; i++) { 5449 rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags); 5450 if (rc) { 5451 netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n", 5452 i, rc); 5453 return rc; 5454 } 5455 } 5456 return 0; 5457 } 5458 5459 static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp) 5460 { 5461 int i; 5462 5463 for (i = 0; i < bp->nr_vnics; i++) 5464 bnxt_hwrm_vnic_set_rss(bp, i, false); 5465 } 5466 5467 static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path, 5468 bool irq_re_init) 5469 { 5470 if (bp->vnic_info) { 5471 bnxt_hwrm_clear_vnic_filter(bp); 5472 /* clear all RSS setting before free vnic ctx */ 5473 bnxt_hwrm_clear_vnic_rss(bp); 5474 bnxt_hwrm_vnic_ctx_free(bp); 5475 /* before free the vnic, undo the vnic tpa settings */ 5476 if (bp->flags & BNXT_FLAG_TPA) 5477 bnxt_set_tpa(bp, false); 5478 bnxt_hwrm_vnic_free(bp); 5479 } 5480 bnxt_hwrm_ring_free(bp, close_path); 5481 bnxt_hwrm_ring_grp_free(bp); 5482 if (irq_re_init) { 5483 bnxt_hwrm_stat_ctx_free(bp); 5484 bnxt_hwrm_free_tunnel_ports(bp); 5485 } 5486 } 5487 5488 static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode) 5489 { 5490 struct hwrm_func_cfg_input req = {0}; 5491 int rc; 5492 5493 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1); 5494 req.fid = cpu_to_le16(0xffff); 5495 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_EVB_MODE); 5496 if (br_mode == BRIDGE_MODE_VEB) 5497 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEB; 5498 else if (br_mode == BRIDGE_MODE_VEPA) 5499 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEPA; 5500 else 5501 return -EINVAL; 5502 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5503 if (rc) 5504 rc = -EIO; 5505 return rc; 5506 } 5507 5508 static int bnxt_hwrm_set_cache_line_size(struct bnxt *bp, int size) 5509 { 5510 struct hwrm_func_cfg_input req = {0}; 5511 int rc; 5512 5513 if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10803) 5514 return 0; 5515 5516 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1); 5517 req.fid = cpu_to_le16(0xffff); 5518 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_CACHE_LINESIZE); 5519 req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_64; 5520 if (size == 128) 5521 req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_128; 5522 5523 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5524 if (rc) 5525 rc = -EIO; 5526 return rc; 5527 } 5528 5529 static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id) 5530 { 5531 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5532 int rc; 5533 5534 if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) 5535 goto skip_rss_ctx; 5536 5537 /* allocate context for vnic */ 5538 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 0); 5539 if (rc) { 5540 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n", 5541 vnic_id, rc); 5542 goto vnic_setup_err; 5543 } 5544 bp->rsscos_nr_ctxs++; 5545 5546 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 5547 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 1); 5548 if (rc) { 5549 netdev_err(bp->dev, "hwrm vnic %d cos ctx alloc failure rc: %x\n", 5550 vnic_id, rc); 5551 goto vnic_setup_err; 5552 } 5553 bp->rsscos_nr_ctxs++; 5554 } 5555 5556 skip_rss_ctx: 5557 /* configure default vnic, ring grp */ 5558 rc = bnxt_hwrm_vnic_cfg(bp, vnic_id); 5559 if (rc) { 5560 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n", 5561 vnic_id, rc); 5562 goto vnic_setup_err; 5563 } 5564 5565 /* Enable RSS hashing on vnic */ 5566 rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true); 5567 if (rc) { 5568 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n", 5569 vnic_id, rc); 5570 goto vnic_setup_err; 5571 } 5572 5573 if (bp->flags & BNXT_FLAG_AGG_RINGS) { 5574 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id); 5575 if (rc) { 5576 netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n", 5577 vnic_id, rc); 5578 } 5579 } 5580 5581 vnic_setup_err: 5582 return rc; 5583 } 5584 5585 static int bnxt_alloc_rfs_vnics(struct bnxt *bp) 5586 { 5587 #ifdef CONFIG_RFS_ACCEL 5588 int i, rc = 0; 5589 5590 for (i = 0; i < bp->rx_nr_rings; i++) { 5591 struct bnxt_vnic_info *vnic; 5592 u16 vnic_id = i + 1; 5593 u16 ring_id = i; 5594 5595 if (vnic_id >= bp->nr_vnics) 5596 break; 5597 5598 vnic = &bp->vnic_info[vnic_id]; 5599 vnic->flags |= BNXT_VNIC_RFS_FLAG; 5600 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP) 5601 vnic->flags |= BNXT_VNIC_RFS_NEW_RSS_FLAG; 5602 rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, ring_id, 1); 5603 if (rc) { 5604 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n", 5605 vnic_id, rc); 5606 break; 5607 } 5608 rc = bnxt_setup_vnic(bp, vnic_id); 5609 if (rc) 5610 break; 5611 } 5612 return rc; 5613 #else 5614 return 0; 5615 #endif 5616 } 5617 5618 /* Allow PF and VF with default VLAN to be in promiscuous mode */ 5619 static bool bnxt_promisc_ok(struct bnxt *bp) 5620 { 5621 #ifdef CONFIG_BNXT_SRIOV 5622 if (BNXT_VF(bp) && !bp->vf.vlan) 5623 return false; 5624 #endif 5625 return true; 5626 } 5627 5628 static int bnxt_setup_nitroa0_vnic(struct bnxt *bp) 5629 { 5630 unsigned int rc = 0; 5631 5632 rc = bnxt_hwrm_vnic_alloc(bp, 1, bp->rx_nr_rings - 1, 1); 5633 if (rc) { 5634 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n", 5635 rc); 5636 return rc; 5637 } 5638 5639 rc = bnxt_hwrm_vnic_cfg(bp, 1); 5640 if (rc) { 5641 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n", 5642 rc); 5643 return rc; 5644 } 5645 return rc; 5646 } 5647 5648 static int bnxt_cfg_rx_mode(struct bnxt *); 5649 static bool bnxt_mc_list_updated(struct bnxt *, u32 *); 5650 5651 static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init) 5652 { 5653 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 5654 int rc = 0; 5655 unsigned int rx_nr_rings = bp->rx_nr_rings; 5656 5657 if (irq_re_init) { 5658 rc = bnxt_hwrm_stat_ctx_alloc(bp); 5659 if (rc) { 5660 netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n", 5661 rc); 5662 goto err_out; 5663 } 5664 } 5665 5666 rc = bnxt_hwrm_ring_alloc(bp); 5667 if (rc) { 5668 netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc); 5669 goto err_out; 5670 } 5671 5672 rc = bnxt_hwrm_ring_grp_alloc(bp); 5673 if (rc) { 5674 netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc); 5675 goto err_out; 5676 } 5677 5678 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 5679 rx_nr_rings--; 5680 5681 /* default vnic 0 */ 5682 rc = bnxt_hwrm_vnic_alloc(bp, 0, 0, rx_nr_rings); 5683 if (rc) { 5684 netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc); 5685 goto err_out; 5686 } 5687 5688 rc = bnxt_setup_vnic(bp, 0); 5689 if (rc) 5690 goto err_out; 5691 5692 if (bp->flags & BNXT_FLAG_RFS) { 5693 rc = bnxt_alloc_rfs_vnics(bp); 5694 if (rc) 5695 goto err_out; 5696 } 5697 5698 if (bp->flags & BNXT_FLAG_TPA) { 5699 rc = bnxt_set_tpa(bp, true); 5700 if (rc) 5701 goto err_out; 5702 } 5703 5704 if (BNXT_VF(bp)) 5705 bnxt_update_vf_mac(bp); 5706 5707 /* Filter for default vnic 0 */ 5708 rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr); 5709 if (rc) { 5710 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc); 5711 goto err_out; 5712 } 5713 vnic->uc_filter_count = 1; 5714 5715 vnic->rx_mask = CFA_L2_SET_RX_MASK_REQ_MASK_BCAST; 5716 5717 if ((bp->dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp)) 5718 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 5719 5720 if (bp->dev->flags & IFF_ALLMULTI) { 5721 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 5722 vnic->mc_list_count = 0; 5723 } else { 5724 u32 mask = 0; 5725 5726 bnxt_mc_list_updated(bp, &mask); 5727 vnic->rx_mask |= mask; 5728 } 5729 5730 rc = bnxt_cfg_rx_mode(bp); 5731 if (rc) 5732 goto err_out; 5733 5734 rc = bnxt_hwrm_set_coal(bp); 5735 if (rc) 5736 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n", 5737 rc); 5738 5739 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 5740 rc = bnxt_setup_nitroa0_vnic(bp); 5741 if (rc) 5742 netdev_err(bp->dev, "Special vnic setup failure for NS2 A0 rc: %x\n", 5743 rc); 5744 } 5745 5746 if (BNXT_VF(bp)) { 5747 bnxt_hwrm_func_qcfg(bp); 5748 netdev_update_features(bp->dev); 5749 } 5750 5751 return 0; 5752 5753 err_out: 5754 bnxt_hwrm_resource_free(bp, 0, true); 5755 5756 return rc; 5757 } 5758 5759 static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init) 5760 { 5761 bnxt_hwrm_resource_free(bp, 1, irq_re_init); 5762 return 0; 5763 } 5764 5765 static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init) 5766 { 5767 bnxt_init_cp_rings(bp); 5768 bnxt_init_rx_rings(bp); 5769 bnxt_init_tx_rings(bp); 5770 bnxt_init_ring_grps(bp, irq_re_init); 5771 bnxt_init_vnics(bp); 5772 5773 return bnxt_init_chip(bp, irq_re_init); 5774 } 5775 5776 static int bnxt_set_real_num_queues(struct bnxt *bp) 5777 { 5778 int rc; 5779 struct net_device *dev = bp->dev; 5780 5781 rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings - 5782 bp->tx_nr_rings_xdp); 5783 if (rc) 5784 return rc; 5785 5786 rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings); 5787 if (rc) 5788 return rc; 5789 5790 #ifdef CONFIG_RFS_ACCEL 5791 if (bp->flags & BNXT_FLAG_RFS) 5792 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings); 5793 #endif 5794 5795 return rc; 5796 } 5797 5798 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max, 5799 bool shared) 5800 { 5801 int _rx = *rx, _tx = *tx; 5802 5803 if (shared) { 5804 *rx = min_t(int, _rx, max); 5805 *tx = min_t(int, _tx, max); 5806 } else { 5807 if (max < 2) 5808 return -ENOMEM; 5809 5810 while (_rx + _tx > max) { 5811 if (_rx > _tx && _rx > 1) 5812 _rx--; 5813 else if (_tx > 1) 5814 _tx--; 5815 } 5816 *rx = _rx; 5817 *tx = _tx; 5818 } 5819 return 0; 5820 } 5821 5822 static void bnxt_setup_msix(struct bnxt *bp) 5823 { 5824 const int len = sizeof(bp->irq_tbl[0].name); 5825 struct net_device *dev = bp->dev; 5826 int tcs, i; 5827 5828 tcs = netdev_get_num_tc(dev); 5829 if (tcs > 1) { 5830 int i, off, count; 5831 5832 for (i = 0; i < tcs; i++) { 5833 count = bp->tx_nr_rings_per_tc; 5834 off = i * count; 5835 netdev_set_tc_queue(dev, i, count, off); 5836 } 5837 } 5838 5839 for (i = 0; i < bp->cp_nr_rings; i++) { 5840 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 5841 char *attr; 5842 5843 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 5844 attr = "TxRx"; 5845 else if (i < bp->rx_nr_rings) 5846 attr = "rx"; 5847 else 5848 attr = "tx"; 5849 5850 snprintf(bp->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name, 5851 attr, i); 5852 bp->irq_tbl[map_idx].handler = bnxt_msix; 5853 } 5854 } 5855 5856 static void bnxt_setup_inta(struct bnxt *bp) 5857 { 5858 const int len = sizeof(bp->irq_tbl[0].name); 5859 5860 if (netdev_get_num_tc(bp->dev)) 5861 netdev_reset_tc(bp->dev); 5862 5863 snprintf(bp->irq_tbl[0].name, len, "%s-%s-%d", bp->dev->name, "TxRx", 5864 0); 5865 bp->irq_tbl[0].handler = bnxt_inta; 5866 } 5867 5868 static int bnxt_setup_int_mode(struct bnxt *bp) 5869 { 5870 int rc; 5871 5872 if (bp->flags & BNXT_FLAG_USING_MSIX) 5873 bnxt_setup_msix(bp); 5874 else 5875 bnxt_setup_inta(bp); 5876 5877 rc = bnxt_set_real_num_queues(bp); 5878 return rc; 5879 } 5880 5881 #ifdef CONFIG_RFS_ACCEL 5882 static unsigned int bnxt_get_max_func_rss_ctxs(struct bnxt *bp) 5883 { 5884 return bp->hw_resc.max_rsscos_ctxs; 5885 } 5886 5887 static unsigned int bnxt_get_max_func_vnics(struct bnxt *bp) 5888 { 5889 return bp->hw_resc.max_vnics; 5890 } 5891 #endif 5892 5893 unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp) 5894 { 5895 return bp->hw_resc.max_stat_ctxs; 5896 } 5897 5898 void bnxt_set_max_func_stat_ctxs(struct bnxt *bp, unsigned int max) 5899 { 5900 bp->hw_resc.max_stat_ctxs = max; 5901 } 5902 5903 unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp) 5904 { 5905 return bp->hw_resc.max_cp_rings; 5906 } 5907 5908 void bnxt_set_max_func_cp_rings(struct bnxt *bp, unsigned int max) 5909 { 5910 bp->hw_resc.max_cp_rings = max; 5911 } 5912 5913 unsigned int bnxt_get_max_func_irqs(struct bnxt *bp) 5914 { 5915 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 5916 5917 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_cp_rings); 5918 } 5919 5920 void bnxt_set_max_func_irqs(struct bnxt *bp, unsigned int max_irqs) 5921 { 5922 bp->hw_resc.max_irqs = max_irqs; 5923 } 5924 5925 int bnxt_get_avail_msix(struct bnxt *bp, int num) 5926 { 5927 int max_cp = bnxt_get_max_func_cp_rings(bp); 5928 int max_irq = bnxt_get_max_func_irqs(bp); 5929 int total_req = bp->cp_nr_rings + num; 5930 int max_idx, avail_msix; 5931 5932 max_idx = min_t(int, bp->total_irqs, max_cp); 5933 avail_msix = max_idx - bp->cp_nr_rings; 5934 if (!(bp->flags & BNXT_FLAG_NEW_RM) || avail_msix >= num) 5935 return avail_msix; 5936 5937 if (max_irq < total_req) { 5938 num = max_irq - bp->cp_nr_rings; 5939 if (num <= 0) 5940 return 0; 5941 } 5942 return num; 5943 } 5944 5945 static int bnxt_get_num_msix(struct bnxt *bp) 5946 { 5947 if (!(bp->flags & BNXT_FLAG_NEW_RM)) 5948 return bnxt_get_max_func_irqs(bp); 5949 5950 return bnxt_cp_rings_in_use(bp); 5951 } 5952 5953 static int bnxt_init_msix(struct bnxt *bp) 5954 { 5955 int i, total_vecs, max, rc = 0, min = 1, ulp_msix; 5956 struct msix_entry *msix_ent; 5957 5958 total_vecs = bnxt_get_num_msix(bp); 5959 max = bnxt_get_max_func_irqs(bp); 5960 if (total_vecs > max) 5961 total_vecs = max; 5962 5963 if (!total_vecs) 5964 return 0; 5965 5966 msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL); 5967 if (!msix_ent) 5968 return -ENOMEM; 5969 5970 for (i = 0; i < total_vecs; i++) { 5971 msix_ent[i].entry = i; 5972 msix_ent[i].vector = 0; 5973 } 5974 5975 if (!(bp->flags & BNXT_FLAG_SHARED_RINGS)) 5976 min = 2; 5977 5978 total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs); 5979 ulp_msix = bnxt_get_ulp_msix_num(bp); 5980 if (total_vecs < 0 || total_vecs < ulp_msix) { 5981 rc = -ENODEV; 5982 goto msix_setup_exit; 5983 } 5984 5985 bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL); 5986 if (bp->irq_tbl) { 5987 for (i = 0; i < total_vecs; i++) 5988 bp->irq_tbl[i].vector = msix_ent[i].vector; 5989 5990 bp->total_irqs = total_vecs; 5991 /* Trim rings based upon num of vectors allocated */ 5992 rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings, 5993 total_vecs - ulp_msix, min == 1); 5994 if (rc) 5995 goto msix_setup_exit; 5996 5997 bp->cp_nr_rings = (min == 1) ? 5998 max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) : 5999 bp->tx_nr_rings + bp->rx_nr_rings; 6000 6001 } else { 6002 rc = -ENOMEM; 6003 goto msix_setup_exit; 6004 } 6005 bp->flags |= BNXT_FLAG_USING_MSIX; 6006 kfree(msix_ent); 6007 return 0; 6008 6009 msix_setup_exit: 6010 netdev_err(bp->dev, "bnxt_init_msix err: %x\n", rc); 6011 kfree(bp->irq_tbl); 6012 bp->irq_tbl = NULL; 6013 pci_disable_msix(bp->pdev); 6014 kfree(msix_ent); 6015 return rc; 6016 } 6017 6018 static int bnxt_init_inta(struct bnxt *bp) 6019 { 6020 bp->irq_tbl = kcalloc(1, sizeof(struct bnxt_irq), GFP_KERNEL); 6021 if (!bp->irq_tbl) 6022 return -ENOMEM; 6023 6024 bp->total_irqs = 1; 6025 bp->rx_nr_rings = 1; 6026 bp->tx_nr_rings = 1; 6027 bp->cp_nr_rings = 1; 6028 bp->flags |= BNXT_FLAG_SHARED_RINGS; 6029 bp->irq_tbl[0].vector = bp->pdev->irq; 6030 return 0; 6031 } 6032 6033 static int bnxt_init_int_mode(struct bnxt *bp) 6034 { 6035 int rc = 0; 6036 6037 if (bp->flags & BNXT_FLAG_MSIX_CAP) 6038 rc = bnxt_init_msix(bp); 6039 6040 if (!(bp->flags & BNXT_FLAG_USING_MSIX) && BNXT_PF(bp)) { 6041 /* fallback to INTA */ 6042 rc = bnxt_init_inta(bp); 6043 } 6044 return rc; 6045 } 6046 6047 static void bnxt_clear_int_mode(struct bnxt *bp) 6048 { 6049 if (bp->flags & BNXT_FLAG_USING_MSIX) 6050 pci_disable_msix(bp->pdev); 6051 6052 kfree(bp->irq_tbl); 6053 bp->irq_tbl = NULL; 6054 bp->flags &= ~BNXT_FLAG_USING_MSIX; 6055 } 6056 6057 int bnxt_reserve_rings(struct bnxt *bp) 6058 { 6059 int tcs = netdev_get_num_tc(bp->dev); 6060 int rc; 6061 6062 if (!bnxt_need_reserve_rings(bp)) 6063 return 0; 6064 6065 rc = __bnxt_reserve_rings(bp); 6066 if (rc) { 6067 netdev_err(bp->dev, "ring reservation failure rc: %d\n", rc); 6068 return rc; 6069 } 6070 if ((bp->flags & BNXT_FLAG_NEW_RM) && 6071 (bnxt_get_num_msix(bp) != bp->total_irqs)) { 6072 bnxt_ulp_irq_stop(bp); 6073 bnxt_clear_int_mode(bp); 6074 rc = bnxt_init_int_mode(bp); 6075 bnxt_ulp_irq_restart(bp, rc); 6076 if (rc) 6077 return rc; 6078 } 6079 if (tcs && (bp->tx_nr_rings_per_tc * tcs != bp->tx_nr_rings)) { 6080 netdev_err(bp->dev, "tx ring reservation failure\n"); 6081 netdev_reset_tc(bp->dev); 6082 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 6083 return -ENOMEM; 6084 } 6085 bp->num_stat_ctxs = bp->cp_nr_rings; 6086 return 0; 6087 } 6088 6089 static void bnxt_free_irq(struct bnxt *bp) 6090 { 6091 struct bnxt_irq *irq; 6092 int i; 6093 6094 #ifdef CONFIG_RFS_ACCEL 6095 free_irq_cpu_rmap(bp->dev->rx_cpu_rmap); 6096 bp->dev->rx_cpu_rmap = NULL; 6097 #endif 6098 if (!bp->irq_tbl || !bp->bnapi) 6099 return; 6100 6101 for (i = 0; i < bp->cp_nr_rings; i++) { 6102 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 6103 6104 irq = &bp->irq_tbl[map_idx]; 6105 if (irq->requested) { 6106 if (irq->have_cpumask) { 6107 irq_set_affinity_hint(irq->vector, NULL); 6108 free_cpumask_var(irq->cpu_mask); 6109 irq->have_cpumask = 0; 6110 } 6111 free_irq(irq->vector, bp->bnapi[i]); 6112 } 6113 6114 irq->requested = 0; 6115 } 6116 } 6117 6118 static int bnxt_request_irq(struct bnxt *bp) 6119 { 6120 int i, j, rc = 0; 6121 unsigned long flags = 0; 6122 #ifdef CONFIG_RFS_ACCEL 6123 struct cpu_rmap *rmap; 6124 #endif 6125 6126 rc = bnxt_setup_int_mode(bp); 6127 if (rc) { 6128 netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n", 6129 rc); 6130 return rc; 6131 } 6132 #ifdef CONFIG_RFS_ACCEL 6133 rmap = bp->dev->rx_cpu_rmap; 6134 #endif 6135 if (!(bp->flags & BNXT_FLAG_USING_MSIX)) 6136 flags = IRQF_SHARED; 6137 6138 for (i = 0, j = 0; i < bp->cp_nr_rings; i++) { 6139 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 6140 struct bnxt_irq *irq = &bp->irq_tbl[map_idx]; 6141 6142 #ifdef CONFIG_RFS_ACCEL 6143 if (rmap && bp->bnapi[i]->rx_ring) { 6144 rc = irq_cpu_rmap_add(rmap, irq->vector); 6145 if (rc) 6146 netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n", 6147 j); 6148 j++; 6149 } 6150 #endif 6151 rc = request_irq(irq->vector, irq->handler, flags, irq->name, 6152 bp->bnapi[i]); 6153 if (rc) 6154 break; 6155 6156 irq->requested = 1; 6157 6158 if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) { 6159 int numa_node = dev_to_node(&bp->pdev->dev); 6160 6161 irq->have_cpumask = 1; 6162 cpumask_set_cpu(cpumask_local_spread(i, numa_node), 6163 irq->cpu_mask); 6164 rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask); 6165 if (rc) { 6166 netdev_warn(bp->dev, 6167 "Set affinity failed, IRQ = %d\n", 6168 irq->vector); 6169 break; 6170 } 6171 } 6172 } 6173 return rc; 6174 } 6175 6176 static void bnxt_del_napi(struct bnxt *bp) 6177 { 6178 int i; 6179 6180 if (!bp->bnapi) 6181 return; 6182 6183 for (i = 0; i < bp->cp_nr_rings; i++) { 6184 struct bnxt_napi *bnapi = bp->bnapi[i]; 6185 6186 napi_hash_del(&bnapi->napi); 6187 netif_napi_del(&bnapi->napi); 6188 } 6189 /* We called napi_hash_del() before netif_napi_del(), we need 6190 * to respect an RCU grace period before freeing napi structures. 6191 */ 6192 synchronize_net(); 6193 } 6194 6195 static void bnxt_init_napi(struct bnxt *bp) 6196 { 6197 int i; 6198 unsigned int cp_nr_rings = bp->cp_nr_rings; 6199 struct bnxt_napi *bnapi; 6200 6201 if (bp->flags & BNXT_FLAG_USING_MSIX) { 6202 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 6203 cp_nr_rings--; 6204 for (i = 0; i < cp_nr_rings; i++) { 6205 bnapi = bp->bnapi[i]; 6206 netif_napi_add(bp->dev, &bnapi->napi, 6207 bnxt_poll, 64); 6208 } 6209 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 6210 bnapi = bp->bnapi[cp_nr_rings]; 6211 netif_napi_add(bp->dev, &bnapi->napi, 6212 bnxt_poll_nitroa0, 64); 6213 } 6214 } else { 6215 bnapi = bp->bnapi[0]; 6216 netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll, 64); 6217 } 6218 } 6219 6220 static void bnxt_disable_napi(struct bnxt *bp) 6221 { 6222 int i; 6223 6224 if (!bp->bnapi) 6225 return; 6226 6227 for (i = 0; i < bp->cp_nr_rings; i++) { 6228 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring; 6229 6230 if (bp->bnapi[i]->rx_ring) 6231 cancel_work_sync(&cpr->dim.work); 6232 6233 napi_disable(&bp->bnapi[i]->napi); 6234 } 6235 } 6236 6237 static void bnxt_enable_napi(struct bnxt *bp) 6238 { 6239 int i; 6240 6241 for (i = 0; i < bp->cp_nr_rings; i++) { 6242 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring; 6243 bp->bnapi[i]->in_reset = false; 6244 6245 if (bp->bnapi[i]->rx_ring) { 6246 INIT_WORK(&cpr->dim.work, bnxt_dim_work); 6247 cpr->dim.mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE; 6248 } 6249 napi_enable(&bp->bnapi[i]->napi); 6250 } 6251 } 6252 6253 void bnxt_tx_disable(struct bnxt *bp) 6254 { 6255 int i; 6256 struct bnxt_tx_ring_info *txr; 6257 6258 if (bp->tx_ring) { 6259 for (i = 0; i < bp->tx_nr_rings; i++) { 6260 txr = &bp->tx_ring[i]; 6261 txr->dev_state = BNXT_DEV_STATE_CLOSING; 6262 } 6263 } 6264 /* Stop all TX queues */ 6265 netif_tx_disable(bp->dev); 6266 netif_carrier_off(bp->dev); 6267 } 6268 6269 void bnxt_tx_enable(struct bnxt *bp) 6270 { 6271 int i; 6272 struct bnxt_tx_ring_info *txr; 6273 6274 for (i = 0; i < bp->tx_nr_rings; i++) { 6275 txr = &bp->tx_ring[i]; 6276 txr->dev_state = 0; 6277 } 6278 netif_tx_wake_all_queues(bp->dev); 6279 if (bp->link_info.link_up) 6280 netif_carrier_on(bp->dev); 6281 } 6282 6283 static void bnxt_report_link(struct bnxt *bp) 6284 { 6285 if (bp->link_info.link_up) { 6286 const char *duplex; 6287 const char *flow_ctrl; 6288 u32 speed; 6289 u16 fec; 6290 6291 netif_carrier_on(bp->dev); 6292 if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL) 6293 duplex = "full"; 6294 else 6295 duplex = "half"; 6296 if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH) 6297 flow_ctrl = "ON - receive & transmit"; 6298 else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX) 6299 flow_ctrl = "ON - transmit"; 6300 else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX) 6301 flow_ctrl = "ON - receive"; 6302 else 6303 flow_ctrl = "none"; 6304 speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed); 6305 netdev_info(bp->dev, "NIC Link is Up, %u Mbps %s duplex, Flow control: %s\n", 6306 speed, duplex, flow_ctrl); 6307 if (bp->flags & BNXT_FLAG_EEE_CAP) 6308 netdev_info(bp->dev, "EEE is %s\n", 6309 bp->eee.eee_active ? "active" : 6310 "not active"); 6311 fec = bp->link_info.fec_cfg; 6312 if (!(fec & PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED)) 6313 netdev_info(bp->dev, "FEC autoneg %s encodings: %s\n", 6314 (fec & BNXT_FEC_AUTONEG) ? "on" : "off", 6315 (fec & BNXT_FEC_ENC_BASE_R) ? "BaseR" : 6316 (fec & BNXT_FEC_ENC_RS) ? "RS" : "None"); 6317 } else { 6318 netif_carrier_off(bp->dev); 6319 netdev_err(bp->dev, "NIC Link is Down\n"); 6320 } 6321 } 6322 6323 static int bnxt_hwrm_phy_qcaps(struct bnxt *bp) 6324 { 6325 int rc = 0; 6326 struct hwrm_port_phy_qcaps_input req = {0}; 6327 struct hwrm_port_phy_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 6328 struct bnxt_link_info *link_info = &bp->link_info; 6329 6330 if (bp->hwrm_spec_code < 0x10201) 6331 return 0; 6332 6333 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCAPS, -1, -1); 6334 6335 mutex_lock(&bp->hwrm_cmd_lock); 6336 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6337 if (rc) 6338 goto hwrm_phy_qcaps_exit; 6339 6340 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) { 6341 struct ethtool_eee *eee = &bp->eee; 6342 u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode); 6343 6344 bp->flags |= BNXT_FLAG_EEE_CAP; 6345 eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0); 6346 bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) & 6347 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK; 6348 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) & 6349 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK; 6350 } 6351 if (resp->supported_speeds_auto_mode) 6352 link_info->support_auto_speeds = 6353 le16_to_cpu(resp->supported_speeds_auto_mode); 6354 6355 bp->port_count = resp->port_cnt; 6356 6357 hwrm_phy_qcaps_exit: 6358 mutex_unlock(&bp->hwrm_cmd_lock); 6359 return rc; 6360 } 6361 6362 static int bnxt_update_link(struct bnxt *bp, bool chng_link_state) 6363 { 6364 int rc = 0; 6365 struct bnxt_link_info *link_info = &bp->link_info; 6366 struct hwrm_port_phy_qcfg_input req = {0}; 6367 struct hwrm_port_phy_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 6368 u8 link_up = link_info->link_up; 6369 u16 diff; 6370 6371 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCFG, -1, -1); 6372 6373 mutex_lock(&bp->hwrm_cmd_lock); 6374 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6375 if (rc) { 6376 mutex_unlock(&bp->hwrm_cmd_lock); 6377 return rc; 6378 } 6379 6380 memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp)); 6381 link_info->phy_link_status = resp->link; 6382 link_info->duplex = resp->duplex_cfg; 6383 if (bp->hwrm_spec_code >= 0x10800) 6384 link_info->duplex = resp->duplex_state; 6385 link_info->pause = resp->pause; 6386 link_info->auto_mode = resp->auto_mode; 6387 link_info->auto_pause_setting = resp->auto_pause; 6388 link_info->lp_pause = resp->link_partner_adv_pause; 6389 link_info->force_pause_setting = resp->force_pause; 6390 link_info->duplex_setting = resp->duplex_cfg; 6391 if (link_info->phy_link_status == BNXT_LINK_LINK) 6392 link_info->link_speed = le16_to_cpu(resp->link_speed); 6393 else 6394 link_info->link_speed = 0; 6395 link_info->force_link_speed = le16_to_cpu(resp->force_link_speed); 6396 link_info->support_speeds = le16_to_cpu(resp->support_speeds); 6397 link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask); 6398 link_info->lp_auto_link_speeds = 6399 le16_to_cpu(resp->link_partner_adv_speeds); 6400 link_info->preemphasis = le32_to_cpu(resp->preemphasis); 6401 link_info->phy_ver[0] = resp->phy_maj; 6402 link_info->phy_ver[1] = resp->phy_min; 6403 link_info->phy_ver[2] = resp->phy_bld; 6404 link_info->media_type = resp->media_type; 6405 link_info->phy_type = resp->phy_type; 6406 link_info->transceiver = resp->xcvr_pkg_type; 6407 link_info->phy_addr = resp->eee_config_phy_addr & 6408 PORT_PHY_QCFG_RESP_PHY_ADDR_MASK; 6409 link_info->module_status = resp->module_status; 6410 6411 if (bp->flags & BNXT_FLAG_EEE_CAP) { 6412 struct ethtool_eee *eee = &bp->eee; 6413 u16 fw_speeds; 6414 6415 eee->eee_active = 0; 6416 if (resp->eee_config_phy_addr & 6417 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) { 6418 eee->eee_active = 1; 6419 fw_speeds = le16_to_cpu( 6420 resp->link_partner_adv_eee_link_speed_mask); 6421 eee->lp_advertised = 6422 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0); 6423 } 6424 6425 /* Pull initial EEE config */ 6426 if (!chng_link_state) { 6427 if (resp->eee_config_phy_addr & 6428 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED) 6429 eee->eee_enabled = 1; 6430 6431 fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask); 6432 eee->advertised = 6433 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0); 6434 6435 if (resp->eee_config_phy_addr & 6436 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) { 6437 __le32 tmr; 6438 6439 eee->tx_lpi_enabled = 1; 6440 tmr = resp->xcvr_identifier_type_tx_lpi_timer; 6441 eee->tx_lpi_timer = le32_to_cpu(tmr) & 6442 PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK; 6443 } 6444 } 6445 } 6446 6447 link_info->fec_cfg = PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED; 6448 if (bp->hwrm_spec_code >= 0x10504) 6449 link_info->fec_cfg = le16_to_cpu(resp->fec_cfg); 6450 6451 /* TODO: need to add more logic to report VF link */ 6452 if (chng_link_state) { 6453 if (link_info->phy_link_status == BNXT_LINK_LINK) 6454 link_info->link_up = 1; 6455 else 6456 link_info->link_up = 0; 6457 if (link_up != link_info->link_up) 6458 bnxt_report_link(bp); 6459 } else { 6460 /* alwasy link down if not require to update link state */ 6461 link_info->link_up = 0; 6462 } 6463 mutex_unlock(&bp->hwrm_cmd_lock); 6464 6465 if (!BNXT_SINGLE_PF(bp)) 6466 return 0; 6467 6468 diff = link_info->support_auto_speeds ^ link_info->advertising; 6469 if ((link_info->support_auto_speeds | diff) != 6470 link_info->support_auto_speeds) { 6471 /* An advertised speed is no longer supported, so we need to 6472 * update the advertisement settings. Caller holds RTNL 6473 * so we can modify link settings. 6474 */ 6475 link_info->advertising = link_info->support_auto_speeds; 6476 if (link_info->autoneg & BNXT_AUTONEG_SPEED) 6477 bnxt_hwrm_set_link_setting(bp, true, false); 6478 } 6479 return 0; 6480 } 6481 6482 static void bnxt_get_port_module_status(struct bnxt *bp) 6483 { 6484 struct bnxt_link_info *link_info = &bp->link_info; 6485 struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp; 6486 u8 module_status; 6487 6488 if (bnxt_update_link(bp, true)) 6489 return; 6490 6491 module_status = link_info->module_status; 6492 switch (module_status) { 6493 case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX: 6494 case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN: 6495 case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG: 6496 netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n", 6497 bp->pf.port_id); 6498 if (bp->hwrm_spec_code >= 0x10201) { 6499 netdev_warn(bp->dev, "Module part number %s\n", 6500 resp->phy_vendor_partnumber); 6501 } 6502 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX) 6503 netdev_warn(bp->dev, "TX is disabled\n"); 6504 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN) 6505 netdev_warn(bp->dev, "SFP+ module is shutdown\n"); 6506 } 6507 } 6508 6509 static void 6510 bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req) 6511 { 6512 if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) { 6513 if (bp->hwrm_spec_code >= 0x10201) 6514 req->auto_pause = 6515 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE; 6516 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX) 6517 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX; 6518 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX) 6519 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX; 6520 req->enables |= 6521 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE); 6522 } else { 6523 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX) 6524 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX; 6525 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX) 6526 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX; 6527 req->enables |= 6528 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE); 6529 if (bp->hwrm_spec_code >= 0x10201) { 6530 req->auto_pause = req->force_pause; 6531 req->enables |= cpu_to_le32( 6532 PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE); 6533 } 6534 } 6535 } 6536 6537 static void bnxt_hwrm_set_link_common(struct bnxt *bp, 6538 struct hwrm_port_phy_cfg_input *req) 6539 { 6540 u8 autoneg = bp->link_info.autoneg; 6541 u16 fw_link_speed = bp->link_info.req_link_speed; 6542 u16 advertising = bp->link_info.advertising; 6543 6544 if (autoneg & BNXT_AUTONEG_SPEED) { 6545 req->auto_mode |= 6546 PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK; 6547 6548 req->enables |= cpu_to_le32( 6549 PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK); 6550 req->auto_link_speed_mask = cpu_to_le16(advertising); 6551 6552 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE); 6553 req->flags |= 6554 cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG); 6555 } else { 6556 req->force_link_speed = cpu_to_le16(fw_link_speed); 6557 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE); 6558 } 6559 6560 /* tell chimp that the setting takes effect immediately */ 6561 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY); 6562 } 6563 6564 int bnxt_hwrm_set_pause(struct bnxt *bp) 6565 { 6566 struct hwrm_port_phy_cfg_input req = {0}; 6567 int rc; 6568 6569 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1); 6570 bnxt_hwrm_set_pause_common(bp, &req); 6571 6572 if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) || 6573 bp->link_info.force_link_chng) 6574 bnxt_hwrm_set_link_common(bp, &req); 6575 6576 mutex_lock(&bp->hwrm_cmd_lock); 6577 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6578 if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) { 6579 /* since changing of pause setting doesn't trigger any link 6580 * change event, the driver needs to update the current pause 6581 * result upon successfully return of the phy_cfg command 6582 */ 6583 bp->link_info.pause = 6584 bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl; 6585 bp->link_info.auto_pause_setting = 0; 6586 if (!bp->link_info.force_link_chng) 6587 bnxt_report_link(bp); 6588 } 6589 bp->link_info.force_link_chng = false; 6590 mutex_unlock(&bp->hwrm_cmd_lock); 6591 return rc; 6592 } 6593 6594 static void bnxt_hwrm_set_eee(struct bnxt *bp, 6595 struct hwrm_port_phy_cfg_input *req) 6596 { 6597 struct ethtool_eee *eee = &bp->eee; 6598 6599 if (eee->eee_enabled) { 6600 u16 eee_speeds; 6601 u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE; 6602 6603 if (eee->tx_lpi_enabled) 6604 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE; 6605 else 6606 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE; 6607 6608 req->flags |= cpu_to_le32(flags); 6609 eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised); 6610 req->eee_link_speed_mask = cpu_to_le16(eee_speeds); 6611 req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer); 6612 } else { 6613 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE); 6614 } 6615 } 6616 6617 int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee) 6618 { 6619 struct hwrm_port_phy_cfg_input req = {0}; 6620 6621 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1); 6622 if (set_pause) 6623 bnxt_hwrm_set_pause_common(bp, &req); 6624 6625 bnxt_hwrm_set_link_common(bp, &req); 6626 6627 if (set_eee) 6628 bnxt_hwrm_set_eee(bp, &req); 6629 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6630 } 6631 6632 static int bnxt_hwrm_shutdown_link(struct bnxt *bp) 6633 { 6634 struct hwrm_port_phy_cfg_input req = {0}; 6635 6636 if (!BNXT_SINGLE_PF(bp)) 6637 return 0; 6638 6639 if (pci_num_vf(bp->pdev)) 6640 return 0; 6641 6642 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1); 6643 req.flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN); 6644 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6645 } 6646 6647 static int bnxt_hwrm_port_led_qcaps(struct bnxt *bp) 6648 { 6649 struct hwrm_port_led_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 6650 struct hwrm_port_led_qcaps_input req = {0}; 6651 struct bnxt_pf_info *pf = &bp->pf; 6652 int rc; 6653 6654 if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10601) 6655 return 0; 6656 6657 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_LED_QCAPS, -1, -1); 6658 req.port_id = cpu_to_le16(pf->port_id); 6659 mutex_lock(&bp->hwrm_cmd_lock); 6660 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6661 if (rc) { 6662 mutex_unlock(&bp->hwrm_cmd_lock); 6663 return rc; 6664 } 6665 if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) { 6666 int i; 6667 6668 bp->num_leds = resp->num_leds; 6669 memcpy(bp->leds, &resp->led0_id, sizeof(bp->leds[0]) * 6670 bp->num_leds); 6671 for (i = 0; i < bp->num_leds; i++) { 6672 struct bnxt_led_info *led = &bp->leds[i]; 6673 __le16 caps = led->led_state_caps; 6674 6675 if (!led->led_group_id || 6676 !BNXT_LED_ALT_BLINK_CAP(caps)) { 6677 bp->num_leds = 0; 6678 break; 6679 } 6680 } 6681 } 6682 mutex_unlock(&bp->hwrm_cmd_lock); 6683 return 0; 6684 } 6685 6686 int bnxt_hwrm_alloc_wol_fltr(struct bnxt *bp) 6687 { 6688 struct hwrm_wol_filter_alloc_input req = {0}; 6689 struct hwrm_wol_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr; 6690 int rc; 6691 6692 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_ALLOC, -1, -1); 6693 req.port_id = cpu_to_le16(bp->pf.port_id); 6694 req.wol_type = WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT; 6695 req.enables = cpu_to_le32(WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS); 6696 memcpy(req.mac_address, bp->dev->dev_addr, ETH_ALEN); 6697 mutex_lock(&bp->hwrm_cmd_lock); 6698 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6699 if (!rc) 6700 bp->wol_filter_id = resp->wol_filter_id; 6701 mutex_unlock(&bp->hwrm_cmd_lock); 6702 return rc; 6703 } 6704 6705 int bnxt_hwrm_free_wol_fltr(struct bnxt *bp) 6706 { 6707 struct hwrm_wol_filter_free_input req = {0}; 6708 int rc; 6709 6710 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_FREE, -1, -1); 6711 req.port_id = cpu_to_le16(bp->pf.port_id); 6712 req.enables = cpu_to_le32(WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID); 6713 req.wol_filter_id = bp->wol_filter_id; 6714 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6715 return rc; 6716 } 6717 6718 static u16 bnxt_hwrm_get_wol_fltrs(struct bnxt *bp, u16 handle) 6719 { 6720 struct hwrm_wol_filter_qcfg_input req = {0}; 6721 struct hwrm_wol_filter_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 6722 u16 next_handle = 0; 6723 int rc; 6724 6725 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_QCFG, -1, -1); 6726 req.port_id = cpu_to_le16(bp->pf.port_id); 6727 req.handle = cpu_to_le16(handle); 6728 mutex_lock(&bp->hwrm_cmd_lock); 6729 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6730 if (!rc) { 6731 next_handle = le16_to_cpu(resp->next_handle); 6732 if (next_handle != 0) { 6733 if (resp->wol_type == 6734 WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT) { 6735 bp->wol = 1; 6736 bp->wol_filter_id = resp->wol_filter_id; 6737 } 6738 } 6739 } 6740 mutex_unlock(&bp->hwrm_cmd_lock); 6741 return next_handle; 6742 } 6743 6744 static void bnxt_get_wol_settings(struct bnxt *bp) 6745 { 6746 u16 handle = 0; 6747 6748 if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_WOL_CAP)) 6749 return; 6750 6751 do { 6752 handle = bnxt_hwrm_get_wol_fltrs(bp, handle); 6753 } while (handle && handle != 0xffff); 6754 } 6755 6756 static bool bnxt_eee_config_ok(struct bnxt *bp) 6757 { 6758 struct ethtool_eee *eee = &bp->eee; 6759 struct bnxt_link_info *link_info = &bp->link_info; 6760 6761 if (!(bp->flags & BNXT_FLAG_EEE_CAP)) 6762 return true; 6763 6764 if (eee->eee_enabled) { 6765 u32 advertising = 6766 _bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0); 6767 6768 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) { 6769 eee->eee_enabled = 0; 6770 return false; 6771 } 6772 if (eee->advertised & ~advertising) { 6773 eee->advertised = advertising & eee->supported; 6774 return false; 6775 } 6776 } 6777 return true; 6778 } 6779 6780 static int bnxt_update_phy_setting(struct bnxt *bp) 6781 { 6782 int rc; 6783 bool update_link = false; 6784 bool update_pause = false; 6785 bool update_eee = false; 6786 struct bnxt_link_info *link_info = &bp->link_info; 6787 6788 rc = bnxt_update_link(bp, true); 6789 if (rc) { 6790 netdev_err(bp->dev, "failed to update link (rc: %x)\n", 6791 rc); 6792 return rc; 6793 } 6794 if (!BNXT_SINGLE_PF(bp)) 6795 return 0; 6796 6797 if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) && 6798 (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) != 6799 link_info->req_flow_ctrl) 6800 update_pause = true; 6801 if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) && 6802 link_info->force_pause_setting != link_info->req_flow_ctrl) 6803 update_pause = true; 6804 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) { 6805 if (BNXT_AUTO_MODE(link_info->auto_mode)) 6806 update_link = true; 6807 if (link_info->req_link_speed != link_info->force_link_speed) 6808 update_link = true; 6809 if (link_info->req_duplex != link_info->duplex_setting) 6810 update_link = true; 6811 } else { 6812 if (link_info->auto_mode == BNXT_LINK_AUTO_NONE) 6813 update_link = true; 6814 if (link_info->advertising != link_info->auto_link_speeds) 6815 update_link = true; 6816 } 6817 6818 /* The last close may have shutdown the link, so need to call 6819 * PHY_CFG to bring it back up. 6820 */ 6821 if (!netif_carrier_ok(bp->dev)) 6822 update_link = true; 6823 6824 if (!bnxt_eee_config_ok(bp)) 6825 update_eee = true; 6826 6827 if (update_link) 6828 rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee); 6829 else if (update_pause) 6830 rc = bnxt_hwrm_set_pause(bp); 6831 if (rc) { 6832 netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n", 6833 rc); 6834 return rc; 6835 } 6836 6837 return rc; 6838 } 6839 6840 /* Common routine to pre-map certain register block to different GRC window. 6841 * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows 6842 * in PF and 3 windows in VF that can be customized to map in different 6843 * register blocks. 6844 */ 6845 static void bnxt_preset_reg_win(struct bnxt *bp) 6846 { 6847 if (BNXT_PF(bp)) { 6848 /* CAG registers map to GRC window #4 */ 6849 writel(BNXT_CAG_REG_BASE, 6850 bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12); 6851 } 6852 } 6853 6854 static int bnxt_init_dflt_ring_mode(struct bnxt *bp); 6855 6856 static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) 6857 { 6858 int rc = 0; 6859 6860 bnxt_preset_reg_win(bp); 6861 netif_carrier_off(bp->dev); 6862 if (irq_re_init) { 6863 /* Reserve rings now if none were reserved at driver probe. */ 6864 rc = bnxt_init_dflt_ring_mode(bp); 6865 if (rc) { 6866 netdev_err(bp->dev, "Failed to reserve default rings at open\n"); 6867 return rc; 6868 } 6869 rc = bnxt_reserve_rings(bp); 6870 if (rc) 6871 return rc; 6872 } 6873 if ((bp->flags & BNXT_FLAG_RFS) && 6874 !(bp->flags & BNXT_FLAG_USING_MSIX)) { 6875 /* disable RFS if falling back to INTA */ 6876 bp->dev->hw_features &= ~NETIF_F_NTUPLE; 6877 bp->flags &= ~BNXT_FLAG_RFS; 6878 } 6879 6880 rc = bnxt_alloc_mem(bp, irq_re_init); 6881 if (rc) { 6882 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc); 6883 goto open_err_free_mem; 6884 } 6885 6886 if (irq_re_init) { 6887 bnxt_init_napi(bp); 6888 rc = bnxt_request_irq(bp); 6889 if (rc) { 6890 netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc); 6891 goto open_err; 6892 } 6893 } 6894 6895 bnxt_enable_napi(bp); 6896 bnxt_debug_dev_init(bp); 6897 6898 rc = bnxt_init_nic(bp, irq_re_init); 6899 if (rc) { 6900 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc); 6901 goto open_err; 6902 } 6903 6904 if (link_re_init) { 6905 mutex_lock(&bp->link_lock); 6906 rc = bnxt_update_phy_setting(bp); 6907 mutex_unlock(&bp->link_lock); 6908 if (rc) 6909 netdev_warn(bp->dev, "failed to update phy settings\n"); 6910 } 6911 6912 if (irq_re_init) 6913 udp_tunnel_get_rx_info(bp->dev); 6914 6915 set_bit(BNXT_STATE_OPEN, &bp->state); 6916 bnxt_enable_int(bp); 6917 /* Enable TX queues */ 6918 bnxt_tx_enable(bp); 6919 mod_timer(&bp->timer, jiffies + bp->current_interval); 6920 /* Poll link status and check for SFP+ module status */ 6921 bnxt_get_port_module_status(bp); 6922 6923 /* VF-reps may need to be re-opened after the PF is re-opened */ 6924 if (BNXT_PF(bp)) 6925 bnxt_vf_reps_open(bp); 6926 return 0; 6927 6928 open_err: 6929 bnxt_debug_dev_exit(bp); 6930 bnxt_disable_napi(bp); 6931 bnxt_del_napi(bp); 6932 6933 open_err_free_mem: 6934 bnxt_free_skbs(bp); 6935 bnxt_free_irq(bp); 6936 bnxt_free_mem(bp, true); 6937 return rc; 6938 } 6939 6940 /* rtnl_lock held */ 6941 int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) 6942 { 6943 int rc = 0; 6944 6945 rc = __bnxt_open_nic(bp, irq_re_init, link_re_init); 6946 if (rc) { 6947 netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc); 6948 dev_close(bp->dev); 6949 } 6950 return rc; 6951 } 6952 6953 /* rtnl_lock held, open the NIC half way by allocating all resources, but 6954 * NAPI, IRQ, and TX are not enabled. This is mainly used for offline 6955 * self tests. 6956 */ 6957 int bnxt_half_open_nic(struct bnxt *bp) 6958 { 6959 int rc = 0; 6960 6961 rc = bnxt_alloc_mem(bp, false); 6962 if (rc) { 6963 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc); 6964 goto half_open_err; 6965 } 6966 rc = bnxt_init_nic(bp, false); 6967 if (rc) { 6968 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc); 6969 goto half_open_err; 6970 } 6971 return 0; 6972 6973 half_open_err: 6974 bnxt_free_skbs(bp); 6975 bnxt_free_mem(bp, false); 6976 dev_close(bp->dev); 6977 return rc; 6978 } 6979 6980 /* rtnl_lock held, this call can only be made after a previous successful 6981 * call to bnxt_half_open_nic(). 6982 */ 6983 void bnxt_half_close_nic(struct bnxt *bp) 6984 { 6985 bnxt_hwrm_resource_free(bp, false, false); 6986 bnxt_free_skbs(bp); 6987 bnxt_free_mem(bp, false); 6988 } 6989 6990 static int bnxt_open(struct net_device *dev) 6991 { 6992 struct bnxt *bp = netdev_priv(dev); 6993 6994 return __bnxt_open_nic(bp, true, true); 6995 } 6996 6997 static bool bnxt_drv_busy(struct bnxt *bp) 6998 { 6999 return (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state) || 7000 test_bit(BNXT_STATE_READ_STATS, &bp->state)); 7001 } 7002 7003 static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init, 7004 bool link_re_init) 7005 { 7006 /* Close the VF-reps before closing PF */ 7007 if (BNXT_PF(bp)) 7008 bnxt_vf_reps_close(bp); 7009 7010 /* Change device state to avoid TX queue wake up's */ 7011 bnxt_tx_disable(bp); 7012 7013 clear_bit(BNXT_STATE_OPEN, &bp->state); 7014 smp_mb__after_atomic(); 7015 while (bnxt_drv_busy(bp)) 7016 msleep(20); 7017 7018 /* Flush rings and and disable interrupts */ 7019 bnxt_shutdown_nic(bp, irq_re_init); 7020 7021 /* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */ 7022 7023 bnxt_debug_dev_exit(bp); 7024 bnxt_disable_napi(bp); 7025 del_timer_sync(&bp->timer); 7026 bnxt_free_skbs(bp); 7027 7028 if (irq_re_init) { 7029 bnxt_free_irq(bp); 7030 bnxt_del_napi(bp); 7031 } 7032 bnxt_free_mem(bp, irq_re_init); 7033 } 7034 7035 int bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) 7036 { 7037 int rc = 0; 7038 7039 #ifdef CONFIG_BNXT_SRIOV 7040 if (bp->sriov_cfg) { 7041 rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait, 7042 !bp->sriov_cfg, 7043 BNXT_SRIOV_CFG_WAIT_TMO); 7044 if (rc) 7045 netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete!\n"); 7046 } 7047 #endif 7048 __bnxt_close_nic(bp, irq_re_init, link_re_init); 7049 return rc; 7050 } 7051 7052 static int bnxt_close(struct net_device *dev) 7053 { 7054 struct bnxt *bp = netdev_priv(dev); 7055 7056 bnxt_close_nic(bp, true, true); 7057 bnxt_hwrm_shutdown_link(bp); 7058 return 0; 7059 } 7060 7061 /* rtnl_lock held */ 7062 static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) 7063 { 7064 switch (cmd) { 7065 case SIOCGMIIPHY: 7066 /* fallthru */ 7067 case SIOCGMIIREG: { 7068 if (!netif_running(dev)) 7069 return -EAGAIN; 7070 7071 return 0; 7072 } 7073 7074 case SIOCSMIIREG: 7075 if (!netif_running(dev)) 7076 return -EAGAIN; 7077 7078 return 0; 7079 7080 default: 7081 /* do nothing */ 7082 break; 7083 } 7084 return -EOPNOTSUPP; 7085 } 7086 7087 static void 7088 bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) 7089 { 7090 u32 i; 7091 struct bnxt *bp = netdev_priv(dev); 7092 7093 set_bit(BNXT_STATE_READ_STATS, &bp->state); 7094 /* Make sure bnxt_close_nic() sees that we are reading stats before 7095 * we check the BNXT_STATE_OPEN flag. 7096 */ 7097 smp_mb__after_atomic(); 7098 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 7099 clear_bit(BNXT_STATE_READ_STATS, &bp->state); 7100 return; 7101 } 7102 7103 /* TODO check if we need to synchronize with bnxt_close path */ 7104 for (i = 0; i < bp->cp_nr_rings; i++) { 7105 struct bnxt_napi *bnapi = bp->bnapi[i]; 7106 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 7107 struct ctx_hw_stats *hw_stats = cpr->hw_stats; 7108 7109 stats->rx_packets += le64_to_cpu(hw_stats->rx_ucast_pkts); 7110 stats->rx_packets += le64_to_cpu(hw_stats->rx_mcast_pkts); 7111 stats->rx_packets += le64_to_cpu(hw_stats->rx_bcast_pkts); 7112 7113 stats->tx_packets += le64_to_cpu(hw_stats->tx_ucast_pkts); 7114 stats->tx_packets += le64_to_cpu(hw_stats->tx_mcast_pkts); 7115 stats->tx_packets += le64_to_cpu(hw_stats->tx_bcast_pkts); 7116 7117 stats->rx_bytes += le64_to_cpu(hw_stats->rx_ucast_bytes); 7118 stats->rx_bytes += le64_to_cpu(hw_stats->rx_mcast_bytes); 7119 stats->rx_bytes += le64_to_cpu(hw_stats->rx_bcast_bytes); 7120 7121 stats->tx_bytes += le64_to_cpu(hw_stats->tx_ucast_bytes); 7122 stats->tx_bytes += le64_to_cpu(hw_stats->tx_mcast_bytes); 7123 stats->tx_bytes += le64_to_cpu(hw_stats->tx_bcast_bytes); 7124 7125 stats->rx_missed_errors += 7126 le64_to_cpu(hw_stats->rx_discard_pkts); 7127 7128 stats->multicast += le64_to_cpu(hw_stats->rx_mcast_pkts); 7129 7130 stats->tx_dropped += le64_to_cpu(hw_stats->tx_drop_pkts); 7131 } 7132 7133 if (bp->flags & BNXT_FLAG_PORT_STATS) { 7134 struct rx_port_stats *rx = bp->hw_rx_port_stats; 7135 struct tx_port_stats *tx = bp->hw_tx_port_stats; 7136 7137 stats->rx_crc_errors = le64_to_cpu(rx->rx_fcs_err_frames); 7138 stats->rx_frame_errors = le64_to_cpu(rx->rx_align_err_frames); 7139 stats->rx_length_errors = le64_to_cpu(rx->rx_undrsz_frames) + 7140 le64_to_cpu(rx->rx_ovrsz_frames) + 7141 le64_to_cpu(rx->rx_runt_frames); 7142 stats->rx_errors = le64_to_cpu(rx->rx_false_carrier_frames) + 7143 le64_to_cpu(rx->rx_jbr_frames); 7144 stats->collisions = le64_to_cpu(tx->tx_total_collisions); 7145 stats->tx_fifo_errors = le64_to_cpu(tx->tx_fifo_underruns); 7146 stats->tx_errors = le64_to_cpu(tx->tx_err); 7147 } 7148 clear_bit(BNXT_STATE_READ_STATS, &bp->state); 7149 } 7150 7151 static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask) 7152 { 7153 struct net_device *dev = bp->dev; 7154 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 7155 struct netdev_hw_addr *ha; 7156 u8 *haddr; 7157 int mc_count = 0; 7158 bool update = false; 7159 int off = 0; 7160 7161 netdev_for_each_mc_addr(ha, dev) { 7162 if (mc_count >= BNXT_MAX_MC_ADDRS) { 7163 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 7164 vnic->mc_list_count = 0; 7165 return false; 7166 } 7167 haddr = ha->addr; 7168 if (!ether_addr_equal(haddr, vnic->mc_list + off)) { 7169 memcpy(vnic->mc_list + off, haddr, ETH_ALEN); 7170 update = true; 7171 } 7172 off += ETH_ALEN; 7173 mc_count++; 7174 } 7175 if (mc_count) 7176 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST; 7177 7178 if (mc_count != vnic->mc_list_count) { 7179 vnic->mc_list_count = mc_count; 7180 update = true; 7181 } 7182 return update; 7183 } 7184 7185 static bool bnxt_uc_list_updated(struct bnxt *bp) 7186 { 7187 struct net_device *dev = bp->dev; 7188 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 7189 struct netdev_hw_addr *ha; 7190 int off = 0; 7191 7192 if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1)) 7193 return true; 7194 7195 netdev_for_each_uc_addr(ha, dev) { 7196 if (!ether_addr_equal(ha->addr, vnic->uc_list + off)) 7197 return true; 7198 7199 off += ETH_ALEN; 7200 } 7201 return false; 7202 } 7203 7204 static void bnxt_set_rx_mode(struct net_device *dev) 7205 { 7206 struct bnxt *bp = netdev_priv(dev); 7207 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 7208 u32 mask = vnic->rx_mask; 7209 bool mc_update = false; 7210 bool uc_update; 7211 7212 if (!netif_running(dev)) 7213 return; 7214 7215 mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS | 7216 CFA_L2_SET_RX_MASK_REQ_MASK_MCAST | 7217 CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST); 7218 7219 if ((dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp)) 7220 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 7221 7222 uc_update = bnxt_uc_list_updated(bp); 7223 7224 if (dev->flags & IFF_ALLMULTI) { 7225 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 7226 vnic->mc_list_count = 0; 7227 } else { 7228 mc_update = bnxt_mc_list_updated(bp, &mask); 7229 } 7230 7231 if (mask != vnic->rx_mask || uc_update || mc_update) { 7232 vnic->rx_mask = mask; 7233 7234 set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event); 7235 bnxt_queue_sp_work(bp); 7236 } 7237 } 7238 7239 static int bnxt_cfg_rx_mode(struct bnxt *bp) 7240 { 7241 struct net_device *dev = bp->dev; 7242 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 7243 struct netdev_hw_addr *ha; 7244 int i, off = 0, rc; 7245 bool uc_update; 7246 7247 netif_addr_lock_bh(dev); 7248 uc_update = bnxt_uc_list_updated(bp); 7249 netif_addr_unlock_bh(dev); 7250 7251 if (!uc_update) 7252 goto skip_uc; 7253 7254 mutex_lock(&bp->hwrm_cmd_lock); 7255 for (i = 1; i < vnic->uc_filter_count; i++) { 7256 struct hwrm_cfa_l2_filter_free_input req = {0}; 7257 7258 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_FREE, -1, 7259 -1); 7260 7261 req.l2_filter_id = vnic->fw_l2_filter_id[i]; 7262 7263 rc = _hwrm_send_message(bp, &req, sizeof(req), 7264 HWRM_CMD_TIMEOUT); 7265 } 7266 mutex_unlock(&bp->hwrm_cmd_lock); 7267 7268 vnic->uc_filter_count = 1; 7269 7270 netif_addr_lock_bh(dev); 7271 if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) { 7272 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 7273 } else { 7274 netdev_for_each_uc_addr(ha, dev) { 7275 memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN); 7276 off += ETH_ALEN; 7277 vnic->uc_filter_count++; 7278 } 7279 } 7280 netif_addr_unlock_bh(dev); 7281 7282 for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) { 7283 rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off); 7284 if (rc) { 7285 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", 7286 rc); 7287 vnic->uc_filter_count = i; 7288 return rc; 7289 } 7290 } 7291 7292 skip_uc: 7293 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0); 7294 if (rc) 7295 netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %x\n", 7296 rc); 7297 7298 return rc; 7299 } 7300 7301 static bool bnxt_can_reserve_rings(struct bnxt *bp) 7302 { 7303 #ifdef CONFIG_BNXT_SRIOV 7304 if ((bp->flags & BNXT_FLAG_NEW_RM) && BNXT_VF(bp)) { 7305 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 7306 7307 /* No minimum rings were provisioned by the PF. Don't 7308 * reserve rings by default when device is down. 7309 */ 7310 if (hw_resc->min_tx_rings || hw_resc->resv_tx_rings) 7311 return true; 7312 7313 if (!netif_running(bp->dev)) 7314 return false; 7315 } 7316 #endif 7317 return true; 7318 } 7319 7320 /* If the chip and firmware supports RFS */ 7321 static bool bnxt_rfs_supported(struct bnxt *bp) 7322 { 7323 if (BNXT_PF(bp) && !BNXT_CHIP_TYPE_NITRO_A0(bp)) 7324 return true; 7325 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP) 7326 return true; 7327 return false; 7328 } 7329 7330 /* If runtime conditions support RFS */ 7331 static bool bnxt_rfs_capable(struct bnxt *bp) 7332 { 7333 #ifdef CONFIG_RFS_ACCEL 7334 int vnics, max_vnics, max_rss_ctxs; 7335 7336 if (!(bp->flags & BNXT_FLAG_MSIX_CAP) || !bnxt_can_reserve_rings(bp)) 7337 return false; 7338 7339 vnics = 1 + bp->rx_nr_rings; 7340 max_vnics = bnxt_get_max_func_vnics(bp); 7341 max_rss_ctxs = bnxt_get_max_func_rss_ctxs(bp); 7342 7343 /* RSS contexts not a limiting factor */ 7344 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP) 7345 max_rss_ctxs = max_vnics; 7346 if (vnics > max_vnics || vnics > max_rss_ctxs) { 7347 if (bp->rx_nr_rings > 1) 7348 netdev_warn(bp->dev, 7349 "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n", 7350 min(max_rss_ctxs - 1, max_vnics - 1)); 7351 return false; 7352 } 7353 7354 if (!(bp->flags & BNXT_FLAG_NEW_RM)) 7355 return true; 7356 7357 if (vnics == bp->hw_resc.resv_vnics) 7358 return true; 7359 7360 bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, vnics); 7361 if (vnics <= bp->hw_resc.resv_vnics) 7362 return true; 7363 7364 netdev_warn(bp->dev, "Unable to reserve resources to support NTUPLE filters.\n"); 7365 bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 1); 7366 return false; 7367 #else 7368 return false; 7369 #endif 7370 } 7371 7372 static netdev_features_t bnxt_fix_features(struct net_device *dev, 7373 netdev_features_t features) 7374 { 7375 struct bnxt *bp = netdev_priv(dev); 7376 7377 if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp)) 7378 features &= ~NETIF_F_NTUPLE; 7379 7380 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS) 7381 features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); 7382 7383 if (!(features & NETIF_F_GRO)) 7384 features &= ~NETIF_F_GRO_HW; 7385 7386 if (features & NETIF_F_GRO_HW) 7387 features &= ~NETIF_F_LRO; 7388 7389 /* Both CTAG and STAG VLAN accelaration on the RX side have to be 7390 * turned on or off together. 7391 */ 7392 if ((features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)) != 7393 (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)) { 7394 if (dev->features & NETIF_F_HW_VLAN_CTAG_RX) 7395 features &= ~(NETIF_F_HW_VLAN_CTAG_RX | 7396 NETIF_F_HW_VLAN_STAG_RX); 7397 else 7398 features |= NETIF_F_HW_VLAN_CTAG_RX | 7399 NETIF_F_HW_VLAN_STAG_RX; 7400 } 7401 #ifdef CONFIG_BNXT_SRIOV 7402 if (BNXT_VF(bp)) { 7403 if (bp->vf.vlan) { 7404 features &= ~(NETIF_F_HW_VLAN_CTAG_RX | 7405 NETIF_F_HW_VLAN_STAG_RX); 7406 } 7407 } 7408 #endif 7409 return features; 7410 } 7411 7412 static int bnxt_set_features(struct net_device *dev, netdev_features_t features) 7413 { 7414 struct bnxt *bp = netdev_priv(dev); 7415 u32 flags = bp->flags; 7416 u32 changes; 7417 int rc = 0; 7418 bool re_init = false; 7419 bool update_tpa = false; 7420 7421 flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS; 7422 if (features & NETIF_F_GRO_HW) 7423 flags |= BNXT_FLAG_GRO; 7424 else if (features & NETIF_F_LRO) 7425 flags |= BNXT_FLAG_LRO; 7426 7427 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS) 7428 flags &= ~BNXT_FLAG_TPA; 7429 7430 if (features & NETIF_F_HW_VLAN_CTAG_RX) 7431 flags |= BNXT_FLAG_STRIP_VLAN; 7432 7433 if (features & NETIF_F_NTUPLE) 7434 flags |= BNXT_FLAG_RFS; 7435 7436 changes = flags ^ bp->flags; 7437 if (changes & BNXT_FLAG_TPA) { 7438 update_tpa = true; 7439 if ((bp->flags & BNXT_FLAG_TPA) == 0 || 7440 (flags & BNXT_FLAG_TPA) == 0) 7441 re_init = true; 7442 } 7443 7444 if (changes & ~BNXT_FLAG_TPA) 7445 re_init = true; 7446 7447 if (flags != bp->flags) { 7448 u32 old_flags = bp->flags; 7449 7450 bp->flags = flags; 7451 7452 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 7453 if (update_tpa) 7454 bnxt_set_ring_params(bp); 7455 return rc; 7456 } 7457 7458 if (re_init) { 7459 bnxt_close_nic(bp, false, false); 7460 if (update_tpa) 7461 bnxt_set_ring_params(bp); 7462 7463 return bnxt_open_nic(bp, false, false); 7464 } 7465 if (update_tpa) { 7466 rc = bnxt_set_tpa(bp, 7467 (flags & BNXT_FLAG_TPA) ? 7468 true : false); 7469 if (rc) 7470 bp->flags = old_flags; 7471 } 7472 } 7473 return rc; 7474 } 7475 7476 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi) 7477 { 7478 struct bnxt_tx_ring_info *txr = bnapi->tx_ring; 7479 int i = bnapi->index; 7480 7481 if (!txr) 7482 return; 7483 7484 netdev_info(bnapi->bp->dev, "[%d]: tx{fw_ring: %d prod: %x cons: %x}\n", 7485 i, txr->tx_ring_struct.fw_ring_id, txr->tx_prod, 7486 txr->tx_cons); 7487 } 7488 7489 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi) 7490 { 7491 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 7492 int i = bnapi->index; 7493 7494 if (!rxr) 7495 return; 7496 7497 netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n", 7498 i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod, 7499 rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod, 7500 rxr->rx_sw_agg_prod); 7501 } 7502 7503 static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi) 7504 { 7505 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 7506 int i = bnapi->index; 7507 7508 netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n", 7509 i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons); 7510 } 7511 7512 static void bnxt_dbg_dump_states(struct bnxt *bp) 7513 { 7514 int i; 7515 struct bnxt_napi *bnapi; 7516 7517 for (i = 0; i < bp->cp_nr_rings; i++) { 7518 bnapi = bp->bnapi[i]; 7519 if (netif_msg_drv(bp)) { 7520 bnxt_dump_tx_sw_state(bnapi); 7521 bnxt_dump_rx_sw_state(bnapi); 7522 bnxt_dump_cp_sw_state(bnapi); 7523 } 7524 } 7525 } 7526 7527 static void bnxt_reset_task(struct bnxt *bp, bool silent) 7528 { 7529 if (!silent) 7530 bnxt_dbg_dump_states(bp); 7531 if (netif_running(bp->dev)) { 7532 int rc; 7533 7534 if (!silent) 7535 bnxt_ulp_stop(bp); 7536 bnxt_close_nic(bp, false, false); 7537 rc = bnxt_open_nic(bp, false, false); 7538 if (!silent && !rc) 7539 bnxt_ulp_start(bp); 7540 } 7541 } 7542 7543 static void bnxt_tx_timeout(struct net_device *dev) 7544 { 7545 struct bnxt *bp = netdev_priv(dev); 7546 7547 netdev_err(bp->dev, "TX timeout detected, starting reset task!\n"); 7548 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event); 7549 bnxt_queue_sp_work(bp); 7550 } 7551 7552 #ifdef CONFIG_NET_POLL_CONTROLLER 7553 static void bnxt_poll_controller(struct net_device *dev) 7554 { 7555 struct bnxt *bp = netdev_priv(dev); 7556 int i; 7557 7558 /* Only process tx rings/combined rings in netpoll mode. */ 7559 for (i = 0; i < bp->tx_nr_rings; i++) { 7560 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 7561 7562 napi_schedule(&txr->bnapi->napi); 7563 } 7564 } 7565 #endif 7566 7567 static void bnxt_timer(struct timer_list *t) 7568 { 7569 struct bnxt *bp = from_timer(bp, t, timer); 7570 struct net_device *dev = bp->dev; 7571 7572 if (!netif_running(dev)) 7573 return; 7574 7575 if (atomic_read(&bp->intr_sem) != 0) 7576 goto bnxt_restart_timer; 7577 7578 if (bp->link_info.link_up && (bp->flags & BNXT_FLAG_PORT_STATS) && 7579 bp->stats_coal_ticks) { 7580 set_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event); 7581 bnxt_queue_sp_work(bp); 7582 } 7583 7584 if (bnxt_tc_flower_enabled(bp)) { 7585 set_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event); 7586 bnxt_queue_sp_work(bp); 7587 } 7588 bnxt_restart_timer: 7589 mod_timer(&bp->timer, jiffies + bp->current_interval); 7590 } 7591 7592 static void bnxt_rtnl_lock_sp(struct bnxt *bp) 7593 { 7594 /* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK 7595 * set. If the device is being closed, bnxt_close() may be holding 7596 * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear. So we 7597 * must clear BNXT_STATE_IN_SP_TASK before holding rtnl(). 7598 */ 7599 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 7600 rtnl_lock(); 7601 } 7602 7603 static void bnxt_rtnl_unlock_sp(struct bnxt *bp) 7604 { 7605 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 7606 rtnl_unlock(); 7607 } 7608 7609 /* Only called from bnxt_sp_task() */ 7610 static void bnxt_reset(struct bnxt *bp, bool silent) 7611 { 7612 bnxt_rtnl_lock_sp(bp); 7613 if (test_bit(BNXT_STATE_OPEN, &bp->state)) 7614 bnxt_reset_task(bp, silent); 7615 bnxt_rtnl_unlock_sp(bp); 7616 } 7617 7618 static void bnxt_cfg_ntp_filters(struct bnxt *); 7619 7620 static void bnxt_sp_task(struct work_struct *work) 7621 { 7622 struct bnxt *bp = container_of(work, struct bnxt, sp_task); 7623 7624 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 7625 smp_mb__after_atomic(); 7626 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 7627 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 7628 return; 7629 } 7630 7631 if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event)) 7632 bnxt_cfg_rx_mode(bp); 7633 7634 if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event)) 7635 bnxt_cfg_ntp_filters(bp); 7636 if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event)) 7637 bnxt_hwrm_exec_fwd_req(bp); 7638 if (test_and_clear_bit(BNXT_VXLAN_ADD_PORT_SP_EVENT, &bp->sp_event)) { 7639 bnxt_hwrm_tunnel_dst_port_alloc( 7640 bp, bp->vxlan_port, 7641 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN); 7642 } 7643 if (test_and_clear_bit(BNXT_VXLAN_DEL_PORT_SP_EVENT, &bp->sp_event)) { 7644 bnxt_hwrm_tunnel_dst_port_free( 7645 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN); 7646 } 7647 if (test_and_clear_bit(BNXT_GENEVE_ADD_PORT_SP_EVENT, &bp->sp_event)) { 7648 bnxt_hwrm_tunnel_dst_port_alloc( 7649 bp, bp->nge_port, 7650 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE); 7651 } 7652 if (test_and_clear_bit(BNXT_GENEVE_DEL_PORT_SP_EVENT, &bp->sp_event)) { 7653 bnxt_hwrm_tunnel_dst_port_free( 7654 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE); 7655 } 7656 if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event)) { 7657 bnxt_hwrm_port_qstats(bp); 7658 bnxt_hwrm_port_qstats_ext(bp); 7659 } 7660 7661 if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) { 7662 int rc; 7663 7664 mutex_lock(&bp->link_lock); 7665 if (test_and_clear_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, 7666 &bp->sp_event)) 7667 bnxt_hwrm_phy_qcaps(bp); 7668 7669 rc = bnxt_update_link(bp, true); 7670 mutex_unlock(&bp->link_lock); 7671 if (rc) 7672 netdev_err(bp->dev, "SP task can't update link (rc: %x)\n", 7673 rc); 7674 } 7675 if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event)) { 7676 mutex_lock(&bp->link_lock); 7677 bnxt_get_port_module_status(bp); 7678 mutex_unlock(&bp->link_lock); 7679 } 7680 7681 if (test_and_clear_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event)) 7682 bnxt_tc_flow_stats_work(bp); 7683 7684 /* These functions below will clear BNXT_STATE_IN_SP_TASK. They 7685 * must be the last functions to be called before exiting. 7686 */ 7687 if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event)) 7688 bnxt_reset(bp, false); 7689 7690 if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event)) 7691 bnxt_reset(bp, true); 7692 7693 smp_mb__before_atomic(); 7694 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 7695 } 7696 7697 /* Under rtnl_lock */ 7698 int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs, 7699 int tx_xdp) 7700 { 7701 int max_rx, max_tx, tx_sets = 1; 7702 int tx_rings_needed; 7703 int rx_rings = rx; 7704 int cp, vnics, rc; 7705 7706 if (tcs) 7707 tx_sets = tcs; 7708 7709 rc = bnxt_get_max_rings(bp, &max_rx, &max_tx, sh); 7710 if (rc) 7711 return rc; 7712 7713 if (max_rx < rx) 7714 return -ENOMEM; 7715 7716 tx_rings_needed = tx * tx_sets + tx_xdp; 7717 if (max_tx < tx_rings_needed) 7718 return -ENOMEM; 7719 7720 vnics = 1; 7721 if (bp->flags & BNXT_FLAG_RFS) 7722 vnics += rx_rings; 7723 7724 if (bp->flags & BNXT_FLAG_AGG_RINGS) 7725 rx_rings <<= 1; 7726 cp = sh ? max_t(int, tx_rings_needed, rx) : tx_rings_needed + rx; 7727 if (bp->flags & BNXT_FLAG_NEW_RM) 7728 cp += bnxt_get_ulp_msix_num(bp); 7729 return bnxt_hwrm_check_rings(bp, tx_rings_needed, rx_rings, rx, cp, 7730 vnics); 7731 } 7732 7733 static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev) 7734 { 7735 if (bp->bar2) { 7736 pci_iounmap(pdev, bp->bar2); 7737 bp->bar2 = NULL; 7738 } 7739 7740 if (bp->bar1) { 7741 pci_iounmap(pdev, bp->bar1); 7742 bp->bar1 = NULL; 7743 } 7744 7745 if (bp->bar0) { 7746 pci_iounmap(pdev, bp->bar0); 7747 bp->bar0 = NULL; 7748 } 7749 } 7750 7751 static void bnxt_cleanup_pci(struct bnxt *bp) 7752 { 7753 bnxt_unmap_bars(bp, bp->pdev); 7754 pci_release_regions(bp->pdev); 7755 pci_disable_device(bp->pdev); 7756 } 7757 7758 static void bnxt_init_dflt_coal(struct bnxt *bp) 7759 { 7760 struct bnxt_coal *coal; 7761 7762 /* Tick values in micro seconds. 7763 * 1 coal_buf x bufs_per_record = 1 completion record. 7764 */ 7765 coal = &bp->rx_coal; 7766 coal->coal_ticks = 14; 7767 coal->coal_bufs = 30; 7768 coal->coal_ticks_irq = 1; 7769 coal->coal_bufs_irq = 2; 7770 coal->idle_thresh = 50; 7771 coal->bufs_per_record = 2; 7772 coal->budget = 64; /* NAPI budget */ 7773 7774 coal = &bp->tx_coal; 7775 coal->coal_ticks = 28; 7776 coal->coal_bufs = 30; 7777 coal->coal_ticks_irq = 2; 7778 coal->coal_bufs_irq = 2; 7779 coal->bufs_per_record = 1; 7780 7781 bp->stats_coal_ticks = BNXT_DEF_STATS_COAL_TICKS; 7782 } 7783 7784 static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev) 7785 { 7786 int rc; 7787 struct bnxt *bp = netdev_priv(dev); 7788 7789 SET_NETDEV_DEV(dev, &pdev->dev); 7790 7791 /* enable device (incl. PCI PM wakeup), and bus-mastering */ 7792 rc = pci_enable_device(pdev); 7793 if (rc) { 7794 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n"); 7795 goto init_err; 7796 } 7797 7798 if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) { 7799 dev_err(&pdev->dev, 7800 "Cannot find PCI device base address, aborting\n"); 7801 rc = -ENODEV; 7802 goto init_err_disable; 7803 } 7804 7805 rc = pci_request_regions(pdev, DRV_MODULE_NAME); 7806 if (rc) { 7807 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n"); 7808 goto init_err_disable; 7809 } 7810 7811 if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 && 7812 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) { 7813 dev_err(&pdev->dev, "System does not support DMA, aborting\n"); 7814 goto init_err_disable; 7815 } 7816 7817 pci_set_master(pdev); 7818 7819 bp->dev = dev; 7820 bp->pdev = pdev; 7821 7822 bp->bar0 = pci_ioremap_bar(pdev, 0); 7823 if (!bp->bar0) { 7824 dev_err(&pdev->dev, "Cannot map device registers, aborting\n"); 7825 rc = -ENOMEM; 7826 goto init_err_release; 7827 } 7828 7829 bp->bar1 = pci_ioremap_bar(pdev, 2); 7830 if (!bp->bar1) { 7831 dev_err(&pdev->dev, "Cannot map doorbell registers, aborting\n"); 7832 rc = -ENOMEM; 7833 goto init_err_release; 7834 } 7835 7836 bp->bar2 = pci_ioremap_bar(pdev, 4); 7837 if (!bp->bar2) { 7838 dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n"); 7839 rc = -ENOMEM; 7840 goto init_err_release; 7841 } 7842 7843 pci_enable_pcie_error_reporting(pdev); 7844 7845 INIT_WORK(&bp->sp_task, bnxt_sp_task); 7846 7847 spin_lock_init(&bp->ntp_fltr_lock); 7848 7849 bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE; 7850 bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE; 7851 7852 bnxt_init_dflt_coal(bp); 7853 7854 timer_setup(&bp->timer, bnxt_timer, 0); 7855 bp->current_interval = BNXT_TIMER_INTERVAL; 7856 7857 clear_bit(BNXT_STATE_OPEN, &bp->state); 7858 return 0; 7859 7860 init_err_release: 7861 bnxt_unmap_bars(bp, pdev); 7862 pci_release_regions(pdev); 7863 7864 init_err_disable: 7865 pci_disable_device(pdev); 7866 7867 init_err: 7868 return rc; 7869 } 7870 7871 /* rtnl_lock held */ 7872 static int bnxt_change_mac_addr(struct net_device *dev, void *p) 7873 { 7874 struct sockaddr *addr = p; 7875 struct bnxt *bp = netdev_priv(dev); 7876 int rc = 0; 7877 7878 if (!is_valid_ether_addr(addr->sa_data)) 7879 return -EADDRNOTAVAIL; 7880 7881 if (ether_addr_equal(addr->sa_data, dev->dev_addr)) 7882 return 0; 7883 7884 rc = bnxt_approve_mac(bp, addr->sa_data); 7885 if (rc) 7886 return rc; 7887 7888 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len); 7889 if (netif_running(dev)) { 7890 bnxt_close_nic(bp, false, false); 7891 rc = bnxt_open_nic(bp, false, false); 7892 } 7893 7894 return rc; 7895 } 7896 7897 /* rtnl_lock held */ 7898 static int bnxt_change_mtu(struct net_device *dev, int new_mtu) 7899 { 7900 struct bnxt *bp = netdev_priv(dev); 7901 7902 if (netif_running(dev)) 7903 bnxt_close_nic(bp, false, false); 7904 7905 dev->mtu = new_mtu; 7906 bnxt_set_ring_params(bp); 7907 7908 if (netif_running(dev)) 7909 return bnxt_open_nic(bp, false, false); 7910 7911 return 0; 7912 } 7913 7914 int bnxt_setup_mq_tc(struct net_device *dev, u8 tc) 7915 { 7916 struct bnxt *bp = netdev_priv(dev); 7917 bool sh = false; 7918 int rc; 7919 7920 if (tc > bp->max_tc) { 7921 netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n", 7922 tc, bp->max_tc); 7923 return -EINVAL; 7924 } 7925 7926 if (netdev_get_num_tc(dev) == tc) 7927 return 0; 7928 7929 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 7930 sh = true; 7931 7932 rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings, 7933 sh, tc, bp->tx_nr_rings_xdp); 7934 if (rc) 7935 return rc; 7936 7937 /* Needs to close the device and do hw resource re-allocations */ 7938 if (netif_running(bp->dev)) 7939 bnxt_close_nic(bp, true, false); 7940 7941 if (tc) { 7942 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc; 7943 netdev_set_num_tc(dev, tc); 7944 } else { 7945 bp->tx_nr_rings = bp->tx_nr_rings_per_tc; 7946 netdev_reset_tc(dev); 7947 } 7948 bp->tx_nr_rings += bp->tx_nr_rings_xdp; 7949 bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) : 7950 bp->tx_nr_rings + bp->rx_nr_rings; 7951 bp->num_stat_ctxs = bp->cp_nr_rings; 7952 7953 if (netif_running(bp->dev)) 7954 return bnxt_open_nic(bp, true, false); 7955 7956 return 0; 7957 } 7958 7959 static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 7960 void *cb_priv) 7961 { 7962 struct bnxt *bp = cb_priv; 7963 7964 if (!bnxt_tc_flower_enabled(bp) || 7965 !tc_cls_can_offload_and_chain0(bp->dev, type_data)) 7966 return -EOPNOTSUPP; 7967 7968 switch (type) { 7969 case TC_SETUP_CLSFLOWER: 7970 return bnxt_tc_setup_flower(bp, bp->pf.fw_fid, type_data); 7971 default: 7972 return -EOPNOTSUPP; 7973 } 7974 } 7975 7976 static int bnxt_setup_tc_block(struct net_device *dev, 7977 struct tc_block_offload *f) 7978 { 7979 struct bnxt *bp = netdev_priv(dev); 7980 7981 if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS) 7982 return -EOPNOTSUPP; 7983 7984 switch (f->command) { 7985 case TC_BLOCK_BIND: 7986 return tcf_block_cb_register(f->block, bnxt_setup_tc_block_cb, 7987 bp, bp); 7988 case TC_BLOCK_UNBIND: 7989 tcf_block_cb_unregister(f->block, bnxt_setup_tc_block_cb, bp); 7990 return 0; 7991 default: 7992 return -EOPNOTSUPP; 7993 } 7994 } 7995 7996 static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type, 7997 void *type_data) 7998 { 7999 switch (type) { 8000 case TC_SETUP_BLOCK: 8001 return bnxt_setup_tc_block(dev, type_data); 8002 case TC_SETUP_QDISC_MQPRIO: { 8003 struct tc_mqprio_qopt *mqprio = type_data; 8004 8005 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 8006 8007 return bnxt_setup_mq_tc(dev, mqprio->num_tc); 8008 } 8009 default: 8010 return -EOPNOTSUPP; 8011 } 8012 } 8013 8014 #ifdef CONFIG_RFS_ACCEL 8015 static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1, 8016 struct bnxt_ntuple_filter *f2) 8017 { 8018 struct flow_keys *keys1 = &f1->fkeys; 8019 struct flow_keys *keys2 = &f2->fkeys; 8020 8021 if (keys1->addrs.v4addrs.src == keys2->addrs.v4addrs.src && 8022 keys1->addrs.v4addrs.dst == keys2->addrs.v4addrs.dst && 8023 keys1->ports.ports == keys2->ports.ports && 8024 keys1->basic.ip_proto == keys2->basic.ip_proto && 8025 keys1->basic.n_proto == keys2->basic.n_proto && 8026 keys1->control.flags == keys2->control.flags && 8027 ether_addr_equal(f1->src_mac_addr, f2->src_mac_addr) && 8028 ether_addr_equal(f1->dst_mac_addr, f2->dst_mac_addr)) 8029 return true; 8030 8031 return false; 8032 } 8033 8034 static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb, 8035 u16 rxq_index, u32 flow_id) 8036 { 8037 struct bnxt *bp = netdev_priv(dev); 8038 struct bnxt_ntuple_filter *fltr, *new_fltr; 8039 struct flow_keys *fkeys; 8040 struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb); 8041 int rc = 0, idx, bit_id, l2_idx = 0; 8042 struct hlist_head *head; 8043 8044 if (!ether_addr_equal(dev->dev_addr, eth->h_dest)) { 8045 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 8046 int off = 0, j; 8047 8048 netif_addr_lock_bh(dev); 8049 for (j = 0; j < vnic->uc_filter_count; j++, off += ETH_ALEN) { 8050 if (ether_addr_equal(eth->h_dest, 8051 vnic->uc_list + off)) { 8052 l2_idx = j + 1; 8053 break; 8054 } 8055 } 8056 netif_addr_unlock_bh(dev); 8057 if (!l2_idx) 8058 return -EINVAL; 8059 } 8060 new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC); 8061 if (!new_fltr) 8062 return -ENOMEM; 8063 8064 fkeys = &new_fltr->fkeys; 8065 if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) { 8066 rc = -EPROTONOSUPPORT; 8067 goto err_free; 8068 } 8069 8070 if ((fkeys->basic.n_proto != htons(ETH_P_IP) && 8071 fkeys->basic.n_proto != htons(ETH_P_IPV6)) || 8072 ((fkeys->basic.ip_proto != IPPROTO_TCP) && 8073 (fkeys->basic.ip_proto != IPPROTO_UDP))) { 8074 rc = -EPROTONOSUPPORT; 8075 goto err_free; 8076 } 8077 if (fkeys->basic.n_proto == htons(ETH_P_IPV6) && 8078 bp->hwrm_spec_code < 0x10601) { 8079 rc = -EPROTONOSUPPORT; 8080 goto err_free; 8081 } 8082 if ((fkeys->control.flags & FLOW_DIS_ENCAPSULATION) && 8083 bp->hwrm_spec_code < 0x10601) { 8084 rc = -EPROTONOSUPPORT; 8085 goto err_free; 8086 } 8087 8088 memcpy(new_fltr->dst_mac_addr, eth->h_dest, ETH_ALEN); 8089 memcpy(new_fltr->src_mac_addr, eth->h_source, ETH_ALEN); 8090 8091 idx = skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK; 8092 head = &bp->ntp_fltr_hash_tbl[idx]; 8093 rcu_read_lock(); 8094 hlist_for_each_entry_rcu(fltr, head, hash) { 8095 if (bnxt_fltr_match(fltr, new_fltr)) { 8096 rcu_read_unlock(); 8097 rc = 0; 8098 goto err_free; 8099 } 8100 } 8101 rcu_read_unlock(); 8102 8103 spin_lock_bh(&bp->ntp_fltr_lock); 8104 bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap, 8105 BNXT_NTP_FLTR_MAX_FLTR, 0); 8106 if (bit_id < 0) { 8107 spin_unlock_bh(&bp->ntp_fltr_lock); 8108 rc = -ENOMEM; 8109 goto err_free; 8110 } 8111 8112 new_fltr->sw_id = (u16)bit_id; 8113 new_fltr->flow_id = flow_id; 8114 new_fltr->l2_fltr_idx = l2_idx; 8115 new_fltr->rxq = rxq_index; 8116 hlist_add_head_rcu(&new_fltr->hash, head); 8117 bp->ntp_fltr_count++; 8118 spin_unlock_bh(&bp->ntp_fltr_lock); 8119 8120 set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event); 8121 bnxt_queue_sp_work(bp); 8122 8123 return new_fltr->sw_id; 8124 8125 err_free: 8126 kfree(new_fltr); 8127 return rc; 8128 } 8129 8130 static void bnxt_cfg_ntp_filters(struct bnxt *bp) 8131 { 8132 int i; 8133 8134 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) { 8135 struct hlist_head *head; 8136 struct hlist_node *tmp; 8137 struct bnxt_ntuple_filter *fltr; 8138 int rc; 8139 8140 head = &bp->ntp_fltr_hash_tbl[i]; 8141 hlist_for_each_entry_safe(fltr, tmp, head, hash) { 8142 bool del = false; 8143 8144 if (test_bit(BNXT_FLTR_VALID, &fltr->state)) { 8145 if (rps_may_expire_flow(bp->dev, fltr->rxq, 8146 fltr->flow_id, 8147 fltr->sw_id)) { 8148 bnxt_hwrm_cfa_ntuple_filter_free(bp, 8149 fltr); 8150 del = true; 8151 } 8152 } else { 8153 rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp, 8154 fltr); 8155 if (rc) 8156 del = true; 8157 else 8158 set_bit(BNXT_FLTR_VALID, &fltr->state); 8159 } 8160 8161 if (del) { 8162 spin_lock_bh(&bp->ntp_fltr_lock); 8163 hlist_del_rcu(&fltr->hash); 8164 bp->ntp_fltr_count--; 8165 spin_unlock_bh(&bp->ntp_fltr_lock); 8166 synchronize_rcu(); 8167 clear_bit(fltr->sw_id, bp->ntp_fltr_bmap); 8168 kfree(fltr); 8169 } 8170 } 8171 } 8172 if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event)) 8173 netdev_info(bp->dev, "Receive PF driver unload event!"); 8174 } 8175 8176 #else 8177 8178 static void bnxt_cfg_ntp_filters(struct bnxt *bp) 8179 { 8180 } 8181 8182 #endif /* CONFIG_RFS_ACCEL */ 8183 8184 static void bnxt_udp_tunnel_add(struct net_device *dev, 8185 struct udp_tunnel_info *ti) 8186 { 8187 struct bnxt *bp = netdev_priv(dev); 8188 8189 if (ti->sa_family != AF_INET6 && ti->sa_family != AF_INET) 8190 return; 8191 8192 if (!netif_running(dev)) 8193 return; 8194 8195 switch (ti->type) { 8196 case UDP_TUNNEL_TYPE_VXLAN: 8197 if (bp->vxlan_port_cnt && bp->vxlan_port != ti->port) 8198 return; 8199 8200 bp->vxlan_port_cnt++; 8201 if (bp->vxlan_port_cnt == 1) { 8202 bp->vxlan_port = ti->port; 8203 set_bit(BNXT_VXLAN_ADD_PORT_SP_EVENT, &bp->sp_event); 8204 bnxt_queue_sp_work(bp); 8205 } 8206 break; 8207 case UDP_TUNNEL_TYPE_GENEVE: 8208 if (bp->nge_port_cnt && bp->nge_port != ti->port) 8209 return; 8210 8211 bp->nge_port_cnt++; 8212 if (bp->nge_port_cnt == 1) { 8213 bp->nge_port = ti->port; 8214 set_bit(BNXT_GENEVE_ADD_PORT_SP_EVENT, &bp->sp_event); 8215 } 8216 break; 8217 default: 8218 return; 8219 } 8220 8221 bnxt_queue_sp_work(bp); 8222 } 8223 8224 static void bnxt_udp_tunnel_del(struct net_device *dev, 8225 struct udp_tunnel_info *ti) 8226 { 8227 struct bnxt *bp = netdev_priv(dev); 8228 8229 if (ti->sa_family != AF_INET6 && ti->sa_family != AF_INET) 8230 return; 8231 8232 if (!netif_running(dev)) 8233 return; 8234 8235 switch (ti->type) { 8236 case UDP_TUNNEL_TYPE_VXLAN: 8237 if (!bp->vxlan_port_cnt || bp->vxlan_port != ti->port) 8238 return; 8239 bp->vxlan_port_cnt--; 8240 8241 if (bp->vxlan_port_cnt != 0) 8242 return; 8243 8244 set_bit(BNXT_VXLAN_DEL_PORT_SP_EVENT, &bp->sp_event); 8245 break; 8246 case UDP_TUNNEL_TYPE_GENEVE: 8247 if (!bp->nge_port_cnt || bp->nge_port != ti->port) 8248 return; 8249 bp->nge_port_cnt--; 8250 8251 if (bp->nge_port_cnt != 0) 8252 return; 8253 8254 set_bit(BNXT_GENEVE_DEL_PORT_SP_EVENT, &bp->sp_event); 8255 break; 8256 default: 8257 return; 8258 } 8259 8260 bnxt_queue_sp_work(bp); 8261 } 8262 8263 static int bnxt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 8264 struct net_device *dev, u32 filter_mask, 8265 int nlflags) 8266 { 8267 struct bnxt *bp = netdev_priv(dev); 8268 8269 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bp->br_mode, 0, 0, 8270 nlflags, filter_mask, NULL); 8271 } 8272 8273 static int bnxt_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh, 8274 u16 flags) 8275 { 8276 struct bnxt *bp = netdev_priv(dev); 8277 struct nlattr *attr, *br_spec; 8278 int rem, rc = 0; 8279 8280 if (bp->hwrm_spec_code < 0x10708 || !BNXT_SINGLE_PF(bp)) 8281 return -EOPNOTSUPP; 8282 8283 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 8284 if (!br_spec) 8285 return -EINVAL; 8286 8287 nla_for_each_nested(attr, br_spec, rem) { 8288 u16 mode; 8289 8290 if (nla_type(attr) != IFLA_BRIDGE_MODE) 8291 continue; 8292 8293 if (nla_len(attr) < sizeof(mode)) 8294 return -EINVAL; 8295 8296 mode = nla_get_u16(attr); 8297 if (mode == bp->br_mode) 8298 break; 8299 8300 rc = bnxt_hwrm_set_br_mode(bp, mode); 8301 if (!rc) 8302 bp->br_mode = mode; 8303 break; 8304 } 8305 return rc; 8306 } 8307 8308 static int bnxt_get_phys_port_name(struct net_device *dev, char *buf, 8309 size_t len) 8310 { 8311 struct bnxt *bp = netdev_priv(dev); 8312 int rc; 8313 8314 /* The PF and it's VF-reps only support the switchdev framework */ 8315 if (!BNXT_PF(bp)) 8316 return -EOPNOTSUPP; 8317 8318 rc = snprintf(buf, len, "p%d", bp->pf.port_id); 8319 8320 if (rc >= len) 8321 return -EOPNOTSUPP; 8322 return 0; 8323 } 8324 8325 int bnxt_port_attr_get(struct bnxt *bp, struct switchdev_attr *attr) 8326 { 8327 if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV) 8328 return -EOPNOTSUPP; 8329 8330 /* The PF and it's VF-reps only support the switchdev framework */ 8331 if (!BNXT_PF(bp)) 8332 return -EOPNOTSUPP; 8333 8334 switch (attr->id) { 8335 case SWITCHDEV_ATTR_ID_PORT_PARENT_ID: 8336 attr->u.ppid.id_len = sizeof(bp->switch_id); 8337 memcpy(attr->u.ppid.id, bp->switch_id, attr->u.ppid.id_len); 8338 break; 8339 default: 8340 return -EOPNOTSUPP; 8341 } 8342 return 0; 8343 } 8344 8345 static int bnxt_swdev_port_attr_get(struct net_device *dev, 8346 struct switchdev_attr *attr) 8347 { 8348 return bnxt_port_attr_get(netdev_priv(dev), attr); 8349 } 8350 8351 static const struct switchdev_ops bnxt_switchdev_ops = { 8352 .switchdev_port_attr_get = bnxt_swdev_port_attr_get 8353 }; 8354 8355 static const struct net_device_ops bnxt_netdev_ops = { 8356 .ndo_open = bnxt_open, 8357 .ndo_start_xmit = bnxt_start_xmit, 8358 .ndo_stop = bnxt_close, 8359 .ndo_get_stats64 = bnxt_get_stats64, 8360 .ndo_set_rx_mode = bnxt_set_rx_mode, 8361 .ndo_do_ioctl = bnxt_ioctl, 8362 .ndo_validate_addr = eth_validate_addr, 8363 .ndo_set_mac_address = bnxt_change_mac_addr, 8364 .ndo_change_mtu = bnxt_change_mtu, 8365 .ndo_fix_features = bnxt_fix_features, 8366 .ndo_set_features = bnxt_set_features, 8367 .ndo_tx_timeout = bnxt_tx_timeout, 8368 #ifdef CONFIG_BNXT_SRIOV 8369 .ndo_get_vf_config = bnxt_get_vf_config, 8370 .ndo_set_vf_mac = bnxt_set_vf_mac, 8371 .ndo_set_vf_vlan = bnxt_set_vf_vlan, 8372 .ndo_set_vf_rate = bnxt_set_vf_bw, 8373 .ndo_set_vf_link_state = bnxt_set_vf_link_state, 8374 .ndo_set_vf_spoofchk = bnxt_set_vf_spoofchk, 8375 .ndo_set_vf_trust = bnxt_set_vf_trust, 8376 #endif 8377 #ifdef CONFIG_NET_POLL_CONTROLLER 8378 .ndo_poll_controller = bnxt_poll_controller, 8379 #endif 8380 .ndo_setup_tc = bnxt_setup_tc, 8381 #ifdef CONFIG_RFS_ACCEL 8382 .ndo_rx_flow_steer = bnxt_rx_flow_steer, 8383 #endif 8384 .ndo_udp_tunnel_add = bnxt_udp_tunnel_add, 8385 .ndo_udp_tunnel_del = bnxt_udp_tunnel_del, 8386 .ndo_bpf = bnxt_xdp, 8387 .ndo_bridge_getlink = bnxt_bridge_getlink, 8388 .ndo_bridge_setlink = bnxt_bridge_setlink, 8389 .ndo_get_phys_port_name = bnxt_get_phys_port_name 8390 }; 8391 8392 static void bnxt_remove_one(struct pci_dev *pdev) 8393 { 8394 struct net_device *dev = pci_get_drvdata(pdev); 8395 struct bnxt *bp = netdev_priv(dev); 8396 8397 if (BNXT_PF(bp)) { 8398 bnxt_sriov_disable(bp); 8399 bnxt_dl_unregister(bp); 8400 } 8401 8402 pci_disable_pcie_error_reporting(pdev); 8403 unregister_netdev(dev); 8404 bnxt_shutdown_tc(bp); 8405 bnxt_cancel_sp_work(bp); 8406 bp->sp_event = 0; 8407 8408 bnxt_clear_int_mode(bp); 8409 bnxt_hwrm_func_drv_unrgtr(bp); 8410 bnxt_free_hwrm_resources(bp); 8411 bnxt_free_hwrm_short_cmd_req(bp); 8412 bnxt_ethtool_free(bp); 8413 bnxt_dcb_free(bp); 8414 kfree(bp->edev); 8415 bp->edev = NULL; 8416 bnxt_cleanup_pci(bp); 8417 free_netdev(dev); 8418 } 8419 8420 static int bnxt_probe_phy(struct bnxt *bp) 8421 { 8422 int rc = 0; 8423 struct bnxt_link_info *link_info = &bp->link_info; 8424 8425 rc = bnxt_hwrm_phy_qcaps(bp); 8426 if (rc) { 8427 netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n", 8428 rc); 8429 return rc; 8430 } 8431 mutex_init(&bp->link_lock); 8432 8433 rc = bnxt_update_link(bp, false); 8434 if (rc) { 8435 netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n", 8436 rc); 8437 return rc; 8438 } 8439 8440 /* Older firmware does not have supported_auto_speeds, so assume 8441 * that all supported speeds can be autonegotiated. 8442 */ 8443 if (link_info->auto_link_speeds && !link_info->support_auto_speeds) 8444 link_info->support_auto_speeds = link_info->support_speeds; 8445 8446 /*initialize the ethool setting copy with NVM settings */ 8447 if (BNXT_AUTO_MODE(link_info->auto_mode)) { 8448 link_info->autoneg = BNXT_AUTONEG_SPEED; 8449 if (bp->hwrm_spec_code >= 0x10201) { 8450 if (link_info->auto_pause_setting & 8451 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE) 8452 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL; 8453 } else { 8454 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL; 8455 } 8456 link_info->advertising = link_info->auto_link_speeds; 8457 } else { 8458 link_info->req_link_speed = link_info->force_link_speed; 8459 link_info->req_duplex = link_info->duplex_setting; 8460 } 8461 if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) 8462 link_info->req_flow_ctrl = 8463 link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH; 8464 else 8465 link_info->req_flow_ctrl = link_info->force_pause_setting; 8466 return rc; 8467 } 8468 8469 static int bnxt_get_max_irq(struct pci_dev *pdev) 8470 { 8471 u16 ctrl; 8472 8473 if (!pdev->msix_cap) 8474 return 1; 8475 8476 pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl); 8477 return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1; 8478 } 8479 8480 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, 8481 int *max_cp) 8482 { 8483 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 8484 int max_ring_grps = 0; 8485 8486 *max_tx = hw_resc->max_tx_rings; 8487 *max_rx = hw_resc->max_rx_rings; 8488 *max_cp = min_t(int, hw_resc->max_irqs, hw_resc->max_cp_rings); 8489 *max_cp = min_t(int, *max_cp, hw_resc->max_stat_ctxs); 8490 max_ring_grps = hw_resc->max_hw_ring_grps; 8491 if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) { 8492 *max_cp -= 1; 8493 *max_rx -= 2; 8494 } 8495 if (bp->flags & BNXT_FLAG_AGG_RINGS) 8496 *max_rx >>= 1; 8497 *max_rx = min_t(int, *max_rx, max_ring_grps); 8498 } 8499 8500 int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared) 8501 { 8502 int rx, tx, cp; 8503 8504 _bnxt_get_max_rings(bp, &rx, &tx, &cp); 8505 if (!rx || !tx || !cp) 8506 return -ENOMEM; 8507 8508 *max_rx = rx; 8509 *max_tx = tx; 8510 return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared); 8511 } 8512 8513 static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx, 8514 bool shared) 8515 { 8516 int rc; 8517 8518 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared); 8519 if (rc && (bp->flags & BNXT_FLAG_AGG_RINGS)) { 8520 /* Not enough rings, try disabling agg rings. */ 8521 bp->flags &= ~BNXT_FLAG_AGG_RINGS; 8522 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared); 8523 if (rc) 8524 return rc; 8525 bp->flags |= BNXT_FLAG_NO_AGG_RINGS; 8526 bp->dev->hw_features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); 8527 bp->dev->features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); 8528 bnxt_set_ring_params(bp); 8529 } 8530 8531 if (bp->flags & BNXT_FLAG_ROCE_CAP) { 8532 int max_cp, max_stat, max_irq; 8533 8534 /* Reserve minimum resources for RoCE */ 8535 max_cp = bnxt_get_max_func_cp_rings(bp); 8536 max_stat = bnxt_get_max_func_stat_ctxs(bp); 8537 max_irq = bnxt_get_max_func_irqs(bp); 8538 if (max_cp <= BNXT_MIN_ROCE_CP_RINGS || 8539 max_irq <= BNXT_MIN_ROCE_CP_RINGS || 8540 max_stat <= BNXT_MIN_ROCE_STAT_CTXS) 8541 return 0; 8542 8543 max_cp -= BNXT_MIN_ROCE_CP_RINGS; 8544 max_irq -= BNXT_MIN_ROCE_CP_RINGS; 8545 max_stat -= BNXT_MIN_ROCE_STAT_CTXS; 8546 max_cp = min_t(int, max_cp, max_irq); 8547 max_cp = min_t(int, max_cp, max_stat); 8548 rc = bnxt_trim_rings(bp, max_rx, max_tx, max_cp, shared); 8549 if (rc) 8550 rc = 0; 8551 } 8552 return rc; 8553 } 8554 8555 /* In initial default shared ring setting, each shared ring must have a 8556 * RX/TX ring pair. 8557 */ 8558 static void bnxt_trim_dflt_sh_rings(struct bnxt *bp) 8559 { 8560 bp->cp_nr_rings = min_t(int, bp->tx_nr_rings_per_tc, bp->rx_nr_rings); 8561 bp->rx_nr_rings = bp->cp_nr_rings; 8562 bp->tx_nr_rings_per_tc = bp->cp_nr_rings; 8563 bp->tx_nr_rings = bp->tx_nr_rings_per_tc; 8564 } 8565 8566 static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh) 8567 { 8568 int dflt_rings, max_rx_rings, max_tx_rings, rc; 8569 8570 if (!bnxt_can_reserve_rings(bp)) 8571 return 0; 8572 8573 if (sh) 8574 bp->flags |= BNXT_FLAG_SHARED_RINGS; 8575 dflt_rings = netif_get_num_default_rss_queues(); 8576 /* Reduce default rings on multi-port cards so that total default 8577 * rings do not exceed CPU count. 8578 */ 8579 if (bp->port_count > 1) { 8580 int max_rings = 8581 max_t(int, num_online_cpus() / bp->port_count, 1); 8582 8583 dflt_rings = min_t(int, dflt_rings, max_rings); 8584 } 8585 rc = bnxt_get_dflt_rings(bp, &max_rx_rings, &max_tx_rings, sh); 8586 if (rc) 8587 return rc; 8588 bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings); 8589 bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings); 8590 if (sh) 8591 bnxt_trim_dflt_sh_rings(bp); 8592 else 8593 bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings; 8594 bp->tx_nr_rings = bp->tx_nr_rings_per_tc; 8595 8596 rc = __bnxt_reserve_rings(bp); 8597 if (rc) 8598 netdev_warn(bp->dev, "Unable to reserve tx rings\n"); 8599 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 8600 if (sh) 8601 bnxt_trim_dflt_sh_rings(bp); 8602 8603 /* Rings may have been trimmed, re-reserve the trimmed rings. */ 8604 if (bnxt_need_reserve_rings(bp)) { 8605 rc = __bnxt_reserve_rings(bp); 8606 if (rc) 8607 netdev_warn(bp->dev, "2nd rings reservation failed.\n"); 8608 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 8609 } 8610 bp->num_stat_ctxs = bp->cp_nr_rings; 8611 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 8612 bp->rx_nr_rings++; 8613 bp->cp_nr_rings++; 8614 } 8615 return rc; 8616 } 8617 8618 static int bnxt_init_dflt_ring_mode(struct bnxt *bp) 8619 { 8620 int rc; 8621 8622 if (bp->tx_nr_rings) 8623 return 0; 8624 8625 rc = bnxt_set_dflt_rings(bp, true); 8626 if (rc) { 8627 netdev_err(bp->dev, "Not enough rings available.\n"); 8628 return rc; 8629 } 8630 rc = bnxt_init_int_mode(bp); 8631 if (rc) 8632 return rc; 8633 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 8634 if (bnxt_rfs_supported(bp) && bnxt_rfs_capable(bp)) { 8635 bp->flags |= BNXT_FLAG_RFS; 8636 bp->dev->features |= NETIF_F_NTUPLE; 8637 } 8638 return 0; 8639 } 8640 8641 int bnxt_restore_pf_fw_resources(struct bnxt *bp) 8642 { 8643 int rc; 8644 8645 ASSERT_RTNL(); 8646 bnxt_hwrm_func_qcaps(bp); 8647 8648 if (netif_running(bp->dev)) 8649 __bnxt_close_nic(bp, true, false); 8650 8651 bnxt_ulp_irq_stop(bp); 8652 bnxt_clear_int_mode(bp); 8653 rc = bnxt_init_int_mode(bp); 8654 bnxt_ulp_irq_restart(bp, rc); 8655 8656 if (netif_running(bp->dev)) { 8657 if (rc) 8658 dev_close(bp->dev); 8659 else 8660 rc = bnxt_open_nic(bp, true, false); 8661 } 8662 8663 return rc; 8664 } 8665 8666 static int bnxt_init_mac_addr(struct bnxt *bp) 8667 { 8668 int rc = 0; 8669 8670 if (BNXT_PF(bp)) { 8671 memcpy(bp->dev->dev_addr, bp->pf.mac_addr, ETH_ALEN); 8672 } else { 8673 #ifdef CONFIG_BNXT_SRIOV 8674 struct bnxt_vf_info *vf = &bp->vf; 8675 8676 if (is_valid_ether_addr(vf->mac_addr)) { 8677 /* overwrite netdev dev_addr with admin VF MAC */ 8678 memcpy(bp->dev->dev_addr, vf->mac_addr, ETH_ALEN); 8679 } else { 8680 eth_hw_addr_random(bp->dev); 8681 } 8682 rc = bnxt_approve_mac(bp, bp->dev->dev_addr); 8683 #endif 8684 } 8685 return rc; 8686 } 8687 8688 static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) 8689 { 8690 static int version_printed; 8691 struct net_device *dev; 8692 struct bnxt *bp; 8693 int rc, max_irqs; 8694 8695 if (pci_is_bridge(pdev)) 8696 return -ENODEV; 8697 8698 if (version_printed++ == 0) 8699 pr_info("%s", version); 8700 8701 max_irqs = bnxt_get_max_irq(pdev); 8702 dev = alloc_etherdev_mq(sizeof(*bp), max_irqs); 8703 if (!dev) 8704 return -ENOMEM; 8705 8706 bp = netdev_priv(dev); 8707 8708 if (bnxt_vf_pciid(ent->driver_data)) 8709 bp->flags |= BNXT_FLAG_VF; 8710 8711 if (pdev->msix_cap) 8712 bp->flags |= BNXT_FLAG_MSIX_CAP; 8713 8714 rc = bnxt_init_board(pdev, dev); 8715 if (rc < 0) 8716 goto init_err_free; 8717 8718 dev->netdev_ops = &bnxt_netdev_ops; 8719 dev->watchdog_timeo = BNXT_TX_TIMEOUT; 8720 dev->ethtool_ops = &bnxt_ethtool_ops; 8721 SWITCHDEV_SET_OPS(dev, &bnxt_switchdev_ops); 8722 pci_set_drvdata(pdev, dev); 8723 8724 rc = bnxt_alloc_hwrm_resources(bp); 8725 if (rc) 8726 goto init_err_pci_clean; 8727 8728 mutex_init(&bp->hwrm_cmd_lock); 8729 rc = bnxt_hwrm_ver_get(bp); 8730 if (rc) 8731 goto init_err_pci_clean; 8732 8733 if (bp->flags & BNXT_FLAG_SHORT_CMD) { 8734 rc = bnxt_alloc_hwrm_short_cmd_req(bp); 8735 if (rc) 8736 goto init_err_pci_clean; 8737 } 8738 8739 rc = bnxt_hwrm_func_reset(bp); 8740 if (rc) 8741 goto init_err_pci_clean; 8742 8743 bnxt_hwrm_fw_set_time(bp); 8744 8745 dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG | 8746 NETIF_F_TSO | NETIF_F_TSO6 | 8747 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE | 8748 NETIF_F_GSO_IPXIP4 | 8749 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM | 8750 NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH | 8751 NETIF_F_RXCSUM | NETIF_F_GRO; 8752 8753 if (!BNXT_CHIP_TYPE_NITRO_A0(bp)) 8754 dev->hw_features |= NETIF_F_LRO; 8755 8756 dev->hw_enc_features = 8757 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG | 8758 NETIF_F_TSO | NETIF_F_TSO6 | 8759 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE | 8760 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM | 8761 NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL; 8762 dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM | 8763 NETIF_F_GSO_GRE_CSUM; 8764 dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA; 8765 dev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX | 8766 NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX; 8767 if (!BNXT_CHIP_TYPE_NITRO_A0(bp)) 8768 dev->hw_features |= NETIF_F_GRO_HW; 8769 dev->features |= dev->hw_features | NETIF_F_HIGHDMA; 8770 if (dev->features & NETIF_F_GRO_HW) 8771 dev->features &= ~NETIF_F_LRO; 8772 dev->priv_flags |= IFF_UNICAST_FLT; 8773 8774 #ifdef CONFIG_BNXT_SRIOV 8775 init_waitqueue_head(&bp->sriov_cfg_wait); 8776 mutex_init(&bp->sriov_lock); 8777 #endif 8778 bp->gro_func = bnxt_gro_func_5730x; 8779 if (BNXT_CHIP_P4_PLUS(bp)) 8780 bp->gro_func = bnxt_gro_func_5731x; 8781 else 8782 bp->flags |= BNXT_FLAG_DOUBLE_DB; 8783 8784 rc = bnxt_hwrm_func_drv_rgtr(bp); 8785 if (rc) 8786 goto init_err_pci_clean; 8787 8788 rc = bnxt_hwrm_func_rgtr_async_events(bp, NULL, 0); 8789 if (rc) 8790 goto init_err_pci_clean; 8791 8792 bp->ulp_probe = bnxt_ulp_probe; 8793 8794 /* Get the MAX capabilities for this function */ 8795 rc = bnxt_hwrm_func_qcaps(bp); 8796 if (rc) { 8797 netdev_err(bp->dev, "hwrm query capability failure rc: %x\n", 8798 rc); 8799 rc = -1; 8800 goto init_err_pci_clean; 8801 } 8802 rc = bnxt_init_mac_addr(bp); 8803 if (rc) { 8804 dev_err(&pdev->dev, "Unable to initialize mac address.\n"); 8805 rc = -EADDRNOTAVAIL; 8806 goto init_err_pci_clean; 8807 } 8808 rc = bnxt_hwrm_queue_qportcfg(bp); 8809 if (rc) { 8810 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %x\n", 8811 rc); 8812 rc = -1; 8813 goto init_err_pci_clean; 8814 } 8815 8816 bnxt_hwrm_func_qcfg(bp); 8817 bnxt_hwrm_port_led_qcaps(bp); 8818 bnxt_ethtool_init(bp); 8819 bnxt_dcb_init(bp); 8820 8821 /* MTU range: 60 - FW defined max */ 8822 dev->min_mtu = ETH_ZLEN; 8823 dev->max_mtu = bp->max_mtu; 8824 8825 rc = bnxt_probe_phy(bp); 8826 if (rc) 8827 goto init_err_pci_clean; 8828 8829 bnxt_set_rx_skb_mode(bp, false); 8830 bnxt_set_tpa_flags(bp); 8831 bnxt_set_ring_params(bp); 8832 bnxt_set_max_func_irqs(bp, max_irqs); 8833 rc = bnxt_set_dflt_rings(bp, true); 8834 if (rc) { 8835 netdev_err(bp->dev, "Not enough rings available.\n"); 8836 rc = -ENOMEM; 8837 goto init_err_pci_clean; 8838 } 8839 8840 /* Default RSS hash cfg. */ 8841 bp->rss_hash_cfg = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 | 8842 VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 | 8843 VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 | 8844 VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6; 8845 if (BNXT_CHIP_P4_PLUS(bp) && bp->hwrm_spec_code >= 0x10501) { 8846 bp->flags |= BNXT_FLAG_UDP_RSS_CAP; 8847 bp->rss_hash_cfg |= VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 | 8848 VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6; 8849 } 8850 8851 bnxt_hwrm_vnic_qcaps(bp); 8852 if (bnxt_rfs_supported(bp)) { 8853 dev->hw_features |= NETIF_F_NTUPLE; 8854 if (bnxt_rfs_capable(bp)) { 8855 bp->flags |= BNXT_FLAG_RFS; 8856 dev->features |= NETIF_F_NTUPLE; 8857 } 8858 } 8859 8860 if (dev->hw_features & NETIF_F_HW_VLAN_CTAG_RX) 8861 bp->flags |= BNXT_FLAG_STRIP_VLAN; 8862 8863 rc = bnxt_init_int_mode(bp); 8864 if (rc) 8865 goto init_err_pci_clean; 8866 8867 /* No TC has been set yet and rings may have been trimmed due to 8868 * limited MSIX, so we re-initialize the TX rings per TC. 8869 */ 8870 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 8871 8872 bnxt_get_wol_settings(bp); 8873 if (bp->flags & BNXT_FLAG_WOL_CAP) 8874 device_set_wakeup_enable(&pdev->dev, bp->wol); 8875 else 8876 device_set_wakeup_capable(&pdev->dev, false); 8877 8878 bnxt_hwrm_set_cache_line_size(bp, cache_line_size()); 8879 8880 if (BNXT_PF(bp)) { 8881 if (!bnxt_pf_wq) { 8882 bnxt_pf_wq = 8883 create_singlethread_workqueue("bnxt_pf_wq"); 8884 if (!bnxt_pf_wq) { 8885 dev_err(&pdev->dev, "Unable to create workqueue.\n"); 8886 goto init_err_pci_clean; 8887 } 8888 } 8889 bnxt_init_tc(bp); 8890 } 8891 8892 rc = register_netdev(dev); 8893 if (rc) 8894 goto init_err_cleanup_tc; 8895 8896 if (BNXT_PF(bp)) 8897 bnxt_dl_register(bp); 8898 8899 netdev_info(dev, "%s found at mem %lx, node addr %pM\n", 8900 board_info[ent->driver_data].name, 8901 (long)pci_resource_start(pdev, 0), dev->dev_addr); 8902 pcie_print_link_status(pdev); 8903 8904 return 0; 8905 8906 init_err_cleanup_tc: 8907 bnxt_shutdown_tc(bp); 8908 bnxt_clear_int_mode(bp); 8909 8910 init_err_pci_clean: 8911 bnxt_cleanup_pci(bp); 8912 8913 init_err_free: 8914 free_netdev(dev); 8915 return rc; 8916 } 8917 8918 static void bnxt_shutdown(struct pci_dev *pdev) 8919 { 8920 struct net_device *dev = pci_get_drvdata(pdev); 8921 struct bnxt *bp; 8922 8923 if (!dev) 8924 return; 8925 8926 rtnl_lock(); 8927 bp = netdev_priv(dev); 8928 if (!bp) 8929 goto shutdown_exit; 8930 8931 if (netif_running(dev)) 8932 dev_close(dev); 8933 8934 bnxt_ulp_shutdown(bp); 8935 8936 if (system_state == SYSTEM_POWER_OFF) { 8937 bnxt_clear_int_mode(bp); 8938 pci_wake_from_d3(pdev, bp->wol); 8939 pci_set_power_state(pdev, PCI_D3hot); 8940 } 8941 8942 shutdown_exit: 8943 rtnl_unlock(); 8944 } 8945 8946 #ifdef CONFIG_PM_SLEEP 8947 static int bnxt_suspend(struct device *device) 8948 { 8949 struct pci_dev *pdev = to_pci_dev(device); 8950 struct net_device *dev = pci_get_drvdata(pdev); 8951 struct bnxt *bp = netdev_priv(dev); 8952 int rc = 0; 8953 8954 rtnl_lock(); 8955 if (netif_running(dev)) { 8956 netif_device_detach(dev); 8957 rc = bnxt_close(dev); 8958 } 8959 bnxt_hwrm_func_drv_unrgtr(bp); 8960 rtnl_unlock(); 8961 return rc; 8962 } 8963 8964 static int bnxt_resume(struct device *device) 8965 { 8966 struct pci_dev *pdev = to_pci_dev(device); 8967 struct net_device *dev = pci_get_drvdata(pdev); 8968 struct bnxt *bp = netdev_priv(dev); 8969 int rc = 0; 8970 8971 rtnl_lock(); 8972 if (bnxt_hwrm_ver_get(bp) || bnxt_hwrm_func_drv_rgtr(bp)) { 8973 rc = -ENODEV; 8974 goto resume_exit; 8975 } 8976 rc = bnxt_hwrm_func_reset(bp); 8977 if (rc) { 8978 rc = -EBUSY; 8979 goto resume_exit; 8980 } 8981 bnxt_get_wol_settings(bp); 8982 if (netif_running(dev)) { 8983 rc = bnxt_open(dev); 8984 if (!rc) 8985 netif_device_attach(dev); 8986 } 8987 8988 resume_exit: 8989 rtnl_unlock(); 8990 return rc; 8991 } 8992 8993 static SIMPLE_DEV_PM_OPS(bnxt_pm_ops, bnxt_suspend, bnxt_resume); 8994 #define BNXT_PM_OPS (&bnxt_pm_ops) 8995 8996 #else 8997 8998 #define BNXT_PM_OPS NULL 8999 9000 #endif /* CONFIG_PM_SLEEP */ 9001 9002 /** 9003 * bnxt_io_error_detected - called when PCI error is detected 9004 * @pdev: Pointer to PCI device 9005 * @state: The current pci connection state 9006 * 9007 * This function is called after a PCI bus error affecting 9008 * this device has been detected. 9009 */ 9010 static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev, 9011 pci_channel_state_t state) 9012 { 9013 struct net_device *netdev = pci_get_drvdata(pdev); 9014 struct bnxt *bp = netdev_priv(netdev); 9015 9016 netdev_info(netdev, "PCI I/O error detected\n"); 9017 9018 rtnl_lock(); 9019 netif_device_detach(netdev); 9020 9021 bnxt_ulp_stop(bp); 9022 9023 if (state == pci_channel_io_perm_failure) { 9024 rtnl_unlock(); 9025 return PCI_ERS_RESULT_DISCONNECT; 9026 } 9027 9028 if (netif_running(netdev)) 9029 bnxt_close(netdev); 9030 9031 pci_disable_device(pdev); 9032 rtnl_unlock(); 9033 9034 /* Request a slot slot reset. */ 9035 return PCI_ERS_RESULT_NEED_RESET; 9036 } 9037 9038 /** 9039 * bnxt_io_slot_reset - called after the pci bus has been reset. 9040 * @pdev: Pointer to PCI device 9041 * 9042 * Restart the card from scratch, as if from a cold-boot. 9043 * At this point, the card has exprienced a hard reset, 9044 * followed by fixups by BIOS, and has its config space 9045 * set up identically to what it was at cold boot. 9046 */ 9047 static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev) 9048 { 9049 struct net_device *netdev = pci_get_drvdata(pdev); 9050 struct bnxt *bp = netdev_priv(netdev); 9051 int err = 0; 9052 pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT; 9053 9054 netdev_info(bp->dev, "PCI Slot Reset\n"); 9055 9056 rtnl_lock(); 9057 9058 if (pci_enable_device(pdev)) { 9059 dev_err(&pdev->dev, 9060 "Cannot re-enable PCI device after reset.\n"); 9061 } else { 9062 pci_set_master(pdev); 9063 9064 err = bnxt_hwrm_func_reset(bp); 9065 if (!err && netif_running(netdev)) 9066 err = bnxt_open(netdev); 9067 9068 if (!err) { 9069 result = PCI_ERS_RESULT_RECOVERED; 9070 bnxt_ulp_start(bp); 9071 } 9072 } 9073 9074 if (result != PCI_ERS_RESULT_RECOVERED && netif_running(netdev)) 9075 dev_close(netdev); 9076 9077 rtnl_unlock(); 9078 9079 err = pci_cleanup_aer_uncorrect_error_status(pdev); 9080 if (err) { 9081 dev_err(&pdev->dev, 9082 "pci_cleanup_aer_uncorrect_error_status failed 0x%0x\n", 9083 err); /* non-fatal, continue */ 9084 } 9085 9086 return PCI_ERS_RESULT_RECOVERED; 9087 } 9088 9089 /** 9090 * bnxt_io_resume - called when traffic can start flowing again. 9091 * @pdev: Pointer to PCI device 9092 * 9093 * This callback is called when the error recovery driver tells 9094 * us that its OK to resume normal operation. 9095 */ 9096 static void bnxt_io_resume(struct pci_dev *pdev) 9097 { 9098 struct net_device *netdev = pci_get_drvdata(pdev); 9099 9100 rtnl_lock(); 9101 9102 netif_device_attach(netdev); 9103 9104 rtnl_unlock(); 9105 } 9106 9107 static const struct pci_error_handlers bnxt_err_handler = { 9108 .error_detected = bnxt_io_error_detected, 9109 .slot_reset = bnxt_io_slot_reset, 9110 .resume = bnxt_io_resume 9111 }; 9112 9113 static struct pci_driver bnxt_pci_driver = { 9114 .name = DRV_MODULE_NAME, 9115 .id_table = bnxt_pci_tbl, 9116 .probe = bnxt_init_one, 9117 .remove = bnxt_remove_one, 9118 .shutdown = bnxt_shutdown, 9119 .driver.pm = BNXT_PM_OPS, 9120 .err_handler = &bnxt_err_handler, 9121 #if defined(CONFIG_BNXT_SRIOV) 9122 .sriov_configure = bnxt_sriov_configure, 9123 #endif 9124 }; 9125 9126 static int __init bnxt_init(void) 9127 { 9128 bnxt_debug_init(); 9129 return pci_register_driver(&bnxt_pci_driver); 9130 } 9131 9132 static void __exit bnxt_exit(void) 9133 { 9134 pci_unregister_driver(&bnxt_pci_driver); 9135 if (bnxt_pf_wq) 9136 destroy_workqueue(bnxt_pf_wq); 9137 bnxt_debug_exit(); 9138 } 9139 9140 module_init(bnxt_init); 9141 module_exit(bnxt_exit); 9142