1 /* Broadcom NetXtreme-C/E network driver. 2 * 3 * Copyright (c) 2014-2016 Broadcom Corporation 4 * Copyright (c) 2016-2018 Broadcom Limited 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License as published by 8 * the Free Software Foundation. 9 */ 10 11 #include <linux/module.h> 12 13 #include <linux/stringify.h> 14 #include <linux/kernel.h> 15 #include <linux/timer.h> 16 #include <linux/errno.h> 17 #include <linux/ioport.h> 18 #include <linux/slab.h> 19 #include <linux/vmalloc.h> 20 #include <linux/interrupt.h> 21 #include <linux/pci.h> 22 #include <linux/netdevice.h> 23 #include <linux/etherdevice.h> 24 #include <linux/skbuff.h> 25 #include <linux/dma-mapping.h> 26 #include <linux/bitops.h> 27 #include <linux/io.h> 28 #include <linux/irq.h> 29 #include <linux/delay.h> 30 #include <asm/byteorder.h> 31 #include <asm/page.h> 32 #include <linux/time.h> 33 #include <linux/mii.h> 34 #include <linux/if.h> 35 #include <linux/if_vlan.h> 36 #include <linux/if_bridge.h> 37 #include <linux/rtc.h> 38 #include <linux/bpf.h> 39 #include <net/ip.h> 40 #include <net/tcp.h> 41 #include <net/udp.h> 42 #include <net/checksum.h> 43 #include <net/ip6_checksum.h> 44 #include <net/udp_tunnel.h> 45 #include <linux/workqueue.h> 46 #include <linux/prefetch.h> 47 #include <linux/cache.h> 48 #include <linux/log2.h> 49 #include <linux/aer.h> 50 #include <linux/bitmap.h> 51 #include <linux/cpu_rmap.h> 52 #include <linux/cpumask.h> 53 #include <net/pkt_cls.h> 54 55 #include "bnxt_hsi.h" 56 #include "bnxt.h" 57 #include "bnxt_ulp.h" 58 #include "bnxt_sriov.h" 59 #include "bnxt_ethtool.h" 60 #include "bnxt_dcb.h" 61 #include "bnxt_xdp.h" 62 #include "bnxt_vfr.h" 63 #include "bnxt_tc.h" 64 #include "bnxt_devlink.h" 65 #include "bnxt_debugfs.h" 66 67 #define BNXT_TX_TIMEOUT (5 * HZ) 68 69 static const char version[] = 70 "Broadcom NetXtreme-C/E driver " DRV_MODULE_NAME " v" DRV_MODULE_VERSION "\n"; 71 72 MODULE_LICENSE("GPL"); 73 MODULE_DESCRIPTION("Broadcom BCM573xx network driver"); 74 MODULE_VERSION(DRV_MODULE_VERSION); 75 76 #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN) 77 #define BNXT_RX_DMA_OFFSET NET_SKB_PAD 78 #define BNXT_RX_COPY_THRESH 256 79 80 #define BNXT_TX_PUSH_THRESH 164 81 82 enum board_idx { 83 BCM57301, 84 BCM57302, 85 BCM57304, 86 BCM57417_NPAR, 87 BCM58700, 88 BCM57311, 89 BCM57312, 90 BCM57402, 91 BCM57404, 92 BCM57406, 93 BCM57402_NPAR, 94 BCM57407, 95 BCM57412, 96 BCM57414, 97 BCM57416, 98 BCM57417, 99 BCM57412_NPAR, 100 BCM57314, 101 BCM57417_SFP, 102 BCM57416_SFP, 103 BCM57404_NPAR, 104 BCM57406_NPAR, 105 BCM57407_SFP, 106 BCM57407_NPAR, 107 BCM57414_NPAR, 108 BCM57416_NPAR, 109 BCM57452, 110 BCM57454, 111 BCM5745x_NPAR, 112 BCM58802, 113 BCM58804, 114 BCM58808, 115 NETXTREME_E_VF, 116 NETXTREME_C_VF, 117 NETXTREME_S_VF, 118 }; 119 120 /* indexed by enum above */ 121 static const struct { 122 char *name; 123 } board_info[] = { 124 [BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" }, 125 [BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" }, 126 [BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" }, 127 [BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" }, 128 [BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" }, 129 [BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" }, 130 [BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" }, 131 [BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" }, 132 [BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" }, 133 [BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" }, 134 [BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" }, 135 [BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" }, 136 [BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" }, 137 [BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" }, 138 [BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" }, 139 [BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" }, 140 [BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" }, 141 [BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" }, 142 [BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" }, 143 [BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" }, 144 [BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" }, 145 [BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" }, 146 [BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" }, 147 [BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" }, 148 [BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" }, 149 [BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" }, 150 [BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" }, 151 [BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" }, 152 [BCM5745x_NPAR] = { "Broadcom BCM5745x NetXtreme-E Ethernet Partition" }, 153 [BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" }, 154 [BCM58804] = { "Broadcom BCM58804 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" }, 155 [BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" }, 156 [NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" }, 157 [NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" }, 158 [NETXTREME_S_VF] = { "Broadcom NetXtreme-S Ethernet Virtual Function" }, 159 }; 160 161 static const struct pci_device_id bnxt_pci_tbl[] = { 162 { PCI_VDEVICE(BROADCOM, 0x1604), .driver_data = BCM5745x_NPAR }, 163 { PCI_VDEVICE(BROADCOM, 0x1605), .driver_data = BCM5745x_NPAR }, 164 { PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 }, 165 { PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR }, 166 { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 }, 167 { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 }, 168 { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 }, 169 { PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR }, 170 { PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 }, 171 { PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 }, 172 { PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 }, 173 { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 }, 174 { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 }, 175 { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 }, 176 { PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR }, 177 { PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 }, 178 { PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 }, 179 { PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 }, 180 { PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 }, 181 { PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 }, 182 { PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR }, 183 { PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 }, 184 { PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP }, 185 { PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP }, 186 { PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR }, 187 { PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR }, 188 { PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP }, 189 { PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR }, 190 { PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR }, 191 { PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR }, 192 { PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR }, 193 { PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR }, 194 { PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR }, 195 { PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 }, 196 { PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 }, 197 { PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 }, 198 { PCI_VDEVICE(BROADCOM, 0xd804), .driver_data = BCM58804 }, 199 #ifdef CONFIG_BNXT_SRIOV 200 { PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF }, 201 { PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF }, 202 { PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF }, 203 { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF }, 204 { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF }, 205 { PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF }, 206 { PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF }, 207 { PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF }, 208 { PCI_VDEVICE(BROADCOM, 0xd800), .driver_data = NETXTREME_S_VF }, 209 #endif 210 { 0 } 211 }; 212 213 MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl); 214 215 static const u16 bnxt_vf_req_snif[] = { 216 HWRM_FUNC_CFG, 217 HWRM_FUNC_VF_CFG, 218 HWRM_PORT_PHY_QCFG, 219 HWRM_CFA_L2_FILTER_ALLOC, 220 }; 221 222 static const u16 bnxt_async_events_arr[] = { 223 ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE, 224 ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD, 225 ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED, 226 ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE, 227 ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE, 228 }; 229 230 static struct workqueue_struct *bnxt_pf_wq; 231 232 static bool bnxt_vf_pciid(enum board_idx idx) 233 { 234 return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF || 235 idx == NETXTREME_S_VF); 236 } 237 238 #define DB_CP_REARM_FLAGS (DB_KEY_CP | DB_IDX_VALID) 239 #define DB_CP_FLAGS (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS) 240 #define DB_CP_IRQ_DIS_FLAGS (DB_KEY_CP | DB_IRQ_DIS) 241 242 #define BNXT_CP_DB_REARM(db, raw_cons) \ 243 writel(DB_CP_REARM_FLAGS | RING_CMP(raw_cons), db) 244 245 #define BNXT_CP_DB(db, raw_cons) \ 246 writel(DB_CP_FLAGS | RING_CMP(raw_cons), db) 247 248 #define BNXT_CP_DB_IRQ_DIS(db) \ 249 writel(DB_CP_IRQ_DIS_FLAGS, db) 250 251 const u16 bnxt_lhint_arr[] = { 252 TX_BD_FLAGS_LHINT_512_AND_SMALLER, 253 TX_BD_FLAGS_LHINT_512_TO_1023, 254 TX_BD_FLAGS_LHINT_1024_TO_2047, 255 TX_BD_FLAGS_LHINT_1024_TO_2047, 256 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 257 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 258 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 259 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 260 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 261 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 262 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 263 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 264 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 265 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 266 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 267 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 268 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 269 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 270 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 271 }; 272 273 static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb) 274 { 275 struct metadata_dst *md_dst = skb_metadata_dst(skb); 276 277 if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX) 278 return 0; 279 280 return md_dst->u.port_info.port_id; 281 } 282 283 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev) 284 { 285 struct bnxt *bp = netdev_priv(dev); 286 struct tx_bd *txbd; 287 struct tx_bd_ext *txbd1; 288 struct netdev_queue *txq; 289 int i; 290 dma_addr_t mapping; 291 unsigned int length, pad = 0; 292 u32 len, free_size, vlan_tag_flags, cfa_action, flags; 293 u16 prod, last_frag; 294 struct pci_dev *pdev = bp->pdev; 295 struct bnxt_tx_ring_info *txr; 296 struct bnxt_sw_tx_bd *tx_buf; 297 298 i = skb_get_queue_mapping(skb); 299 if (unlikely(i >= bp->tx_nr_rings)) { 300 dev_kfree_skb_any(skb); 301 return NETDEV_TX_OK; 302 } 303 304 txq = netdev_get_tx_queue(dev, i); 305 txr = &bp->tx_ring[bp->tx_ring_map[i]]; 306 prod = txr->tx_prod; 307 308 free_size = bnxt_tx_avail(bp, txr); 309 if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) { 310 netif_tx_stop_queue(txq); 311 return NETDEV_TX_BUSY; 312 } 313 314 length = skb->len; 315 len = skb_headlen(skb); 316 last_frag = skb_shinfo(skb)->nr_frags; 317 318 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 319 320 txbd->tx_bd_opaque = prod; 321 322 tx_buf = &txr->tx_buf_ring[prod]; 323 tx_buf->skb = skb; 324 tx_buf->nr_frags = last_frag; 325 326 vlan_tag_flags = 0; 327 cfa_action = bnxt_xmit_get_cfa_action(skb); 328 if (skb_vlan_tag_present(skb)) { 329 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN | 330 skb_vlan_tag_get(skb); 331 /* Currently supports 8021Q, 8021AD vlan offloads 332 * QINQ1, QINQ2, QINQ3 vlan headers are deprecated 333 */ 334 if (skb->vlan_proto == htons(ETH_P_8021Q)) 335 vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT; 336 } 337 338 if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh) { 339 struct tx_push_buffer *tx_push_buf = txr->tx_push; 340 struct tx_push_bd *tx_push = &tx_push_buf->push_bd; 341 struct tx_bd_ext *tx_push1 = &tx_push->txbd2; 342 void *pdata = tx_push_buf->data; 343 u64 *end; 344 int j, push_len; 345 346 /* Set COAL_NOW to be ready quickly for the next push */ 347 tx_push->tx_bd_len_flags_type = 348 cpu_to_le32((length << TX_BD_LEN_SHIFT) | 349 TX_BD_TYPE_LONG_TX_BD | 350 TX_BD_FLAGS_LHINT_512_AND_SMALLER | 351 TX_BD_FLAGS_COAL_NOW | 352 TX_BD_FLAGS_PACKET_END | 353 (2 << TX_BD_FLAGS_BD_CNT_SHIFT)); 354 355 if (skb->ip_summed == CHECKSUM_PARTIAL) 356 tx_push1->tx_bd_hsize_lflags = 357 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM); 358 else 359 tx_push1->tx_bd_hsize_lflags = 0; 360 361 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags); 362 tx_push1->tx_bd_cfa_action = 363 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT); 364 365 end = pdata + length; 366 end = PTR_ALIGN(end, 8) - 1; 367 *end = 0; 368 369 skb_copy_from_linear_data(skb, pdata, len); 370 pdata += len; 371 for (j = 0; j < last_frag; j++) { 372 skb_frag_t *frag = &skb_shinfo(skb)->frags[j]; 373 void *fptr; 374 375 fptr = skb_frag_address_safe(frag); 376 if (!fptr) 377 goto normal_tx; 378 379 memcpy(pdata, fptr, skb_frag_size(frag)); 380 pdata += skb_frag_size(frag); 381 } 382 383 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type; 384 txbd->tx_bd_haddr = txr->data_mapping; 385 prod = NEXT_TX(prod); 386 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 387 memcpy(txbd, tx_push1, sizeof(*txbd)); 388 prod = NEXT_TX(prod); 389 tx_push->doorbell = 390 cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | prod); 391 txr->tx_prod = prod; 392 393 tx_buf->is_push = 1; 394 netdev_tx_sent_queue(txq, skb->len); 395 wmb(); /* Sync is_push and byte queue before pushing data */ 396 397 push_len = (length + sizeof(*tx_push) + 7) / 8; 398 if (push_len > 16) { 399 __iowrite64_copy(txr->tx_doorbell, tx_push_buf, 16); 400 __iowrite32_copy(txr->tx_doorbell + 4, tx_push_buf + 1, 401 (push_len - 16) << 1); 402 } else { 403 __iowrite64_copy(txr->tx_doorbell, tx_push_buf, 404 push_len); 405 } 406 407 goto tx_done; 408 } 409 410 normal_tx: 411 if (length < BNXT_MIN_PKT_SIZE) { 412 pad = BNXT_MIN_PKT_SIZE - length; 413 if (skb_pad(skb, pad)) { 414 /* SKB already freed. */ 415 tx_buf->skb = NULL; 416 return NETDEV_TX_OK; 417 } 418 length = BNXT_MIN_PKT_SIZE; 419 } 420 421 mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE); 422 423 if (unlikely(dma_mapping_error(&pdev->dev, mapping))) { 424 dev_kfree_skb_any(skb); 425 tx_buf->skb = NULL; 426 return NETDEV_TX_OK; 427 } 428 429 dma_unmap_addr_set(tx_buf, mapping, mapping); 430 flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD | 431 ((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT); 432 433 txbd->tx_bd_haddr = cpu_to_le64(mapping); 434 435 prod = NEXT_TX(prod); 436 txbd1 = (struct tx_bd_ext *) 437 &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 438 439 txbd1->tx_bd_hsize_lflags = 0; 440 if (skb_is_gso(skb)) { 441 u32 hdr_len; 442 443 if (skb->encapsulation) 444 hdr_len = skb_inner_network_offset(skb) + 445 skb_inner_network_header_len(skb) + 446 inner_tcp_hdrlen(skb); 447 else 448 hdr_len = skb_transport_offset(skb) + 449 tcp_hdrlen(skb); 450 451 txbd1->tx_bd_hsize_lflags = cpu_to_le32(TX_BD_FLAGS_LSO | 452 TX_BD_FLAGS_T_IPID | 453 (hdr_len << (TX_BD_HSIZE_SHIFT - 1))); 454 length = skb_shinfo(skb)->gso_size; 455 txbd1->tx_bd_mss = cpu_to_le32(length); 456 length += hdr_len; 457 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { 458 txbd1->tx_bd_hsize_lflags = 459 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM); 460 txbd1->tx_bd_mss = 0; 461 } 462 463 length >>= 9; 464 flags |= bnxt_lhint_arr[length]; 465 txbd->tx_bd_len_flags_type = cpu_to_le32(flags); 466 467 txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags); 468 txbd1->tx_bd_cfa_action = 469 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT); 470 for (i = 0; i < last_frag; i++) { 471 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 472 473 prod = NEXT_TX(prod); 474 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 475 476 len = skb_frag_size(frag); 477 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len, 478 DMA_TO_DEVICE); 479 480 if (unlikely(dma_mapping_error(&pdev->dev, mapping))) 481 goto tx_dma_error; 482 483 tx_buf = &txr->tx_buf_ring[prod]; 484 dma_unmap_addr_set(tx_buf, mapping, mapping); 485 486 txbd->tx_bd_haddr = cpu_to_le64(mapping); 487 488 flags = len << TX_BD_LEN_SHIFT; 489 txbd->tx_bd_len_flags_type = cpu_to_le32(flags); 490 } 491 492 flags &= ~TX_BD_LEN; 493 txbd->tx_bd_len_flags_type = 494 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags | 495 TX_BD_FLAGS_PACKET_END); 496 497 netdev_tx_sent_queue(txq, skb->len); 498 499 /* Sync BD data before updating doorbell */ 500 wmb(); 501 502 prod = NEXT_TX(prod); 503 txr->tx_prod = prod; 504 505 if (!skb->xmit_more || netif_xmit_stopped(txq)) 506 bnxt_db_write(bp, txr->tx_doorbell, DB_KEY_TX | prod); 507 508 tx_done: 509 510 mmiowb(); 511 512 if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) { 513 if (skb->xmit_more && !tx_buf->is_push) 514 bnxt_db_write(bp, txr->tx_doorbell, DB_KEY_TX | prod); 515 516 netif_tx_stop_queue(txq); 517 518 /* netif_tx_stop_queue() must be done before checking 519 * tx index in bnxt_tx_avail() below, because in 520 * bnxt_tx_int(), we update tx index before checking for 521 * netif_tx_queue_stopped(). 522 */ 523 smp_mb(); 524 if (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh) 525 netif_tx_wake_queue(txq); 526 } 527 return NETDEV_TX_OK; 528 529 tx_dma_error: 530 last_frag = i; 531 532 /* start back at beginning and unmap skb */ 533 prod = txr->tx_prod; 534 tx_buf = &txr->tx_buf_ring[prod]; 535 tx_buf->skb = NULL; 536 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping), 537 skb_headlen(skb), PCI_DMA_TODEVICE); 538 prod = NEXT_TX(prod); 539 540 /* unmap remaining mapped pages */ 541 for (i = 0; i < last_frag; i++) { 542 prod = NEXT_TX(prod); 543 tx_buf = &txr->tx_buf_ring[prod]; 544 dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping), 545 skb_frag_size(&skb_shinfo(skb)->frags[i]), 546 PCI_DMA_TODEVICE); 547 } 548 549 dev_kfree_skb_any(skb); 550 return NETDEV_TX_OK; 551 } 552 553 static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts) 554 { 555 struct bnxt_tx_ring_info *txr = bnapi->tx_ring; 556 struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index); 557 u16 cons = txr->tx_cons; 558 struct pci_dev *pdev = bp->pdev; 559 int i; 560 unsigned int tx_bytes = 0; 561 562 for (i = 0; i < nr_pkts; i++) { 563 struct bnxt_sw_tx_bd *tx_buf; 564 struct sk_buff *skb; 565 int j, last; 566 567 tx_buf = &txr->tx_buf_ring[cons]; 568 cons = NEXT_TX(cons); 569 skb = tx_buf->skb; 570 tx_buf->skb = NULL; 571 572 if (tx_buf->is_push) { 573 tx_buf->is_push = 0; 574 goto next_tx_int; 575 } 576 577 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping), 578 skb_headlen(skb), PCI_DMA_TODEVICE); 579 last = tx_buf->nr_frags; 580 581 for (j = 0; j < last; j++) { 582 cons = NEXT_TX(cons); 583 tx_buf = &txr->tx_buf_ring[cons]; 584 dma_unmap_page( 585 &pdev->dev, 586 dma_unmap_addr(tx_buf, mapping), 587 skb_frag_size(&skb_shinfo(skb)->frags[j]), 588 PCI_DMA_TODEVICE); 589 } 590 591 next_tx_int: 592 cons = NEXT_TX(cons); 593 594 tx_bytes += skb->len; 595 dev_kfree_skb_any(skb); 596 } 597 598 netdev_tx_completed_queue(txq, nr_pkts, tx_bytes); 599 txr->tx_cons = cons; 600 601 /* Need to make the tx_cons update visible to bnxt_start_xmit() 602 * before checking for netif_tx_queue_stopped(). Without the 603 * memory barrier, there is a small possibility that bnxt_start_xmit() 604 * will miss it and cause the queue to be stopped forever. 605 */ 606 smp_mb(); 607 608 if (unlikely(netif_tx_queue_stopped(txq)) && 609 (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)) { 610 __netif_tx_lock(txq, smp_processor_id()); 611 if (netif_tx_queue_stopped(txq) && 612 bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh && 613 txr->dev_state != BNXT_DEV_STATE_CLOSING) 614 netif_tx_wake_queue(txq); 615 __netif_tx_unlock(txq); 616 } 617 } 618 619 static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping, 620 gfp_t gfp) 621 { 622 struct device *dev = &bp->pdev->dev; 623 struct page *page; 624 625 page = alloc_page(gfp); 626 if (!page) 627 return NULL; 628 629 *mapping = dma_map_page_attrs(dev, page, 0, PAGE_SIZE, bp->rx_dir, 630 DMA_ATTR_WEAK_ORDERING); 631 if (dma_mapping_error(dev, *mapping)) { 632 __free_page(page); 633 return NULL; 634 } 635 *mapping += bp->rx_dma_offset; 636 return page; 637 } 638 639 static inline u8 *__bnxt_alloc_rx_data(struct bnxt *bp, dma_addr_t *mapping, 640 gfp_t gfp) 641 { 642 u8 *data; 643 struct pci_dev *pdev = bp->pdev; 644 645 data = kmalloc(bp->rx_buf_size, gfp); 646 if (!data) 647 return NULL; 648 649 *mapping = dma_map_single_attrs(&pdev->dev, data + bp->rx_dma_offset, 650 bp->rx_buf_use_size, bp->rx_dir, 651 DMA_ATTR_WEAK_ORDERING); 652 653 if (dma_mapping_error(&pdev->dev, *mapping)) { 654 kfree(data); 655 data = NULL; 656 } 657 return data; 658 } 659 660 int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, 661 u16 prod, gfp_t gfp) 662 { 663 struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 664 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod]; 665 dma_addr_t mapping; 666 667 if (BNXT_RX_PAGE_MODE(bp)) { 668 struct page *page = __bnxt_alloc_rx_page(bp, &mapping, gfp); 669 670 if (!page) 671 return -ENOMEM; 672 673 rx_buf->data = page; 674 rx_buf->data_ptr = page_address(page) + bp->rx_offset; 675 } else { 676 u8 *data = __bnxt_alloc_rx_data(bp, &mapping, gfp); 677 678 if (!data) 679 return -ENOMEM; 680 681 rx_buf->data = data; 682 rx_buf->data_ptr = data + bp->rx_offset; 683 } 684 rx_buf->mapping = mapping; 685 686 rxbd->rx_bd_haddr = cpu_to_le64(mapping); 687 return 0; 688 } 689 690 void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data) 691 { 692 u16 prod = rxr->rx_prod; 693 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf; 694 struct rx_bd *cons_bd, *prod_bd; 695 696 prod_rx_buf = &rxr->rx_buf_ring[prod]; 697 cons_rx_buf = &rxr->rx_buf_ring[cons]; 698 699 prod_rx_buf->data = data; 700 prod_rx_buf->data_ptr = cons_rx_buf->data_ptr; 701 702 prod_rx_buf->mapping = cons_rx_buf->mapping; 703 704 prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 705 cons_bd = &rxr->rx_desc_ring[RX_RING(cons)][RX_IDX(cons)]; 706 707 prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr; 708 } 709 710 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx) 711 { 712 u16 next, max = rxr->rx_agg_bmap_size; 713 714 next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx); 715 if (next >= max) 716 next = find_first_zero_bit(rxr->rx_agg_bmap, max); 717 return next; 718 } 719 720 static inline int bnxt_alloc_rx_page(struct bnxt *bp, 721 struct bnxt_rx_ring_info *rxr, 722 u16 prod, gfp_t gfp) 723 { 724 struct rx_bd *rxbd = 725 &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 726 struct bnxt_sw_rx_agg_bd *rx_agg_buf; 727 struct pci_dev *pdev = bp->pdev; 728 struct page *page; 729 dma_addr_t mapping; 730 u16 sw_prod = rxr->rx_sw_agg_prod; 731 unsigned int offset = 0; 732 733 if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) { 734 page = rxr->rx_page; 735 if (!page) { 736 page = alloc_page(gfp); 737 if (!page) 738 return -ENOMEM; 739 rxr->rx_page = page; 740 rxr->rx_page_offset = 0; 741 } 742 offset = rxr->rx_page_offset; 743 rxr->rx_page_offset += BNXT_RX_PAGE_SIZE; 744 if (rxr->rx_page_offset == PAGE_SIZE) 745 rxr->rx_page = NULL; 746 else 747 get_page(page); 748 } else { 749 page = alloc_page(gfp); 750 if (!page) 751 return -ENOMEM; 752 } 753 754 mapping = dma_map_page_attrs(&pdev->dev, page, offset, 755 BNXT_RX_PAGE_SIZE, PCI_DMA_FROMDEVICE, 756 DMA_ATTR_WEAK_ORDERING); 757 if (dma_mapping_error(&pdev->dev, mapping)) { 758 __free_page(page); 759 return -EIO; 760 } 761 762 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap))) 763 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod); 764 765 __set_bit(sw_prod, rxr->rx_agg_bmap); 766 rx_agg_buf = &rxr->rx_agg_ring[sw_prod]; 767 rxr->rx_sw_agg_prod = NEXT_RX_AGG(sw_prod); 768 769 rx_agg_buf->page = page; 770 rx_agg_buf->offset = offset; 771 rx_agg_buf->mapping = mapping; 772 rxbd->rx_bd_haddr = cpu_to_le64(mapping); 773 rxbd->rx_bd_opaque = sw_prod; 774 return 0; 775 } 776 777 static void bnxt_reuse_rx_agg_bufs(struct bnxt_napi *bnapi, u16 cp_cons, 778 u32 agg_bufs) 779 { 780 struct bnxt *bp = bnapi->bp; 781 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 782 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 783 u16 prod = rxr->rx_agg_prod; 784 u16 sw_prod = rxr->rx_sw_agg_prod; 785 u32 i; 786 787 for (i = 0; i < agg_bufs; i++) { 788 u16 cons; 789 struct rx_agg_cmp *agg; 790 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf; 791 struct rx_bd *prod_bd; 792 struct page *page; 793 794 agg = (struct rx_agg_cmp *) 795 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 796 cons = agg->rx_agg_cmp_opaque; 797 __clear_bit(cons, rxr->rx_agg_bmap); 798 799 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap))) 800 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod); 801 802 __set_bit(sw_prod, rxr->rx_agg_bmap); 803 prod_rx_buf = &rxr->rx_agg_ring[sw_prod]; 804 cons_rx_buf = &rxr->rx_agg_ring[cons]; 805 806 /* It is possible for sw_prod to be equal to cons, so 807 * set cons_rx_buf->page to NULL first. 808 */ 809 page = cons_rx_buf->page; 810 cons_rx_buf->page = NULL; 811 prod_rx_buf->page = page; 812 prod_rx_buf->offset = cons_rx_buf->offset; 813 814 prod_rx_buf->mapping = cons_rx_buf->mapping; 815 816 prod_bd = &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 817 818 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping); 819 prod_bd->rx_bd_opaque = sw_prod; 820 821 prod = NEXT_RX_AGG(prod); 822 sw_prod = NEXT_RX_AGG(sw_prod); 823 cp_cons = NEXT_CMP(cp_cons); 824 } 825 rxr->rx_agg_prod = prod; 826 rxr->rx_sw_agg_prod = sw_prod; 827 } 828 829 static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp, 830 struct bnxt_rx_ring_info *rxr, 831 u16 cons, void *data, u8 *data_ptr, 832 dma_addr_t dma_addr, 833 unsigned int offset_and_len) 834 { 835 unsigned int payload = offset_and_len >> 16; 836 unsigned int len = offset_and_len & 0xffff; 837 struct skb_frag_struct *frag; 838 struct page *page = data; 839 u16 prod = rxr->rx_prod; 840 struct sk_buff *skb; 841 int off, err; 842 843 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC); 844 if (unlikely(err)) { 845 bnxt_reuse_rx_data(rxr, cons, data); 846 return NULL; 847 } 848 dma_addr -= bp->rx_dma_offset; 849 dma_unmap_page_attrs(&bp->pdev->dev, dma_addr, PAGE_SIZE, bp->rx_dir, 850 DMA_ATTR_WEAK_ORDERING); 851 852 if (unlikely(!payload)) 853 payload = eth_get_headlen(data_ptr, len); 854 855 skb = napi_alloc_skb(&rxr->bnapi->napi, payload); 856 if (!skb) { 857 __free_page(page); 858 return NULL; 859 } 860 861 off = (void *)data_ptr - page_address(page); 862 skb_add_rx_frag(skb, 0, page, off, len, PAGE_SIZE); 863 memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN, 864 payload + NET_IP_ALIGN); 865 866 frag = &skb_shinfo(skb)->frags[0]; 867 skb_frag_size_sub(frag, payload); 868 frag->page_offset += payload; 869 skb->data_len -= payload; 870 skb->tail += payload; 871 872 return skb; 873 } 874 875 static struct sk_buff *bnxt_rx_skb(struct bnxt *bp, 876 struct bnxt_rx_ring_info *rxr, u16 cons, 877 void *data, u8 *data_ptr, 878 dma_addr_t dma_addr, 879 unsigned int offset_and_len) 880 { 881 u16 prod = rxr->rx_prod; 882 struct sk_buff *skb; 883 int err; 884 885 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC); 886 if (unlikely(err)) { 887 bnxt_reuse_rx_data(rxr, cons, data); 888 return NULL; 889 } 890 891 skb = build_skb(data, 0); 892 dma_unmap_single_attrs(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size, 893 bp->rx_dir, DMA_ATTR_WEAK_ORDERING); 894 if (!skb) { 895 kfree(data); 896 return NULL; 897 } 898 899 skb_reserve(skb, bp->rx_offset); 900 skb_put(skb, offset_and_len & 0xffff); 901 return skb; 902 } 903 904 static struct sk_buff *bnxt_rx_pages(struct bnxt *bp, struct bnxt_napi *bnapi, 905 struct sk_buff *skb, u16 cp_cons, 906 u32 agg_bufs) 907 { 908 struct pci_dev *pdev = bp->pdev; 909 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 910 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 911 u16 prod = rxr->rx_agg_prod; 912 u32 i; 913 914 for (i = 0; i < agg_bufs; i++) { 915 u16 cons, frag_len; 916 struct rx_agg_cmp *agg; 917 struct bnxt_sw_rx_agg_bd *cons_rx_buf; 918 struct page *page; 919 dma_addr_t mapping; 920 921 agg = (struct rx_agg_cmp *) 922 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 923 cons = agg->rx_agg_cmp_opaque; 924 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) & 925 RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT; 926 927 cons_rx_buf = &rxr->rx_agg_ring[cons]; 928 skb_fill_page_desc(skb, i, cons_rx_buf->page, 929 cons_rx_buf->offset, frag_len); 930 __clear_bit(cons, rxr->rx_agg_bmap); 931 932 /* It is possible for bnxt_alloc_rx_page() to allocate 933 * a sw_prod index that equals the cons index, so we 934 * need to clear the cons entry now. 935 */ 936 mapping = cons_rx_buf->mapping; 937 page = cons_rx_buf->page; 938 cons_rx_buf->page = NULL; 939 940 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) { 941 struct skb_shared_info *shinfo; 942 unsigned int nr_frags; 943 944 shinfo = skb_shinfo(skb); 945 nr_frags = --shinfo->nr_frags; 946 __skb_frag_set_page(&shinfo->frags[nr_frags], NULL); 947 948 dev_kfree_skb(skb); 949 950 cons_rx_buf->page = page; 951 952 /* Update prod since possibly some pages have been 953 * allocated already. 954 */ 955 rxr->rx_agg_prod = prod; 956 bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs - i); 957 return NULL; 958 } 959 960 dma_unmap_page_attrs(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE, 961 PCI_DMA_FROMDEVICE, 962 DMA_ATTR_WEAK_ORDERING); 963 964 skb->data_len += frag_len; 965 skb->len += frag_len; 966 skb->truesize += PAGE_SIZE; 967 968 prod = NEXT_RX_AGG(prod); 969 cp_cons = NEXT_CMP(cp_cons); 970 } 971 rxr->rx_agg_prod = prod; 972 return skb; 973 } 974 975 static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 976 u8 agg_bufs, u32 *raw_cons) 977 { 978 u16 last; 979 struct rx_agg_cmp *agg; 980 981 *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs); 982 last = RING_CMP(*raw_cons); 983 agg = (struct rx_agg_cmp *) 984 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)]; 985 return RX_AGG_CMP_VALID(agg, *raw_cons); 986 } 987 988 static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data, 989 unsigned int len, 990 dma_addr_t mapping) 991 { 992 struct bnxt *bp = bnapi->bp; 993 struct pci_dev *pdev = bp->pdev; 994 struct sk_buff *skb; 995 996 skb = napi_alloc_skb(&bnapi->napi, len); 997 if (!skb) 998 return NULL; 999 1000 dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copy_thresh, 1001 bp->rx_dir); 1002 1003 memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN, 1004 len + NET_IP_ALIGN); 1005 1006 dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copy_thresh, 1007 bp->rx_dir); 1008 1009 skb_put(skb, len); 1010 return skb; 1011 } 1012 1013 static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_napi *bnapi, 1014 u32 *raw_cons, void *cmp) 1015 { 1016 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 1017 struct rx_cmp *rxcmp = cmp; 1018 u32 tmp_raw_cons = *raw_cons; 1019 u8 cmp_type, agg_bufs = 0; 1020 1021 cmp_type = RX_CMP_TYPE(rxcmp); 1022 1023 if (cmp_type == CMP_TYPE_RX_L2_CMP) { 1024 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) & 1025 RX_CMP_AGG_BUFS) >> 1026 RX_CMP_AGG_BUFS_SHIFT; 1027 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) { 1028 struct rx_tpa_end_cmp *tpa_end = cmp; 1029 1030 agg_bufs = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) & 1031 RX_TPA_END_CMP_AGG_BUFS) >> 1032 RX_TPA_END_CMP_AGG_BUFS_SHIFT; 1033 } 1034 1035 if (agg_bufs) { 1036 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons)) 1037 return -EBUSY; 1038 } 1039 *raw_cons = tmp_raw_cons; 1040 return 0; 1041 } 1042 1043 static void bnxt_queue_sp_work(struct bnxt *bp) 1044 { 1045 if (BNXT_PF(bp)) 1046 queue_work(bnxt_pf_wq, &bp->sp_task); 1047 else 1048 schedule_work(&bp->sp_task); 1049 } 1050 1051 static void bnxt_cancel_sp_work(struct bnxt *bp) 1052 { 1053 if (BNXT_PF(bp)) 1054 flush_workqueue(bnxt_pf_wq); 1055 else 1056 cancel_work_sync(&bp->sp_task); 1057 } 1058 1059 static void bnxt_sched_reset(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) 1060 { 1061 if (!rxr->bnapi->in_reset) { 1062 rxr->bnapi->in_reset = true; 1063 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event); 1064 bnxt_queue_sp_work(bp); 1065 } 1066 rxr->rx_next_cons = 0xffff; 1067 } 1068 1069 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, 1070 struct rx_tpa_start_cmp *tpa_start, 1071 struct rx_tpa_start_cmp_ext *tpa_start1) 1072 { 1073 u8 agg_id = TPA_START_AGG_ID(tpa_start); 1074 u16 cons, prod; 1075 struct bnxt_tpa_info *tpa_info; 1076 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf; 1077 struct rx_bd *prod_bd; 1078 dma_addr_t mapping; 1079 1080 cons = tpa_start->rx_tpa_start_cmp_opaque; 1081 prod = rxr->rx_prod; 1082 cons_rx_buf = &rxr->rx_buf_ring[cons]; 1083 prod_rx_buf = &rxr->rx_buf_ring[prod]; 1084 tpa_info = &rxr->rx_tpa[agg_id]; 1085 1086 if (unlikely(cons != rxr->rx_next_cons)) { 1087 bnxt_sched_reset(bp, rxr); 1088 return; 1089 } 1090 /* Store cfa_code in tpa_info to use in tpa_end 1091 * completion processing. 1092 */ 1093 tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1); 1094 prod_rx_buf->data = tpa_info->data; 1095 prod_rx_buf->data_ptr = tpa_info->data_ptr; 1096 1097 mapping = tpa_info->mapping; 1098 prod_rx_buf->mapping = mapping; 1099 1100 prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 1101 1102 prod_bd->rx_bd_haddr = cpu_to_le64(mapping); 1103 1104 tpa_info->data = cons_rx_buf->data; 1105 tpa_info->data_ptr = cons_rx_buf->data_ptr; 1106 cons_rx_buf->data = NULL; 1107 tpa_info->mapping = cons_rx_buf->mapping; 1108 1109 tpa_info->len = 1110 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >> 1111 RX_TPA_START_CMP_LEN_SHIFT; 1112 if (likely(TPA_START_HASH_VALID(tpa_start))) { 1113 u32 hash_type = TPA_START_HASH_TYPE(tpa_start); 1114 1115 tpa_info->hash_type = PKT_HASH_TYPE_L4; 1116 tpa_info->gso_type = SKB_GSO_TCPV4; 1117 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */ 1118 if (hash_type == 3) 1119 tpa_info->gso_type = SKB_GSO_TCPV6; 1120 tpa_info->rss_hash = 1121 le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash); 1122 } else { 1123 tpa_info->hash_type = PKT_HASH_TYPE_NONE; 1124 tpa_info->gso_type = 0; 1125 if (netif_msg_rx_err(bp)) 1126 netdev_warn(bp->dev, "TPA packet without valid hash\n"); 1127 } 1128 tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2); 1129 tpa_info->metadata = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata); 1130 tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info); 1131 1132 rxr->rx_prod = NEXT_RX(prod); 1133 cons = NEXT_RX(cons); 1134 rxr->rx_next_cons = NEXT_RX(cons); 1135 cons_rx_buf = &rxr->rx_buf_ring[cons]; 1136 1137 bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data); 1138 rxr->rx_prod = NEXT_RX(rxr->rx_prod); 1139 cons_rx_buf->data = NULL; 1140 } 1141 1142 static void bnxt_abort_tpa(struct bnxt *bp, struct bnxt_napi *bnapi, 1143 u16 cp_cons, u32 agg_bufs) 1144 { 1145 if (agg_bufs) 1146 bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs); 1147 } 1148 1149 static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info, 1150 int payload_off, int tcp_ts, 1151 struct sk_buff *skb) 1152 { 1153 #ifdef CONFIG_INET 1154 struct tcphdr *th; 1155 int len, nw_off; 1156 u16 outer_ip_off, inner_ip_off, inner_mac_off; 1157 u32 hdr_info = tpa_info->hdr_info; 1158 bool loopback = false; 1159 1160 inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info); 1161 inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info); 1162 outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info); 1163 1164 /* If the packet is an internal loopback packet, the offsets will 1165 * have an extra 4 bytes. 1166 */ 1167 if (inner_mac_off == 4) { 1168 loopback = true; 1169 } else if (inner_mac_off > 4) { 1170 __be16 proto = *((__be16 *)(skb->data + inner_ip_off - 1171 ETH_HLEN - 2)); 1172 1173 /* We only support inner iPv4/ipv6. If we don't see the 1174 * correct protocol ID, it must be a loopback packet where 1175 * the offsets are off by 4. 1176 */ 1177 if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6)) 1178 loopback = true; 1179 } 1180 if (loopback) { 1181 /* internal loopback packet, subtract all offsets by 4 */ 1182 inner_ip_off -= 4; 1183 inner_mac_off -= 4; 1184 outer_ip_off -= 4; 1185 } 1186 1187 nw_off = inner_ip_off - ETH_HLEN; 1188 skb_set_network_header(skb, nw_off); 1189 if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) { 1190 struct ipv6hdr *iph = ipv6_hdr(skb); 1191 1192 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr)); 1193 len = skb->len - skb_transport_offset(skb); 1194 th = tcp_hdr(skb); 1195 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0); 1196 } else { 1197 struct iphdr *iph = ip_hdr(skb); 1198 1199 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr)); 1200 len = skb->len - skb_transport_offset(skb); 1201 th = tcp_hdr(skb); 1202 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0); 1203 } 1204 1205 if (inner_mac_off) { /* tunnel */ 1206 struct udphdr *uh = NULL; 1207 __be16 proto = *((__be16 *)(skb->data + outer_ip_off - 1208 ETH_HLEN - 2)); 1209 1210 if (proto == htons(ETH_P_IP)) { 1211 struct iphdr *iph = (struct iphdr *)skb->data; 1212 1213 if (iph->protocol == IPPROTO_UDP) 1214 uh = (struct udphdr *)(iph + 1); 1215 } else { 1216 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data; 1217 1218 if (iph->nexthdr == IPPROTO_UDP) 1219 uh = (struct udphdr *)(iph + 1); 1220 } 1221 if (uh) { 1222 if (uh->check) 1223 skb_shinfo(skb)->gso_type |= 1224 SKB_GSO_UDP_TUNNEL_CSUM; 1225 else 1226 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL; 1227 } 1228 } 1229 #endif 1230 return skb; 1231 } 1232 1233 #define BNXT_IPV4_HDR_SIZE (sizeof(struct iphdr) + sizeof(struct tcphdr)) 1234 #define BNXT_IPV6_HDR_SIZE (sizeof(struct ipv6hdr) + sizeof(struct tcphdr)) 1235 1236 static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info, 1237 int payload_off, int tcp_ts, 1238 struct sk_buff *skb) 1239 { 1240 #ifdef CONFIG_INET 1241 struct tcphdr *th; 1242 int len, nw_off, tcp_opt_len = 0; 1243 1244 if (tcp_ts) 1245 tcp_opt_len = 12; 1246 1247 if (tpa_info->gso_type == SKB_GSO_TCPV4) { 1248 struct iphdr *iph; 1249 1250 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len - 1251 ETH_HLEN; 1252 skb_set_network_header(skb, nw_off); 1253 iph = ip_hdr(skb); 1254 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr)); 1255 len = skb->len - skb_transport_offset(skb); 1256 th = tcp_hdr(skb); 1257 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0); 1258 } else if (tpa_info->gso_type == SKB_GSO_TCPV6) { 1259 struct ipv6hdr *iph; 1260 1261 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len - 1262 ETH_HLEN; 1263 skb_set_network_header(skb, nw_off); 1264 iph = ipv6_hdr(skb); 1265 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr)); 1266 len = skb->len - skb_transport_offset(skb); 1267 th = tcp_hdr(skb); 1268 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0); 1269 } else { 1270 dev_kfree_skb_any(skb); 1271 return NULL; 1272 } 1273 1274 if (nw_off) { /* tunnel */ 1275 struct udphdr *uh = NULL; 1276 1277 if (skb->protocol == htons(ETH_P_IP)) { 1278 struct iphdr *iph = (struct iphdr *)skb->data; 1279 1280 if (iph->protocol == IPPROTO_UDP) 1281 uh = (struct udphdr *)(iph + 1); 1282 } else { 1283 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data; 1284 1285 if (iph->nexthdr == IPPROTO_UDP) 1286 uh = (struct udphdr *)(iph + 1); 1287 } 1288 if (uh) { 1289 if (uh->check) 1290 skb_shinfo(skb)->gso_type |= 1291 SKB_GSO_UDP_TUNNEL_CSUM; 1292 else 1293 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL; 1294 } 1295 } 1296 #endif 1297 return skb; 1298 } 1299 1300 static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp, 1301 struct bnxt_tpa_info *tpa_info, 1302 struct rx_tpa_end_cmp *tpa_end, 1303 struct rx_tpa_end_cmp_ext *tpa_end1, 1304 struct sk_buff *skb) 1305 { 1306 #ifdef CONFIG_INET 1307 int payload_off; 1308 u16 segs; 1309 1310 segs = TPA_END_TPA_SEGS(tpa_end); 1311 if (segs == 1) 1312 return skb; 1313 1314 NAPI_GRO_CB(skb)->count = segs; 1315 skb_shinfo(skb)->gso_size = 1316 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len); 1317 skb_shinfo(skb)->gso_type = tpa_info->gso_type; 1318 payload_off = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) & 1319 RX_TPA_END_CMP_PAYLOAD_OFFSET) >> 1320 RX_TPA_END_CMP_PAYLOAD_OFFSET_SHIFT; 1321 skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb); 1322 if (likely(skb)) 1323 tcp_gro_complete(skb); 1324 #endif 1325 return skb; 1326 } 1327 1328 /* Given the cfa_code of a received packet determine which 1329 * netdev (vf-rep or PF) the packet is destined to. 1330 */ 1331 static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code) 1332 { 1333 struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code); 1334 1335 /* if vf-rep dev is NULL, the must belongs to the PF */ 1336 return dev ? dev : bp->dev; 1337 } 1338 1339 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp, 1340 struct bnxt_napi *bnapi, 1341 u32 *raw_cons, 1342 struct rx_tpa_end_cmp *tpa_end, 1343 struct rx_tpa_end_cmp_ext *tpa_end1, 1344 u8 *event) 1345 { 1346 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 1347 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1348 u8 agg_id = TPA_END_AGG_ID(tpa_end); 1349 u8 *data_ptr, agg_bufs; 1350 u16 cp_cons = RING_CMP(*raw_cons); 1351 unsigned int len; 1352 struct bnxt_tpa_info *tpa_info; 1353 dma_addr_t mapping; 1354 struct sk_buff *skb; 1355 void *data; 1356 1357 if (unlikely(bnapi->in_reset)) { 1358 int rc = bnxt_discard_rx(bp, bnapi, raw_cons, tpa_end); 1359 1360 if (rc < 0) 1361 return ERR_PTR(-EBUSY); 1362 return NULL; 1363 } 1364 1365 tpa_info = &rxr->rx_tpa[agg_id]; 1366 data = tpa_info->data; 1367 data_ptr = tpa_info->data_ptr; 1368 prefetch(data_ptr); 1369 len = tpa_info->len; 1370 mapping = tpa_info->mapping; 1371 1372 agg_bufs = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) & 1373 RX_TPA_END_CMP_AGG_BUFS) >> RX_TPA_END_CMP_AGG_BUFS_SHIFT; 1374 1375 if (agg_bufs) { 1376 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons)) 1377 return ERR_PTR(-EBUSY); 1378 1379 *event |= BNXT_AGG_EVENT; 1380 cp_cons = NEXT_CMP(cp_cons); 1381 } 1382 1383 if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) { 1384 bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs); 1385 if (agg_bufs > MAX_SKB_FRAGS) 1386 netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n", 1387 agg_bufs, (int)MAX_SKB_FRAGS); 1388 return NULL; 1389 } 1390 1391 if (len <= bp->rx_copy_thresh) { 1392 skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping); 1393 if (!skb) { 1394 bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs); 1395 return NULL; 1396 } 1397 } else { 1398 u8 *new_data; 1399 dma_addr_t new_mapping; 1400 1401 new_data = __bnxt_alloc_rx_data(bp, &new_mapping, GFP_ATOMIC); 1402 if (!new_data) { 1403 bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs); 1404 return NULL; 1405 } 1406 1407 tpa_info->data = new_data; 1408 tpa_info->data_ptr = new_data + bp->rx_offset; 1409 tpa_info->mapping = new_mapping; 1410 1411 skb = build_skb(data, 0); 1412 dma_unmap_single_attrs(&bp->pdev->dev, mapping, 1413 bp->rx_buf_use_size, bp->rx_dir, 1414 DMA_ATTR_WEAK_ORDERING); 1415 1416 if (!skb) { 1417 kfree(data); 1418 bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs); 1419 return NULL; 1420 } 1421 skb_reserve(skb, bp->rx_offset); 1422 skb_put(skb, len); 1423 } 1424 1425 if (agg_bufs) { 1426 skb = bnxt_rx_pages(bp, bnapi, skb, cp_cons, agg_bufs); 1427 if (!skb) { 1428 /* Page reuse already handled by bnxt_rx_pages(). */ 1429 return NULL; 1430 } 1431 } 1432 1433 skb->protocol = 1434 eth_type_trans(skb, bnxt_get_pkt_dev(bp, tpa_info->cfa_code)); 1435 1436 if (tpa_info->hash_type != PKT_HASH_TYPE_NONE) 1437 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type); 1438 1439 if ((tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) && 1440 (skb->dev->features & NETIF_F_HW_VLAN_CTAG_RX)) { 1441 u16 vlan_proto = tpa_info->metadata >> 1442 RX_CMP_FLAGS2_METADATA_TPID_SFT; 1443 u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_TCI_MASK; 1444 1445 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag); 1446 } 1447 1448 skb_checksum_none_assert(skb); 1449 if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) { 1450 skb->ip_summed = CHECKSUM_UNNECESSARY; 1451 skb->csum_level = 1452 (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3; 1453 } 1454 1455 if (TPA_END_GRO(tpa_end)) 1456 skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb); 1457 1458 return skb; 1459 } 1460 1461 static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi, 1462 struct sk_buff *skb) 1463 { 1464 if (skb->dev != bp->dev) { 1465 /* this packet belongs to a vf-rep */ 1466 bnxt_vf_rep_rx(bp, skb); 1467 return; 1468 } 1469 skb_record_rx_queue(skb, bnapi->index); 1470 napi_gro_receive(&bnapi->napi, skb); 1471 } 1472 1473 /* returns the following: 1474 * 1 - 1 packet successfully received 1475 * 0 - successful TPA_START, packet not completed yet 1476 * -EBUSY - completion ring does not have all the agg buffers yet 1477 * -ENOMEM - packet aborted due to out of memory 1478 * -EIO - packet aborted due to hw error indicated in BD 1479 */ 1480 static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_napi *bnapi, u32 *raw_cons, 1481 u8 *event) 1482 { 1483 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 1484 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1485 struct net_device *dev = bp->dev; 1486 struct rx_cmp *rxcmp; 1487 struct rx_cmp_ext *rxcmp1; 1488 u32 tmp_raw_cons = *raw_cons; 1489 u16 cfa_code, cons, prod, cp_cons = RING_CMP(tmp_raw_cons); 1490 struct bnxt_sw_rx_bd *rx_buf; 1491 unsigned int len; 1492 u8 *data_ptr, agg_bufs, cmp_type; 1493 dma_addr_t dma_addr; 1494 struct sk_buff *skb; 1495 void *data; 1496 int rc = 0; 1497 u32 misc; 1498 1499 rxcmp = (struct rx_cmp *) 1500 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1501 1502 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons); 1503 cp_cons = RING_CMP(tmp_raw_cons); 1504 rxcmp1 = (struct rx_cmp_ext *) 1505 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1506 1507 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons)) 1508 return -EBUSY; 1509 1510 cmp_type = RX_CMP_TYPE(rxcmp); 1511 1512 prod = rxr->rx_prod; 1513 1514 if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) { 1515 bnxt_tpa_start(bp, rxr, (struct rx_tpa_start_cmp *)rxcmp, 1516 (struct rx_tpa_start_cmp_ext *)rxcmp1); 1517 1518 *event |= BNXT_RX_EVENT; 1519 goto next_rx_no_prod_no_len; 1520 1521 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) { 1522 skb = bnxt_tpa_end(bp, bnapi, &tmp_raw_cons, 1523 (struct rx_tpa_end_cmp *)rxcmp, 1524 (struct rx_tpa_end_cmp_ext *)rxcmp1, event); 1525 1526 if (IS_ERR(skb)) 1527 return -EBUSY; 1528 1529 rc = -ENOMEM; 1530 if (likely(skb)) { 1531 bnxt_deliver_skb(bp, bnapi, skb); 1532 rc = 1; 1533 } 1534 *event |= BNXT_RX_EVENT; 1535 goto next_rx_no_prod_no_len; 1536 } 1537 1538 cons = rxcmp->rx_cmp_opaque; 1539 rx_buf = &rxr->rx_buf_ring[cons]; 1540 data = rx_buf->data; 1541 data_ptr = rx_buf->data_ptr; 1542 if (unlikely(cons != rxr->rx_next_cons)) { 1543 int rc1 = bnxt_discard_rx(bp, bnapi, raw_cons, rxcmp); 1544 1545 bnxt_sched_reset(bp, rxr); 1546 return rc1; 1547 } 1548 prefetch(data_ptr); 1549 1550 misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1); 1551 agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT; 1552 1553 if (agg_bufs) { 1554 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons)) 1555 return -EBUSY; 1556 1557 cp_cons = NEXT_CMP(cp_cons); 1558 *event |= BNXT_AGG_EVENT; 1559 } 1560 *event |= BNXT_RX_EVENT; 1561 1562 rx_buf->data = NULL; 1563 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) { 1564 bnxt_reuse_rx_data(rxr, cons, data); 1565 if (agg_bufs) 1566 bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs); 1567 1568 rc = -EIO; 1569 goto next_rx; 1570 } 1571 1572 len = le32_to_cpu(rxcmp->rx_cmp_len_flags_type) >> RX_CMP_LEN_SHIFT; 1573 dma_addr = rx_buf->mapping; 1574 1575 if (bnxt_rx_xdp(bp, rxr, cons, data, &data_ptr, &len, event)) { 1576 rc = 1; 1577 goto next_rx; 1578 } 1579 1580 if (len <= bp->rx_copy_thresh) { 1581 skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr); 1582 bnxt_reuse_rx_data(rxr, cons, data); 1583 if (!skb) { 1584 rc = -ENOMEM; 1585 goto next_rx; 1586 } 1587 } else { 1588 u32 payload; 1589 1590 if (rx_buf->data_ptr == data_ptr) 1591 payload = misc & RX_CMP_PAYLOAD_OFFSET; 1592 else 1593 payload = 0; 1594 skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr, 1595 payload | len); 1596 if (!skb) { 1597 rc = -ENOMEM; 1598 goto next_rx; 1599 } 1600 } 1601 1602 if (agg_bufs) { 1603 skb = bnxt_rx_pages(bp, bnapi, skb, cp_cons, agg_bufs); 1604 if (!skb) { 1605 rc = -ENOMEM; 1606 goto next_rx; 1607 } 1608 } 1609 1610 if (RX_CMP_HASH_VALID(rxcmp)) { 1611 u32 hash_type = RX_CMP_HASH_TYPE(rxcmp); 1612 enum pkt_hash_types type = PKT_HASH_TYPE_L4; 1613 1614 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */ 1615 if (hash_type != 1 && hash_type != 3) 1616 type = PKT_HASH_TYPE_L3; 1617 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type); 1618 } 1619 1620 cfa_code = RX_CMP_CFA_CODE(rxcmp1); 1621 skb->protocol = eth_type_trans(skb, bnxt_get_pkt_dev(bp, cfa_code)); 1622 1623 if ((rxcmp1->rx_cmp_flags2 & 1624 cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) && 1625 (skb->dev->features & NETIF_F_HW_VLAN_CTAG_RX)) { 1626 u32 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data); 1627 u16 vtag = meta_data & RX_CMP_FLAGS2_METADATA_TCI_MASK; 1628 u16 vlan_proto = meta_data >> RX_CMP_FLAGS2_METADATA_TPID_SFT; 1629 1630 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag); 1631 } 1632 1633 skb_checksum_none_assert(skb); 1634 if (RX_CMP_L4_CS_OK(rxcmp1)) { 1635 if (dev->features & NETIF_F_RXCSUM) { 1636 skb->ip_summed = CHECKSUM_UNNECESSARY; 1637 skb->csum_level = RX_CMP_ENCAP(rxcmp1); 1638 } 1639 } else { 1640 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) { 1641 if (dev->features & NETIF_F_RXCSUM) 1642 cpr->rx_l4_csum_errors++; 1643 } 1644 } 1645 1646 bnxt_deliver_skb(bp, bnapi, skb); 1647 rc = 1; 1648 1649 next_rx: 1650 rxr->rx_prod = NEXT_RX(prod); 1651 rxr->rx_next_cons = NEXT_RX(cons); 1652 1653 cpr->rx_packets += 1; 1654 cpr->rx_bytes += len; 1655 1656 next_rx_no_prod_no_len: 1657 *raw_cons = tmp_raw_cons; 1658 1659 return rc; 1660 } 1661 1662 /* In netpoll mode, if we are using a combined completion ring, we need to 1663 * discard the rx packets and recycle the buffers. 1664 */ 1665 static int bnxt_force_rx_discard(struct bnxt *bp, struct bnxt_napi *bnapi, 1666 u32 *raw_cons, u8 *event) 1667 { 1668 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 1669 u32 tmp_raw_cons = *raw_cons; 1670 struct rx_cmp_ext *rxcmp1; 1671 struct rx_cmp *rxcmp; 1672 u16 cp_cons; 1673 u8 cmp_type; 1674 1675 cp_cons = RING_CMP(tmp_raw_cons); 1676 rxcmp = (struct rx_cmp *) 1677 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1678 1679 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons); 1680 cp_cons = RING_CMP(tmp_raw_cons); 1681 rxcmp1 = (struct rx_cmp_ext *) 1682 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1683 1684 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons)) 1685 return -EBUSY; 1686 1687 cmp_type = RX_CMP_TYPE(rxcmp); 1688 if (cmp_type == CMP_TYPE_RX_L2_CMP) { 1689 rxcmp1->rx_cmp_cfa_code_errors_v2 |= 1690 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR); 1691 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) { 1692 struct rx_tpa_end_cmp_ext *tpa_end1; 1693 1694 tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1; 1695 tpa_end1->rx_tpa_end_cmp_errors_v2 |= 1696 cpu_to_le32(RX_TPA_END_CMP_ERRORS); 1697 } 1698 return bnxt_rx_pkt(bp, bnapi, raw_cons, event); 1699 } 1700 1701 #define BNXT_GET_EVENT_PORT(data) \ 1702 ((data) & \ 1703 ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK) 1704 1705 static int bnxt_async_event_process(struct bnxt *bp, 1706 struct hwrm_async_event_cmpl *cmpl) 1707 { 1708 u16 event_id = le16_to_cpu(cmpl->event_id); 1709 1710 /* TODO CHIMP_FW: Define event id's for link change, error etc */ 1711 switch (event_id) { 1712 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: { 1713 u32 data1 = le32_to_cpu(cmpl->event_data1); 1714 struct bnxt_link_info *link_info = &bp->link_info; 1715 1716 if (BNXT_VF(bp)) 1717 goto async_event_process_exit; 1718 1719 /* print unsupported speed warning in forced speed mode only */ 1720 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED) && 1721 (data1 & 0x20000)) { 1722 u16 fw_speed = link_info->force_link_speed; 1723 u32 speed = bnxt_fw_to_ethtool_speed(fw_speed); 1724 1725 if (speed != SPEED_UNKNOWN) 1726 netdev_warn(bp->dev, "Link speed %d no longer supported\n", 1727 speed); 1728 } 1729 set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event); 1730 /* fall through */ 1731 } 1732 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE: 1733 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event); 1734 break; 1735 case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD: 1736 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event); 1737 break; 1738 case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: { 1739 u32 data1 = le32_to_cpu(cmpl->event_data1); 1740 u16 port_id = BNXT_GET_EVENT_PORT(data1); 1741 1742 if (BNXT_VF(bp)) 1743 break; 1744 1745 if (bp->pf.port_id != port_id) 1746 break; 1747 1748 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event); 1749 break; 1750 } 1751 case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE: 1752 if (BNXT_PF(bp)) 1753 goto async_event_process_exit; 1754 set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event); 1755 break; 1756 default: 1757 goto async_event_process_exit; 1758 } 1759 bnxt_queue_sp_work(bp); 1760 async_event_process_exit: 1761 bnxt_ulp_async_events(bp, cmpl); 1762 return 0; 1763 } 1764 1765 static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp) 1766 { 1767 u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id; 1768 struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp; 1769 struct hwrm_fwd_req_cmpl *fwd_req_cmpl = 1770 (struct hwrm_fwd_req_cmpl *)txcmp; 1771 1772 switch (cmpl_type) { 1773 case CMPL_BASE_TYPE_HWRM_DONE: 1774 seq_id = le16_to_cpu(h_cmpl->sequence_id); 1775 if (seq_id == bp->hwrm_intr_seq_id) 1776 bp->hwrm_intr_seq_id = HWRM_SEQ_ID_INVALID; 1777 else 1778 netdev_err(bp->dev, "Invalid hwrm seq id %d\n", seq_id); 1779 break; 1780 1781 case CMPL_BASE_TYPE_HWRM_FWD_REQ: 1782 vf_id = le16_to_cpu(fwd_req_cmpl->source_id); 1783 1784 if ((vf_id < bp->pf.first_vf_id) || 1785 (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) { 1786 netdev_err(bp->dev, "Msg contains invalid VF id %x\n", 1787 vf_id); 1788 return -EINVAL; 1789 } 1790 1791 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap); 1792 set_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event); 1793 bnxt_queue_sp_work(bp); 1794 break; 1795 1796 case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT: 1797 bnxt_async_event_process(bp, 1798 (struct hwrm_async_event_cmpl *)txcmp); 1799 1800 default: 1801 break; 1802 } 1803 1804 return 0; 1805 } 1806 1807 static irqreturn_t bnxt_msix(int irq, void *dev_instance) 1808 { 1809 struct bnxt_napi *bnapi = dev_instance; 1810 struct bnxt *bp = bnapi->bp; 1811 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 1812 u32 cons = RING_CMP(cpr->cp_raw_cons); 1813 1814 cpr->event_ctr++; 1815 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]); 1816 napi_schedule(&bnapi->napi); 1817 return IRQ_HANDLED; 1818 } 1819 1820 static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr) 1821 { 1822 u32 raw_cons = cpr->cp_raw_cons; 1823 u16 cons = RING_CMP(raw_cons); 1824 struct tx_cmp *txcmp; 1825 1826 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]; 1827 1828 return TX_CMP_VALID(txcmp, raw_cons); 1829 } 1830 1831 static irqreturn_t bnxt_inta(int irq, void *dev_instance) 1832 { 1833 struct bnxt_napi *bnapi = dev_instance; 1834 struct bnxt *bp = bnapi->bp; 1835 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 1836 u32 cons = RING_CMP(cpr->cp_raw_cons); 1837 u32 int_status; 1838 1839 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]); 1840 1841 if (!bnxt_has_work(bp, cpr)) { 1842 int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS); 1843 /* return if erroneous interrupt */ 1844 if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id))) 1845 return IRQ_NONE; 1846 } 1847 1848 /* disable ring IRQ */ 1849 BNXT_CP_DB_IRQ_DIS(cpr->cp_doorbell); 1850 1851 /* Return here if interrupt is shared and is disabled. */ 1852 if (unlikely(atomic_read(&bp->intr_sem) != 0)) 1853 return IRQ_HANDLED; 1854 1855 napi_schedule(&bnapi->napi); 1856 return IRQ_HANDLED; 1857 } 1858 1859 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_napi *bnapi, int budget) 1860 { 1861 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 1862 u32 raw_cons = cpr->cp_raw_cons; 1863 u32 cons; 1864 int tx_pkts = 0; 1865 int rx_pkts = 0; 1866 u8 event = 0; 1867 struct tx_cmp *txcmp; 1868 1869 while (1) { 1870 int rc; 1871 1872 cons = RING_CMP(raw_cons); 1873 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]; 1874 1875 if (!TX_CMP_VALID(txcmp, raw_cons)) 1876 break; 1877 1878 /* The valid test of the entry must be done first before 1879 * reading any further. 1880 */ 1881 dma_rmb(); 1882 if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) { 1883 tx_pkts++; 1884 /* return full budget so NAPI will complete. */ 1885 if (unlikely(tx_pkts > bp->tx_wake_thresh)) 1886 rx_pkts = budget; 1887 } else if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) { 1888 if (likely(budget)) 1889 rc = bnxt_rx_pkt(bp, bnapi, &raw_cons, &event); 1890 else 1891 rc = bnxt_force_rx_discard(bp, bnapi, &raw_cons, 1892 &event); 1893 if (likely(rc >= 0)) 1894 rx_pkts += rc; 1895 /* Increment rx_pkts when rc is -ENOMEM to count towards 1896 * the NAPI budget. Otherwise, we may potentially loop 1897 * here forever if we consistently cannot allocate 1898 * buffers. 1899 */ 1900 else if (rc == -ENOMEM && budget) 1901 rx_pkts++; 1902 else if (rc == -EBUSY) /* partial completion */ 1903 break; 1904 } else if (unlikely((TX_CMP_TYPE(txcmp) == 1905 CMPL_BASE_TYPE_HWRM_DONE) || 1906 (TX_CMP_TYPE(txcmp) == 1907 CMPL_BASE_TYPE_HWRM_FWD_REQ) || 1908 (TX_CMP_TYPE(txcmp) == 1909 CMPL_BASE_TYPE_HWRM_ASYNC_EVENT))) { 1910 bnxt_hwrm_handler(bp, txcmp); 1911 } 1912 raw_cons = NEXT_RAW_CMP(raw_cons); 1913 1914 if (rx_pkts == budget) 1915 break; 1916 } 1917 1918 if (event & BNXT_TX_EVENT) { 1919 struct bnxt_tx_ring_info *txr = bnapi->tx_ring; 1920 void __iomem *db = txr->tx_doorbell; 1921 u16 prod = txr->tx_prod; 1922 1923 /* Sync BD data before updating doorbell */ 1924 wmb(); 1925 1926 bnxt_db_write_relaxed(bp, db, DB_KEY_TX | prod); 1927 } 1928 1929 cpr->cp_raw_cons = raw_cons; 1930 /* ACK completion ring before freeing tx ring and producing new 1931 * buffers in rx/agg rings to prevent overflowing the completion 1932 * ring. 1933 */ 1934 BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons); 1935 1936 if (tx_pkts) 1937 bnapi->tx_int(bp, bnapi, tx_pkts); 1938 1939 if (event & BNXT_RX_EVENT) { 1940 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1941 1942 bnxt_db_write(bp, rxr->rx_doorbell, DB_KEY_RX | rxr->rx_prod); 1943 if (event & BNXT_AGG_EVENT) 1944 bnxt_db_write(bp, rxr->rx_agg_doorbell, 1945 DB_KEY_RX | rxr->rx_agg_prod); 1946 } 1947 return rx_pkts; 1948 } 1949 1950 static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget) 1951 { 1952 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi); 1953 struct bnxt *bp = bnapi->bp; 1954 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 1955 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1956 struct tx_cmp *txcmp; 1957 struct rx_cmp_ext *rxcmp1; 1958 u32 cp_cons, tmp_raw_cons; 1959 u32 raw_cons = cpr->cp_raw_cons; 1960 u32 rx_pkts = 0; 1961 u8 event = 0; 1962 1963 while (1) { 1964 int rc; 1965 1966 cp_cons = RING_CMP(raw_cons); 1967 txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1968 1969 if (!TX_CMP_VALID(txcmp, raw_cons)) 1970 break; 1971 1972 if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) { 1973 tmp_raw_cons = NEXT_RAW_CMP(raw_cons); 1974 cp_cons = RING_CMP(tmp_raw_cons); 1975 rxcmp1 = (struct rx_cmp_ext *) 1976 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1977 1978 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons)) 1979 break; 1980 1981 /* force an error to recycle the buffer */ 1982 rxcmp1->rx_cmp_cfa_code_errors_v2 |= 1983 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR); 1984 1985 rc = bnxt_rx_pkt(bp, bnapi, &raw_cons, &event); 1986 if (likely(rc == -EIO) && budget) 1987 rx_pkts++; 1988 else if (rc == -EBUSY) /* partial completion */ 1989 break; 1990 } else if (unlikely(TX_CMP_TYPE(txcmp) == 1991 CMPL_BASE_TYPE_HWRM_DONE)) { 1992 bnxt_hwrm_handler(bp, txcmp); 1993 } else { 1994 netdev_err(bp->dev, 1995 "Invalid completion received on special ring\n"); 1996 } 1997 raw_cons = NEXT_RAW_CMP(raw_cons); 1998 1999 if (rx_pkts == budget) 2000 break; 2001 } 2002 2003 cpr->cp_raw_cons = raw_cons; 2004 BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons); 2005 bnxt_db_write(bp, rxr->rx_doorbell, DB_KEY_RX | rxr->rx_prod); 2006 2007 if (event & BNXT_AGG_EVENT) 2008 bnxt_db_write(bp, rxr->rx_agg_doorbell, 2009 DB_KEY_RX | rxr->rx_agg_prod); 2010 2011 if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) { 2012 napi_complete_done(napi, rx_pkts); 2013 BNXT_CP_DB_REARM(cpr->cp_doorbell, cpr->cp_raw_cons); 2014 } 2015 return rx_pkts; 2016 } 2017 2018 static int bnxt_poll(struct napi_struct *napi, int budget) 2019 { 2020 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi); 2021 struct bnxt *bp = bnapi->bp; 2022 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2023 int work_done = 0; 2024 2025 while (1) { 2026 work_done += bnxt_poll_work(bp, bnapi, budget - work_done); 2027 2028 if (work_done >= budget) 2029 break; 2030 2031 if (!bnxt_has_work(bp, cpr)) { 2032 if (napi_complete_done(napi, work_done)) 2033 BNXT_CP_DB_REARM(cpr->cp_doorbell, 2034 cpr->cp_raw_cons); 2035 break; 2036 } 2037 } 2038 if (bp->flags & BNXT_FLAG_DIM) { 2039 struct net_dim_sample dim_sample; 2040 2041 net_dim_sample(cpr->event_ctr, 2042 cpr->rx_packets, 2043 cpr->rx_bytes, 2044 &dim_sample); 2045 net_dim(&cpr->dim, dim_sample); 2046 } 2047 mmiowb(); 2048 return work_done; 2049 } 2050 2051 static void bnxt_free_tx_skbs(struct bnxt *bp) 2052 { 2053 int i, max_idx; 2054 struct pci_dev *pdev = bp->pdev; 2055 2056 if (!bp->tx_ring) 2057 return; 2058 2059 max_idx = bp->tx_nr_pages * TX_DESC_CNT; 2060 for (i = 0; i < bp->tx_nr_rings; i++) { 2061 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 2062 int j; 2063 2064 for (j = 0; j < max_idx;) { 2065 struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j]; 2066 struct sk_buff *skb = tx_buf->skb; 2067 int k, last; 2068 2069 if (!skb) { 2070 j++; 2071 continue; 2072 } 2073 2074 tx_buf->skb = NULL; 2075 2076 if (tx_buf->is_push) { 2077 dev_kfree_skb(skb); 2078 j += 2; 2079 continue; 2080 } 2081 2082 dma_unmap_single(&pdev->dev, 2083 dma_unmap_addr(tx_buf, mapping), 2084 skb_headlen(skb), 2085 PCI_DMA_TODEVICE); 2086 2087 last = tx_buf->nr_frags; 2088 j += 2; 2089 for (k = 0; k < last; k++, j++) { 2090 int ring_idx = j & bp->tx_ring_mask; 2091 skb_frag_t *frag = &skb_shinfo(skb)->frags[k]; 2092 2093 tx_buf = &txr->tx_buf_ring[ring_idx]; 2094 dma_unmap_page( 2095 &pdev->dev, 2096 dma_unmap_addr(tx_buf, mapping), 2097 skb_frag_size(frag), PCI_DMA_TODEVICE); 2098 } 2099 dev_kfree_skb(skb); 2100 } 2101 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i)); 2102 } 2103 } 2104 2105 static void bnxt_free_rx_skbs(struct bnxt *bp) 2106 { 2107 int i, max_idx, max_agg_idx; 2108 struct pci_dev *pdev = bp->pdev; 2109 2110 if (!bp->rx_ring) 2111 return; 2112 2113 max_idx = bp->rx_nr_pages * RX_DESC_CNT; 2114 max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT; 2115 for (i = 0; i < bp->rx_nr_rings; i++) { 2116 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 2117 int j; 2118 2119 if (rxr->rx_tpa) { 2120 for (j = 0; j < MAX_TPA; j++) { 2121 struct bnxt_tpa_info *tpa_info = 2122 &rxr->rx_tpa[j]; 2123 u8 *data = tpa_info->data; 2124 2125 if (!data) 2126 continue; 2127 2128 dma_unmap_single_attrs(&pdev->dev, 2129 tpa_info->mapping, 2130 bp->rx_buf_use_size, 2131 bp->rx_dir, 2132 DMA_ATTR_WEAK_ORDERING); 2133 2134 tpa_info->data = NULL; 2135 2136 kfree(data); 2137 } 2138 } 2139 2140 for (j = 0; j < max_idx; j++) { 2141 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[j]; 2142 dma_addr_t mapping = rx_buf->mapping; 2143 void *data = rx_buf->data; 2144 2145 if (!data) 2146 continue; 2147 2148 rx_buf->data = NULL; 2149 2150 if (BNXT_RX_PAGE_MODE(bp)) { 2151 mapping -= bp->rx_dma_offset; 2152 dma_unmap_page_attrs(&pdev->dev, mapping, 2153 PAGE_SIZE, bp->rx_dir, 2154 DMA_ATTR_WEAK_ORDERING); 2155 __free_page(data); 2156 } else { 2157 dma_unmap_single_attrs(&pdev->dev, mapping, 2158 bp->rx_buf_use_size, 2159 bp->rx_dir, 2160 DMA_ATTR_WEAK_ORDERING); 2161 kfree(data); 2162 } 2163 } 2164 2165 for (j = 0; j < max_agg_idx; j++) { 2166 struct bnxt_sw_rx_agg_bd *rx_agg_buf = 2167 &rxr->rx_agg_ring[j]; 2168 struct page *page = rx_agg_buf->page; 2169 2170 if (!page) 2171 continue; 2172 2173 dma_unmap_page_attrs(&pdev->dev, rx_agg_buf->mapping, 2174 BNXT_RX_PAGE_SIZE, 2175 PCI_DMA_FROMDEVICE, 2176 DMA_ATTR_WEAK_ORDERING); 2177 2178 rx_agg_buf->page = NULL; 2179 __clear_bit(j, rxr->rx_agg_bmap); 2180 2181 __free_page(page); 2182 } 2183 if (rxr->rx_page) { 2184 __free_page(rxr->rx_page); 2185 rxr->rx_page = NULL; 2186 } 2187 } 2188 } 2189 2190 static void bnxt_free_skbs(struct bnxt *bp) 2191 { 2192 bnxt_free_tx_skbs(bp); 2193 bnxt_free_rx_skbs(bp); 2194 } 2195 2196 static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_struct *ring) 2197 { 2198 struct pci_dev *pdev = bp->pdev; 2199 int i; 2200 2201 for (i = 0; i < ring->nr_pages; i++) { 2202 if (!ring->pg_arr[i]) 2203 continue; 2204 2205 dma_free_coherent(&pdev->dev, ring->page_size, 2206 ring->pg_arr[i], ring->dma_arr[i]); 2207 2208 ring->pg_arr[i] = NULL; 2209 } 2210 if (ring->pg_tbl) { 2211 dma_free_coherent(&pdev->dev, ring->nr_pages * 8, 2212 ring->pg_tbl, ring->pg_tbl_map); 2213 ring->pg_tbl = NULL; 2214 } 2215 if (ring->vmem_size && *ring->vmem) { 2216 vfree(*ring->vmem); 2217 *ring->vmem = NULL; 2218 } 2219 } 2220 2221 static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_struct *ring) 2222 { 2223 int i; 2224 struct pci_dev *pdev = bp->pdev; 2225 2226 if (ring->nr_pages > 1) { 2227 ring->pg_tbl = dma_alloc_coherent(&pdev->dev, 2228 ring->nr_pages * 8, 2229 &ring->pg_tbl_map, 2230 GFP_KERNEL); 2231 if (!ring->pg_tbl) 2232 return -ENOMEM; 2233 } 2234 2235 for (i = 0; i < ring->nr_pages; i++) { 2236 ring->pg_arr[i] = dma_alloc_coherent(&pdev->dev, 2237 ring->page_size, 2238 &ring->dma_arr[i], 2239 GFP_KERNEL); 2240 if (!ring->pg_arr[i]) 2241 return -ENOMEM; 2242 2243 if (ring->nr_pages > 1) 2244 ring->pg_tbl[i] = cpu_to_le64(ring->dma_arr[i]); 2245 } 2246 2247 if (ring->vmem_size) { 2248 *ring->vmem = vzalloc(ring->vmem_size); 2249 if (!(*ring->vmem)) 2250 return -ENOMEM; 2251 } 2252 return 0; 2253 } 2254 2255 static void bnxt_free_rx_rings(struct bnxt *bp) 2256 { 2257 int i; 2258 2259 if (!bp->rx_ring) 2260 return; 2261 2262 for (i = 0; i < bp->rx_nr_rings; i++) { 2263 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 2264 struct bnxt_ring_struct *ring; 2265 2266 if (rxr->xdp_prog) 2267 bpf_prog_put(rxr->xdp_prog); 2268 2269 if (xdp_rxq_info_is_reg(&rxr->xdp_rxq)) 2270 xdp_rxq_info_unreg(&rxr->xdp_rxq); 2271 2272 kfree(rxr->rx_tpa); 2273 rxr->rx_tpa = NULL; 2274 2275 kfree(rxr->rx_agg_bmap); 2276 rxr->rx_agg_bmap = NULL; 2277 2278 ring = &rxr->rx_ring_struct; 2279 bnxt_free_ring(bp, ring); 2280 2281 ring = &rxr->rx_agg_ring_struct; 2282 bnxt_free_ring(bp, ring); 2283 } 2284 } 2285 2286 static int bnxt_alloc_rx_rings(struct bnxt *bp) 2287 { 2288 int i, rc, agg_rings = 0, tpa_rings = 0; 2289 2290 if (!bp->rx_ring) 2291 return -ENOMEM; 2292 2293 if (bp->flags & BNXT_FLAG_AGG_RINGS) 2294 agg_rings = 1; 2295 2296 if (bp->flags & BNXT_FLAG_TPA) 2297 tpa_rings = 1; 2298 2299 for (i = 0; i < bp->rx_nr_rings; i++) { 2300 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 2301 struct bnxt_ring_struct *ring; 2302 2303 ring = &rxr->rx_ring_struct; 2304 2305 rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i); 2306 if (rc < 0) 2307 return rc; 2308 2309 rc = bnxt_alloc_ring(bp, ring); 2310 if (rc) 2311 return rc; 2312 2313 if (agg_rings) { 2314 u16 mem_size; 2315 2316 ring = &rxr->rx_agg_ring_struct; 2317 rc = bnxt_alloc_ring(bp, ring); 2318 if (rc) 2319 return rc; 2320 2321 ring->grp_idx = i; 2322 rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1; 2323 mem_size = rxr->rx_agg_bmap_size / 8; 2324 rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL); 2325 if (!rxr->rx_agg_bmap) 2326 return -ENOMEM; 2327 2328 if (tpa_rings) { 2329 rxr->rx_tpa = kcalloc(MAX_TPA, 2330 sizeof(struct bnxt_tpa_info), 2331 GFP_KERNEL); 2332 if (!rxr->rx_tpa) 2333 return -ENOMEM; 2334 } 2335 } 2336 } 2337 return 0; 2338 } 2339 2340 static void bnxt_free_tx_rings(struct bnxt *bp) 2341 { 2342 int i; 2343 struct pci_dev *pdev = bp->pdev; 2344 2345 if (!bp->tx_ring) 2346 return; 2347 2348 for (i = 0; i < bp->tx_nr_rings; i++) { 2349 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 2350 struct bnxt_ring_struct *ring; 2351 2352 if (txr->tx_push) { 2353 dma_free_coherent(&pdev->dev, bp->tx_push_size, 2354 txr->tx_push, txr->tx_push_mapping); 2355 txr->tx_push = NULL; 2356 } 2357 2358 ring = &txr->tx_ring_struct; 2359 2360 bnxt_free_ring(bp, ring); 2361 } 2362 } 2363 2364 static int bnxt_alloc_tx_rings(struct bnxt *bp) 2365 { 2366 int i, j, rc; 2367 struct pci_dev *pdev = bp->pdev; 2368 2369 bp->tx_push_size = 0; 2370 if (bp->tx_push_thresh) { 2371 int push_size; 2372 2373 push_size = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) + 2374 bp->tx_push_thresh); 2375 2376 if (push_size > 256) { 2377 push_size = 0; 2378 bp->tx_push_thresh = 0; 2379 } 2380 2381 bp->tx_push_size = push_size; 2382 } 2383 2384 for (i = 0, j = 0; i < bp->tx_nr_rings; i++) { 2385 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 2386 struct bnxt_ring_struct *ring; 2387 u8 qidx; 2388 2389 ring = &txr->tx_ring_struct; 2390 2391 rc = bnxt_alloc_ring(bp, ring); 2392 if (rc) 2393 return rc; 2394 2395 ring->grp_idx = txr->bnapi->index; 2396 if (bp->tx_push_size) { 2397 dma_addr_t mapping; 2398 2399 /* One pre-allocated DMA buffer to backup 2400 * TX push operation 2401 */ 2402 txr->tx_push = dma_alloc_coherent(&pdev->dev, 2403 bp->tx_push_size, 2404 &txr->tx_push_mapping, 2405 GFP_KERNEL); 2406 2407 if (!txr->tx_push) 2408 return -ENOMEM; 2409 2410 mapping = txr->tx_push_mapping + 2411 sizeof(struct tx_push_bd); 2412 txr->data_mapping = cpu_to_le64(mapping); 2413 2414 memset(txr->tx_push, 0, sizeof(struct tx_push_bd)); 2415 } 2416 qidx = bp->tc_to_qidx[j]; 2417 ring->queue_id = bp->q_info[qidx].queue_id; 2418 if (i < bp->tx_nr_rings_xdp) 2419 continue; 2420 if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1)) 2421 j++; 2422 } 2423 return 0; 2424 } 2425 2426 static void bnxt_free_cp_rings(struct bnxt *bp) 2427 { 2428 int i; 2429 2430 if (!bp->bnapi) 2431 return; 2432 2433 for (i = 0; i < bp->cp_nr_rings; i++) { 2434 struct bnxt_napi *bnapi = bp->bnapi[i]; 2435 struct bnxt_cp_ring_info *cpr; 2436 struct bnxt_ring_struct *ring; 2437 2438 if (!bnapi) 2439 continue; 2440 2441 cpr = &bnapi->cp_ring; 2442 ring = &cpr->cp_ring_struct; 2443 2444 bnxt_free_ring(bp, ring); 2445 } 2446 } 2447 2448 static int bnxt_alloc_cp_rings(struct bnxt *bp) 2449 { 2450 int i, rc, ulp_base_vec, ulp_msix; 2451 2452 ulp_msix = bnxt_get_ulp_msix_num(bp); 2453 ulp_base_vec = bnxt_get_ulp_msix_base(bp); 2454 for (i = 0; i < bp->cp_nr_rings; i++) { 2455 struct bnxt_napi *bnapi = bp->bnapi[i]; 2456 struct bnxt_cp_ring_info *cpr; 2457 struct bnxt_ring_struct *ring; 2458 2459 if (!bnapi) 2460 continue; 2461 2462 cpr = &bnapi->cp_ring; 2463 ring = &cpr->cp_ring_struct; 2464 2465 rc = bnxt_alloc_ring(bp, ring); 2466 if (rc) 2467 return rc; 2468 2469 if (ulp_msix && i >= ulp_base_vec) 2470 ring->map_idx = i + ulp_msix; 2471 else 2472 ring->map_idx = i; 2473 } 2474 return 0; 2475 } 2476 2477 static void bnxt_init_ring_struct(struct bnxt *bp) 2478 { 2479 int i; 2480 2481 for (i = 0; i < bp->cp_nr_rings; i++) { 2482 struct bnxt_napi *bnapi = bp->bnapi[i]; 2483 struct bnxt_cp_ring_info *cpr; 2484 struct bnxt_rx_ring_info *rxr; 2485 struct bnxt_tx_ring_info *txr; 2486 struct bnxt_ring_struct *ring; 2487 2488 if (!bnapi) 2489 continue; 2490 2491 cpr = &bnapi->cp_ring; 2492 ring = &cpr->cp_ring_struct; 2493 ring->nr_pages = bp->cp_nr_pages; 2494 ring->page_size = HW_CMPD_RING_SIZE; 2495 ring->pg_arr = (void **)cpr->cp_desc_ring; 2496 ring->dma_arr = cpr->cp_desc_mapping; 2497 ring->vmem_size = 0; 2498 2499 rxr = bnapi->rx_ring; 2500 if (!rxr) 2501 goto skip_rx; 2502 2503 ring = &rxr->rx_ring_struct; 2504 ring->nr_pages = bp->rx_nr_pages; 2505 ring->page_size = HW_RXBD_RING_SIZE; 2506 ring->pg_arr = (void **)rxr->rx_desc_ring; 2507 ring->dma_arr = rxr->rx_desc_mapping; 2508 ring->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages; 2509 ring->vmem = (void **)&rxr->rx_buf_ring; 2510 2511 ring = &rxr->rx_agg_ring_struct; 2512 ring->nr_pages = bp->rx_agg_nr_pages; 2513 ring->page_size = HW_RXBD_RING_SIZE; 2514 ring->pg_arr = (void **)rxr->rx_agg_desc_ring; 2515 ring->dma_arr = rxr->rx_agg_desc_mapping; 2516 ring->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages; 2517 ring->vmem = (void **)&rxr->rx_agg_ring; 2518 2519 skip_rx: 2520 txr = bnapi->tx_ring; 2521 if (!txr) 2522 continue; 2523 2524 ring = &txr->tx_ring_struct; 2525 ring->nr_pages = bp->tx_nr_pages; 2526 ring->page_size = HW_RXBD_RING_SIZE; 2527 ring->pg_arr = (void **)txr->tx_desc_ring; 2528 ring->dma_arr = txr->tx_desc_mapping; 2529 ring->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages; 2530 ring->vmem = (void **)&txr->tx_buf_ring; 2531 } 2532 } 2533 2534 static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type) 2535 { 2536 int i; 2537 u32 prod; 2538 struct rx_bd **rx_buf_ring; 2539 2540 rx_buf_ring = (struct rx_bd **)ring->pg_arr; 2541 for (i = 0, prod = 0; i < ring->nr_pages; i++) { 2542 int j; 2543 struct rx_bd *rxbd; 2544 2545 rxbd = rx_buf_ring[i]; 2546 if (!rxbd) 2547 continue; 2548 2549 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) { 2550 rxbd->rx_bd_len_flags_type = cpu_to_le32(type); 2551 rxbd->rx_bd_opaque = prod; 2552 } 2553 } 2554 } 2555 2556 static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr) 2557 { 2558 struct net_device *dev = bp->dev; 2559 struct bnxt_rx_ring_info *rxr; 2560 struct bnxt_ring_struct *ring; 2561 u32 prod, type; 2562 int i; 2563 2564 type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) | 2565 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP; 2566 2567 if (NET_IP_ALIGN == 2) 2568 type |= RX_BD_FLAGS_SOP; 2569 2570 rxr = &bp->rx_ring[ring_nr]; 2571 ring = &rxr->rx_ring_struct; 2572 bnxt_init_rxbd_pages(ring, type); 2573 2574 if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) { 2575 rxr->xdp_prog = bpf_prog_add(bp->xdp_prog, 1); 2576 if (IS_ERR(rxr->xdp_prog)) { 2577 int rc = PTR_ERR(rxr->xdp_prog); 2578 2579 rxr->xdp_prog = NULL; 2580 return rc; 2581 } 2582 } 2583 prod = rxr->rx_prod; 2584 for (i = 0; i < bp->rx_ring_size; i++) { 2585 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL) != 0) { 2586 netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n", 2587 ring_nr, i, bp->rx_ring_size); 2588 break; 2589 } 2590 prod = NEXT_RX(prod); 2591 } 2592 rxr->rx_prod = prod; 2593 ring->fw_ring_id = INVALID_HW_RING_ID; 2594 2595 ring = &rxr->rx_agg_ring_struct; 2596 ring->fw_ring_id = INVALID_HW_RING_ID; 2597 2598 if (!(bp->flags & BNXT_FLAG_AGG_RINGS)) 2599 return 0; 2600 2601 type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) | 2602 RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP; 2603 2604 bnxt_init_rxbd_pages(ring, type); 2605 2606 prod = rxr->rx_agg_prod; 2607 for (i = 0; i < bp->rx_agg_ring_size; i++) { 2608 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL) != 0) { 2609 netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n", 2610 ring_nr, i, bp->rx_ring_size); 2611 break; 2612 } 2613 prod = NEXT_RX_AGG(prod); 2614 } 2615 rxr->rx_agg_prod = prod; 2616 2617 if (bp->flags & BNXT_FLAG_TPA) { 2618 if (rxr->rx_tpa) { 2619 u8 *data; 2620 dma_addr_t mapping; 2621 2622 for (i = 0; i < MAX_TPA; i++) { 2623 data = __bnxt_alloc_rx_data(bp, &mapping, 2624 GFP_KERNEL); 2625 if (!data) 2626 return -ENOMEM; 2627 2628 rxr->rx_tpa[i].data = data; 2629 rxr->rx_tpa[i].data_ptr = data + bp->rx_offset; 2630 rxr->rx_tpa[i].mapping = mapping; 2631 } 2632 } else { 2633 netdev_err(bp->dev, "No resource allocated for LRO/GRO\n"); 2634 return -ENOMEM; 2635 } 2636 } 2637 2638 return 0; 2639 } 2640 2641 static void bnxt_init_cp_rings(struct bnxt *bp) 2642 { 2643 int i; 2644 2645 for (i = 0; i < bp->cp_nr_rings; i++) { 2646 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring; 2647 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 2648 2649 ring->fw_ring_id = INVALID_HW_RING_ID; 2650 cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks; 2651 cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs; 2652 } 2653 } 2654 2655 static int bnxt_init_rx_rings(struct bnxt *bp) 2656 { 2657 int i, rc = 0; 2658 2659 if (BNXT_RX_PAGE_MODE(bp)) { 2660 bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM; 2661 bp->rx_dma_offset = XDP_PACKET_HEADROOM; 2662 } else { 2663 bp->rx_offset = BNXT_RX_OFFSET; 2664 bp->rx_dma_offset = BNXT_RX_DMA_OFFSET; 2665 } 2666 2667 for (i = 0; i < bp->rx_nr_rings; i++) { 2668 rc = bnxt_init_one_rx_ring(bp, i); 2669 if (rc) 2670 break; 2671 } 2672 2673 return rc; 2674 } 2675 2676 static int bnxt_init_tx_rings(struct bnxt *bp) 2677 { 2678 u16 i; 2679 2680 bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2, 2681 MAX_SKB_FRAGS + 1); 2682 2683 for (i = 0; i < bp->tx_nr_rings; i++) { 2684 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 2685 struct bnxt_ring_struct *ring = &txr->tx_ring_struct; 2686 2687 ring->fw_ring_id = INVALID_HW_RING_ID; 2688 } 2689 2690 return 0; 2691 } 2692 2693 static void bnxt_free_ring_grps(struct bnxt *bp) 2694 { 2695 kfree(bp->grp_info); 2696 bp->grp_info = NULL; 2697 } 2698 2699 static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init) 2700 { 2701 int i; 2702 2703 if (irq_re_init) { 2704 bp->grp_info = kcalloc(bp->cp_nr_rings, 2705 sizeof(struct bnxt_ring_grp_info), 2706 GFP_KERNEL); 2707 if (!bp->grp_info) 2708 return -ENOMEM; 2709 } 2710 for (i = 0; i < bp->cp_nr_rings; i++) { 2711 if (irq_re_init) 2712 bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID; 2713 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID; 2714 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID; 2715 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID; 2716 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID; 2717 } 2718 return 0; 2719 } 2720 2721 static void bnxt_free_vnics(struct bnxt *bp) 2722 { 2723 kfree(bp->vnic_info); 2724 bp->vnic_info = NULL; 2725 bp->nr_vnics = 0; 2726 } 2727 2728 static int bnxt_alloc_vnics(struct bnxt *bp) 2729 { 2730 int num_vnics = 1; 2731 2732 #ifdef CONFIG_RFS_ACCEL 2733 if (bp->flags & BNXT_FLAG_RFS) 2734 num_vnics += bp->rx_nr_rings; 2735 #endif 2736 2737 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 2738 num_vnics++; 2739 2740 bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info), 2741 GFP_KERNEL); 2742 if (!bp->vnic_info) 2743 return -ENOMEM; 2744 2745 bp->nr_vnics = num_vnics; 2746 return 0; 2747 } 2748 2749 static void bnxt_init_vnics(struct bnxt *bp) 2750 { 2751 int i; 2752 2753 for (i = 0; i < bp->nr_vnics; i++) { 2754 struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; 2755 2756 vnic->fw_vnic_id = INVALID_HW_RING_ID; 2757 vnic->fw_rss_cos_lb_ctx[0] = INVALID_HW_RING_ID; 2758 vnic->fw_rss_cos_lb_ctx[1] = INVALID_HW_RING_ID; 2759 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID; 2760 2761 if (bp->vnic_info[i].rss_hash_key) { 2762 if (i == 0) 2763 prandom_bytes(vnic->rss_hash_key, 2764 HW_HASH_KEY_SIZE); 2765 else 2766 memcpy(vnic->rss_hash_key, 2767 bp->vnic_info[0].rss_hash_key, 2768 HW_HASH_KEY_SIZE); 2769 } 2770 } 2771 } 2772 2773 static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg) 2774 { 2775 int pages; 2776 2777 pages = ring_size / desc_per_pg; 2778 2779 if (!pages) 2780 return 1; 2781 2782 pages++; 2783 2784 while (pages & (pages - 1)) 2785 pages++; 2786 2787 return pages; 2788 } 2789 2790 void bnxt_set_tpa_flags(struct bnxt *bp) 2791 { 2792 bp->flags &= ~BNXT_FLAG_TPA; 2793 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS) 2794 return; 2795 if (bp->dev->features & NETIF_F_LRO) 2796 bp->flags |= BNXT_FLAG_LRO; 2797 else if (bp->dev->features & NETIF_F_GRO_HW) 2798 bp->flags |= BNXT_FLAG_GRO; 2799 } 2800 2801 /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must 2802 * be set on entry. 2803 */ 2804 void bnxt_set_ring_params(struct bnxt *bp) 2805 { 2806 u32 ring_size, rx_size, rx_space; 2807 u32 agg_factor = 0, agg_ring_size = 0; 2808 2809 /* 8 for CRC and VLAN */ 2810 rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8); 2811 2812 rx_space = rx_size + NET_SKB_PAD + 2813 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 2814 2815 bp->rx_copy_thresh = BNXT_RX_COPY_THRESH; 2816 ring_size = bp->rx_ring_size; 2817 bp->rx_agg_ring_size = 0; 2818 bp->rx_agg_nr_pages = 0; 2819 2820 if (bp->flags & BNXT_FLAG_TPA) 2821 agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE); 2822 2823 bp->flags &= ~BNXT_FLAG_JUMBO; 2824 if (rx_space > PAGE_SIZE && !(bp->flags & BNXT_FLAG_NO_AGG_RINGS)) { 2825 u32 jumbo_factor; 2826 2827 bp->flags |= BNXT_FLAG_JUMBO; 2828 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT; 2829 if (jumbo_factor > agg_factor) 2830 agg_factor = jumbo_factor; 2831 } 2832 agg_ring_size = ring_size * agg_factor; 2833 2834 if (agg_ring_size) { 2835 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size, 2836 RX_DESC_CNT); 2837 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) { 2838 u32 tmp = agg_ring_size; 2839 2840 bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES; 2841 agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1; 2842 netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n", 2843 tmp, agg_ring_size); 2844 } 2845 bp->rx_agg_ring_size = agg_ring_size; 2846 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1; 2847 rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN); 2848 rx_space = rx_size + NET_SKB_PAD + 2849 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 2850 } 2851 2852 bp->rx_buf_use_size = rx_size; 2853 bp->rx_buf_size = rx_space; 2854 2855 bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT); 2856 bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1; 2857 2858 ring_size = bp->tx_ring_size; 2859 bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT); 2860 bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1; 2861 2862 ring_size = bp->rx_ring_size * (2 + agg_factor) + bp->tx_ring_size; 2863 bp->cp_ring_size = ring_size; 2864 2865 bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT); 2866 if (bp->cp_nr_pages > MAX_CP_PAGES) { 2867 bp->cp_nr_pages = MAX_CP_PAGES; 2868 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1; 2869 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n", 2870 ring_size, bp->cp_ring_size); 2871 } 2872 bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT; 2873 bp->cp_ring_mask = bp->cp_bit - 1; 2874 } 2875 2876 /* Changing allocation mode of RX rings. 2877 * TODO: Update when extending xdp_rxq_info to support allocation modes. 2878 */ 2879 int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode) 2880 { 2881 if (page_mode) { 2882 if (bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU) 2883 return -EOPNOTSUPP; 2884 bp->dev->max_mtu = 2885 min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU); 2886 bp->flags &= ~BNXT_FLAG_AGG_RINGS; 2887 bp->flags |= BNXT_FLAG_NO_AGG_RINGS | BNXT_FLAG_RX_PAGE_MODE; 2888 bp->rx_dir = DMA_BIDIRECTIONAL; 2889 bp->rx_skb_func = bnxt_rx_page_skb; 2890 /* Disable LRO or GRO_HW */ 2891 netdev_update_features(bp->dev); 2892 } else { 2893 bp->dev->max_mtu = bp->max_mtu; 2894 bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE; 2895 bp->rx_dir = DMA_FROM_DEVICE; 2896 bp->rx_skb_func = bnxt_rx_skb; 2897 } 2898 return 0; 2899 } 2900 2901 static void bnxt_free_vnic_attributes(struct bnxt *bp) 2902 { 2903 int i; 2904 struct bnxt_vnic_info *vnic; 2905 struct pci_dev *pdev = bp->pdev; 2906 2907 if (!bp->vnic_info) 2908 return; 2909 2910 for (i = 0; i < bp->nr_vnics; i++) { 2911 vnic = &bp->vnic_info[i]; 2912 2913 kfree(vnic->fw_grp_ids); 2914 vnic->fw_grp_ids = NULL; 2915 2916 kfree(vnic->uc_list); 2917 vnic->uc_list = NULL; 2918 2919 if (vnic->mc_list) { 2920 dma_free_coherent(&pdev->dev, vnic->mc_list_size, 2921 vnic->mc_list, vnic->mc_list_mapping); 2922 vnic->mc_list = NULL; 2923 } 2924 2925 if (vnic->rss_table) { 2926 dma_free_coherent(&pdev->dev, PAGE_SIZE, 2927 vnic->rss_table, 2928 vnic->rss_table_dma_addr); 2929 vnic->rss_table = NULL; 2930 } 2931 2932 vnic->rss_hash_key = NULL; 2933 vnic->flags = 0; 2934 } 2935 } 2936 2937 static int bnxt_alloc_vnic_attributes(struct bnxt *bp) 2938 { 2939 int i, rc = 0, size; 2940 struct bnxt_vnic_info *vnic; 2941 struct pci_dev *pdev = bp->pdev; 2942 int max_rings; 2943 2944 for (i = 0; i < bp->nr_vnics; i++) { 2945 vnic = &bp->vnic_info[i]; 2946 2947 if (vnic->flags & BNXT_VNIC_UCAST_FLAG) { 2948 int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN; 2949 2950 if (mem_size > 0) { 2951 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL); 2952 if (!vnic->uc_list) { 2953 rc = -ENOMEM; 2954 goto out; 2955 } 2956 } 2957 } 2958 2959 if (vnic->flags & BNXT_VNIC_MCAST_FLAG) { 2960 vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN; 2961 vnic->mc_list = 2962 dma_alloc_coherent(&pdev->dev, 2963 vnic->mc_list_size, 2964 &vnic->mc_list_mapping, 2965 GFP_KERNEL); 2966 if (!vnic->mc_list) { 2967 rc = -ENOMEM; 2968 goto out; 2969 } 2970 } 2971 2972 if (vnic->flags & BNXT_VNIC_RSS_FLAG) 2973 max_rings = bp->rx_nr_rings; 2974 else 2975 max_rings = 1; 2976 2977 vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL); 2978 if (!vnic->fw_grp_ids) { 2979 rc = -ENOMEM; 2980 goto out; 2981 } 2982 2983 if ((bp->flags & BNXT_FLAG_NEW_RSS_CAP) && 2984 !(vnic->flags & BNXT_VNIC_RSS_FLAG)) 2985 continue; 2986 2987 /* Allocate rss table and hash key */ 2988 vnic->rss_table = dma_alloc_coherent(&pdev->dev, PAGE_SIZE, 2989 &vnic->rss_table_dma_addr, 2990 GFP_KERNEL); 2991 if (!vnic->rss_table) { 2992 rc = -ENOMEM; 2993 goto out; 2994 } 2995 2996 size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16)); 2997 2998 vnic->rss_hash_key = ((void *)vnic->rss_table) + size; 2999 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size; 3000 } 3001 return 0; 3002 3003 out: 3004 return rc; 3005 } 3006 3007 static void bnxt_free_hwrm_resources(struct bnxt *bp) 3008 { 3009 struct pci_dev *pdev = bp->pdev; 3010 3011 dma_free_coherent(&pdev->dev, PAGE_SIZE, bp->hwrm_cmd_resp_addr, 3012 bp->hwrm_cmd_resp_dma_addr); 3013 3014 bp->hwrm_cmd_resp_addr = NULL; 3015 } 3016 3017 static int bnxt_alloc_hwrm_resources(struct bnxt *bp) 3018 { 3019 struct pci_dev *pdev = bp->pdev; 3020 3021 bp->hwrm_cmd_resp_addr = dma_alloc_coherent(&pdev->dev, PAGE_SIZE, 3022 &bp->hwrm_cmd_resp_dma_addr, 3023 GFP_KERNEL); 3024 if (!bp->hwrm_cmd_resp_addr) 3025 return -ENOMEM; 3026 3027 return 0; 3028 } 3029 3030 static void bnxt_free_hwrm_short_cmd_req(struct bnxt *bp) 3031 { 3032 if (bp->hwrm_short_cmd_req_addr) { 3033 struct pci_dev *pdev = bp->pdev; 3034 3035 dma_free_coherent(&pdev->dev, BNXT_HWRM_MAX_REQ_LEN, 3036 bp->hwrm_short_cmd_req_addr, 3037 bp->hwrm_short_cmd_req_dma_addr); 3038 bp->hwrm_short_cmd_req_addr = NULL; 3039 } 3040 } 3041 3042 static int bnxt_alloc_hwrm_short_cmd_req(struct bnxt *bp) 3043 { 3044 struct pci_dev *pdev = bp->pdev; 3045 3046 bp->hwrm_short_cmd_req_addr = 3047 dma_alloc_coherent(&pdev->dev, BNXT_HWRM_MAX_REQ_LEN, 3048 &bp->hwrm_short_cmd_req_dma_addr, 3049 GFP_KERNEL); 3050 if (!bp->hwrm_short_cmd_req_addr) 3051 return -ENOMEM; 3052 3053 return 0; 3054 } 3055 3056 static void bnxt_free_stats(struct bnxt *bp) 3057 { 3058 u32 size, i; 3059 struct pci_dev *pdev = bp->pdev; 3060 3061 bp->flags &= ~BNXT_FLAG_PORT_STATS; 3062 bp->flags &= ~BNXT_FLAG_PORT_STATS_EXT; 3063 3064 if (bp->hw_rx_port_stats) { 3065 dma_free_coherent(&pdev->dev, bp->hw_port_stats_size, 3066 bp->hw_rx_port_stats, 3067 bp->hw_rx_port_stats_map); 3068 bp->hw_rx_port_stats = NULL; 3069 } 3070 3071 if (bp->hw_rx_port_stats_ext) { 3072 dma_free_coherent(&pdev->dev, sizeof(struct rx_port_stats_ext), 3073 bp->hw_rx_port_stats_ext, 3074 bp->hw_rx_port_stats_ext_map); 3075 bp->hw_rx_port_stats_ext = NULL; 3076 } 3077 3078 if (!bp->bnapi) 3079 return; 3080 3081 size = sizeof(struct ctx_hw_stats); 3082 3083 for (i = 0; i < bp->cp_nr_rings; i++) { 3084 struct bnxt_napi *bnapi = bp->bnapi[i]; 3085 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 3086 3087 if (cpr->hw_stats) { 3088 dma_free_coherent(&pdev->dev, size, cpr->hw_stats, 3089 cpr->hw_stats_map); 3090 cpr->hw_stats = NULL; 3091 } 3092 } 3093 } 3094 3095 static int bnxt_alloc_stats(struct bnxt *bp) 3096 { 3097 u32 size, i; 3098 struct pci_dev *pdev = bp->pdev; 3099 3100 size = sizeof(struct ctx_hw_stats); 3101 3102 for (i = 0; i < bp->cp_nr_rings; i++) { 3103 struct bnxt_napi *bnapi = bp->bnapi[i]; 3104 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 3105 3106 cpr->hw_stats = dma_alloc_coherent(&pdev->dev, size, 3107 &cpr->hw_stats_map, 3108 GFP_KERNEL); 3109 if (!cpr->hw_stats) 3110 return -ENOMEM; 3111 3112 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID; 3113 } 3114 3115 if (BNXT_PF(bp) && bp->chip_num != CHIP_NUM_58700) { 3116 bp->hw_port_stats_size = sizeof(struct rx_port_stats) + 3117 sizeof(struct tx_port_stats) + 1024; 3118 3119 bp->hw_rx_port_stats = 3120 dma_alloc_coherent(&pdev->dev, bp->hw_port_stats_size, 3121 &bp->hw_rx_port_stats_map, 3122 GFP_KERNEL); 3123 if (!bp->hw_rx_port_stats) 3124 return -ENOMEM; 3125 3126 bp->hw_tx_port_stats = (void *)(bp->hw_rx_port_stats + 1) + 3127 512; 3128 bp->hw_tx_port_stats_map = bp->hw_rx_port_stats_map + 3129 sizeof(struct rx_port_stats) + 512; 3130 bp->flags |= BNXT_FLAG_PORT_STATS; 3131 3132 /* Display extended statistics only if FW supports it */ 3133 if (bp->hwrm_spec_code < 0x10804 || 3134 bp->hwrm_spec_code == 0x10900) 3135 return 0; 3136 3137 bp->hw_rx_port_stats_ext = 3138 dma_zalloc_coherent(&pdev->dev, 3139 sizeof(struct rx_port_stats_ext), 3140 &bp->hw_rx_port_stats_ext_map, 3141 GFP_KERNEL); 3142 if (!bp->hw_rx_port_stats_ext) 3143 return 0; 3144 3145 bp->flags |= BNXT_FLAG_PORT_STATS_EXT; 3146 } 3147 return 0; 3148 } 3149 3150 static void bnxt_clear_ring_indices(struct bnxt *bp) 3151 { 3152 int i; 3153 3154 if (!bp->bnapi) 3155 return; 3156 3157 for (i = 0; i < bp->cp_nr_rings; i++) { 3158 struct bnxt_napi *bnapi = bp->bnapi[i]; 3159 struct bnxt_cp_ring_info *cpr; 3160 struct bnxt_rx_ring_info *rxr; 3161 struct bnxt_tx_ring_info *txr; 3162 3163 if (!bnapi) 3164 continue; 3165 3166 cpr = &bnapi->cp_ring; 3167 cpr->cp_raw_cons = 0; 3168 3169 txr = bnapi->tx_ring; 3170 if (txr) { 3171 txr->tx_prod = 0; 3172 txr->tx_cons = 0; 3173 } 3174 3175 rxr = bnapi->rx_ring; 3176 if (rxr) { 3177 rxr->rx_prod = 0; 3178 rxr->rx_agg_prod = 0; 3179 rxr->rx_sw_agg_prod = 0; 3180 rxr->rx_next_cons = 0; 3181 } 3182 } 3183 } 3184 3185 static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool irq_reinit) 3186 { 3187 #ifdef CONFIG_RFS_ACCEL 3188 int i; 3189 3190 /* Under rtnl_lock and all our NAPIs have been disabled. It's 3191 * safe to delete the hash table. 3192 */ 3193 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) { 3194 struct hlist_head *head; 3195 struct hlist_node *tmp; 3196 struct bnxt_ntuple_filter *fltr; 3197 3198 head = &bp->ntp_fltr_hash_tbl[i]; 3199 hlist_for_each_entry_safe(fltr, tmp, head, hash) { 3200 hlist_del(&fltr->hash); 3201 kfree(fltr); 3202 } 3203 } 3204 if (irq_reinit) { 3205 kfree(bp->ntp_fltr_bmap); 3206 bp->ntp_fltr_bmap = NULL; 3207 } 3208 bp->ntp_fltr_count = 0; 3209 #endif 3210 } 3211 3212 static int bnxt_alloc_ntp_fltrs(struct bnxt *bp) 3213 { 3214 #ifdef CONFIG_RFS_ACCEL 3215 int i, rc = 0; 3216 3217 if (!(bp->flags & BNXT_FLAG_RFS)) 3218 return 0; 3219 3220 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) 3221 INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]); 3222 3223 bp->ntp_fltr_count = 0; 3224 bp->ntp_fltr_bmap = kcalloc(BITS_TO_LONGS(BNXT_NTP_FLTR_MAX_FLTR), 3225 sizeof(long), 3226 GFP_KERNEL); 3227 3228 if (!bp->ntp_fltr_bmap) 3229 rc = -ENOMEM; 3230 3231 return rc; 3232 #else 3233 return 0; 3234 #endif 3235 } 3236 3237 static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init) 3238 { 3239 bnxt_free_vnic_attributes(bp); 3240 bnxt_free_tx_rings(bp); 3241 bnxt_free_rx_rings(bp); 3242 bnxt_free_cp_rings(bp); 3243 bnxt_free_ntp_fltrs(bp, irq_re_init); 3244 if (irq_re_init) { 3245 bnxt_free_stats(bp); 3246 bnxt_free_ring_grps(bp); 3247 bnxt_free_vnics(bp); 3248 kfree(bp->tx_ring_map); 3249 bp->tx_ring_map = NULL; 3250 kfree(bp->tx_ring); 3251 bp->tx_ring = NULL; 3252 kfree(bp->rx_ring); 3253 bp->rx_ring = NULL; 3254 kfree(bp->bnapi); 3255 bp->bnapi = NULL; 3256 } else { 3257 bnxt_clear_ring_indices(bp); 3258 } 3259 } 3260 3261 static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init) 3262 { 3263 int i, j, rc, size, arr_size; 3264 void *bnapi; 3265 3266 if (irq_re_init) { 3267 /* Allocate bnapi mem pointer array and mem block for 3268 * all queues 3269 */ 3270 arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) * 3271 bp->cp_nr_rings); 3272 size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi)); 3273 bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL); 3274 if (!bnapi) 3275 return -ENOMEM; 3276 3277 bp->bnapi = bnapi; 3278 bnapi += arr_size; 3279 for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) { 3280 bp->bnapi[i] = bnapi; 3281 bp->bnapi[i]->index = i; 3282 bp->bnapi[i]->bp = bp; 3283 } 3284 3285 bp->rx_ring = kcalloc(bp->rx_nr_rings, 3286 sizeof(struct bnxt_rx_ring_info), 3287 GFP_KERNEL); 3288 if (!bp->rx_ring) 3289 return -ENOMEM; 3290 3291 for (i = 0; i < bp->rx_nr_rings; i++) { 3292 bp->rx_ring[i].bnapi = bp->bnapi[i]; 3293 bp->bnapi[i]->rx_ring = &bp->rx_ring[i]; 3294 } 3295 3296 bp->tx_ring = kcalloc(bp->tx_nr_rings, 3297 sizeof(struct bnxt_tx_ring_info), 3298 GFP_KERNEL); 3299 if (!bp->tx_ring) 3300 return -ENOMEM; 3301 3302 bp->tx_ring_map = kcalloc(bp->tx_nr_rings, sizeof(u16), 3303 GFP_KERNEL); 3304 3305 if (!bp->tx_ring_map) 3306 return -ENOMEM; 3307 3308 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 3309 j = 0; 3310 else 3311 j = bp->rx_nr_rings; 3312 3313 for (i = 0; i < bp->tx_nr_rings; i++, j++) { 3314 bp->tx_ring[i].bnapi = bp->bnapi[j]; 3315 bp->bnapi[j]->tx_ring = &bp->tx_ring[i]; 3316 bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i; 3317 if (i >= bp->tx_nr_rings_xdp) { 3318 bp->tx_ring[i].txq_index = i - 3319 bp->tx_nr_rings_xdp; 3320 bp->bnapi[j]->tx_int = bnxt_tx_int; 3321 } else { 3322 bp->bnapi[j]->flags |= BNXT_NAPI_FLAG_XDP; 3323 bp->bnapi[j]->tx_int = bnxt_tx_int_xdp; 3324 } 3325 } 3326 3327 rc = bnxt_alloc_stats(bp); 3328 if (rc) 3329 goto alloc_mem_err; 3330 3331 rc = bnxt_alloc_ntp_fltrs(bp); 3332 if (rc) 3333 goto alloc_mem_err; 3334 3335 rc = bnxt_alloc_vnics(bp); 3336 if (rc) 3337 goto alloc_mem_err; 3338 } 3339 3340 bnxt_init_ring_struct(bp); 3341 3342 rc = bnxt_alloc_rx_rings(bp); 3343 if (rc) 3344 goto alloc_mem_err; 3345 3346 rc = bnxt_alloc_tx_rings(bp); 3347 if (rc) 3348 goto alloc_mem_err; 3349 3350 rc = bnxt_alloc_cp_rings(bp); 3351 if (rc) 3352 goto alloc_mem_err; 3353 3354 bp->vnic_info[0].flags |= BNXT_VNIC_RSS_FLAG | BNXT_VNIC_MCAST_FLAG | 3355 BNXT_VNIC_UCAST_FLAG; 3356 rc = bnxt_alloc_vnic_attributes(bp); 3357 if (rc) 3358 goto alloc_mem_err; 3359 return 0; 3360 3361 alloc_mem_err: 3362 bnxt_free_mem(bp, true); 3363 return rc; 3364 } 3365 3366 static void bnxt_disable_int(struct bnxt *bp) 3367 { 3368 int i; 3369 3370 if (!bp->bnapi) 3371 return; 3372 3373 for (i = 0; i < bp->cp_nr_rings; i++) { 3374 struct bnxt_napi *bnapi = bp->bnapi[i]; 3375 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 3376 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 3377 3378 if (ring->fw_ring_id != INVALID_HW_RING_ID) 3379 BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons); 3380 } 3381 } 3382 3383 static int bnxt_cp_num_to_irq_num(struct bnxt *bp, int n) 3384 { 3385 struct bnxt_napi *bnapi = bp->bnapi[n]; 3386 struct bnxt_cp_ring_info *cpr; 3387 3388 cpr = &bnapi->cp_ring; 3389 return cpr->cp_ring_struct.map_idx; 3390 } 3391 3392 static void bnxt_disable_int_sync(struct bnxt *bp) 3393 { 3394 int i; 3395 3396 atomic_inc(&bp->intr_sem); 3397 3398 bnxt_disable_int(bp); 3399 for (i = 0; i < bp->cp_nr_rings; i++) { 3400 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 3401 3402 synchronize_irq(bp->irq_tbl[map_idx].vector); 3403 } 3404 } 3405 3406 static void bnxt_enable_int(struct bnxt *bp) 3407 { 3408 int i; 3409 3410 atomic_set(&bp->intr_sem, 0); 3411 for (i = 0; i < bp->cp_nr_rings; i++) { 3412 struct bnxt_napi *bnapi = bp->bnapi[i]; 3413 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 3414 3415 BNXT_CP_DB_REARM(cpr->cp_doorbell, cpr->cp_raw_cons); 3416 } 3417 } 3418 3419 void bnxt_hwrm_cmd_hdr_init(struct bnxt *bp, void *request, u16 req_type, 3420 u16 cmpl_ring, u16 target_id) 3421 { 3422 struct input *req = request; 3423 3424 req->req_type = cpu_to_le16(req_type); 3425 req->cmpl_ring = cpu_to_le16(cmpl_ring); 3426 req->target_id = cpu_to_le16(target_id); 3427 req->resp_addr = cpu_to_le64(bp->hwrm_cmd_resp_dma_addr); 3428 } 3429 3430 static int bnxt_hwrm_do_send_msg(struct bnxt *bp, void *msg, u32 msg_len, 3431 int timeout, bool silent) 3432 { 3433 int i, intr_process, rc, tmo_count; 3434 struct input *req = msg; 3435 u32 *data = msg; 3436 __le32 *resp_len; 3437 u8 *valid; 3438 u16 cp_ring_id, len = 0; 3439 struct hwrm_err_output *resp = bp->hwrm_cmd_resp_addr; 3440 u16 max_req_len = BNXT_HWRM_MAX_REQ_LEN; 3441 struct hwrm_short_input short_input = {0}; 3442 3443 req->seq_id = cpu_to_le16(bp->hwrm_cmd_seq++); 3444 memset(resp, 0, PAGE_SIZE); 3445 cp_ring_id = le16_to_cpu(req->cmpl_ring); 3446 intr_process = (cp_ring_id == INVALID_HW_RING_ID) ? 0 : 1; 3447 3448 if (bp->flags & BNXT_FLAG_SHORT_CMD) { 3449 void *short_cmd_req = bp->hwrm_short_cmd_req_addr; 3450 3451 memcpy(short_cmd_req, req, msg_len); 3452 memset(short_cmd_req + msg_len, 0, BNXT_HWRM_MAX_REQ_LEN - 3453 msg_len); 3454 3455 short_input.req_type = req->req_type; 3456 short_input.signature = 3457 cpu_to_le16(SHORT_REQ_SIGNATURE_SHORT_CMD); 3458 short_input.size = cpu_to_le16(msg_len); 3459 short_input.req_addr = 3460 cpu_to_le64(bp->hwrm_short_cmd_req_dma_addr); 3461 3462 data = (u32 *)&short_input; 3463 msg_len = sizeof(short_input); 3464 3465 /* Sync memory write before updating doorbell */ 3466 wmb(); 3467 3468 max_req_len = BNXT_HWRM_SHORT_REQ_LEN; 3469 } 3470 3471 /* Write request msg to hwrm channel */ 3472 __iowrite32_copy(bp->bar0, data, msg_len / 4); 3473 3474 for (i = msg_len; i < max_req_len; i += 4) 3475 writel(0, bp->bar0 + i); 3476 3477 /* currently supports only one outstanding message */ 3478 if (intr_process) 3479 bp->hwrm_intr_seq_id = le16_to_cpu(req->seq_id); 3480 3481 /* Ring channel doorbell */ 3482 writel(1, bp->bar0 + 0x100); 3483 3484 if (!timeout) 3485 timeout = DFLT_HWRM_CMD_TIMEOUT; 3486 /* convert timeout to usec */ 3487 timeout *= 1000; 3488 3489 i = 0; 3490 /* Short timeout for the first few iterations: 3491 * number of loops = number of loops for short timeout + 3492 * number of loops for standard timeout. 3493 */ 3494 tmo_count = HWRM_SHORT_TIMEOUT_COUNTER; 3495 timeout = timeout - HWRM_SHORT_MIN_TIMEOUT * HWRM_SHORT_TIMEOUT_COUNTER; 3496 tmo_count += DIV_ROUND_UP(timeout, HWRM_MIN_TIMEOUT); 3497 resp_len = bp->hwrm_cmd_resp_addr + HWRM_RESP_LEN_OFFSET; 3498 if (intr_process) { 3499 /* Wait until hwrm response cmpl interrupt is processed */ 3500 while (bp->hwrm_intr_seq_id != HWRM_SEQ_ID_INVALID && 3501 i++ < tmo_count) { 3502 /* on first few passes, just barely sleep */ 3503 if (i < HWRM_SHORT_TIMEOUT_COUNTER) 3504 usleep_range(HWRM_SHORT_MIN_TIMEOUT, 3505 HWRM_SHORT_MAX_TIMEOUT); 3506 else 3507 usleep_range(HWRM_MIN_TIMEOUT, 3508 HWRM_MAX_TIMEOUT); 3509 } 3510 3511 if (bp->hwrm_intr_seq_id != HWRM_SEQ_ID_INVALID) { 3512 netdev_err(bp->dev, "Resp cmpl intr err msg: 0x%x\n", 3513 le16_to_cpu(req->req_type)); 3514 return -1; 3515 } 3516 len = (le32_to_cpu(*resp_len) & HWRM_RESP_LEN_MASK) >> 3517 HWRM_RESP_LEN_SFT; 3518 valid = bp->hwrm_cmd_resp_addr + len - 1; 3519 } else { 3520 int j; 3521 3522 /* Check if response len is updated */ 3523 for (i = 0; i < tmo_count; i++) { 3524 len = (le32_to_cpu(*resp_len) & HWRM_RESP_LEN_MASK) >> 3525 HWRM_RESP_LEN_SFT; 3526 if (len) 3527 break; 3528 /* on first few passes, just barely sleep */ 3529 if (i < DFLT_HWRM_CMD_TIMEOUT) 3530 usleep_range(HWRM_SHORT_MIN_TIMEOUT, 3531 HWRM_SHORT_MAX_TIMEOUT); 3532 else 3533 usleep_range(HWRM_MIN_TIMEOUT, 3534 HWRM_MAX_TIMEOUT); 3535 } 3536 3537 if (i >= tmo_count) { 3538 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d\n", 3539 HWRM_TOTAL_TIMEOUT(i), 3540 le16_to_cpu(req->req_type), 3541 le16_to_cpu(req->seq_id), len); 3542 return -1; 3543 } 3544 3545 /* Last byte of resp contains valid bit */ 3546 valid = bp->hwrm_cmd_resp_addr + len - 1; 3547 for (j = 0; j < HWRM_VALID_BIT_DELAY_USEC; j++) { 3548 /* make sure we read from updated DMA memory */ 3549 dma_rmb(); 3550 if (*valid) 3551 break; 3552 udelay(1); 3553 } 3554 3555 if (j >= HWRM_VALID_BIT_DELAY_USEC) { 3556 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d v:%d\n", 3557 HWRM_TOTAL_TIMEOUT(i), 3558 le16_to_cpu(req->req_type), 3559 le16_to_cpu(req->seq_id), len, *valid); 3560 return -1; 3561 } 3562 } 3563 3564 /* Zero valid bit for compatibility. Valid bit in an older spec 3565 * may become a new field in a newer spec. We must make sure that 3566 * a new field not implemented by old spec will read zero. 3567 */ 3568 *valid = 0; 3569 rc = le16_to_cpu(resp->error_code); 3570 if (rc && !silent) 3571 netdev_err(bp->dev, "hwrm req_type 0x%x seq id 0x%x error 0x%x\n", 3572 le16_to_cpu(resp->req_type), 3573 le16_to_cpu(resp->seq_id), rc); 3574 return rc; 3575 } 3576 3577 int _hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout) 3578 { 3579 return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, false); 3580 } 3581 3582 int _hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len, 3583 int timeout) 3584 { 3585 return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true); 3586 } 3587 3588 int hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout) 3589 { 3590 int rc; 3591 3592 mutex_lock(&bp->hwrm_cmd_lock); 3593 rc = _hwrm_send_message(bp, msg, msg_len, timeout); 3594 mutex_unlock(&bp->hwrm_cmd_lock); 3595 return rc; 3596 } 3597 3598 int hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len, 3599 int timeout) 3600 { 3601 int rc; 3602 3603 mutex_lock(&bp->hwrm_cmd_lock); 3604 rc = bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true); 3605 mutex_unlock(&bp->hwrm_cmd_lock); 3606 return rc; 3607 } 3608 3609 int bnxt_hwrm_func_rgtr_async_events(struct bnxt *bp, unsigned long *bmap, 3610 int bmap_size) 3611 { 3612 struct hwrm_func_drv_rgtr_input req = {0}; 3613 DECLARE_BITMAP(async_events_bmap, 256); 3614 u32 *events = (u32 *)async_events_bmap; 3615 int i; 3616 3617 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1); 3618 3619 req.enables = 3620 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD); 3621 3622 memset(async_events_bmap, 0, sizeof(async_events_bmap)); 3623 for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++) 3624 __set_bit(bnxt_async_events_arr[i], async_events_bmap); 3625 3626 if (bmap && bmap_size) { 3627 for (i = 0; i < bmap_size; i++) { 3628 if (test_bit(i, bmap)) 3629 __set_bit(i, async_events_bmap); 3630 } 3631 } 3632 3633 for (i = 0; i < 8; i++) 3634 req.async_event_fwd[i] |= cpu_to_le32(events[i]); 3635 3636 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3637 } 3638 3639 static int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp) 3640 { 3641 struct hwrm_func_drv_rgtr_input req = {0}; 3642 3643 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1); 3644 3645 req.enables = 3646 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE | 3647 FUNC_DRV_RGTR_REQ_ENABLES_VER); 3648 3649 req.os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX); 3650 req.flags = cpu_to_le32(FUNC_DRV_RGTR_REQ_FLAGS_16BIT_VER_MODE); 3651 req.ver_maj_8b = DRV_VER_MAJ; 3652 req.ver_min_8b = DRV_VER_MIN; 3653 req.ver_upd_8b = DRV_VER_UPD; 3654 req.ver_maj = cpu_to_le16(DRV_VER_MAJ); 3655 req.ver_min = cpu_to_le16(DRV_VER_MIN); 3656 req.ver_upd = cpu_to_le16(DRV_VER_UPD); 3657 3658 if (BNXT_PF(bp)) { 3659 u32 data[8]; 3660 int i; 3661 3662 memset(data, 0, sizeof(data)); 3663 for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) { 3664 u16 cmd = bnxt_vf_req_snif[i]; 3665 unsigned int bit, idx; 3666 3667 idx = cmd / 32; 3668 bit = cmd % 32; 3669 data[idx] |= 1 << bit; 3670 } 3671 3672 for (i = 0; i < 8; i++) 3673 req.vf_req_fwd[i] = cpu_to_le32(data[i]); 3674 3675 req.enables |= 3676 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD); 3677 } 3678 3679 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3680 } 3681 3682 static int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp) 3683 { 3684 struct hwrm_func_drv_unrgtr_input req = {0}; 3685 3686 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_UNRGTR, -1, -1); 3687 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3688 } 3689 3690 static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type) 3691 { 3692 u32 rc = 0; 3693 struct hwrm_tunnel_dst_port_free_input req = {0}; 3694 3695 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_FREE, -1, -1); 3696 req.tunnel_type = tunnel_type; 3697 3698 switch (tunnel_type) { 3699 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN: 3700 req.tunnel_dst_port_id = bp->vxlan_fw_dst_port_id; 3701 break; 3702 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE: 3703 req.tunnel_dst_port_id = bp->nge_fw_dst_port_id; 3704 break; 3705 default: 3706 break; 3707 } 3708 3709 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3710 if (rc) 3711 netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n", 3712 rc); 3713 return rc; 3714 } 3715 3716 static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port, 3717 u8 tunnel_type) 3718 { 3719 u32 rc = 0; 3720 struct hwrm_tunnel_dst_port_alloc_input req = {0}; 3721 struct hwrm_tunnel_dst_port_alloc_output *resp = bp->hwrm_cmd_resp_addr; 3722 3723 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_ALLOC, -1, -1); 3724 3725 req.tunnel_type = tunnel_type; 3726 req.tunnel_dst_port_val = port; 3727 3728 mutex_lock(&bp->hwrm_cmd_lock); 3729 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3730 if (rc) { 3731 netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n", 3732 rc); 3733 goto err_out; 3734 } 3735 3736 switch (tunnel_type) { 3737 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN: 3738 bp->vxlan_fw_dst_port_id = resp->tunnel_dst_port_id; 3739 break; 3740 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE: 3741 bp->nge_fw_dst_port_id = resp->tunnel_dst_port_id; 3742 break; 3743 default: 3744 break; 3745 } 3746 3747 err_out: 3748 mutex_unlock(&bp->hwrm_cmd_lock); 3749 return rc; 3750 } 3751 3752 static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id) 3753 { 3754 struct hwrm_cfa_l2_set_rx_mask_input req = {0}; 3755 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 3756 3757 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_SET_RX_MASK, -1, -1); 3758 req.vnic_id = cpu_to_le32(vnic->fw_vnic_id); 3759 3760 req.num_mc_entries = cpu_to_le32(vnic->mc_list_count); 3761 req.mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping); 3762 req.mask = cpu_to_le32(vnic->rx_mask); 3763 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3764 } 3765 3766 #ifdef CONFIG_RFS_ACCEL 3767 static int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp, 3768 struct bnxt_ntuple_filter *fltr) 3769 { 3770 struct hwrm_cfa_ntuple_filter_free_input req = {0}; 3771 3772 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_FREE, -1, -1); 3773 req.ntuple_filter_id = fltr->filter_id; 3774 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3775 } 3776 3777 #define BNXT_NTP_FLTR_FLAGS \ 3778 (CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID | \ 3779 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE | \ 3780 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR | \ 3781 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE | \ 3782 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR | \ 3783 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK | \ 3784 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR | \ 3785 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK | \ 3786 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL | \ 3787 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT | \ 3788 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK | \ 3789 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT | \ 3790 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK | \ 3791 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID) 3792 3793 #define BNXT_NTP_TUNNEL_FLTR_FLAG \ 3794 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE 3795 3796 static int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp, 3797 struct bnxt_ntuple_filter *fltr) 3798 { 3799 int rc = 0; 3800 struct hwrm_cfa_ntuple_filter_alloc_input req = {0}; 3801 struct hwrm_cfa_ntuple_filter_alloc_output *resp = 3802 bp->hwrm_cmd_resp_addr; 3803 struct flow_keys *keys = &fltr->fkeys; 3804 struct bnxt_vnic_info *vnic = &bp->vnic_info[fltr->rxq + 1]; 3805 3806 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_ALLOC, -1, -1); 3807 req.l2_filter_id = bp->vnic_info[0].fw_l2_filter_id[fltr->l2_fltr_idx]; 3808 3809 req.enables = cpu_to_le32(BNXT_NTP_FLTR_FLAGS); 3810 3811 req.ethertype = htons(ETH_P_IP); 3812 memcpy(req.src_macaddr, fltr->src_mac_addr, ETH_ALEN); 3813 req.ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4; 3814 req.ip_protocol = keys->basic.ip_proto; 3815 3816 if (keys->basic.n_proto == htons(ETH_P_IPV6)) { 3817 int i; 3818 3819 req.ethertype = htons(ETH_P_IPV6); 3820 req.ip_addr_type = 3821 CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6; 3822 *(struct in6_addr *)&req.src_ipaddr[0] = 3823 keys->addrs.v6addrs.src; 3824 *(struct in6_addr *)&req.dst_ipaddr[0] = 3825 keys->addrs.v6addrs.dst; 3826 for (i = 0; i < 4; i++) { 3827 req.src_ipaddr_mask[i] = cpu_to_be32(0xffffffff); 3828 req.dst_ipaddr_mask[i] = cpu_to_be32(0xffffffff); 3829 } 3830 } else { 3831 req.src_ipaddr[0] = keys->addrs.v4addrs.src; 3832 req.src_ipaddr_mask[0] = cpu_to_be32(0xffffffff); 3833 req.dst_ipaddr[0] = keys->addrs.v4addrs.dst; 3834 req.dst_ipaddr_mask[0] = cpu_to_be32(0xffffffff); 3835 } 3836 if (keys->control.flags & FLOW_DIS_ENCAPSULATION) { 3837 req.enables |= cpu_to_le32(BNXT_NTP_TUNNEL_FLTR_FLAG); 3838 req.tunnel_type = 3839 CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL; 3840 } 3841 3842 req.src_port = keys->ports.src; 3843 req.src_port_mask = cpu_to_be16(0xffff); 3844 req.dst_port = keys->ports.dst; 3845 req.dst_port_mask = cpu_to_be16(0xffff); 3846 3847 req.dst_id = cpu_to_le16(vnic->fw_vnic_id); 3848 mutex_lock(&bp->hwrm_cmd_lock); 3849 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3850 if (!rc) 3851 fltr->filter_id = resp->ntuple_filter_id; 3852 mutex_unlock(&bp->hwrm_cmd_lock); 3853 return rc; 3854 } 3855 #endif 3856 3857 static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx, 3858 u8 *mac_addr) 3859 { 3860 u32 rc = 0; 3861 struct hwrm_cfa_l2_filter_alloc_input req = {0}; 3862 struct hwrm_cfa_l2_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr; 3863 3864 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_ALLOC, -1, -1); 3865 req.flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX); 3866 if (!BNXT_CHIP_TYPE_NITRO_A0(bp)) 3867 req.flags |= 3868 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST); 3869 req.dst_id = cpu_to_le16(bp->vnic_info[vnic_id].fw_vnic_id); 3870 req.enables = 3871 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR | 3872 CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID | 3873 CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK); 3874 memcpy(req.l2_addr, mac_addr, ETH_ALEN); 3875 req.l2_addr_mask[0] = 0xff; 3876 req.l2_addr_mask[1] = 0xff; 3877 req.l2_addr_mask[2] = 0xff; 3878 req.l2_addr_mask[3] = 0xff; 3879 req.l2_addr_mask[4] = 0xff; 3880 req.l2_addr_mask[5] = 0xff; 3881 3882 mutex_lock(&bp->hwrm_cmd_lock); 3883 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3884 if (!rc) 3885 bp->vnic_info[vnic_id].fw_l2_filter_id[idx] = 3886 resp->l2_filter_id; 3887 mutex_unlock(&bp->hwrm_cmd_lock); 3888 return rc; 3889 } 3890 3891 static int bnxt_hwrm_clear_vnic_filter(struct bnxt *bp) 3892 { 3893 u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */ 3894 int rc = 0; 3895 3896 /* Any associated ntuple filters will also be cleared by firmware. */ 3897 mutex_lock(&bp->hwrm_cmd_lock); 3898 for (i = 0; i < num_of_vnics; i++) { 3899 struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; 3900 3901 for (j = 0; j < vnic->uc_filter_count; j++) { 3902 struct hwrm_cfa_l2_filter_free_input req = {0}; 3903 3904 bnxt_hwrm_cmd_hdr_init(bp, &req, 3905 HWRM_CFA_L2_FILTER_FREE, -1, -1); 3906 3907 req.l2_filter_id = vnic->fw_l2_filter_id[j]; 3908 3909 rc = _hwrm_send_message(bp, &req, sizeof(req), 3910 HWRM_CMD_TIMEOUT); 3911 } 3912 vnic->uc_filter_count = 0; 3913 } 3914 mutex_unlock(&bp->hwrm_cmd_lock); 3915 3916 return rc; 3917 } 3918 3919 static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags) 3920 { 3921 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 3922 struct hwrm_vnic_tpa_cfg_input req = {0}; 3923 3924 if (vnic->fw_vnic_id == INVALID_HW_RING_ID) 3925 return 0; 3926 3927 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_TPA_CFG, -1, -1); 3928 3929 if (tpa_flags) { 3930 u16 mss = bp->dev->mtu - 40; 3931 u32 nsegs, n, segs = 0, flags; 3932 3933 flags = VNIC_TPA_CFG_REQ_FLAGS_TPA | 3934 VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA | 3935 VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE | 3936 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN | 3937 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ; 3938 if (tpa_flags & BNXT_FLAG_GRO) 3939 flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO; 3940 3941 req.flags = cpu_to_le32(flags); 3942 3943 req.enables = 3944 cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS | 3945 VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS | 3946 VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN); 3947 3948 /* Number of segs are log2 units, and first packet is not 3949 * included as part of this units. 3950 */ 3951 if (mss <= BNXT_RX_PAGE_SIZE) { 3952 n = BNXT_RX_PAGE_SIZE / mss; 3953 nsegs = (MAX_SKB_FRAGS - 1) * n; 3954 } else { 3955 n = mss / BNXT_RX_PAGE_SIZE; 3956 if (mss & (BNXT_RX_PAGE_SIZE - 1)) 3957 n++; 3958 nsegs = (MAX_SKB_FRAGS - n) / n; 3959 } 3960 3961 segs = ilog2(nsegs); 3962 req.max_agg_segs = cpu_to_le16(segs); 3963 req.max_aggs = cpu_to_le16(VNIC_TPA_CFG_REQ_MAX_AGGS_MAX); 3964 3965 req.min_agg_len = cpu_to_le32(512); 3966 } 3967 req.vnic_id = cpu_to_le16(vnic->fw_vnic_id); 3968 3969 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3970 } 3971 3972 static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss) 3973 { 3974 u32 i, j, max_rings; 3975 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 3976 struct hwrm_vnic_rss_cfg_input req = {0}; 3977 3978 if (vnic->fw_rss_cos_lb_ctx[0] == INVALID_HW_RING_ID) 3979 return 0; 3980 3981 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1); 3982 if (set_rss) { 3983 req.hash_type = cpu_to_le32(bp->rss_hash_cfg); 3984 if (vnic->flags & BNXT_VNIC_RSS_FLAG) { 3985 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 3986 max_rings = bp->rx_nr_rings - 1; 3987 else 3988 max_rings = bp->rx_nr_rings; 3989 } else { 3990 max_rings = 1; 3991 } 3992 3993 /* Fill the RSS indirection table with ring group ids */ 3994 for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++, j++) { 3995 if (j == max_rings) 3996 j = 0; 3997 vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]); 3998 } 3999 4000 req.ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr); 4001 req.hash_key_tbl_addr = 4002 cpu_to_le64(vnic->rss_hash_key_dma_addr); 4003 } 4004 req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]); 4005 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4006 } 4007 4008 static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id) 4009 { 4010 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 4011 struct hwrm_vnic_plcmodes_cfg_input req = {0}; 4012 4013 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_PLCMODES_CFG, -1, -1); 4014 req.flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT | 4015 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 | 4016 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6); 4017 req.enables = 4018 cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID | 4019 VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID); 4020 /* thresholds not implemented in firmware yet */ 4021 req.jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh); 4022 req.hds_threshold = cpu_to_le16(bp->rx_copy_thresh); 4023 req.vnic_id = cpu_to_le32(vnic->fw_vnic_id); 4024 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4025 } 4026 4027 static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id, 4028 u16 ctx_idx) 4029 { 4030 struct hwrm_vnic_rss_cos_lb_ctx_free_input req = {0}; 4031 4032 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_FREE, -1, -1); 4033 req.rss_cos_lb_ctx_id = 4034 cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx]); 4035 4036 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4037 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID; 4038 } 4039 4040 static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp) 4041 { 4042 int i, j; 4043 4044 for (i = 0; i < bp->nr_vnics; i++) { 4045 struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; 4046 4047 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) { 4048 if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID) 4049 bnxt_hwrm_vnic_ctx_free_one(bp, i, j); 4050 } 4051 } 4052 bp->rsscos_nr_ctxs = 0; 4053 } 4054 4055 static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id, u16 ctx_idx) 4056 { 4057 int rc; 4058 struct hwrm_vnic_rss_cos_lb_ctx_alloc_input req = {0}; 4059 struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp = 4060 bp->hwrm_cmd_resp_addr; 4061 4062 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC, -1, 4063 -1); 4064 4065 mutex_lock(&bp->hwrm_cmd_lock); 4066 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4067 if (!rc) 4068 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = 4069 le16_to_cpu(resp->rss_cos_lb_ctx_id); 4070 mutex_unlock(&bp->hwrm_cmd_lock); 4071 4072 return rc; 4073 } 4074 4075 static u32 bnxt_get_roce_vnic_mode(struct bnxt *bp) 4076 { 4077 if (bp->flags & BNXT_FLAG_ROCE_MIRROR_CAP) 4078 return VNIC_CFG_REQ_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE; 4079 return VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE; 4080 } 4081 4082 int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id) 4083 { 4084 unsigned int ring = 0, grp_idx; 4085 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 4086 struct hwrm_vnic_cfg_input req = {0}; 4087 u16 def_vlan = 0; 4088 4089 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_CFG, -1, -1); 4090 4091 req.enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP); 4092 /* Only RSS support for now TBD: COS & LB */ 4093 if (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID) { 4094 req.rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]); 4095 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE | 4096 VNIC_CFG_REQ_ENABLES_MRU); 4097 } else if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) { 4098 req.rss_rule = 4099 cpu_to_le16(bp->vnic_info[0].fw_rss_cos_lb_ctx[0]); 4100 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE | 4101 VNIC_CFG_REQ_ENABLES_MRU); 4102 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE); 4103 } else { 4104 req.rss_rule = cpu_to_le16(0xffff); 4105 } 4106 4107 if (BNXT_CHIP_TYPE_NITRO_A0(bp) && 4108 (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID)) { 4109 req.cos_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[1]); 4110 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_COS_RULE); 4111 } else { 4112 req.cos_rule = cpu_to_le16(0xffff); 4113 } 4114 4115 if (vnic->flags & BNXT_VNIC_RSS_FLAG) 4116 ring = 0; 4117 else if (vnic->flags & BNXT_VNIC_RFS_FLAG) 4118 ring = vnic_id - 1; 4119 else if ((vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp)) 4120 ring = bp->rx_nr_rings - 1; 4121 4122 grp_idx = bp->rx_ring[ring].bnapi->index; 4123 req.vnic_id = cpu_to_le16(vnic->fw_vnic_id); 4124 req.dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id); 4125 4126 req.lb_rule = cpu_to_le16(0xffff); 4127 req.mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + ETH_FCS_LEN + 4128 VLAN_HLEN); 4129 4130 #ifdef CONFIG_BNXT_SRIOV 4131 if (BNXT_VF(bp)) 4132 def_vlan = bp->vf.vlan; 4133 #endif 4134 if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan) 4135 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE); 4136 if (!vnic_id && bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP)) 4137 req.flags |= cpu_to_le32(bnxt_get_roce_vnic_mode(bp)); 4138 4139 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4140 } 4141 4142 static int bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id) 4143 { 4144 u32 rc = 0; 4145 4146 if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) { 4147 struct hwrm_vnic_free_input req = {0}; 4148 4149 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_FREE, -1, -1); 4150 req.vnic_id = 4151 cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id); 4152 4153 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4154 if (rc) 4155 return rc; 4156 bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID; 4157 } 4158 return rc; 4159 } 4160 4161 static void bnxt_hwrm_vnic_free(struct bnxt *bp) 4162 { 4163 u16 i; 4164 4165 for (i = 0; i < bp->nr_vnics; i++) 4166 bnxt_hwrm_vnic_free_one(bp, i); 4167 } 4168 4169 static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id, 4170 unsigned int start_rx_ring_idx, 4171 unsigned int nr_rings) 4172 { 4173 int rc = 0; 4174 unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings; 4175 struct hwrm_vnic_alloc_input req = {0}; 4176 struct hwrm_vnic_alloc_output *resp = bp->hwrm_cmd_resp_addr; 4177 4178 /* map ring groups to this vnic */ 4179 for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) { 4180 grp_idx = bp->rx_ring[i].bnapi->index; 4181 if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) { 4182 netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n", 4183 j, nr_rings); 4184 break; 4185 } 4186 bp->vnic_info[vnic_id].fw_grp_ids[j] = 4187 bp->grp_info[grp_idx].fw_grp_id; 4188 } 4189 4190 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[0] = INVALID_HW_RING_ID; 4191 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[1] = INVALID_HW_RING_ID; 4192 if (vnic_id == 0) 4193 req.flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT); 4194 4195 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_ALLOC, -1, -1); 4196 4197 mutex_lock(&bp->hwrm_cmd_lock); 4198 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4199 if (!rc) 4200 bp->vnic_info[vnic_id].fw_vnic_id = le32_to_cpu(resp->vnic_id); 4201 mutex_unlock(&bp->hwrm_cmd_lock); 4202 return rc; 4203 } 4204 4205 static int bnxt_hwrm_vnic_qcaps(struct bnxt *bp) 4206 { 4207 struct hwrm_vnic_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 4208 struct hwrm_vnic_qcaps_input req = {0}; 4209 int rc; 4210 4211 if (bp->hwrm_spec_code < 0x10600) 4212 return 0; 4213 4214 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_QCAPS, -1, -1); 4215 mutex_lock(&bp->hwrm_cmd_lock); 4216 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4217 if (!rc) { 4218 u32 flags = le32_to_cpu(resp->flags); 4219 4220 if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP) 4221 bp->flags |= BNXT_FLAG_NEW_RSS_CAP; 4222 if (flags & 4223 VNIC_QCAPS_RESP_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_CAP) 4224 bp->flags |= BNXT_FLAG_ROCE_MIRROR_CAP; 4225 } 4226 mutex_unlock(&bp->hwrm_cmd_lock); 4227 return rc; 4228 } 4229 4230 static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp) 4231 { 4232 u16 i; 4233 u32 rc = 0; 4234 4235 mutex_lock(&bp->hwrm_cmd_lock); 4236 for (i = 0; i < bp->rx_nr_rings; i++) { 4237 struct hwrm_ring_grp_alloc_input req = {0}; 4238 struct hwrm_ring_grp_alloc_output *resp = 4239 bp->hwrm_cmd_resp_addr; 4240 unsigned int grp_idx = bp->rx_ring[i].bnapi->index; 4241 4242 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_ALLOC, -1, -1); 4243 4244 req.cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id); 4245 req.rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id); 4246 req.ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id); 4247 req.sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx); 4248 4249 rc = _hwrm_send_message(bp, &req, sizeof(req), 4250 HWRM_CMD_TIMEOUT); 4251 if (rc) 4252 break; 4253 4254 bp->grp_info[grp_idx].fw_grp_id = 4255 le32_to_cpu(resp->ring_group_id); 4256 } 4257 mutex_unlock(&bp->hwrm_cmd_lock); 4258 return rc; 4259 } 4260 4261 static int bnxt_hwrm_ring_grp_free(struct bnxt *bp) 4262 { 4263 u16 i; 4264 u32 rc = 0; 4265 struct hwrm_ring_grp_free_input req = {0}; 4266 4267 if (!bp->grp_info) 4268 return 0; 4269 4270 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_FREE, -1, -1); 4271 4272 mutex_lock(&bp->hwrm_cmd_lock); 4273 for (i = 0; i < bp->cp_nr_rings; i++) { 4274 if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID) 4275 continue; 4276 req.ring_group_id = 4277 cpu_to_le32(bp->grp_info[i].fw_grp_id); 4278 4279 rc = _hwrm_send_message(bp, &req, sizeof(req), 4280 HWRM_CMD_TIMEOUT); 4281 if (rc) 4282 break; 4283 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID; 4284 } 4285 mutex_unlock(&bp->hwrm_cmd_lock); 4286 return rc; 4287 } 4288 4289 static int hwrm_ring_alloc_send_msg(struct bnxt *bp, 4290 struct bnxt_ring_struct *ring, 4291 u32 ring_type, u32 map_index) 4292 { 4293 int rc = 0, err = 0; 4294 struct hwrm_ring_alloc_input req = {0}; 4295 struct hwrm_ring_alloc_output *resp = bp->hwrm_cmd_resp_addr; 4296 struct bnxt_ring_grp_info *grp_info; 4297 u16 ring_id; 4298 4299 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_ALLOC, -1, -1); 4300 4301 req.enables = 0; 4302 if (ring->nr_pages > 1) { 4303 req.page_tbl_addr = cpu_to_le64(ring->pg_tbl_map); 4304 /* Page size is in log2 units */ 4305 req.page_size = BNXT_PAGE_SHIFT; 4306 req.page_tbl_depth = 1; 4307 } else { 4308 req.page_tbl_addr = cpu_to_le64(ring->dma_arr[0]); 4309 } 4310 req.fbo = 0; 4311 /* Association of ring index with doorbell index and MSIX number */ 4312 req.logical_id = cpu_to_le16(map_index); 4313 4314 switch (ring_type) { 4315 case HWRM_RING_ALLOC_TX: 4316 req.ring_type = RING_ALLOC_REQ_RING_TYPE_TX; 4317 /* Association of transmit ring with completion ring */ 4318 grp_info = &bp->grp_info[ring->grp_idx]; 4319 req.cmpl_ring_id = cpu_to_le16(grp_info->cp_fw_ring_id); 4320 req.length = cpu_to_le32(bp->tx_ring_mask + 1); 4321 req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx); 4322 req.queue_id = cpu_to_le16(ring->queue_id); 4323 break; 4324 case HWRM_RING_ALLOC_RX: 4325 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX; 4326 req.length = cpu_to_le32(bp->rx_ring_mask + 1); 4327 break; 4328 case HWRM_RING_ALLOC_AGG: 4329 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX; 4330 req.length = cpu_to_le32(bp->rx_agg_ring_mask + 1); 4331 break; 4332 case HWRM_RING_ALLOC_CMPL: 4333 req.ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL; 4334 req.length = cpu_to_le32(bp->cp_ring_mask + 1); 4335 if (bp->flags & BNXT_FLAG_USING_MSIX) 4336 req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX; 4337 break; 4338 default: 4339 netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n", 4340 ring_type); 4341 return -1; 4342 } 4343 4344 mutex_lock(&bp->hwrm_cmd_lock); 4345 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4346 err = le16_to_cpu(resp->error_code); 4347 ring_id = le16_to_cpu(resp->ring_id); 4348 mutex_unlock(&bp->hwrm_cmd_lock); 4349 4350 if (rc || err) { 4351 netdev_err(bp->dev, "hwrm_ring_alloc type %d failed. rc:%x err:%x\n", 4352 ring_type, rc, err); 4353 return -EIO; 4354 } 4355 ring->fw_ring_id = ring_id; 4356 return rc; 4357 } 4358 4359 static int bnxt_hwrm_set_async_event_cr(struct bnxt *bp, int idx) 4360 { 4361 int rc; 4362 4363 if (BNXT_PF(bp)) { 4364 struct hwrm_func_cfg_input req = {0}; 4365 4366 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1); 4367 req.fid = cpu_to_le16(0xffff); 4368 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR); 4369 req.async_event_cr = cpu_to_le16(idx); 4370 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4371 } else { 4372 struct hwrm_func_vf_cfg_input req = {0}; 4373 4374 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_VF_CFG, -1, -1); 4375 req.enables = 4376 cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR); 4377 req.async_event_cr = cpu_to_le16(idx); 4378 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4379 } 4380 return rc; 4381 } 4382 4383 static int bnxt_hwrm_ring_alloc(struct bnxt *bp) 4384 { 4385 int i, rc = 0; 4386 4387 for (i = 0; i < bp->cp_nr_rings; i++) { 4388 struct bnxt_napi *bnapi = bp->bnapi[i]; 4389 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 4390 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 4391 u32 map_idx = ring->map_idx; 4392 4393 cpr->cp_doorbell = bp->bar1 + map_idx * 0x80; 4394 rc = hwrm_ring_alloc_send_msg(bp, ring, HWRM_RING_ALLOC_CMPL, 4395 map_idx); 4396 if (rc) 4397 goto err_out; 4398 BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons); 4399 bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id; 4400 4401 if (!i) { 4402 rc = bnxt_hwrm_set_async_event_cr(bp, ring->fw_ring_id); 4403 if (rc) 4404 netdev_warn(bp->dev, "Failed to set async event completion ring.\n"); 4405 } 4406 } 4407 4408 for (i = 0; i < bp->tx_nr_rings; i++) { 4409 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 4410 struct bnxt_ring_struct *ring = &txr->tx_ring_struct; 4411 u32 map_idx = i; 4412 4413 rc = hwrm_ring_alloc_send_msg(bp, ring, HWRM_RING_ALLOC_TX, 4414 map_idx); 4415 if (rc) 4416 goto err_out; 4417 txr->tx_doorbell = bp->bar1 + map_idx * 0x80; 4418 } 4419 4420 for (i = 0; i < bp->rx_nr_rings; i++) { 4421 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 4422 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct; 4423 u32 map_idx = rxr->bnapi->index; 4424 4425 rc = hwrm_ring_alloc_send_msg(bp, ring, HWRM_RING_ALLOC_RX, 4426 map_idx); 4427 if (rc) 4428 goto err_out; 4429 rxr->rx_doorbell = bp->bar1 + map_idx * 0x80; 4430 writel(DB_KEY_RX | rxr->rx_prod, rxr->rx_doorbell); 4431 bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id; 4432 } 4433 4434 if (bp->flags & BNXT_FLAG_AGG_RINGS) { 4435 for (i = 0; i < bp->rx_nr_rings; i++) { 4436 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 4437 struct bnxt_ring_struct *ring = 4438 &rxr->rx_agg_ring_struct; 4439 u32 grp_idx = ring->grp_idx; 4440 u32 map_idx = grp_idx + bp->rx_nr_rings; 4441 4442 rc = hwrm_ring_alloc_send_msg(bp, ring, 4443 HWRM_RING_ALLOC_AGG, 4444 map_idx); 4445 if (rc) 4446 goto err_out; 4447 4448 rxr->rx_agg_doorbell = bp->bar1 + map_idx * 0x80; 4449 writel(DB_KEY_RX | rxr->rx_agg_prod, 4450 rxr->rx_agg_doorbell); 4451 bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id; 4452 } 4453 } 4454 err_out: 4455 return rc; 4456 } 4457 4458 static int hwrm_ring_free_send_msg(struct bnxt *bp, 4459 struct bnxt_ring_struct *ring, 4460 u32 ring_type, int cmpl_ring_id) 4461 { 4462 int rc; 4463 struct hwrm_ring_free_input req = {0}; 4464 struct hwrm_ring_free_output *resp = bp->hwrm_cmd_resp_addr; 4465 u16 error_code; 4466 4467 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_FREE, cmpl_ring_id, -1); 4468 req.ring_type = ring_type; 4469 req.ring_id = cpu_to_le16(ring->fw_ring_id); 4470 4471 mutex_lock(&bp->hwrm_cmd_lock); 4472 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4473 error_code = le16_to_cpu(resp->error_code); 4474 mutex_unlock(&bp->hwrm_cmd_lock); 4475 4476 if (rc || error_code) { 4477 netdev_err(bp->dev, "hwrm_ring_free type %d failed. rc:%x err:%x\n", 4478 ring_type, rc, error_code); 4479 return -EIO; 4480 } 4481 return 0; 4482 } 4483 4484 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path) 4485 { 4486 int i; 4487 4488 if (!bp->bnapi) 4489 return; 4490 4491 for (i = 0; i < bp->tx_nr_rings; i++) { 4492 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 4493 struct bnxt_ring_struct *ring = &txr->tx_ring_struct; 4494 u32 grp_idx = txr->bnapi->index; 4495 u32 cmpl_ring_id = bp->grp_info[grp_idx].cp_fw_ring_id; 4496 4497 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 4498 hwrm_ring_free_send_msg(bp, ring, 4499 RING_FREE_REQ_RING_TYPE_TX, 4500 close_path ? cmpl_ring_id : 4501 INVALID_HW_RING_ID); 4502 ring->fw_ring_id = INVALID_HW_RING_ID; 4503 } 4504 } 4505 4506 for (i = 0; i < bp->rx_nr_rings; i++) { 4507 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 4508 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct; 4509 u32 grp_idx = rxr->bnapi->index; 4510 u32 cmpl_ring_id = bp->grp_info[grp_idx].cp_fw_ring_id; 4511 4512 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 4513 hwrm_ring_free_send_msg(bp, ring, 4514 RING_FREE_REQ_RING_TYPE_RX, 4515 close_path ? cmpl_ring_id : 4516 INVALID_HW_RING_ID); 4517 ring->fw_ring_id = INVALID_HW_RING_ID; 4518 bp->grp_info[grp_idx].rx_fw_ring_id = 4519 INVALID_HW_RING_ID; 4520 } 4521 } 4522 4523 for (i = 0; i < bp->rx_nr_rings; i++) { 4524 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 4525 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct; 4526 u32 grp_idx = rxr->bnapi->index; 4527 u32 cmpl_ring_id = bp->grp_info[grp_idx].cp_fw_ring_id; 4528 4529 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 4530 hwrm_ring_free_send_msg(bp, ring, 4531 RING_FREE_REQ_RING_TYPE_RX, 4532 close_path ? cmpl_ring_id : 4533 INVALID_HW_RING_ID); 4534 ring->fw_ring_id = INVALID_HW_RING_ID; 4535 bp->grp_info[grp_idx].agg_fw_ring_id = 4536 INVALID_HW_RING_ID; 4537 } 4538 } 4539 4540 /* The completion rings are about to be freed. After that the 4541 * IRQ doorbell will not work anymore. So we need to disable 4542 * IRQ here. 4543 */ 4544 bnxt_disable_int_sync(bp); 4545 4546 for (i = 0; i < bp->cp_nr_rings; i++) { 4547 struct bnxt_napi *bnapi = bp->bnapi[i]; 4548 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 4549 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 4550 4551 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 4552 hwrm_ring_free_send_msg(bp, ring, 4553 RING_FREE_REQ_RING_TYPE_L2_CMPL, 4554 INVALID_HW_RING_ID); 4555 ring->fw_ring_id = INVALID_HW_RING_ID; 4556 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID; 4557 } 4558 } 4559 } 4560 4561 static int bnxt_hwrm_get_rings(struct bnxt *bp) 4562 { 4563 struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 4564 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 4565 struct hwrm_func_qcfg_input req = {0}; 4566 int rc; 4567 4568 if (bp->hwrm_spec_code < 0x10601) 4569 return 0; 4570 4571 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1); 4572 req.fid = cpu_to_le16(0xffff); 4573 mutex_lock(&bp->hwrm_cmd_lock); 4574 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4575 if (rc) { 4576 mutex_unlock(&bp->hwrm_cmd_lock); 4577 return -EIO; 4578 } 4579 4580 hw_resc->resv_tx_rings = le16_to_cpu(resp->alloc_tx_rings); 4581 if (bp->flags & BNXT_FLAG_NEW_RM) { 4582 u16 cp, stats; 4583 4584 hw_resc->resv_rx_rings = le16_to_cpu(resp->alloc_rx_rings); 4585 hw_resc->resv_hw_ring_grps = 4586 le32_to_cpu(resp->alloc_hw_ring_grps); 4587 hw_resc->resv_vnics = le16_to_cpu(resp->alloc_vnics); 4588 cp = le16_to_cpu(resp->alloc_cmpl_rings); 4589 stats = le16_to_cpu(resp->alloc_stat_ctx); 4590 cp = min_t(u16, cp, stats); 4591 hw_resc->resv_cp_rings = cp; 4592 } 4593 mutex_unlock(&bp->hwrm_cmd_lock); 4594 return 0; 4595 } 4596 4597 /* Caller must hold bp->hwrm_cmd_lock */ 4598 int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings) 4599 { 4600 struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 4601 struct hwrm_func_qcfg_input req = {0}; 4602 int rc; 4603 4604 if (bp->hwrm_spec_code < 0x10601) 4605 return 0; 4606 4607 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1); 4608 req.fid = cpu_to_le16(fid); 4609 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4610 if (!rc) 4611 *tx_rings = le16_to_cpu(resp->alloc_tx_rings); 4612 4613 return rc; 4614 } 4615 4616 static void 4617 __bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, struct hwrm_func_cfg_input *req, 4618 int tx_rings, int rx_rings, int ring_grps, 4619 int cp_rings, int vnics) 4620 { 4621 u32 enables = 0; 4622 4623 bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_CFG, -1, -1); 4624 req->fid = cpu_to_le16(0xffff); 4625 enables |= tx_rings ? FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS : 0; 4626 req->num_tx_rings = cpu_to_le16(tx_rings); 4627 if (bp->flags & BNXT_FLAG_NEW_RM) { 4628 enables |= rx_rings ? FUNC_CFG_REQ_ENABLES_NUM_RX_RINGS : 0; 4629 enables |= cp_rings ? FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS | 4630 FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0; 4631 enables |= ring_grps ? 4632 FUNC_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0; 4633 enables |= vnics ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0; 4634 4635 req->num_rx_rings = cpu_to_le16(rx_rings); 4636 req->num_hw_ring_grps = cpu_to_le16(ring_grps); 4637 req->num_cmpl_rings = cpu_to_le16(cp_rings); 4638 req->num_stat_ctxs = req->num_cmpl_rings; 4639 req->num_vnics = cpu_to_le16(vnics); 4640 } 4641 req->enables = cpu_to_le32(enables); 4642 } 4643 4644 static void 4645 __bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, 4646 struct hwrm_func_vf_cfg_input *req, int tx_rings, 4647 int rx_rings, int ring_grps, int cp_rings, 4648 int vnics) 4649 { 4650 u32 enables = 0; 4651 4652 bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_VF_CFG, -1, -1); 4653 enables |= tx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_TX_RINGS : 0; 4654 enables |= rx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_RX_RINGS : 0; 4655 enables |= cp_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS | 4656 FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0; 4657 enables |= ring_grps ? FUNC_VF_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0; 4658 enables |= vnics ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0; 4659 4660 req->num_tx_rings = cpu_to_le16(tx_rings); 4661 req->num_rx_rings = cpu_to_le16(rx_rings); 4662 req->num_hw_ring_grps = cpu_to_le16(ring_grps); 4663 req->num_cmpl_rings = cpu_to_le16(cp_rings); 4664 req->num_stat_ctxs = req->num_cmpl_rings; 4665 req->num_vnics = cpu_to_le16(vnics); 4666 4667 req->enables = cpu_to_le32(enables); 4668 } 4669 4670 static int 4671 bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 4672 int ring_grps, int cp_rings, int vnics) 4673 { 4674 struct hwrm_func_cfg_input req = {0}; 4675 int rc; 4676 4677 __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps, 4678 cp_rings, vnics); 4679 if (!req.enables) 4680 return 0; 4681 4682 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4683 if (rc) 4684 return -ENOMEM; 4685 4686 if (bp->hwrm_spec_code < 0x10601) 4687 bp->hw_resc.resv_tx_rings = tx_rings; 4688 4689 rc = bnxt_hwrm_get_rings(bp); 4690 return rc; 4691 } 4692 4693 static int 4694 bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 4695 int ring_grps, int cp_rings, int vnics) 4696 { 4697 struct hwrm_func_vf_cfg_input req = {0}; 4698 int rc; 4699 4700 if (!(bp->flags & BNXT_FLAG_NEW_RM)) { 4701 bp->hw_resc.resv_tx_rings = tx_rings; 4702 return 0; 4703 } 4704 4705 __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps, 4706 cp_rings, vnics); 4707 req.enables |= cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS | 4708 FUNC_VF_CFG_REQ_ENABLES_NUM_L2_CTXS); 4709 req.num_rsscos_ctxs = cpu_to_le16(BNXT_VF_MAX_RSS_CTX); 4710 req.num_l2_ctxs = cpu_to_le16(BNXT_VF_MAX_L2_CTX); 4711 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4712 if (rc) 4713 return -ENOMEM; 4714 4715 rc = bnxt_hwrm_get_rings(bp); 4716 return rc; 4717 } 4718 4719 static int bnxt_hwrm_reserve_rings(struct bnxt *bp, int tx, int rx, int grp, 4720 int cp, int vnic) 4721 { 4722 if (BNXT_PF(bp)) 4723 return bnxt_hwrm_reserve_pf_rings(bp, tx, rx, grp, cp, vnic); 4724 else 4725 return bnxt_hwrm_reserve_vf_rings(bp, tx, rx, grp, cp, vnic); 4726 } 4727 4728 static int bnxt_cp_rings_in_use(struct bnxt *bp) 4729 { 4730 int cp = bp->cp_nr_rings; 4731 int ulp_msix, ulp_base; 4732 4733 ulp_msix = bnxt_get_ulp_msix_num(bp); 4734 if (ulp_msix) { 4735 ulp_base = bnxt_get_ulp_msix_base(bp); 4736 cp += ulp_msix; 4737 if ((ulp_base + ulp_msix) > cp) 4738 cp = ulp_base + ulp_msix; 4739 } 4740 return cp; 4741 } 4742 4743 static bool bnxt_need_reserve_rings(struct bnxt *bp) 4744 { 4745 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 4746 int cp = bnxt_cp_rings_in_use(bp); 4747 int rx = bp->rx_nr_rings; 4748 int vnic = 1, grp = rx; 4749 4750 if (bp->hwrm_spec_code < 0x10601) 4751 return false; 4752 4753 if (hw_resc->resv_tx_rings != bp->tx_nr_rings) 4754 return true; 4755 4756 if (bp->flags & BNXT_FLAG_RFS) 4757 vnic = rx + 1; 4758 if (bp->flags & BNXT_FLAG_AGG_RINGS) 4759 rx <<= 1; 4760 if ((bp->flags & BNXT_FLAG_NEW_RM) && 4761 (hw_resc->resv_rx_rings != rx || hw_resc->resv_cp_rings != cp || 4762 hw_resc->resv_hw_ring_grps != grp || hw_resc->resv_vnics != vnic)) 4763 return true; 4764 return false; 4765 } 4766 4767 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max, 4768 bool shared); 4769 4770 static int __bnxt_reserve_rings(struct bnxt *bp) 4771 { 4772 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 4773 int cp = bnxt_cp_rings_in_use(bp); 4774 int tx = bp->tx_nr_rings; 4775 int rx = bp->rx_nr_rings; 4776 int grp, rx_rings, rc; 4777 bool sh = false; 4778 int vnic = 1; 4779 4780 if (!bnxt_need_reserve_rings(bp)) 4781 return 0; 4782 4783 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 4784 sh = true; 4785 if (bp->flags & BNXT_FLAG_RFS) 4786 vnic = rx + 1; 4787 if (bp->flags & BNXT_FLAG_AGG_RINGS) 4788 rx <<= 1; 4789 grp = bp->rx_nr_rings; 4790 4791 rc = bnxt_hwrm_reserve_rings(bp, tx, rx, grp, cp, vnic); 4792 if (rc) 4793 return rc; 4794 4795 tx = hw_resc->resv_tx_rings; 4796 if (bp->flags & BNXT_FLAG_NEW_RM) { 4797 rx = hw_resc->resv_rx_rings; 4798 cp = hw_resc->resv_cp_rings; 4799 grp = hw_resc->resv_hw_ring_grps; 4800 vnic = hw_resc->resv_vnics; 4801 } 4802 4803 rx_rings = rx; 4804 if (bp->flags & BNXT_FLAG_AGG_RINGS) { 4805 if (rx >= 2) { 4806 rx_rings = rx >> 1; 4807 } else { 4808 if (netif_running(bp->dev)) 4809 return -ENOMEM; 4810 4811 bp->flags &= ~BNXT_FLAG_AGG_RINGS; 4812 bp->flags |= BNXT_FLAG_NO_AGG_RINGS; 4813 bp->dev->hw_features &= ~NETIF_F_LRO; 4814 bp->dev->features &= ~NETIF_F_LRO; 4815 bnxt_set_ring_params(bp); 4816 } 4817 } 4818 rx_rings = min_t(int, rx_rings, grp); 4819 rc = bnxt_trim_rings(bp, &rx_rings, &tx, cp, sh); 4820 if (bp->flags & BNXT_FLAG_AGG_RINGS) 4821 rx = rx_rings << 1; 4822 cp = sh ? max_t(int, tx, rx_rings) : tx + rx_rings; 4823 bp->tx_nr_rings = tx; 4824 bp->rx_nr_rings = rx_rings; 4825 bp->cp_nr_rings = cp; 4826 4827 if (!tx || !rx || !cp || !grp || !vnic) 4828 return -ENOMEM; 4829 4830 return rc; 4831 } 4832 4833 static int bnxt_hwrm_check_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 4834 int ring_grps, int cp_rings, int vnics) 4835 { 4836 struct hwrm_func_vf_cfg_input req = {0}; 4837 u32 flags; 4838 int rc; 4839 4840 if (!(bp->flags & BNXT_FLAG_NEW_RM)) 4841 return 0; 4842 4843 __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps, 4844 cp_rings, vnics); 4845 flags = FUNC_VF_CFG_REQ_FLAGS_TX_ASSETS_TEST | 4846 FUNC_VF_CFG_REQ_FLAGS_RX_ASSETS_TEST | 4847 FUNC_VF_CFG_REQ_FLAGS_CMPL_ASSETS_TEST | 4848 FUNC_VF_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST | 4849 FUNC_VF_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST | 4850 FUNC_VF_CFG_REQ_FLAGS_VNIC_ASSETS_TEST; 4851 4852 req.flags = cpu_to_le32(flags); 4853 rc = hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4854 if (rc) 4855 return -ENOMEM; 4856 return 0; 4857 } 4858 4859 static int bnxt_hwrm_check_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 4860 int ring_grps, int cp_rings, int vnics) 4861 { 4862 struct hwrm_func_cfg_input req = {0}; 4863 u32 flags; 4864 int rc; 4865 4866 __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps, 4867 cp_rings, vnics); 4868 flags = FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST; 4869 if (bp->flags & BNXT_FLAG_NEW_RM) 4870 flags |= FUNC_CFG_REQ_FLAGS_RX_ASSETS_TEST | 4871 FUNC_CFG_REQ_FLAGS_CMPL_ASSETS_TEST | 4872 FUNC_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST | 4873 FUNC_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST | 4874 FUNC_CFG_REQ_FLAGS_VNIC_ASSETS_TEST; 4875 4876 req.flags = cpu_to_le32(flags); 4877 rc = hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4878 if (rc) 4879 return -ENOMEM; 4880 return 0; 4881 } 4882 4883 static int bnxt_hwrm_check_rings(struct bnxt *bp, int tx_rings, int rx_rings, 4884 int ring_grps, int cp_rings, int vnics) 4885 { 4886 if (bp->hwrm_spec_code < 0x10801) 4887 return 0; 4888 4889 if (BNXT_PF(bp)) 4890 return bnxt_hwrm_check_pf_rings(bp, tx_rings, rx_rings, 4891 ring_grps, cp_rings, vnics); 4892 4893 return bnxt_hwrm_check_vf_rings(bp, tx_rings, rx_rings, ring_grps, 4894 cp_rings, vnics); 4895 } 4896 4897 static void bnxt_hwrm_set_coal_params(struct bnxt_coal *hw_coal, 4898 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req) 4899 { 4900 u16 val, tmr, max, flags; 4901 4902 max = hw_coal->bufs_per_record * 128; 4903 if (hw_coal->budget) 4904 max = hw_coal->bufs_per_record * hw_coal->budget; 4905 4906 val = clamp_t(u16, hw_coal->coal_bufs, 1, max); 4907 req->num_cmpl_aggr_int = cpu_to_le16(val); 4908 4909 /* This is a 6-bit value and must not be 0, or we'll get non stop IRQ */ 4910 val = min_t(u16, val, 63); 4911 req->num_cmpl_dma_aggr = cpu_to_le16(val); 4912 4913 /* This is a 6-bit value and must not be 0, or we'll get non stop IRQ */ 4914 val = clamp_t(u16, hw_coal->coal_bufs_irq, 1, 63); 4915 req->num_cmpl_dma_aggr_during_int = cpu_to_le16(val); 4916 4917 tmr = BNXT_USEC_TO_COAL_TIMER(hw_coal->coal_ticks); 4918 tmr = max_t(u16, tmr, 1); 4919 req->int_lat_tmr_max = cpu_to_le16(tmr); 4920 4921 /* min timer set to 1/2 of interrupt timer */ 4922 val = tmr / 2; 4923 req->int_lat_tmr_min = cpu_to_le16(val); 4924 4925 /* buf timer set to 1/4 of interrupt timer */ 4926 val = max_t(u16, tmr / 4, 1); 4927 req->cmpl_aggr_dma_tmr = cpu_to_le16(val); 4928 4929 tmr = BNXT_USEC_TO_COAL_TIMER(hw_coal->coal_ticks_irq); 4930 tmr = max_t(u16, tmr, 1); 4931 req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(tmr); 4932 4933 flags = RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET; 4934 if (hw_coal->idle_thresh && hw_coal->coal_ticks < hw_coal->idle_thresh) 4935 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE; 4936 req->flags = cpu_to_le16(flags); 4937 } 4938 4939 int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi) 4940 { 4941 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0}; 4942 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 4943 struct bnxt_coal coal; 4944 unsigned int grp_idx; 4945 4946 /* Tick values in micro seconds. 4947 * 1 coal_buf x bufs_per_record = 1 completion record. 4948 */ 4949 memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal)); 4950 4951 coal.coal_ticks = cpr->rx_ring_coal.coal_ticks; 4952 coal.coal_bufs = cpr->rx_ring_coal.coal_bufs; 4953 4954 if (!bnapi->rx_ring) 4955 return -ENODEV; 4956 4957 bnxt_hwrm_cmd_hdr_init(bp, &req_rx, 4958 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1); 4959 4960 bnxt_hwrm_set_coal_params(&coal, &req_rx); 4961 4962 grp_idx = bnapi->index; 4963 req_rx.ring_id = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id); 4964 4965 return hwrm_send_message(bp, &req_rx, sizeof(req_rx), 4966 HWRM_CMD_TIMEOUT); 4967 } 4968 4969 int bnxt_hwrm_set_coal(struct bnxt *bp) 4970 { 4971 int i, rc = 0; 4972 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0}, 4973 req_tx = {0}, *req; 4974 4975 bnxt_hwrm_cmd_hdr_init(bp, &req_rx, 4976 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1); 4977 bnxt_hwrm_cmd_hdr_init(bp, &req_tx, 4978 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1); 4979 4980 bnxt_hwrm_set_coal_params(&bp->rx_coal, &req_rx); 4981 bnxt_hwrm_set_coal_params(&bp->tx_coal, &req_tx); 4982 4983 mutex_lock(&bp->hwrm_cmd_lock); 4984 for (i = 0; i < bp->cp_nr_rings; i++) { 4985 struct bnxt_napi *bnapi = bp->bnapi[i]; 4986 4987 req = &req_rx; 4988 if (!bnapi->rx_ring) 4989 req = &req_tx; 4990 req->ring_id = cpu_to_le16(bp->grp_info[i].cp_fw_ring_id); 4991 4992 rc = _hwrm_send_message(bp, req, sizeof(*req), 4993 HWRM_CMD_TIMEOUT); 4994 if (rc) 4995 break; 4996 } 4997 mutex_unlock(&bp->hwrm_cmd_lock); 4998 return rc; 4999 } 5000 5001 static int bnxt_hwrm_stat_ctx_free(struct bnxt *bp) 5002 { 5003 int rc = 0, i; 5004 struct hwrm_stat_ctx_free_input req = {0}; 5005 5006 if (!bp->bnapi) 5007 return 0; 5008 5009 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 5010 return 0; 5011 5012 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_FREE, -1, -1); 5013 5014 mutex_lock(&bp->hwrm_cmd_lock); 5015 for (i = 0; i < bp->cp_nr_rings; i++) { 5016 struct bnxt_napi *bnapi = bp->bnapi[i]; 5017 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 5018 5019 if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) { 5020 req.stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id); 5021 5022 rc = _hwrm_send_message(bp, &req, sizeof(req), 5023 HWRM_CMD_TIMEOUT); 5024 if (rc) 5025 break; 5026 5027 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID; 5028 } 5029 } 5030 mutex_unlock(&bp->hwrm_cmd_lock); 5031 return rc; 5032 } 5033 5034 static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp) 5035 { 5036 int rc = 0, i; 5037 struct hwrm_stat_ctx_alloc_input req = {0}; 5038 struct hwrm_stat_ctx_alloc_output *resp = bp->hwrm_cmd_resp_addr; 5039 5040 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 5041 return 0; 5042 5043 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_ALLOC, -1, -1); 5044 5045 req.update_period_ms = cpu_to_le32(bp->stats_coal_ticks / 1000); 5046 5047 mutex_lock(&bp->hwrm_cmd_lock); 5048 for (i = 0; i < bp->cp_nr_rings; i++) { 5049 struct bnxt_napi *bnapi = bp->bnapi[i]; 5050 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 5051 5052 req.stats_dma_addr = cpu_to_le64(cpr->hw_stats_map); 5053 5054 rc = _hwrm_send_message(bp, &req, sizeof(req), 5055 HWRM_CMD_TIMEOUT); 5056 if (rc) 5057 break; 5058 5059 cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id); 5060 5061 bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id; 5062 } 5063 mutex_unlock(&bp->hwrm_cmd_lock); 5064 return rc; 5065 } 5066 5067 static int bnxt_hwrm_func_qcfg(struct bnxt *bp) 5068 { 5069 struct hwrm_func_qcfg_input req = {0}; 5070 struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 5071 u16 flags; 5072 int rc; 5073 5074 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1); 5075 req.fid = cpu_to_le16(0xffff); 5076 mutex_lock(&bp->hwrm_cmd_lock); 5077 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5078 if (rc) 5079 goto func_qcfg_exit; 5080 5081 #ifdef CONFIG_BNXT_SRIOV 5082 if (BNXT_VF(bp)) { 5083 struct bnxt_vf_info *vf = &bp->vf; 5084 5085 vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK; 5086 } 5087 #endif 5088 flags = le16_to_cpu(resp->flags); 5089 if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED | 5090 FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED)) { 5091 bp->flags |= BNXT_FLAG_FW_LLDP_AGENT; 5092 if (flags & FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED) 5093 bp->flags |= BNXT_FLAG_FW_DCBX_AGENT; 5094 } 5095 if (BNXT_PF(bp) && (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST)) 5096 bp->flags |= BNXT_FLAG_MULTI_HOST; 5097 5098 switch (resp->port_partition_type) { 5099 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0: 5100 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5: 5101 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0: 5102 bp->port_partition_type = resp->port_partition_type; 5103 break; 5104 } 5105 if (bp->hwrm_spec_code < 0x10707 || 5106 resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEB) 5107 bp->br_mode = BRIDGE_MODE_VEB; 5108 else if (resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEPA) 5109 bp->br_mode = BRIDGE_MODE_VEPA; 5110 else 5111 bp->br_mode = BRIDGE_MODE_UNDEF; 5112 5113 bp->max_mtu = le16_to_cpu(resp->max_mtu_configured); 5114 if (!bp->max_mtu) 5115 bp->max_mtu = BNXT_MAX_MTU; 5116 5117 func_qcfg_exit: 5118 mutex_unlock(&bp->hwrm_cmd_lock); 5119 return rc; 5120 } 5121 5122 int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp, bool all) 5123 { 5124 struct hwrm_func_resource_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 5125 struct hwrm_func_resource_qcaps_input req = {0}; 5126 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 5127 int rc; 5128 5129 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESOURCE_QCAPS, -1, -1); 5130 req.fid = cpu_to_le16(0xffff); 5131 5132 mutex_lock(&bp->hwrm_cmd_lock); 5133 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5134 if (rc) { 5135 rc = -EIO; 5136 goto hwrm_func_resc_qcaps_exit; 5137 } 5138 5139 hw_resc->max_tx_sch_inputs = le16_to_cpu(resp->max_tx_scheduler_inputs); 5140 if (!all) 5141 goto hwrm_func_resc_qcaps_exit; 5142 5143 hw_resc->min_rsscos_ctxs = le16_to_cpu(resp->min_rsscos_ctx); 5144 hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx); 5145 hw_resc->min_cp_rings = le16_to_cpu(resp->min_cmpl_rings); 5146 hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings); 5147 hw_resc->min_tx_rings = le16_to_cpu(resp->min_tx_rings); 5148 hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings); 5149 hw_resc->min_rx_rings = le16_to_cpu(resp->min_rx_rings); 5150 hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings); 5151 hw_resc->min_hw_ring_grps = le16_to_cpu(resp->min_hw_ring_grps); 5152 hw_resc->max_hw_ring_grps = le16_to_cpu(resp->max_hw_ring_grps); 5153 hw_resc->min_l2_ctxs = le16_to_cpu(resp->min_l2_ctxs); 5154 hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs); 5155 hw_resc->min_vnics = le16_to_cpu(resp->min_vnics); 5156 hw_resc->max_vnics = le16_to_cpu(resp->max_vnics); 5157 hw_resc->min_stat_ctxs = le16_to_cpu(resp->min_stat_ctx); 5158 hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx); 5159 5160 if (BNXT_PF(bp)) { 5161 struct bnxt_pf_info *pf = &bp->pf; 5162 5163 pf->vf_resv_strategy = 5164 le16_to_cpu(resp->vf_reservation_strategy); 5165 if (pf->vf_resv_strategy > BNXT_VF_RESV_STRATEGY_MINIMAL) 5166 pf->vf_resv_strategy = BNXT_VF_RESV_STRATEGY_MAXIMAL; 5167 } 5168 hwrm_func_resc_qcaps_exit: 5169 mutex_unlock(&bp->hwrm_cmd_lock); 5170 return rc; 5171 } 5172 5173 static int __bnxt_hwrm_func_qcaps(struct bnxt *bp) 5174 { 5175 int rc = 0; 5176 struct hwrm_func_qcaps_input req = {0}; 5177 struct hwrm_func_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 5178 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 5179 u32 flags; 5180 5181 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCAPS, -1, -1); 5182 req.fid = cpu_to_le16(0xffff); 5183 5184 mutex_lock(&bp->hwrm_cmd_lock); 5185 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5186 if (rc) 5187 goto hwrm_func_qcaps_exit; 5188 5189 flags = le32_to_cpu(resp->flags); 5190 if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED) 5191 bp->flags |= BNXT_FLAG_ROCEV1_CAP; 5192 if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED) 5193 bp->flags |= BNXT_FLAG_ROCEV2_CAP; 5194 5195 bp->tx_push_thresh = 0; 5196 if (flags & FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED) 5197 bp->tx_push_thresh = BNXT_TX_PUSH_THRESH; 5198 5199 hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx); 5200 hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings); 5201 hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings); 5202 hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings); 5203 hw_resc->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps); 5204 if (!hw_resc->max_hw_ring_grps) 5205 hw_resc->max_hw_ring_grps = hw_resc->max_tx_rings; 5206 hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs); 5207 hw_resc->max_vnics = le16_to_cpu(resp->max_vnics); 5208 hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx); 5209 5210 if (BNXT_PF(bp)) { 5211 struct bnxt_pf_info *pf = &bp->pf; 5212 5213 pf->fw_fid = le16_to_cpu(resp->fid); 5214 pf->port_id = le16_to_cpu(resp->port_id); 5215 bp->dev->dev_port = pf->port_id; 5216 memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN); 5217 pf->first_vf_id = le16_to_cpu(resp->first_vf_id); 5218 pf->max_vfs = le16_to_cpu(resp->max_vfs); 5219 pf->max_encap_records = le32_to_cpu(resp->max_encap_records); 5220 pf->max_decap_records = le32_to_cpu(resp->max_decap_records); 5221 pf->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows); 5222 pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows); 5223 pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows); 5224 pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows); 5225 if (flags & FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED) 5226 bp->flags |= BNXT_FLAG_WOL_CAP; 5227 } else { 5228 #ifdef CONFIG_BNXT_SRIOV 5229 struct bnxt_vf_info *vf = &bp->vf; 5230 5231 vf->fw_fid = le16_to_cpu(resp->fid); 5232 memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN); 5233 #endif 5234 } 5235 5236 hwrm_func_qcaps_exit: 5237 mutex_unlock(&bp->hwrm_cmd_lock); 5238 return rc; 5239 } 5240 5241 static int bnxt_hwrm_func_qcaps(struct bnxt *bp) 5242 { 5243 int rc; 5244 5245 rc = __bnxt_hwrm_func_qcaps(bp); 5246 if (rc) 5247 return rc; 5248 if (bp->hwrm_spec_code >= 0x10803) { 5249 rc = bnxt_hwrm_func_resc_qcaps(bp, true); 5250 if (!rc) 5251 bp->flags |= BNXT_FLAG_NEW_RM; 5252 } 5253 return 0; 5254 } 5255 5256 static int bnxt_hwrm_func_reset(struct bnxt *bp) 5257 { 5258 struct hwrm_func_reset_input req = {0}; 5259 5260 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESET, -1, -1); 5261 req.enables = 0; 5262 5263 return hwrm_send_message(bp, &req, sizeof(req), HWRM_RESET_TIMEOUT); 5264 } 5265 5266 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp) 5267 { 5268 int rc = 0; 5269 struct hwrm_queue_qportcfg_input req = {0}; 5270 struct hwrm_queue_qportcfg_output *resp = bp->hwrm_cmd_resp_addr; 5271 u8 i, *qptr; 5272 5273 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_QUEUE_QPORTCFG, -1, -1); 5274 5275 mutex_lock(&bp->hwrm_cmd_lock); 5276 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5277 if (rc) 5278 goto qportcfg_exit; 5279 5280 if (!resp->max_configurable_queues) { 5281 rc = -EINVAL; 5282 goto qportcfg_exit; 5283 } 5284 bp->max_tc = resp->max_configurable_queues; 5285 bp->max_lltc = resp->max_configurable_lossless_queues; 5286 if (bp->max_tc > BNXT_MAX_QUEUE) 5287 bp->max_tc = BNXT_MAX_QUEUE; 5288 5289 if (resp->queue_cfg_info & QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG) 5290 bp->max_tc = 1; 5291 5292 if (bp->max_lltc > bp->max_tc) 5293 bp->max_lltc = bp->max_tc; 5294 5295 qptr = &resp->queue_id0; 5296 for (i = 0; i < bp->max_tc; i++) { 5297 bp->q_info[i].queue_id = *qptr++; 5298 bp->q_info[i].queue_profile = *qptr++; 5299 bp->tc_to_qidx[i] = i; 5300 } 5301 5302 qportcfg_exit: 5303 mutex_unlock(&bp->hwrm_cmd_lock); 5304 return rc; 5305 } 5306 5307 static int bnxt_hwrm_ver_get(struct bnxt *bp) 5308 { 5309 int rc; 5310 struct hwrm_ver_get_input req = {0}; 5311 struct hwrm_ver_get_output *resp = bp->hwrm_cmd_resp_addr; 5312 u32 dev_caps_cfg; 5313 5314 bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN; 5315 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VER_GET, -1, -1); 5316 req.hwrm_intf_maj = HWRM_VERSION_MAJOR; 5317 req.hwrm_intf_min = HWRM_VERSION_MINOR; 5318 req.hwrm_intf_upd = HWRM_VERSION_UPDATE; 5319 mutex_lock(&bp->hwrm_cmd_lock); 5320 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5321 if (rc) 5322 goto hwrm_ver_get_exit; 5323 5324 memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output)); 5325 5326 bp->hwrm_spec_code = resp->hwrm_intf_maj_8b << 16 | 5327 resp->hwrm_intf_min_8b << 8 | 5328 resp->hwrm_intf_upd_8b; 5329 if (resp->hwrm_intf_maj_8b < 1) { 5330 netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n", 5331 resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b, 5332 resp->hwrm_intf_upd_8b); 5333 netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n"); 5334 } 5335 snprintf(bp->fw_ver_str, BC_HWRM_STR_LEN, "%d.%d.%d.%d", 5336 resp->hwrm_fw_maj_8b, resp->hwrm_fw_min_8b, 5337 resp->hwrm_fw_bld_8b, resp->hwrm_fw_rsvd_8b); 5338 5339 bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout); 5340 if (!bp->hwrm_cmd_timeout) 5341 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT; 5342 5343 if (resp->hwrm_intf_maj_8b >= 1) 5344 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len); 5345 5346 bp->chip_num = le16_to_cpu(resp->chip_num); 5347 if (bp->chip_num == CHIP_NUM_58700 && !resp->chip_rev && 5348 !resp->chip_metal) 5349 bp->flags |= BNXT_FLAG_CHIP_NITRO_A0; 5350 5351 dev_caps_cfg = le32_to_cpu(resp->dev_caps_cfg); 5352 if ((dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) && 5353 (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED)) 5354 bp->flags |= BNXT_FLAG_SHORT_CMD; 5355 5356 hwrm_ver_get_exit: 5357 mutex_unlock(&bp->hwrm_cmd_lock); 5358 return rc; 5359 } 5360 5361 int bnxt_hwrm_fw_set_time(struct bnxt *bp) 5362 { 5363 struct hwrm_fw_set_time_input req = {0}; 5364 struct tm tm; 5365 time64_t now = ktime_get_real_seconds(); 5366 5367 if ((BNXT_VF(bp) && bp->hwrm_spec_code < 0x10901) || 5368 bp->hwrm_spec_code < 0x10400) 5369 return -EOPNOTSUPP; 5370 5371 time64_to_tm(now, 0, &tm); 5372 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FW_SET_TIME, -1, -1); 5373 req.year = cpu_to_le16(1900 + tm.tm_year); 5374 req.month = 1 + tm.tm_mon; 5375 req.day = tm.tm_mday; 5376 req.hour = tm.tm_hour; 5377 req.minute = tm.tm_min; 5378 req.second = tm.tm_sec; 5379 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5380 } 5381 5382 static int bnxt_hwrm_port_qstats(struct bnxt *bp) 5383 { 5384 int rc; 5385 struct bnxt_pf_info *pf = &bp->pf; 5386 struct hwrm_port_qstats_input req = {0}; 5387 5388 if (!(bp->flags & BNXT_FLAG_PORT_STATS)) 5389 return 0; 5390 5391 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS, -1, -1); 5392 req.port_id = cpu_to_le16(pf->port_id); 5393 req.tx_stat_host_addr = cpu_to_le64(bp->hw_tx_port_stats_map); 5394 req.rx_stat_host_addr = cpu_to_le64(bp->hw_rx_port_stats_map); 5395 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5396 return rc; 5397 } 5398 5399 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp) 5400 { 5401 struct hwrm_port_qstats_ext_input req = {0}; 5402 struct bnxt_pf_info *pf = &bp->pf; 5403 5404 if (!(bp->flags & BNXT_FLAG_PORT_STATS_EXT)) 5405 return 0; 5406 5407 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS_EXT, -1, -1); 5408 req.port_id = cpu_to_le16(pf->port_id); 5409 req.rx_stat_size = cpu_to_le16(sizeof(struct rx_port_stats_ext)); 5410 req.rx_stat_host_addr = cpu_to_le64(bp->hw_rx_port_stats_ext_map); 5411 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5412 } 5413 5414 static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp) 5415 { 5416 if (bp->vxlan_port_cnt) { 5417 bnxt_hwrm_tunnel_dst_port_free( 5418 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN); 5419 } 5420 bp->vxlan_port_cnt = 0; 5421 if (bp->nge_port_cnt) { 5422 bnxt_hwrm_tunnel_dst_port_free( 5423 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE); 5424 } 5425 bp->nge_port_cnt = 0; 5426 } 5427 5428 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa) 5429 { 5430 int rc, i; 5431 u32 tpa_flags = 0; 5432 5433 if (set_tpa) 5434 tpa_flags = bp->flags & BNXT_FLAG_TPA; 5435 for (i = 0; i < bp->nr_vnics; i++) { 5436 rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags); 5437 if (rc) { 5438 netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n", 5439 i, rc); 5440 return rc; 5441 } 5442 } 5443 return 0; 5444 } 5445 5446 static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp) 5447 { 5448 int i; 5449 5450 for (i = 0; i < bp->nr_vnics; i++) 5451 bnxt_hwrm_vnic_set_rss(bp, i, false); 5452 } 5453 5454 static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path, 5455 bool irq_re_init) 5456 { 5457 if (bp->vnic_info) { 5458 bnxt_hwrm_clear_vnic_filter(bp); 5459 /* clear all RSS setting before free vnic ctx */ 5460 bnxt_hwrm_clear_vnic_rss(bp); 5461 bnxt_hwrm_vnic_ctx_free(bp); 5462 /* before free the vnic, undo the vnic tpa settings */ 5463 if (bp->flags & BNXT_FLAG_TPA) 5464 bnxt_set_tpa(bp, false); 5465 bnxt_hwrm_vnic_free(bp); 5466 } 5467 bnxt_hwrm_ring_free(bp, close_path); 5468 bnxt_hwrm_ring_grp_free(bp); 5469 if (irq_re_init) { 5470 bnxt_hwrm_stat_ctx_free(bp); 5471 bnxt_hwrm_free_tunnel_ports(bp); 5472 } 5473 } 5474 5475 static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode) 5476 { 5477 struct hwrm_func_cfg_input req = {0}; 5478 int rc; 5479 5480 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1); 5481 req.fid = cpu_to_le16(0xffff); 5482 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_EVB_MODE); 5483 if (br_mode == BRIDGE_MODE_VEB) 5484 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEB; 5485 else if (br_mode == BRIDGE_MODE_VEPA) 5486 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEPA; 5487 else 5488 return -EINVAL; 5489 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5490 if (rc) 5491 rc = -EIO; 5492 return rc; 5493 } 5494 5495 static int bnxt_hwrm_set_cache_line_size(struct bnxt *bp, int size) 5496 { 5497 struct hwrm_func_cfg_input req = {0}; 5498 int rc; 5499 5500 if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10803) 5501 return 0; 5502 5503 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1); 5504 req.fid = cpu_to_le16(0xffff); 5505 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_CACHE_LINESIZE); 5506 req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_64; 5507 if (size == 128) 5508 req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_128; 5509 5510 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5511 if (rc) 5512 rc = -EIO; 5513 return rc; 5514 } 5515 5516 static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id) 5517 { 5518 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5519 int rc; 5520 5521 if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) 5522 goto skip_rss_ctx; 5523 5524 /* allocate context for vnic */ 5525 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 0); 5526 if (rc) { 5527 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n", 5528 vnic_id, rc); 5529 goto vnic_setup_err; 5530 } 5531 bp->rsscos_nr_ctxs++; 5532 5533 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 5534 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 1); 5535 if (rc) { 5536 netdev_err(bp->dev, "hwrm vnic %d cos ctx alloc failure rc: %x\n", 5537 vnic_id, rc); 5538 goto vnic_setup_err; 5539 } 5540 bp->rsscos_nr_ctxs++; 5541 } 5542 5543 skip_rss_ctx: 5544 /* configure default vnic, ring grp */ 5545 rc = bnxt_hwrm_vnic_cfg(bp, vnic_id); 5546 if (rc) { 5547 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n", 5548 vnic_id, rc); 5549 goto vnic_setup_err; 5550 } 5551 5552 /* Enable RSS hashing on vnic */ 5553 rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true); 5554 if (rc) { 5555 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n", 5556 vnic_id, rc); 5557 goto vnic_setup_err; 5558 } 5559 5560 if (bp->flags & BNXT_FLAG_AGG_RINGS) { 5561 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id); 5562 if (rc) { 5563 netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n", 5564 vnic_id, rc); 5565 } 5566 } 5567 5568 vnic_setup_err: 5569 return rc; 5570 } 5571 5572 static int bnxt_alloc_rfs_vnics(struct bnxt *bp) 5573 { 5574 #ifdef CONFIG_RFS_ACCEL 5575 int i, rc = 0; 5576 5577 for (i = 0; i < bp->rx_nr_rings; i++) { 5578 struct bnxt_vnic_info *vnic; 5579 u16 vnic_id = i + 1; 5580 u16 ring_id = i; 5581 5582 if (vnic_id >= bp->nr_vnics) 5583 break; 5584 5585 vnic = &bp->vnic_info[vnic_id]; 5586 vnic->flags |= BNXT_VNIC_RFS_FLAG; 5587 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP) 5588 vnic->flags |= BNXT_VNIC_RFS_NEW_RSS_FLAG; 5589 rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, ring_id, 1); 5590 if (rc) { 5591 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n", 5592 vnic_id, rc); 5593 break; 5594 } 5595 rc = bnxt_setup_vnic(bp, vnic_id); 5596 if (rc) 5597 break; 5598 } 5599 return rc; 5600 #else 5601 return 0; 5602 #endif 5603 } 5604 5605 /* Allow PF and VF with default VLAN to be in promiscuous mode */ 5606 static bool bnxt_promisc_ok(struct bnxt *bp) 5607 { 5608 #ifdef CONFIG_BNXT_SRIOV 5609 if (BNXT_VF(bp) && !bp->vf.vlan) 5610 return false; 5611 #endif 5612 return true; 5613 } 5614 5615 static int bnxt_setup_nitroa0_vnic(struct bnxt *bp) 5616 { 5617 unsigned int rc = 0; 5618 5619 rc = bnxt_hwrm_vnic_alloc(bp, 1, bp->rx_nr_rings - 1, 1); 5620 if (rc) { 5621 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n", 5622 rc); 5623 return rc; 5624 } 5625 5626 rc = bnxt_hwrm_vnic_cfg(bp, 1); 5627 if (rc) { 5628 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n", 5629 rc); 5630 return rc; 5631 } 5632 return rc; 5633 } 5634 5635 static int bnxt_cfg_rx_mode(struct bnxt *); 5636 static bool bnxt_mc_list_updated(struct bnxt *, u32 *); 5637 5638 static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init) 5639 { 5640 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 5641 int rc = 0; 5642 unsigned int rx_nr_rings = bp->rx_nr_rings; 5643 5644 if (irq_re_init) { 5645 rc = bnxt_hwrm_stat_ctx_alloc(bp); 5646 if (rc) { 5647 netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n", 5648 rc); 5649 goto err_out; 5650 } 5651 } 5652 5653 rc = bnxt_hwrm_ring_alloc(bp); 5654 if (rc) { 5655 netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc); 5656 goto err_out; 5657 } 5658 5659 rc = bnxt_hwrm_ring_grp_alloc(bp); 5660 if (rc) { 5661 netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc); 5662 goto err_out; 5663 } 5664 5665 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 5666 rx_nr_rings--; 5667 5668 /* default vnic 0 */ 5669 rc = bnxt_hwrm_vnic_alloc(bp, 0, 0, rx_nr_rings); 5670 if (rc) { 5671 netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc); 5672 goto err_out; 5673 } 5674 5675 rc = bnxt_setup_vnic(bp, 0); 5676 if (rc) 5677 goto err_out; 5678 5679 if (bp->flags & BNXT_FLAG_RFS) { 5680 rc = bnxt_alloc_rfs_vnics(bp); 5681 if (rc) 5682 goto err_out; 5683 } 5684 5685 if (bp->flags & BNXT_FLAG_TPA) { 5686 rc = bnxt_set_tpa(bp, true); 5687 if (rc) 5688 goto err_out; 5689 } 5690 5691 if (BNXT_VF(bp)) 5692 bnxt_update_vf_mac(bp); 5693 5694 /* Filter for default vnic 0 */ 5695 rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr); 5696 if (rc) { 5697 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc); 5698 goto err_out; 5699 } 5700 vnic->uc_filter_count = 1; 5701 5702 vnic->rx_mask = 0; 5703 if (bp->dev->flags & IFF_BROADCAST) 5704 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST; 5705 5706 if ((bp->dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp)) 5707 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 5708 5709 if (bp->dev->flags & IFF_ALLMULTI) { 5710 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 5711 vnic->mc_list_count = 0; 5712 } else { 5713 u32 mask = 0; 5714 5715 bnxt_mc_list_updated(bp, &mask); 5716 vnic->rx_mask |= mask; 5717 } 5718 5719 rc = bnxt_cfg_rx_mode(bp); 5720 if (rc) 5721 goto err_out; 5722 5723 rc = bnxt_hwrm_set_coal(bp); 5724 if (rc) 5725 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n", 5726 rc); 5727 5728 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 5729 rc = bnxt_setup_nitroa0_vnic(bp); 5730 if (rc) 5731 netdev_err(bp->dev, "Special vnic setup failure for NS2 A0 rc: %x\n", 5732 rc); 5733 } 5734 5735 if (BNXT_VF(bp)) { 5736 bnxt_hwrm_func_qcfg(bp); 5737 netdev_update_features(bp->dev); 5738 } 5739 5740 return 0; 5741 5742 err_out: 5743 bnxt_hwrm_resource_free(bp, 0, true); 5744 5745 return rc; 5746 } 5747 5748 static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init) 5749 { 5750 bnxt_hwrm_resource_free(bp, 1, irq_re_init); 5751 return 0; 5752 } 5753 5754 static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init) 5755 { 5756 bnxt_init_cp_rings(bp); 5757 bnxt_init_rx_rings(bp); 5758 bnxt_init_tx_rings(bp); 5759 bnxt_init_ring_grps(bp, irq_re_init); 5760 bnxt_init_vnics(bp); 5761 5762 return bnxt_init_chip(bp, irq_re_init); 5763 } 5764 5765 static int bnxt_set_real_num_queues(struct bnxt *bp) 5766 { 5767 int rc; 5768 struct net_device *dev = bp->dev; 5769 5770 rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings - 5771 bp->tx_nr_rings_xdp); 5772 if (rc) 5773 return rc; 5774 5775 rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings); 5776 if (rc) 5777 return rc; 5778 5779 #ifdef CONFIG_RFS_ACCEL 5780 if (bp->flags & BNXT_FLAG_RFS) 5781 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings); 5782 #endif 5783 5784 return rc; 5785 } 5786 5787 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max, 5788 bool shared) 5789 { 5790 int _rx = *rx, _tx = *tx; 5791 5792 if (shared) { 5793 *rx = min_t(int, _rx, max); 5794 *tx = min_t(int, _tx, max); 5795 } else { 5796 if (max < 2) 5797 return -ENOMEM; 5798 5799 while (_rx + _tx > max) { 5800 if (_rx > _tx && _rx > 1) 5801 _rx--; 5802 else if (_tx > 1) 5803 _tx--; 5804 } 5805 *rx = _rx; 5806 *tx = _tx; 5807 } 5808 return 0; 5809 } 5810 5811 static void bnxt_setup_msix(struct bnxt *bp) 5812 { 5813 const int len = sizeof(bp->irq_tbl[0].name); 5814 struct net_device *dev = bp->dev; 5815 int tcs, i; 5816 5817 tcs = netdev_get_num_tc(dev); 5818 if (tcs > 1) { 5819 int i, off, count; 5820 5821 for (i = 0; i < tcs; i++) { 5822 count = bp->tx_nr_rings_per_tc; 5823 off = i * count; 5824 netdev_set_tc_queue(dev, i, count, off); 5825 } 5826 } 5827 5828 for (i = 0; i < bp->cp_nr_rings; i++) { 5829 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 5830 char *attr; 5831 5832 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 5833 attr = "TxRx"; 5834 else if (i < bp->rx_nr_rings) 5835 attr = "rx"; 5836 else 5837 attr = "tx"; 5838 5839 snprintf(bp->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name, 5840 attr, i); 5841 bp->irq_tbl[map_idx].handler = bnxt_msix; 5842 } 5843 } 5844 5845 static void bnxt_setup_inta(struct bnxt *bp) 5846 { 5847 const int len = sizeof(bp->irq_tbl[0].name); 5848 5849 if (netdev_get_num_tc(bp->dev)) 5850 netdev_reset_tc(bp->dev); 5851 5852 snprintf(bp->irq_tbl[0].name, len, "%s-%s-%d", bp->dev->name, "TxRx", 5853 0); 5854 bp->irq_tbl[0].handler = bnxt_inta; 5855 } 5856 5857 static int bnxt_setup_int_mode(struct bnxt *bp) 5858 { 5859 int rc; 5860 5861 if (bp->flags & BNXT_FLAG_USING_MSIX) 5862 bnxt_setup_msix(bp); 5863 else 5864 bnxt_setup_inta(bp); 5865 5866 rc = bnxt_set_real_num_queues(bp); 5867 return rc; 5868 } 5869 5870 #ifdef CONFIG_RFS_ACCEL 5871 static unsigned int bnxt_get_max_func_rss_ctxs(struct bnxt *bp) 5872 { 5873 return bp->hw_resc.max_rsscos_ctxs; 5874 } 5875 5876 static unsigned int bnxt_get_max_func_vnics(struct bnxt *bp) 5877 { 5878 return bp->hw_resc.max_vnics; 5879 } 5880 #endif 5881 5882 unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp) 5883 { 5884 return bp->hw_resc.max_stat_ctxs; 5885 } 5886 5887 void bnxt_set_max_func_stat_ctxs(struct bnxt *bp, unsigned int max) 5888 { 5889 bp->hw_resc.max_stat_ctxs = max; 5890 } 5891 5892 unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp) 5893 { 5894 return bp->hw_resc.max_cp_rings; 5895 } 5896 5897 void bnxt_set_max_func_cp_rings(struct bnxt *bp, unsigned int max) 5898 { 5899 bp->hw_resc.max_cp_rings = max; 5900 } 5901 5902 unsigned int bnxt_get_max_func_irqs(struct bnxt *bp) 5903 { 5904 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 5905 5906 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_cp_rings); 5907 } 5908 5909 static void bnxt_set_max_func_irqs(struct bnxt *bp, unsigned int max_irqs) 5910 { 5911 bp->hw_resc.max_irqs = max_irqs; 5912 } 5913 5914 int bnxt_get_avail_msix(struct bnxt *bp, int num) 5915 { 5916 int max_cp = bnxt_get_max_func_cp_rings(bp); 5917 int max_irq = bnxt_get_max_func_irqs(bp); 5918 int total_req = bp->cp_nr_rings + num; 5919 int max_idx, avail_msix; 5920 5921 max_idx = min_t(int, bp->total_irqs, max_cp); 5922 avail_msix = max_idx - bp->cp_nr_rings; 5923 if (!(bp->flags & BNXT_FLAG_NEW_RM) || avail_msix >= num) 5924 return avail_msix; 5925 5926 if (max_irq < total_req) { 5927 num = max_irq - bp->cp_nr_rings; 5928 if (num <= 0) 5929 return 0; 5930 } 5931 return num; 5932 } 5933 5934 static int bnxt_get_num_msix(struct bnxt *bp) 5935 { 5936 if (!(bp->flags & BNXT_FLAG_NEW_RM)) 5937 return bnxt_get_max_func_irqs(bp); 5938 5939 return bnxt_cp_rings_in_use(bp); 5940 } 5941 5942 static int bnxt_init_msix(struct bnxt *bp) 5943 { 5944 int i, total_vecs, max, rc = 0, min = 1, ulp_msix; 5945 struct msix_entry *msix_ent; 5946 5947 total_vecs = bnxt_get_num_msix(bp); 5948 max = bnxt_get_max_func_irqs(bp); 5949 if (total_vecs > max) 5950 total_vecs = max; 5951 5952 if (!total_vecs) 5953 return 0; 5954 5955 msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL); 5956 if (!msix_ent) 5957 return -ENOMEM; 5958 5959 for (i = 0; i < total_vecs; i++) { 5960 msix_ent[i].entry = i; 5961 msix_ent[i].vector = 0; 5962 } 5963 5964 if (!(bp->flags & BNXT_FLAG_SHARED_RINGS)) 5965 min = 2; 5966 5967 total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs); 5968 ulp_msix = bnxt_get_ulp_msix_num(bp); 5969 if (total_vecs < 0 || total_vecs < ulp_msix) { 5970 rc = -ENODEV; 5971 goto msix_setup_exit; 5972 } 5973 5974 bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL); 5975 if (bp->irq_tbl) { 5976 for (i = 0; i < total_vecs; i++) 5977 bp->irq_tbl[i].vector = msix_ent[i].vector; 5978 5979 bp->total_irqs = total_vecs; 5980 /* Trim rings based upon num of vectors allocated */ 5981 rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings, 5982 total_vecs - ulp_msix, min == 1); 5983 if (rc) 5984 goto msix_setup_exit; 5985 5986 bp->cp_nr_rings = (min == 1) ? 5987 max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) : 5988 bp->tx_nr_rings + bp->rx_nr_rings; 5989 5990 } else { 5991 rc = -ENOMEM; 5992 goto msix_setup_exit; 5993 } 5994 bp->flags |= BNXT_FLAG_USING_MSIX; 5995 kfree(msix_ent); 5996 return 0; 5997 5998 msix_setup_exit: 5999 netdev_err(bp->dev, "bnxt_init_msix err: %x\n", rc); 6000 kfree(bp->irq_tbl); 6001 bp->irq_tbl = NULL; 6002 pci_disable_msix(bp->pdev); 6003 kfree(msix_ent); 6004 return rc; 6005 } 6006 6007 static int bnxt_init_inta(struct bnxt *bp) 6008 { 6009 bp->irq_tbl = kcalloc(1, sizeof(struct bnxt_irq), GFP_KERNEL); 6010 if (!bp->irq_tbl) 6011 return -ENOMEM; 6012 6013 bp->total_irqs = 1; 6014 bp->rx_nr_rings = 1; 6015 bp->tx_nr_rings = 1; 6016 bp->cp_nr_rings = 1; 6017 bp->flags |= BNXT_FLAG_SHARED_RINGS; 6018 bp->irq_tbl[0].vector = bp->pdev->irq; 6019 return 0; 6020 } 6021 6022 static int bnxt_init_int_mode(struct bnxt *bp) 6023 { 6024 int rc = 0; 6025 6026 if (bp->flags & BNXT_FLAG_MSIX_CAP) 6027 rc = bnxt_init_msix(bp); 6028 6029 if (!(bp->flags & BNXT_FLAG_USING_MSIX) && BNXT_PF(bp)) { 6030 /* fallback to INTA */ 6031 rc = bnxt_init_inta(bp); 6032 } 6033 return rc; 6034 } 6035 6036 static void bnxt_clear_int_mode(struct bnxt *bp) 6037 { 6038 if (bp->flags & BNXT_FLAG_USING_MSIX) 6039 pci_disable_msix(bp->pdev); 6040 6041 kfree(bp->irq_tbl); 6042 bp->irq_tbl = NULL; 6043 bp->flags &= ~BNXT_FLAG_USING_MSIX; 6044 } 6045 6046 int bnxt_reserve_rings(struct bnxt *bp) 6047 { 6048 int tcs = netdev_get_num_tc(bp->dev); 6049 int rc; 6050 6051 if (!bnxt_need_reserve_rings(bp)) 6052 return 0; 6053 6054 rc = __bnxt_reserve_rings(bp); 6055 if (rc) { 6056 netdev_err(bp->dev, "ring reservation failure rc: %d\n", rc); 6057 return rc; 6058 } 6059 if ((bp->flags & BNXT_FLAG_NEW_RM) && 6060 (bnxt_get_num_msix(bp) != bp->total_irqs)) { 6061 bnxt_ulp_irq_stop(bp); 6062 bnxt_clear_int_mode(bp); 6063 rc = bnxt_init_int_mode(bp); 6064 bnxt_ulp_irq_restart(bp, rc); 6065 if (rc) 6066 return rc; 6067 } 6068 if (tcs && (bp->tx_nr_rings_per_tc * tcs != bp->tx_nr_rings)) { 6069 netdev_err(bp->dev, "tx ring reservation failure\n"); 6070 netdev_reset_tc(bp->dev); 6071 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 6072 return -ENOMEM; 6073 } 6074 bp->num_stat_ctxs = bp->cp_nr_rings; 6075 return 0; 6076 } 6077 6078 static void bnxt_free_irq(struct bnxt *bp) 6079 { 6080 struct bnxt_irq *irq; 6081 int i; 6082 6083 #ifdef CONFIG_RFS_ACCEL 6084 free_irq_cpu_rmap(bp->dev->rx_cpu_rmap); 6085 bp->dev->rx_cpu_rmap = NULL; 6086 #endif 6087 if (!bp->irq_tbl || !bp->bnapi) 6088 return; 6089 6090 for (i = 0; i < bp->cp_nr_rings; i++) { 6091 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 6092 6093 irq = &bp->irq_tbl[map_idx]; 6094 if (irq->requested) { 6095 if (irq->have_cpumask) { 6096 irq_set_affinity_hint(irq->vector, NULL); 6097 free_cpumask_var(irq->cpu_mask); 6098 irq->have_cpumask = 0; 6099 } 6100 free_irq(irq->vector, bp->bnapi[i]); 6101 } 6102 6103 irq->requested = 0; 6104 } 6105 } 6106 6107 static int bnxt_request_irq(struct bnxt *bp) 6108 { 6109 int i, j, rc = 0; 6110 unsigned long flags = 0; 6111 #ifdef CONFIG_RFS_ACCEL 6112 struct cpu_rmap *rmap; 6113 #endif 6114 6115 rc = bnxt_setup_int_mode(bp); 6116 if (rc) { 6117 netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n", 6118 rc); 6119 return rc; 6120 } 6121 #ifdef CONFIG_RFS_ACCEL 6122 rmap = bp->dev->rx_cpu_rmap; 6123 #endif 6124 if (!(bp->flags & BNXT_FLAG_USING_MSIX)) 6125 flags = IRQF_SHARED; 6126 6127 for (i = 0, j = 0; i < bp->cp_nr_rings; i++) { 6128 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 6129 struct bnxt_irq *irq = &bp->irq_tbl[map_idx]; 6130 6131 #ifdef CONFIG_RFS_ACCEL 6132 if (rmap && bp->bnapi[i]->rx_ring) { 6133 rc = irq_cpu_rmap_add(rmap, irq->vector); 6134 if (rc) 6135 netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n", 6136 j); 6137 j++; 6138 } 6139 #endif 6140 rc = request_irq(irq->vector, irq->handler, flags, irq->name, 6141 bp->bnapi[i]); 6142 if (rc) 6143 break; 6144 6145 irq->requested = 1; 6146 6147 if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) { 6148 int numa_node = dev_to_node(&bp->pdev->dev); 6149 6150 irq->have_cpumask = 1; 6151 cpumask_set_cpu(cpumask_local_spread(i, numa_node), 6152 irq->cpu_mask); 6153 rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask); 6154 if (rc) { 6155 netdev_warn(bp->dev, 6156 "Set affinity failed, IRQ = %d\n", 6157 irq->vector); 6158 break; 6159 } 6160 } 6161 } 6162 return rc; 6163 } 6164 6165 static void bnxt_del_napi(struct bnxt *bp) 6166 { 6167 int i; 6168 6169 if (!bp->bnapi) 6170 return; 6171 6172 for (i = 0; i < bp->cp_nr_rings; i++) { 6173 struct bnxt_napi *bnapi = bp->bnapi[i]; 6174 6175 napi_hash_del(&bnapi->napi); 6176 netif_napi_del(&bnapi->napi); 6177 } 6178 /* We called napi_hash_del() before netif_napi_del(), we need 6179 * to respect an RCU grace period before freeing napi structures. 6180 */ 6181 synchronize_net(); 6182 } 6183 6184 static void bnxt_init_napi(struct bnxt *bp) 6185 { 6186 int i; 6187 unsigned int cp_nr_rings = bp->cp_nr_rings; 6188 struct bnxt_napi *bnapi; 6189 6190 if (bp->flags & BNXT_FLAG_USING_MSIX) { 6191 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 6192 cp_nr_rings--; 6193 for (i = 0; i < cp_nr_rings; i++) { 6194 bnapi = bp->bnapi[i]; 6195 netif_napi_add(bp->dev, &bnapi->napi, 6196 bnxt_poll, 64); 6197 } 6198 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 6199 bnapi = bp->bnapi[cp_nr_rings]; 6200 netif_napi_add(bp->dev, &bnapi->napi, 6201 bnxt_poll_nitroa0, 64); 6202 } 6203 } else { 6204 bnapi = bp->bnapi[0]; 6205 netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll, 64); 6206 } 6207 } 6208 6209 static void bnxt_disable_napi(struct bnxt *bp) 6210 { 6211 int i; 6212 6213 if (!bp->bnapi) 6214 return; 6215 6216 for (i = 0; i < bp->cp_nr_rings; i++) { 6217 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring; 6218 6219 if (bp->bnapi[i]->rx_ring) 6220 cancel_work_sync(&cpr->dim.work); 6221 6222 napi_disable(&bp->bnapi[i]->napi); 6223 } 6224 } 6225 6226 static void bnxt_enable_napi(struct bnxt *bp) 6227 { 6228 int i; 6229 6230 for (i = 0; i < bp->cp_nr_rings; i++) { 6231 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring; 6232 bp->bnapi[i]->in_reset = false; 6233 6234 if (bp->bnapi[i]->rx_ring) { 6235 INIT_WORK(&cpr->dim.work, bnxt_dim_work); 6236 cpr->dim.mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE; 6237 } 6238 napi_enable(&bp->bnapi[i]->napi); 6239 } 6240 } 6241 6242 void bnxt_tx_disable(struct bnxt *bp) 6243 { 6244 int i; 6245 struct bnxt_tx_ring_info *txr; 6246 6247 if (bp->tx_ring) { 6248 for (i = 0; i < bp->tx_nr_rings; i++) { 6249 txr = &bp->tx_ring[i]; 6250 txr->dev_state = BNXT_DEV_STATE_CLOSING; 6251 } 6252 } 6253 /* Stop all TX queues */ 6254 netif_tx_disable(bp->dev); 6255 netif_carrier_off(bp->dev); 6256 } 6257 6258 void bnxt_tx_enable(struct bnxt *bp) 6259 { 6260 int i; 6261 struct bnxt_tx_ring_info *txr; 6262 6263 for (i = 0; i < bp->tx_nr_rings; i++) { 6264 txr = &bp->tx_ring[i]; 6265 txr->dev_state = 0; 6266 } 6267 netif_tx_wake_all_queues(bp->dev); 6268 if (bp->link_info.link_up) 6269 netif_carrier_on(bp->dev); 6270 } 6271 6272 static void bnxt_report_link(struct bnxt *bp) 6273 { 6274 if (bp->link_info.link_up) { 6275 const char *duplex; 6276 const char *flow_ctrl; 6277 u32 speed; 6278 u16 fec; 6279 6280 netif_carrier_on(bp->dev); 6281 if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL) 6282 duplex = "full"; 6283 else 6284 duplex = "half"; 6285 if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH) 6286 flow_ctrl = "ON - receive & transmit"; 6287 else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX) 6288 flow_ctrl = "ON - transmit"; 6289 else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX) 6290 flow_ctrl = "ON - receive"; 6291 else 6292 flow_ctrl = "none"; 6293 speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed); 6294 netdev_info(bp->dev, "NIC Link is Up, %u Mbps %s duplex, Flow control: %s\n", 6295 speed, duplex, flow_ctrl); 6296 if (bp->flags & BNXT_FLAG_EEE_CAP) 6297 netdev_info(bp->dev, "EEE is %s\n", 6298 bp->eee.eee_active ? "active" : 6299 "not active"); 6300 fec = bp->link_info.fec_cfg; 6301 if (!(fec & PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED)) 6302 netdev_info(bp->dev, "FEC autoneg %s encodings: %s\n", 6303 (fec & BNXT_FEC_AUTONEG) ? "on" : "off", 6304 (fec & BNXT_FEC_ENC_BASE_R) ? "BaseR" : 6305 (fec & BNXT_FEC_ENC_RS) ? "RS" : "None"); 6306 } else { 6307 netif_carrier_off(bp->dev); 6308 netdev_err(bp->dev, "NIC Link is Down\n"); 6309 } 6310 } 6311 6312 static int bnxt_hwrm_phy_qcaps(struct bnxt *bp) 6313 { 6314 int rc = 0; 6315 struct hwrm_port_phy_qcaps_input req = {0}; 6316 struct hwrm_port_phy_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 6317 struct bnxt_link_info *link_info = &bp->link_info; 6318 6319 if (bp->hwrm_spec_code < 0x10201) 6320 return 0; 6321 6322 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCAPS, -1, -1); 6323 6324 mutex_lock(&bp->hwrm_cmd_lock); 6325 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6326 if (rc) 6327 goto hwrm_phy_qcaps_exit; 6328 6329 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) { 6330 struct ethtool_eee *eee = &bp->eee; 6331 u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode); 6332 6333 bp->flags |= BNXT_FLAG_EEE_CAP; 6334 eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0); 6335 bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) & 6336 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK; 6337 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) & 6338 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK; 6339 } 6340 if (resp->supported_speeds_auto_mode) 6341 link_info->support_auto_speeds = 6342 le16_to_cpu(resp->supported_speeds_auto_mode); 6343 6344 bp->port_count = resp->port_cnt; 6345 6346 hwrm_phy_qcaps_exit: 6347 mutex_unlock(&bp->hwrm_cmd_lock); 6348 return rc; 6349 } 6350 6351 static int bnxt_update_link(struct bnxt *bp, bool chng_link_state) 6352 { 6353 int rc = 0; 6354 struct bnxt_link_info *link_info = &bp->link_info; 6355 struct hwrm_port_phy_qcfg_input req = {0}; 6356 struct hwrm_port_phy_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 6357 u8 link_up = link_info->link_up; 6358 u16 diff; 6359 6360 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCFG, -1, -1); 6361 6362 mutex_lock(&bp->hwrm_cmd_lock); 6363 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6364 if (rc) { 6365 mutex_unlock(&bp->hwrm_cmd_lock); 6366 return rc; 6367 } 6368 6369 memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp)); 6370 link_info->phy_link_status = resp->link; 6371 link_info->duplex = resp->duplex_cfg; 6372 if (bp->hwrm_spec_code >= 0x10800) 6373 link_info->duplex = resp->duplex_state; 6374 link_info->pause = resp->pause; 6375 link_info->auto_mode = resp->auto_mode; 6376 link_info->auto_pause_setting = resp->auto_pause; 6377 link_info->lp_pause = resp->link_partner_adv_pause; 6378 link_info->force_pause_setting = resp->force_pause; 6379 link_info->duplex_setting = resp->duplex_cfg; 6380 if (link_info->phy_link_status == BNXT_LINK_LINK) 6381 link_info->link_speed = le16_to_cpu(resp->link_speed); 6382 else 6383 link_info->link_speed = 0; 6384 link_info->force_link_speed = le16_to_cpu(resp->force_link_speed); 6385 link_info->support_speeds = le16_to_cpu(resp->support_speeds); 6386 link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask); 6387 link_info->lp_auto_link_speeds = 6388 le16_to_cpu(resp->link_partner_adv_speeds); 6389 link_info->preemphasis = le32_to_cpu(resp->preemphasis); 6390 link_info->phy_ver[0] = resp->phy_maj; 6391 link_info->phy_ver[1] = resp->phy_min; 6392 link_info->phy_ver[2] = resp->phy_bld; 6393 link_info->media_type = resp->media_type; 6394 link_info->phy_type = resp->phy_type; 6395 link_info->transceiver = resp->xcvr_pkg_type; 6396 link_info->phy_addr = resp->eee_config_phy_addr & 6397 PORT_PHY_QCFG_RESP_PHY_ADDR_MASK; 6398 link_info->module_status = resp->module_status; 6399 6400 if (bp->flags & BNXT_FLAG_EEE_CAP) { 6401 struct ethtool_eee *eee = &bp->eee; 6402 u16 fw_speeds; 6403 6404 eee->eee_active = 0; 6405 if (resp->eee_config_phy_addr & 6406 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) { 6407 eee->eee_active = 1; 6408 fw_speeds = le16_to_cpu( 6409 resp->link_partner_adv_eee_link_speed_mask); 6410 eee->lp_advertised = 6411 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0); 6412 } 6413 6414 /* Pull initial EEE config */ 6415 if (!chng_link_state) { 6416 if (resp->eee_config_phy_addr & 6417 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED) 6418 eee->eee_enabled = 1; 6419 6420 fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask); 6421 eee->advertised = 6422 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0); 6423 6424 if (resp->eee_config_phy_addr & 6425 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) { 6426 __le32 tmr; 6427 6428 eee->tx_lpi_enabled = 1; 6429 tmr = resp->xcvr_identifier_type_tx_lpi_timer; 6430 eee->tx_lpi_timer = le32_to_cpu(tmr) & 6431 PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK; 6432 } 6433 } 6434 } 6435 6436 link_info->fec_cfg = PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED; 6437 if (bp->hwrm_spec_code >= 0x10504) 6438 link_info->fec_cfg = le16_to_cpu(resp->fec_cfg); 6439 6440 /* TODO: need to add more logic to report VF link */ 6441 if (chng_link_state) { 6442 if (link_info->phy_link_status == BNXT_LINK_LINK) 6443 link_info->link_up = 1; 6444 else 6445 link_info->link_up = 0; 6446 if (link_up != link_info->link_up) 6447 bnxt_report_link(bp); 6448 } else { 6449 /* alwasy link down if not require to update link state */ 6450 link_info->link_up = 0; 6451 } 6452 mutex_unlock(&bp->hwrm_cmd_lock); 6453 6454 if (!BNXT_SINGLE_PF(bp)) 6455 return 0; 6456 6457 diff = link_info->support_auto_speeds ^ link_info->advertising; 6458 if ((link_info->support_auto_speeds | diff) != 6459 link_info->support_auto_speeds) { 6460 /* An advertised speed is no longer supported, so we need to 6461 * update the advertisement settings. Caller holds RTNL 6462 * so we can modify link settings. 6463 */ 6464 link_info->advertising = link_info->support_auto_speeds; 6465 if (link_info->autoneg & BNXT_AUTONEG_SPEED) 6466 bnxt_hwrm_set_link_setting(bp, true, false); 6467 } 6468 return 0; 6469 } 6470 6471 static void bnxt_get_port_module_status(struct bnxt *bp) 6472 { 6473 struct bnxt_link_info *link_info = &bp->link_info; 6474 struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp; 6475 u8 module_status; 6476 6477 if (bnxt_update_link(bp, true)) 6478 return; 6479 6480 module_status = link_info->module_status; 6481 switch (module_status) { 6482 case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX: 6483 case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN: 6484 case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG: 6485 netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n", 6486 bp->pf.port_id); 6487 if (bp->hwrm_spec_code >= 0x10201) { 6488 netdev_warn(bp->dev, "Module part number %s\n", 6489 resp->phy_vendor_partnumber); 6490 } 6491 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX) 6492 netdev_warn(bp->dev, "TX is disabled\n"); 6493 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN) 6494 netdev_warn(bp->dev, "SFP+ module is shutdown\n"); 6495 } 6496 } 6497 6498 static void 6499 bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req) 6500 { 6501 if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) { 6502 if (bp->hwrm_spec_code >= 0x10201) 6503 req->auto_pause = 6504 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE; 6505 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX) 6506 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX; 6507 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX) 6508 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX; 6509 req->enables |= 6510 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE); 6511 } else { 6512 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX) 6513 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX; 6514 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX) 6515 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX; 6516 req->enables |= 6517 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE); 6518 if (bp->hwrm_spec_code >= 0x10201) { 6519 req->auto_pause = req->force_pause; 6520 req->enables |= cpu_to_le32( 6521 PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE); 6522 } 6523 } 6524 } 6525 6526 static void bnxt_hwrm_set_link_common(struct bnxt *bp, 6527 struct hwrm_port_phy_cfg_input *req) 6528 { 6529 u8 autoneg = bp->link_info.autoneg; 6530 u16 fw_link_speed = bp->link_info.req_link_speed; 6531 u16 advertising = bp->link_info.advertising; 6532 6533 if (autoneg & BNXT_AUTONEG_SPEED) { 6534 req->auto_mode |= 6535 PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK; 6536 6537 req->enables |= cpu_to_le32( 6538 PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK); 6539 req->auto_link_speed_mask = cpu_to_le16(advertising); 6540 6541 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE); 6542 req->flags |= 6543 cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG); 6544 } else { 6545 req->force_link_speed = cpu_to_le16(fw_link_speed); 6546 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE); 6547 } 6548 6549 /* tell chimp that the setting takes effect immediately */ 6550 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY); 6551 } 6552 6553 int bnxt_hwrm_set_pause(struct bnxt *bp) 6554 { 6555 struct hwrm_port_phy_cfg_input req = {0}; 6556 int rc; 6557 6558 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1); 6559 bnxt_hwrm_set_pause_common(bp, &req); 6560 6561 if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) || 6562 bp->link_info.force_link_chng) 6563 bnxt_hwrm_set_link_common(bp, &req); 6564 6565 mutex_lock(&bp->hwrm_cmd_lock); 6566 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6567 if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) { 6568 /* since changing of pause setting doesn't trigger any link 6569 * change event, the driver needs to update the current pause 6570 * result upon successfully return of the phy_cfg command 6571 */ 6572 bp->link_info.pause = 6573 bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl; 6574 bp->link_info.auto_pause_setting = 0; 6575 if (!bp->link_info.force_link_chng) 6576 bnxt_report_link(bp); 6577 } 6578 bp->link_info.force_link_chng = false; 6579 mutex_unlock(&bp->hwrm_cmd_lock); 6580 return rc; 6581 } 6582 6583 static void bnxt_hwrm_set_eee(struct bnxt *bp, 6584 struct hwrm_port_phy_cfg_input *req) 6585 { 6586 struct ethtool_eee *eee = &bp->eee; 6587 6588 if (eee->eee_enabled) { 6589 u16 eee_speeds; 6590 u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE; 6591 6592 if (eee->tx_lpi_enabled) 6593 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE; 6594 else 6595 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE; 6596 6597 req->flags |= cpu_to_le32(flags); 6598 eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised); 6599 req->eee_link_speed_mask = cpu_to_le16(eee_speeds); 6600 req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer); 6601 } else { 6602 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE); 6603 } 6604 } 6605 6606 int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee) 6607 { 6608 struct hwrm_port_phy_cfg_input req = {0}; 6609 6610 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1); 6611 if (set_pause) 6612 bnxt_hwrm_set_pause_common(bp, &req); 6613 6614 bnxt_hwrm_set_link_common(bp, &req); 6615 6616 if (set_eee) 6617 bnxt_hwrm_set_eee(bp, &req); 6618 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6619 } 6620 6621 static int bnxt_hwrm_shutdown_link(struct bnxt *bp) 6622 { 6623 struct hwrm_port_phy_cfg_input req = {0}; 6624 6625 if (!BNXT_SINGLE_PF(bp)) 6626 return 0; 6627 6628 if (pci_num_vf(bp->pdev)) 6629 return 0; 6630 6631 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1); 6632 req.flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN); 6633 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6634 } 6635 6636 static int bnxt_hwrm_port_led_qcaps(struct bnxt *bp) 6637 { 6638 struct hwrm_port_led_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 6639 struct hwrm_port_led_qcaps_input req = {0}; 6640 struct bnxt_pf_info *pf = &bp->pf; 6641 int rc; 6642 6643 if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10601) 6644 return 0; 6645 6646 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_LED_QCAPS, -1, -1); 6647 req.port_id = cpu_to_le16(pf->port_id); 6648 mutex_lock(&bp->hwrm_cmd_lock); 6649 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6650 if (rc) { 6651 mutex_unlock(&bp->hwrm_cmd_lock); 6652 return rc; 6653 } 6654 if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) { 6655 int i; 6656 6657 bp->num_leds = resp->num_leds; 6658 memcpy(bp->leds, &resp->led0_id, sizeof(bp->leds[0]) * 6659 bp->num_leds); 6660 for (i = 0; i < bp->num_leds; i++) { 6661 struct bnxt_led_info *led = &bp->leds[i]; 6662 __le16 caps = led->led_state_caps; 6663 6664 if (!led->led_group_id || 6665 !BNXT_LED_ALT_BLINK_CAP(caps)) { 6666 bp->num_leds = 0; 6667 break; 6668 } 6669 } 6670 } 6671 mutex_unlock(&bp->hwrm_cmd_lock); 6672 return 0; 6673 } 6674 6675 int bnxt_hwrm_alloc_wol_fltr(struct bnxt *bp) 6676 { 6677 struct hwrm_wol_filter_alloc_input req = {0}; 6678 struct hwrm_wol_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr; 6679 int rc; 6680 6681 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_ALLOC, -1, -1); 6682 req.port_id = cpu_to_le16(bp->pf.port_id); 6683 req.wol_type = WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT; 6684 req.enables = cpu_to_le32(WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS); 6685 memcpy(req.mac_address, bp->dev->dev_addr, ETH_ALEN); 6686 mutex_lock(&bp->hwrm_cmd_lock); 6687 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6688 if (!rc) 6689 bp->wol_filter_id = resp->wol_filter_id; 6690 mutex_unlock(&bp->hwrm_cmd_lock); 6691 return rc; 6692 } 6693 6694 int bnxt_hwrm_free_wol_fltr(struct bnxt *bp) 6695 { 6696 struct hwrm_wol_filter_free_input req = {0}; 6697 int rc; 6698 6699 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_FREE, -1, -1); 6700 req.port_id = cpu_to_le16(bp->pf.port_id); 6701 req.enables = cpu_to_le32(WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID); 6702 req.wol_filter_id = bp->wol_filter_id; 6703 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6704 return rc; 6705 } 6706 6707 static u16 bnxt_hwrm_get_wol_fltrs(struct bnxt *bp, u16 handle) 6708 { 6709 struct hwrm_wol_filter_qcfg_input req = {0}; 6710 struct hwrm_wol_filter_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 6711 u16 next_handle = 0; 6712 int rc; 6713 6714 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_QCFG, -1, -1); 6715 req.port_id = cpu_to_le16(bp->pf.port_id); 6716 req.handle = cpu_to_le16(handle); 6717 mutex_lock(&bp->hwrm_cmd_lock); 6718 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6719 if (!rc) { 6720 next_handle = le16_to_cpu(resp->next_handle); 6721 if (next_handle != 0) { 6722 if (resp->wol_type == 6723 WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT) { 6724 bp->wol = 1; 6725 bp->wol_filter_id = resp->wol_filter_id; 6726 } 6727 } 6728 } 6729 mutex_unlock(&bp->hwrm_cmd_lock); 6730 return next_handle; 6731 } 6732 6733 static void bnxt_get_wol_settings(struct bnxt *bp) 6734 { 6735 u16 handle = 0; 6736 6737 if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_WOL_CAP)) 6738 return; 6739 6740 do { 6741 handle = bnxt_hwrm_get_wol_fltrs(bp, handle); 6742 } while (handle && handle != 0xffff); 6743 } 6744 6745 static bool bnxt_eee_config_ok(struct bnxt *bp) 6746 { 6747 struct ethtool_eee *eee = &bp->eee; 6748 struct bnxt_link_info *link_info = &bp->link_info; 6749 6750 if (!(bp->flags & BNXT_FLAG_EEE_CAP)) 6751 return true; 6752 6753 if (eee->eee_enabled) { 6754 u32 advertising = 6755 _bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0); 6756 6757 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) { 6758 eee->eee_enabled = 0; 6759 return false; 6760 } 6761 if (eee->advertised & ~advertising) { 6762 eee->advertised = advertising & eee->supported; 6763 return false; 6764 } 6765 } 6766 return true; 6767 } 6768 6769 static int bnxt_update_phy_setting(struct bnxt *bp) 6770 { 6771 int rc; 6772 bool update_link = false; 6773 bool update_pause = false; 6774 bool update_eee = false; 6775 struct bnxt_link_info *link_info = &bp->link_info; 6776 6777 rc = bnxt_update_link(bp, true); 6778 if (rc) { 6779 netdev_err(bp->dev, "failed to update link (rc: %x)\n", 6780 rc); 6781 return rc; 6782 } 6783 if (!BNXT_SINGLE_PF(bp)) 6784 return 0; 6785 6786 if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) && 6787 (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) != 6788 link_info->req_flow_ctrl) 6789 update_pause = true; 6790 if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) && 6791 link_info->force_pause_setting != link_info->req_flow_ctrl) 6792 update_pause = true; 6793 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) { 6794 if (BNXT_AUTO_MODE(link_info->auto_mode)) 6795 update_link = true; 6796 if (link_info->req_link_speed != link_info->force_link_speed) 6797 update_link = true; 6798 if (link_info->req_duplex != link_info->duplex_setting) 6799 update_link = true; 6800 } else { 6801 if (link_info->auto_mode == BNXT_LINK_AUTO_NONE) 6802 update_link = true; 6803 if (link_info->advertising != link_info->auto_link_speeds) 6804 update_link = true; 6805 } 6806 6807 /* The last close may have shutdown the link, so need to call 6808 * PHY_CFG to bring it back up. 6809 */ 6810 if (!netif_carrier_ok(bp->dev)) 6811 update_link = true; 6812 6813 if (!bnxt_eee_config_ok(bp)) 6814 update_eee = true; 6815 6816 if (update_link) 6817 rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee); 6818 else if (update_pause) 6819 rc = bnxt_hwrm_set_pause(bp); 6820 if (rc) { 6821 netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n", 6822 rc); 6823 return rc; 6824 } 6825 6826 return rc; 6827 } 6828 6829 /* Common routine to pre-map certain register block to different GRC window. 6830 * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows 6831 * in PF and 3 windows in VF that can be customized to map in different 6832 * register blocks. 6833 */ 6834 static void bnxt_preset_reg_win(struct bnxt *bp) 6835 { 6836 if (BNXT_PF(bp)) { 6837 /* CAG registers map to GRC window #4 */ 6838 writel(BNXT_CAG_REG_BASE, 6839 bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12); 6840 } 6841 } 6842 6843 static int bnxt_init_dflt_ring_mode(struct bnxt *bp); 6844 6845 static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) 6846 { 6847 int rc = 0; 6848 6849 bnxt_preset_reg_win(bp); 6850 netif_carrier_off(bp->dev); 6851 if (irq_re_init) { 6852 /* Reserve rings now if none were reserved at driver probe. */ 6853 rc = bnxt_init_dflt_ring_mode(bp); 6854 if (rc) { 6855 netdev_err(bp->dev, "Failed to reserve default rings at open\n"); 6856 return rc; 6857 } 6858 rc = bnxt_reserve_rings(bp); 6859 if (rc) 6860 return rc; 6861 } 6862 if ((bp->flags & BNXT_FLAG_RFS) && 6863 !(bp->flags & BNXT_FLAG_USING_MSIX)) { 6864 /* disable RFS if falling back to INTA */ 6865 bp->dev->hw_features &= ~NETIF_F_NTUPLE; 6866 bp->flags &= ~BNXT_FLAG_RFS; 6867 } 6868 6869 rc = bnxt_alloc_mem(bp, irq_re_init); 6870 if (rc) { 6871 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc); 6872 goto open_err_free_mem; 6873 } 6874 6875 if (irq_re_init) { 6876 bnxt_init_napi(bp); 6877 rc = bnxt_request_irq(bp); 6878 if (rc) { 6879 netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc); 6880 goto open_err_irq; 6881 } 6882 } 6883 6884 bnxt_enable_napi(bp); 6885 bnxt_debug_dev_init(bp); 6886 6887 rc = bnxt_init_nic(bp, irq_re_init); 6888 if (rc) { 6889 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc); 6890 goto open_err; 6891 } 6892 6893 if (link_re_init) { 6894 mutex_lock(&bp->link_lock); 6895 rc = bnxt_update_phy_setting(bp); 6896 mutex_unlock(&bp->link_lock); 6897 if (rc) 6898 netdev_warn(bp->dev, "failed to update phy settings\n"); 6899 } 6900 6901 if (irq_re_init) 6902 udp_tunnel_get_rx_info(bp->dev); 6903 6904 set_bit(BNXT_STATE_OPEN, &bp->state); 6905 bnxt_enable_int(bp); 6906 /* Enable TX queues */ 6907 bnxt_tx_enable(bp); 6908 mod_timer(&bp->timer, jiffies + bp->current_interval); 6909 /* Poll link status and check for SFP+ module status */ 6910 bnxt_get_port_module_status(bp); 6911 6912 /* VF-reps may need to be re-opened after the PF is re-opened */ 6913 if (BNXT_PF(bp)) 6914 bnxt_vf_reps_open(bp); 6915 return 0; 6916 6917 open_err: 6918 bnxt_debug_dev_exit(bp); 6919 bnxt_disable_napi(bp); 6920 6921 open_err_irq: 6922 bnxt_del_napi(bp); 6923 6924 open_err_free_mem: 6925 bnxt_free_skbs(bp); 6926 bnxt_free_irq(bp); 6927 bnxt_free_mem(bp, true); 6928 return rc; 6929 } 6930 6931 /* rtnl_lock held */ 6932 int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) 6933 { 6934 int rc = 0; 6935 6936 rc = __bnxt_open_nic(bp, irq_re_init, link_re_init); 6937 if (rc) { 6938 netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc); 6939 dev_close(bp->dev); 6940 } 6941 return rc; 6942 } 6943 6944 /* rtnl_lock held, open the NIC half way by allocating all resources, but 6945 * NAPI, IRQ, and TX are not enabled. This is mainly used for offline 6946 * self tests. 6947 */ 6948 int bnxt_half_open_nic(struct bnxt *bp) 6949 { 6950 int rc = 0; 6951 6952 rc = bnxt_alloc_mem(bp, false); 6953 if (rc) { 6954 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc); 6955 goto half_open_err; 6956 } 6957 rc = bnxt_init_nic(bp, false); 6958 if (rc) { 6959 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc); 6960 goto half_open_err; 6961 } 6962 return 0; 6963 6964 half_open_err: 6965 bnxt_free_skbs(bp); 6966 bnxt_free_mem(bp, false); 6967 dev_close(bp->dev); 6968 return rc; 6969 } 6970 6971 /* rtnl_lock held, this call can only be made after a previous successful 6972 * call to bnxt_half_open_nic(). 6973 */ 6974 void bnxt_half_close_nic(struct bnxt *bp) 6975 { 6976 bnxt_hwrm_resource_free(bp, false, false); 6977 bnxt_free_skbs(bp); 6978 bnxt_free_mem(bp, false); 6979 } 6980 6981 static int bnxt_open(struct net_device *dev) 6982 { 6983 struct bnxt *bp = netdev_priv(dev); 6984 6985 return __bnxt_open_nic(bp, true, true); 6986 } 6987 6988 static bool bnxt_drv_busy(struct bnxt *bp) 6989 { 6990 return (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state) || 6991 test_bit(BNXT_STATE_READ_STATS, &bp->state)); 6992 } 6993 6994 static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init, 6995 bool link_re_init) 6996 { 6997 /* Close the VF-reps before closing PF */ 6998 if (BNXT_PF(bp)) 6999 bnxt_vf_reps_close(bp); 7000 7001 /* Change device state to avoid TX queue wake up's */ 7002 bnxt_tx_disable(bp); 7003 7004 clear_bit(BNXT_STATE_OPEN, &bp->state); 7005 smp_mb__after_atomic(); 7006 while (bnxt_drv_busy(bp)) 7007 msleep(20); 7008 7009 /* Flush rings and and disable interrupts */ 7010 bnxt_shutdown_nic(bp, irq_re_init); 7011 7012 /* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */ 7013 7014 bnxt_debug_dev_exit(bp); 7015 bnxt_disable_napi(bp); 7016 del_timer_sync(&bp->timer); 7017 bnxt_free_skbs(bp); 7018 7019 if (irq_re_init) { 7020 bnxt_free_irq(bp); 7021 bnxt_del_napi(bp); 7022 } 7023 bnxt_free_mem(bp, irq_re_init); 7024 } 7025 7026 int bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) 7027 { 7028 int rc = 0; 7029 7030 #ifdef CONFIG_BNXT_SRIOV 7031 if (bp->sriov_cfg) { 7032 rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait, 7033 !bp->sriov_cfg, 7034 BNXT_SRIOV_CFG_WAIT_TMO); 7035 if (rc) 7036 netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete!\n"); 7037 } 7038 #endif 7039 __bnxt_close_nic(bp, irq_re_init, link_re_init); 7040 return rc; 7041 } 7042 7043 static int bnxt_close(struct net_device *dev) 7044 { 7045 struct bnxt *bp = netdev_priv(dev); 7046 7047 bnxt_close_nic(bp, true, true); 7048 bnxt_hwrm_shutdown_link(bp); 7049 return 0; 7050 } 7051 7052 /* rtnl_lock held */ 7053 static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) 7054 { 7055 switch (cmd) { 7056 case SIOCGMIIPHY: 7057 /* fallthru */ 7058 case SIOCGMIIREG: { 7059 if (!netif_running(dev)) 7060 return -EAGAIN; 7061 7062 return 0; 7063 } 7064 7065 case SIOCSMIIREG: 7066 if (!netif_running(dev)) 7067 return -EAGAIN; 7068 7069 return 0; 7070 7071 default: 7072 /* do nothing */ 7073 break; 7074 } 7075 return -EOPNOTSUPP; 7076 } 7077 7078 static void 7079 bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) 7080 { 7081 u32 i; 7082 struct bnxt *bp = netdev_priv(dev); 7083 7084 set_bit(BNXT_STATE_READ_STATS, &bp->state); 7085 /* Make sure bnxt_close_nic() sees that we are reading stats before 7086 * we check the BNXT_STATE_OPEN flag. 7087 */ 7088 smp_mb__after_atomic(); 7089 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 7090 clear_bit(BNXT_STATE_READ_STATS, &bp->state); 7091 return; 7092 } 7093 7094 /* TODO check if we need to synchronize with bnxt_close path */ 7095 for (i = 0; i < bp->cp_nr_rings; i++) { 7096 struct bnxt_napi *bnapi = bp->bnapi[i]; 7097 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 7098 struct ctx_hw_stats *hw_stats = cpr->hw_stats; 7099 7100 stats->rx_packets += le64_to_cpu(hw_stats->rx_ucast_pkts); 7101 stats->rx_packets += le64_to_cpu(hw_stats->rx_mcast_pkts); 7102 stats->rx_packets += le64_to_cpu(hw_stats->rx_bcast_pkts); 7103 7104 stats->tx_packets += le64_to_cpu(hw_stats->tx_ucast_pkts); 7105 stats->tx_packets += le64_to_cpu(hw_stats->tx_mcast_pkts); 7106 stats->tx_packets += le64_to_cpu(hw_stats->tx_bcast_pkts); 7107 7108 stats->rx_bytes += le64_to_cpu(hw_stats->rx_ucast_bytes); 7109 stats->rx_bytes += le64_to_cpu(hw_stats->rx_mcast_bytes); 7110 stats->rx_bytes += le64_to_cpu(hw_stats->rx_bcast_bytes); 7111 7112 stats->tx_bytes += le64_to_cpu(hw_stats->tx_ucast_bytes); 7113 stats->tx_bytes += le64_to_cpu(hw_stats->tx_mcast_bytes); 7114 stats->tx_bytes += le64_to_cpu(hw_stats->tx_bcast_bytes); 7115 7116 stats->rx_missed_errors += 7117 le64_to_cpu(hw_stats->rx_discard_pkts); 7118 7119 stats->multicast += le64_to_cpu(hw_stats->rx_mcast_pkts); 7120 7121 stats->tx_dropped += le64_to_cpu(hw_stats->tx_drop_pkts); 7122 } 7123 7124 if (bp->flags & BNXT_FLAG_PORT_STATS) { 7125 struct rx_port_stats *rx = bp->hw_rx_port_stats; 7126 struct tx_port_stats *tx = bp->hw_tx_port_stats; 7127 7128 stats->rx_crc_errors = le64_to_cpu(rx->rx_fcs_err_frames); 7129 stats->rx_frame_errors = le64_to_cpu(rx->rx_align_err_frames); 7130 stats->rx_length_errors = le64_to_cpu(rx->rx_undrsz_frames) + 7131 le64_to_cpu(rx->rx_ovrsz_frames) + 7132 le64_to_cpu(rx->rx_runt_frames); 7133 stats->rx_errors = le64_to_cpu(rx->rx_false_carrier_frames) + 7134 le64_to_cpu(rx->rx_jbr_frames); 7135 stats->collisions = le64_to_cpu(tx->tx_total_collisions); 7136 stats->tx_fifo_errors = le64_to_cpu(tx->tx_fifo_underruns); 7137 stats->tx_errors = le64_to_cpu(tx->tx_err); 7138 } 7139 clear_bit(BNXT_STATE_READ_STATS, &bp->state); 7140 } 7141 7142 static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask) 7143 { 7144 struct net_device *dev = bp->dev; 7145 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 7146 struct netdev_hw_addr *ha; 7147 u8 *haddr; 7148 int mc_count = 0; 7149 bool update = false; 7150 int off = 0; 7151 7152 netdev_for_each_mc_addr(ha, dev) { 7153 if (mc_count >= BNXT_MAX_MC_ADDRS) { 7154 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 7155 vnic->mc_list_count = 0; 7156 return false; 7157 } 7158 haddr = ha->addr; 7159 if (!ether_addr_equal(haddr, vnic->mc_list + off)) { 7160 memcpy(vnic->mc_list + off, haddr, ETH_ALEN); 7161 update = true; 7162 } 7163 off += ETH_ALEN; 7164 mc_count++; 7165 } 7166 if (mc_count) 7167 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST; 7168 7169 if (mc_count != vnic->mc_list_count) { 7170 vnic->mc_list_count = mc_count; 7171 update = true; 7172 } 7173 return update; 7174 } 7175 7176 static bool bnxt_uc_list_updated(struct bnxt *bp) 7177 { 7178 struct net_device *dev = bp->dev; 7179 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 7180 struct netdev_hw_addr *ha; 7181 int off = 0; 7182 7183 if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1)) 7184 return true; 7185 7186 netdev_for_each_uc_addr(ha, dev) { 7187 if (!ether_addr_equal(ha->addr, vnic->uc_list + off)) 7188 return true; 7189 7190 off += ETH_ALEN; 7191 } 7192 return false; 7193 } 7194 7195 static void bnxt_set_rx_mode(struct net_device *dev) 7196 { 7197 struct bnxt *bp = netdev_priv(dev); 7198 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 7199 u32 mask = vnic->rx_mask; 7200 bool mc_update = false; 7201 bool uc_update; 7202 7203 if (!netif_running(dev)) 7204 return; 7205 7206 mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS | 7207 CFA_L2_SET_RX_MASK_REQ_MASK_MCAST | 7208 CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST | 7209 CFA_L2_SET_RX_MASK_REQ_MASK_BCAST); 7210 7211 if ((dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp)) 7212 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 7213 7214 uc_update = bnxt_uc_list_updated(bp); 7215 7216 if (dev->flags & IFF_BROADCAST) 7217 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST; 7218 if (dev->flags & IFF_ALLMULTI) { 7219 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 7220 vnic->mc_list_count = 0; 7221 } else { 7222 mc_update = bnxt_mc_list_updated(bp, &mask); 7223 } 7224 7225 if (mask != vnic->rx_mask || uc_update || mc_update) { 7226 vnic->rx_mask = mask; 7227 7228 set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event); 7229 bnxt_queue_sp_work(bp); 7230 } 7231 } 7232 7233 static int bnxt_cfg_rx_mode(struct bnxt *bp) 7234 { 7235 struct net_device *dev = bp->dev; 7236 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 7237 struct netdev_hw_addr *ha; 7238 int i, off = 0, rc; 7239 bool uc_update; 7240 7241 netif_addr_lock_bh(dev); 7242 uc_update = bnxt_uc_list_updated(bp); 7243 netif_addr_unlock_bh(dev); 7244 7245 if (!uc_update) 7246 goto skip_uc; 7247 7248 mutex_lock(&bp->hwrm_cmd_lock); 7249 for (i = 1; i < vnic->uc_filter_count; i++) { 7250 struct hwrm_cfa_l2_filter_free_input req = {0}; 7251 7252 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_FREE, -1, 7253 -1); 7254 7255 req.l2_filter_id = vnic->fw_l2_filter_id[i]; 7256 7257 rc = _hwrm_send_message(bp, &req, sizeof(req), 7258 HWRM_CMD_TIMEOUT); 7259 } 7260 mutex_unlock(&bp->hwrm_cmd_lock); 7261 7262 vnic->uc_filter_count = 1; 7263 7264 netif_addr_lock_bh(dev); 7265 if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) { 7266 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 7267 } else { 7268 netdev_for_each_uc_addr(ha, dev) { 7269 memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN); 7270 off += ETH_ALEN; 7271 vnic->uc_filter_count++; 7272 } 7273 } 7274 netif_addr_unlock_bh(dev); 7275 7276 for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) { 7277 rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off); 7278 if (rc) { 7279 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", 7280 rc); 7281 vnic->uc_filter_count = i; 7282 return rc; 7283 } 7284 } 7285 7286 skip_uc: 7287 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0); 7288 if (rc) 7289 netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %x\n", 7290 rc); 7291 7292 return rc; 7293 } 7294 7295 static bool bnxt_can_reserve_rings(struct bnxt *bp) 7296 { 7297 #ifdef CONFIG_BNXT_SRIOV 7298 if ((bp->flags & BNXT_FLAG_NEW_RM) && BNXT_VF(bp)) { 7299 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 7300 7301 /* No minimum rings were provisioned by the PF. Don't 7302 * reserve rings by default when device is down. 7303 */ 7304 if (hw_resc->min_tx_rings || hw_resc->resv_tx_rings) 7305 return true; 7306 7307 if (!netif_running(bp->dev)) 7308 return false; 7309 } 7310 #endif 7311 return true; 7312 } 7313 7314 /* If the chip and firmware supports RFS */ 7315 static bool bnxt_rfs_supported(struct bnxt *bp) 7316 { 7317 if (BNXT_PF(bp) && !BNXT_CHIP_TYPE_NITRO_A0(bp)) 7318 return true; 7319 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP) 7320 return true; 7321 return false; 7322 } 7323 7324 /* If runtime conditions support RFS */ 7325 static bool bnxt_rfs_capable(struct bnxt *bp) 7326 { 7327 #ifdef CONFIG_RFS_ACCEL 7328 int vnics, max_vnics, max_rss_ctxs; 7329 7330 if (!(bp->flags & BNXT_FLAG_MSIX_CAP) || !bnxt_can_reserve_rings(bp)) 7331 return false; 7332 7333 vnics = 1 + bp->rx_nr_rings; 7334 max_vnics = bnxt_get_max_func_vnics(bp); 7335 max_rss_ctxs = bnxt_get_max_func_rss_ctxs(bp); 7336 7337 /* RSS contexts not a limiting factor */ 7338 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP) 7339 max_rss_ctxs = max_vnics; 7340 if (vnics > max_vnics || vnics > max_rss_ctxs) { 7341 if (bp->rx_nr_rings > 1) 7342 netdev_warn(bp->dev, 7343 "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n", 7344 min(max_rss_ctxs - 1, max_vnics - 1)); 7345 return false; 7346 } 7347 7348 if (!(bp->flags & BNXT_FLAG_NEW_RM)) 7349 return true; 7350 7351 if (vnics == bp->hw_resc.resv_vnics) 7352 return true; 7353 7354 bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, vnics); 7355 if (vnics <= bp->hw_resc.resv_vnics) 7356 return true; 7357 7358 netdev_warn(bp->dev, "Unable to reserve resources to support NTUPLE filters.\n"); 7359 bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 1); 7360 return false; 7361 #else 7362 return false; 7363 #endif 7364 } 7365 7366 static netdev_features_t bnxt_fix_features(struct net_device *dev, 7367 netdev_features_t features) 7368 { 7369 struct bnxt *bp = netdev_priv(dev); 7370 7371 if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp)) 7372 features &= ~NETIF_F_NTUPLE; 7373 7374 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS) 7375 features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); 7376 7377 if (!(features & NETIF_F_GRO)) 7378 features &= ~NETIF_F_GRO_HW; 7379 7380 if (features & NETIF_F_GRO_HW) 7381 features &= ~NETIF_F_LRO; 7382 7383 /* Both CTAG and STAG VLAN accelaration on the RX side have to be 7384 * turned on or off together. 7385 */ 7386 if ((features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)) != 7387 (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)) { 7388 if (dev->features & NETIF_F_HW_VLAN_CTAG_RX) 7389 features &= ~(NETIF_F_HW_VLAN_CTAG_RX | 7390 NETIF_F_HW_VLAN_STAG_RX); 7391 else 7392 features |= NETIF_F_HW_VLAN_CTAG_RX | 7393 NETIF_F_HW_VLAN_STAG_RX; 7394 } 7395 #ifdef CONFIG_BNXT_SRIOV 7396 if (BNXT_VF(bp)) { 7397 if (bp->vf.vlan) { 7398 features &= ~(NETIF_F_HW_VLAN_CTAG_RX | 7399 NETIF_F_HW_VLAN_STAG_RX); 7400 } 7401 } 7402 #endif 7403 return features; 7404 } 7405 7406 static int bnxt_set_features(struct net_device *dev, netdev_features_t features) 7407 { 7408 struct bnxt *bp = netdev_priv(dev); 7409 u32 flags = bp->flags; 7410 u32 changes; 7411 int rc = 0; 7412 bool re_init = false; 7413 bool update_tpa = false; 7414 7415 flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS; 7416 if (features & NETIF_F_GRO_HW) 7417 flags |= BNXT_FLAG_GRO; 7418 else if (features & NETIF_F_LRO) 7419 flags |= BNXT_FLAG_LRO; 7420 7421 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS) 7422 flags &= ~BNXT_FLAG_TPA; 7423 7424 if (features & NETIF_F_HW_VLAN_CTAG_RX) 7425 flags |= BNXT_FLAG_STRIP_VLAN; 7426 7427 if (features & NETIF_F_NTUPLE) 7428 flags |= BNXT_FLAG_RFS; 7429 7430 changes = flags ^ bp->flags; 7431 if (changes & BNXT_FLAG_TPA) { 7432 update_tpa = true; 7433 if ((bp->flags & BNXT_FLAG_TPA) == 0 || 7434 (flags & BNXT_FLAG_TPA) == 0) 7435 re_init = true; 7436 } 7437 7438 if (changes & ~BNXT_FLAG_TPA) 7439 re_init = true; 7440 7441 if (flags != bp->flags) { 7442 u32 old_flags = bp->flags; 7443 7444 bp->flags = flags; 7445 7446 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 7447 if (update_tpa) 7448 bnxt_set_ring_params(bp); 7449 return rc; 7450 } 7451 7452 if (re_init) { 7453 bnxt_close_nic(bp, false, false); 7454 if (update_tpa) 7455 bnxt_set_ring_params(bp); 7456 7457 return bnxt_open_nic(bp, false, false); 7458 } 7459 if (update_tpa) { 7460 rc = bnxt_set_tpa(bp, 7461 (flags & BNXT_FLAG_TPA) ? 7462 true : false); 7463 if (rc) 7464 bp->flags = old_flags; 7465 } 7466 } 7467 return rc; 7468 } 7469 7470 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi) 7471 { 7472 struct bnxt_tx_ring_info *txr = bnapi->tx_ring; 7473 int i = bnapi->index; 7474 7475 if (!txr) 7476 return; 7477 7478 netdev_info(bnapi->bp->dev, "[%d]: tx{fw_ring: %d prod: %x cons: %x}\n", 7479 i, txr->tx_ring_struct.fw_ring_id, txr->tx_prod, 7480 txr->tx_cons); 7481 } 7482 7483 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi) 7484 { 7485 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 7486 int i = bnapi->index; 7487 7488 if (!rxr) 7489 return; 7490 7491 netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n", 7492 i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod, 7493 rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod, 7494 rxr->rx_sw_agg_prod); 7495 } 7496 7497 static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi) 7498 { 7499 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 7500 int i = bnapi->index; 7501 7502 netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n", 7503 i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons); 7504 } 7505 7506 static void bnxt_dbg_dump_states(struct bnxt *bp) 7507 { 7508 int i; 7509 struct bnxt_napi *bnapi; 7510 7511 for (i = 0; i < bp->cp_nr_rings; i++) { 7512 bnapi = bp->bnapi[i]; 7513 if (netif_msg_drv(bp)) { 7514 bnxt_dump_tx_sw_state(bnapi); 7515 bnxt_dump_rx_sw_state(bnapi); 7516 bnxt_dump_cp_sw_state(bnapi); 7517 } 7518 } 7519 } 7520 7521 static void bnxt_reset_task(struct bnxt *bp, bool silent) 7522 { 7523 if (!silent) 7524 bnxt_dbg_dump_states(bp); 7525 if (netif_running(bp->dev)) { 7526 int rc; 7527 7528 if (!silent) 7529 bnxt_ulp_stop(bp); 7530 bnxt_close_nic(bp, false, false); 7531 rc = bnxt_open_nic(bp, false, false); 7532 if (!silent && !rc) 7533 bnxt_ulp_start(bp); 7534 } 7535 } 7536 7537 static void bnxt_tx_timeout(struct net_device *dev) 7538 { 7539 struct bnxt *bp = netdev_priv(dev); 7540 7541 netdev_err(bp->dev, "TX timeout detected, starting reset task!\n"); 7542 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event); 7543 bnxt_queue_sp_work(bp); 7544 } 7545 7546 #ifdef CONFIG_NET_POLL_CONTROLLER 7547 static void bnxt_poll_controller(struct net_device *dev) 7548 { 7549 struct bnxt *bp = netdev_priv(dev); 7550 int i; 7551 7552 /* Only process tx rings/combined rings in netpoll mode. */ 7553 for (i = 0; i < bp->tx_nr_rings; i++) { 7554 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 7555 7556 napi_schedule(&txr->bnapi->napi); 7557 } 7558 } 7559 #endif 7560 7561 static void bnxt_timer(struct timer_list *t) 7562 { 7563 struct bnxt *bp = from_timer(bp, t, timer); 7564 struct net_device *dev = bp->dev; 7565 7566 if (!netif_running(dev)) 7567 return; 7568 7569 if (atomic_read(&bp->intr_sem) != 0) 7570 goto bnxt_restart_timer; 7571 7572 if (bp->link_info.link_up && (bp->flags & BNXT_FLAG_PORT_STATS) && 7573 bp->stats_coal_ticks) { 7574 set_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event); 7575 bnxt_queue_sp_work(bp); 7576 } 7577 7578 if (bnxt_tc_flower_enabled(bp)) { 7579 set_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event); 7580 bnxt_queue_sp_work(bp); 7581 } 7582 bnxt_restart_timer: 7583 mod_timer(&bp->timer, jiffies + bp->current_interval); 7584 } 7585 7586 static void bnxt_rtnl_lock_sp(struct bnxt *bp) 7587 { 7588 /* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK 7589 * set. If the device is being closed, bnxt_close() may be holding 7590 * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear. So we 7591 * must clear BNXT_STATE_IN_SP_TASK before holding rtnl(). 7592 */ 7593 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 7594 rtnl_lock(); 7595 } 7596 7597 static void bnxt_rtnl_unlock_sp(struct bnxt *bp) 7598 { 7599 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 7600 rtnl_unlock(); 7601 } 7602 7603 /* Only called from bnxt_sp_task() */ 7604 static void bnxt_reset(struct bnxt *bp, bool silent) 7605 { 7606 bnxt_rtnl_lock_sp(bp); 7607 if (test_bit(BNXT_STATE_OPEN, &bp->state)) 7608 bnxt_reset_task(bp, silent); 7609 bnxt_rtnl_unlock_sp(bp); 7610 } 7611 7612 static void bnxt_cfg_ntp_filters(struct bnxt *); 7613 7614 static void bnxt_sp_task(struct work_struct *work) 7615 { 7616 struct bnxt *bp = container_of(work, struct bnxt, sp_task); 7617 7618 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 7619 smp_mb__after_atomic(); 7620 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 7621 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 7622 return; 7623 } 7624 7625 if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event)) 7626 bnxt_cfg_rx_mode(bp); 7627 7628 if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event)) 7629 bnxt_cfg_ntp_filters(bp); 7630 if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event)) 7631 bnxt_hwrm_exec_fwd_req(bp); 7632 if (test_and_clear_bit(BNXT_VXLAN_ADD_PORT_SP_EVENT, &bp->sp_event)) { 7633 bnxt_hwrm_tunnel_dst_port_alloc( 7634 bp, bp->vxlan_port, 7635 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN); 7636 } 7637 if (test_and_clear_bit(BNXT_VXLAN_DEL_PORT_SP_EVENT, &bp->sp_event)) { 7638 bnxt_hwrm_tunnel_dst_port_free( 7639 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN); 7640 } 7641 if (test_and_clear_bit(BNXT_GENEVE_ADD_PORT_SP_EVENT, &bp->sp_event)) { 7642 bnxt_hwrm_tunnel_dst_port_alloc( 7643 bp, bp->nge_port, 7644 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE); 7645 } 7646 if (test_and_clear_bit(BNXT_GENEVE_DEL_PORT_SP_EVENT, &bp->sp_event)) { 7647 bnxt_hwrm_tunnel_dst_port_free( 7648 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE); 7649 } 7650 if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event)) { 7651 bnxt_hwrm_port_qstats(bp); 7652 bnxt_hwrm_port_qstats_ext(bp); 7653 } 7654 7655 if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) { 7656 int rc; 7657 7658 mutex_lock(&bp->link_lock); 7659 if (test_and_clear_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, 7660 &bp->sp_event)) 7661 bnxt_hwrm_phy_qcaps(bp); 7662 7663 rc = bnxt_update_link(bp, true); 7664 mutex_unlock(&bp->link_lock); 7665 if (rc) 7666 netdev_err(bp->dev, "SP task can't update link (rc: %x)\n", 7667 rc); 7668 } 7669 if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event)) { 7670 mutex_lock(&bp->link_lock); 7671 bnxt_get_port_module_status(bp); 7672 mutex_unlock(&bp->link_lock); 7673 } 7674 7675 if (test_and_clear_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event)) 7676 bnxt_tc_flow_stats_work(bp); 7677 7678 /* These functions below will clear BNXT_STATE_IN_SP_TASK. They 7679 * must be the last functions to be called before exiting. 7680 */ 7681 if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event)) 7682 bnxt_reset(bp, false); 7683 7684 if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event)) 7685 bnxt_reset(bp, true); 7686 7687 smp_mb__before_atomic(); 7688 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 7689 } 7690 7691 /* Under rtnl_lock */ 7692 int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs, 7693 int tx_xdp) 7694 { 7695 int max_rx, max_tx, tx_sets = 1; 7696 int tx_rings_needed; 7697 int rx_rings = rx; 7698 int cp, vnics, rc; 7699 7700 if (tcs) 7701 tx_sets = tcs; 7702 7703 rc = bnxt_get_max_rings(bp, &max_rx, &max_tx, sh); 7704 if (rc) 7705 return rc; 7706 7707 if (max_rx < rx) 7708 return -ENOMEM; 7709 7710 tx_rings_needed = tx * tx_sets + tx_xdp; 7711 if (max_tx < tx_rings_needed) 7712 return -ENOMEM; 7713 7714 vnics = 1; 7715 if (bp->flags & BNXT_FLAG_RFS) 7716 vnics += rx_rings; 7717 7718 if (bp->flags & BNXT_FLAG_AGG_RINGS) 7719 rx_rings <<= 1; 7720 cp = sh ? max_t(int, tx_rings_needed, rx) : tx_rings_needed + rx; 7721 if (bp->flags & BNXT_FLAG_NEW_RM) 7722 cp += bnxt_get_ulp_msix_num(bp); 7723 return bnxt_hwrm_check_rings(bp, tx_rings_needed, rx_rings, rx, cp, 7724 vnics); 7725 } 7726 7727 static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev) 7728 { 7729 if (bp->bar2) { 7730 pci_iounmap(pdev, bp->bar2); 7731 bp->bar2 = NULL; 7732 } 7733 7734 if (bp->bar1) { 7735 pci_iounmap(pdev, bp->bar1); 7736 bp->bar1 = NULL; 7737 } 7738 7739 if (bp->bar0) { 7740 pci_iounmap(pdev, bp->bar0); 7741 bp->bar0 = NULL; 7742 } 7743 } 7744 7745 static void bnxt_cleanup_pci(struct bnxt *bp) 7746 { 7747 bnxt_unmap_bars(bp, bp->pdev); 7748 pci_release_regions(bp->pdev); 7749 pci_disable_device(bp->pdev); 7750 } 7751 7752 static void bnxt_init_dflt_coal(struct bnxt *bp) 7753 { 7754 struct bnxt_coal *coal; 7755 7756 /* Tick values in micro seconds. 7757 * 1 coal_buf x bufs_per_record = 1 completion record. 7758 */ 7759 coal = &bp->rx_coal; 7760 coal->coal_ticks = 14; 7761 coal->coal_bufs = 30; 7762 coal->coal_ticks_irq = 1; 7763 coal->coal_bufs_irq = 2; 7764 coal->idle_thresh = 50; 7765 coal->bufs_per_record = 2; 7766 coal->budget = 64; /* NAPI budget */ 7767 7768 coal = &bp->tx_coal; 7769 coal->coal_ticks = 28; 7770 coal->coal_bufs = 30; 7771 coal->coal_ticks_irq = 2; 7772 coal->coal_bufs_irq = 2; 7773 coal->bufs_per_record = 1; 7774 7775 bp->stats_coal_ticks = BNXT_DEF_STATS_COAL_TICKS; 7776 } 7777 7778 static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev) 7779 { 7780 int rc; 7781 struct bnxt *bp = netdev_priv(dev); 7782 7783 SET_NETDEV_DEV(dev, &pdev->dev); 7784 7785 /* enable device (incl. PCI PM wakeup), and bus-mastering */ 7786 rc = pci_enable_device(pdev); 7787 if (rc) { 7788 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n"); 7789 goto init_err; 7790 } 7791 7792 if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) { 7793 dev_err(&pdev->dev, 7794 "Cannot find PCI device base address, aborting\n"); 7795 rc = -ENODEV; 7796 goto init_err_disable; 7797 } 7798 7799 rc = pci_request_regions(pdev, DRV_MODULE_NAME); 7800 if (rc) { 7801 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n"); 7802 goto init_err_disable; 7803 } 7804 7805 if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 && 7806 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) { 7807 dev_err(&pdev->dev, "System does not support DMA, aborting\n"); 7808 goto init_err_disable; 7809 } 7810 7811 pci_set_master(pdev); 7812 7813 bp->dev = dev; 7814 bp->pdev = pdev; 7815 7816 bp->bar0 = pci_ioremap_bar(pdev, 0); 7817 if (!bp->bar0) { 7818 dev_err(&pdev->dev, "Cannot map device registers, aborting\n"); 7819 rc = -ENOMEM; 7820 goto init_err_release; 7821 } 7822 7823 bp->bar1 = pci_ioremap_bar(pdev, 2); 7824 if (!bp->bar1) { 7825 dev_err(&pdev->dev, "Cannot map doorbell registers, aborting\n"); 7826 rc = -ENOMEM; 7827 goto init_err_release; 7828 } 7829 7830 bp->bar2 = pci_ioremap_bar(pdev, 4); 7831 if (!bp->bar2) { 7832 dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n"); 7833 rc = -ENOMEM; 7834 goto init_err_release; 7835 } 7836 7837 pci_enable_pcie_error_reporting(pdev); 7838 7839 INIT_WORK(&bp->sp_task, bnxt_sp_task); 7840 7841 spin_lock_init(&bp->ntp_fltr_lock); 7842 7843 bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE; 7844 bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE; 7845 7846 bnxt_init_dflt_coal(bp); 7847 7848 timer_setup(&bp->timer, bnxt_timer, 0); 7849 bp->current_interval = BNXT_TIMER_INTERVAL; 7850 7851 clear_bit(BNXT_STATE_OPEN, &bp->state); 7852 return 0; 7853 7854 init_err_release: 7855 bnxt_unmap_bars(bp, pdev); 7856 pci_release_regions(pdev); 7857 7858 init_err_disable: 7859 pci_disable_device(pdev); 7860 7861 init_err: 7862 return rc; 7863 } 7864 7865 /* rtnl_lock held */ 7866 static int bnxt_change_mac_addr(struct net_device *dev, void *p) 7867 { 7868 struct sockaddr *addr = p; 7869 struct bnxt *bp = netdev_priv(dev); 7870 int rc = 0; 7871 7872 if (!is_valid_ether_addr(addr->sa_data)) 7873 return -EADDRNOTAVAIL; 7874 7875 if (ether_addr_equal(addr->sa_data, dev->dev_addr)) 7876 return 0; 7877 7878 rc = bnxt_approve_mac(bp, addr->sa_data); 7879 if (rc) 7880 return rc; 7881 7882 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len); 7883 if (netif_running(dev)) { 7884 bnxt_close_nic(bp, false, false); 7885 rc = bnxt_open_nic(bp, false, false); 7886 } 7887 7888 return rc; 7889 } 7890 7891 /* rtnl_lock held */ 7892 static int bnxt_change_mtu(struct net_device *dev, int new_mtu) 7893 { 7894 struct bnxt *bp = netdev_priv(dev); 7895 7896 if (netif_running(dev)) 7897 bnxt_close_nic(bp, false, false); 7898 7899 dev->mtu = new_mtu; 7900 bnxt_set_ring_params(bp); 7901 7902 if (netif_running(dev)) 7903 return bnxt_open_nic(bp, false, false); 7904 7905 return 0; 7906 } 7907 7908 int bnxt_setup_mq_tc(struct net_device *dev, u8 tc) 7909 { 7910 struct bnxt *bp = netdev_priv(dev); 7911 bool sh = false; 7912 int rc; 7913 7914 if (tc > bp->max_tc) { 7915 netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n", 7916 tc, bp->max_tc); 7917 return -EINVAL; 7918 } 7919 7920 if (netdev_get_num_tc(dev) == tc) 7921 return 0; 7922 7923 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 7924 sh = true; 7925 7926 rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings, 7927 sh, tc, bp->tx_nr_rings_xdp); 7928 if (rc) 7929 return rc; 7930 7931 /* Needs to close the device and do hw resource re-allocations */ 7932 if (netif_running(bp->dev)) 7933 bnxt_close_nic(bp, true, false); 7934 7935 if (tc) { 7936 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc; 7937 netdev_set_num_tc(dev, tc); 7938 } else { 7939 bp->tx_nr_rings = bp->tx_nr_rings_per_tc; 7940 netdev_reset_tc(dev); 7941 } 7942 bp->tx_nr_rings += bp->tx_nr_rings_xdp; 7943 bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) : 7944 bp->tx_nr_rings + bp->rx_nr_rings; 7945 bp->num_stat_ctxs = bp->cp_nr_rings; 7946 7947 if (netif_running(bp->dev)) 7948 return bnxt_open_nic(bp, true, false); 7949 7950 return 0; 7951 } 7952 7953 static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 7954 void *cb_priv) 7955 { 7956 struct bnxt *bp = cb_priv; 7957 7958 if (!bnxt_tc_flower_enabled(bp) || 7959 !tc_cls_can_offload_and_chain0(bp->dev, type_data)) 7960 return -EOPNOTSUPP; 7961 7962 switch (type) { 7963 case TC_SETUP_CLSFLOWER: 7964 return bnxt_tc_setup_flower(bp, bp->pf.fw_fid, type_data); 7965 default: 7966 return -EOPNOTSUPP; 7967 } 7968 } 7969 7970 static int bnxt_setup_tc_block(struct net_device *dev, 7971 struct tc_block_offload *f) 7972 { 7973 struct bnxt *bp = netdev_priv(dev); 7974 7975 if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS) 7976 return -EOPNOTSUPP; 7977 7978 switch (f->command) { 7979 case TC_BLOCK_BIND: 7980 return tcf_block_cb_register(f->block, bnxt_setup_tc_block_cb, 7981 bp, bp, f->extack); 7982 case TC_BLOCK_UNBIND: 7983 tcf_block_cb_unregister(f->block, bnxt_setup_tc_block_cb, bp); 7984 return 0; 7985 default: 7986 return -EOPNOTSUPP; 7987 } 7988 } 7989 7990 static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type, 7991 void *type_data) 7992 { 7993 switch (type) { 7994 case TC_SETUP_BLOCK: 7995 return bnxt_setup_tc_block(dev, type_data); 7996 case TC_SETUP_QDISC_MQPRIO: { 7997 struct tc_mqprio_qopt *mqprio = type_data; 7998 7999 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 8000 8001 return bnxt_setup_mq_tc(dev, mqprio->num_tc); 8002 } 8003 default: 8004 return -EOPNOTSUPP; 8005 } 8006 } 8007 8008 #ifdef CONFIG_RFS_ACCEL 8009 static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1, 8010 struct bnxt_ntuple_filter *f2) 8011 { 8012 struct flow_keys *keys1 = &f1->fkeys; 8013 struct flow_keys *keys2 = &f2->fkeys; 8014 8015 if (keys1->addrs.v4addrs.src == keys2->addrs.v4addrs.src && 8016 keys1->addrs.v4addrs.dst == keys2->addrs.v4addrs.dst && 8017 keys1->ports.ports == keys2->ports.ports && 8018 keys1->basic.ip_proto == keys2->basic.ip_proto && 8019 keys1->basic.n_proto == keys2->basic.n_proto && 8020 keys1->control.flags == keys2->control.flags && 8021 ether_addr_equal(f1->src_mac_addr, f2->src_mac_addr) && 8022 ether_addr_equal(f1->dst_mac_addr, f2->dst_mac_addr)) 8023 return true; 8024 8025 return false; 8026 } 8027 8028 static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb, 8029 u16 rxq_index, u32 flow_id) 8030 { 8031 struct bnxt *bp = netdev_priv(dev); 8032 struct bnxt_ntuple_filter *fltr, *new_fltr; 8033 struct flow_keys *fkeys; 8034 struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb); 8035 int rc = 0, idx, bit_id, l2_idx = 0; 8036 struct hlist_head *head; 8037 8038 if (!ether_addr_equal(dev->dev_addr, eth->h_dest)) { 8039 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 8040 int off = 0, j; 8041 8042 netif_addr_lock_bh(dev); 8043 for (j = 0; j < vnic->uc_filter_count; j++, off += ETH_ALEN) { 8044 if (ether_addr_equal(eth->h_dest, 8045 vnic->uc_list + off)) { 8046 l2_idx = j + 1; 8047 break; 8048 } 8049 } 8050 netif_addr_unlock_bh(dev); 8051 if (!l2_idx) 8052 return -EINVAL; 8053 } 8054 new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC); 8055 if (!new_fltr) 8056 return -ENOMEM; 8057 8058 fkeys = &new_fltr->fkeys; 8059 if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) { 8060 rc = -EPROTONOSUPPORT; 8061 goto err_free; 8062 } 8063 8064 if ((fkeys->basic.n_proto != htons(ETH_P_IP) && 8065 fkeys->basic.n_proto != htons(ETH_P_IPV6)) || 8066 ((fkeys->basic.ip_proto != IPPROTO_TCP) && 8067 (fkeys->basic.ip_proto != IPPROTO_UDP))) { 8068 rc = -EPROTONOSUPPORT; 8069 goto err_free; 8070 } 8071 if (fkeys->basic.n_proto == htons(ETH_P_IPV6) && 8072 bp->hwrm_spec_code < 0x10601) { 8073 rc = -EPROTONOSUPPORT; 8074 goto err_free; 8075 } 8076 if ((fkeys->control.flags & FLOW_DIS_ENCAPSULATION) && 8077 bp->hwrm_spec_code < 0x10601) { 8078 rc = -EPROTONOSUPPORT; 8079 goto err_free; 8080 } 8081 8082 memcpy(new_fltr->dst_mac_addr, eth->h_dest, ETH_ALEN); 8083 memcpy(new_fltr->src_mac_addr, eth->h_source, ETH_ALEN); 8084 8085 idx = skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK; 8086 head = &bp->ntp_fltr_hash_tbl[idx]; 8087 rcu_read_lock(); 8088 hlist_for_each_entry_rcu(fltr, head, hash) { 8089 if (bnxt_fltr_match(fltr, new_fltr)) { 8090 rcu_read_unlock(); 8091 rc = 0; 8092 goto err_free; 8093 } 8094 } 8095 rcu_read_unlock(); 8096 8097 spin_lock_bh(&bp->ntp_fltr_lock); 8098 bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap, 8099 BNXT_NTP_FLTR_MAX_FLTR, 0); 8100 if (bit_id < 0) { 8101 spin_unlock_bh(&bp->ntp_fltr_lock); 8102 rc = -ENOMEM; 8103 goto err_free; 8104 } 8105 8106 new_fltr->sw_id = (u16)bit_id; 8107 new_fltr->flow_id = flow_id; 8108 new_fltr->l2_fltr_idx = l2_idx; 8109 new_fltr->rxq = rxq_index; 8110 hlist_add_head_rcu(&new_fltr->hash, head); 8111 bp->ntp_fltr_count++; 8112 spin_unlock_bh(&bp->ntp_fltr_lock); 8113 8114 set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event); 8115 bnxt_queue_sp_work(bp); 8116 8117 return new_fltr->sw_id; 8118 8119 err_free: 8120 kfree(new_fltr); 8121 return rc; 8122 } 8123 8124 static void bnxt_cfg_ntp_filters(struct bnxt *bp) 8125 { 8126 int i; 8127 8128 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) { 8129 struct hlist_head *head; 8130 struct hlist_node *tmp; 8131 struct bnxt_ntuple_filter *fltr; 8132 int rc; 8133 8134 head = &bp->ntp_fltr_hash_tbl[i]; 8135 hlist_for_each_entry_safe(fltr, tmp, head, hash) { 8136 bool del = false; 8137 8138 if (test_bit(BNXT_FLTR_VALID, &fltr->state)) { 8139 if (rps_may_expire_flow(bp->dev, fltr->rxq, 8140 fltr->flow_id, 8141 fltr->sw_id)) { 8142 bnxt_hwrm_cfa_ntuple_filter_free(bp, 8143 fltr); 8144 del = true; 8145 } 8146 } else { 8147 rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp, 8148 fltr); 8149 if (rc) 8150 del = true; 8151 else 8152 set_bit(BNXT_FLTR_VALID, &fltr->state); 8153 } 8154 8155 if (del) { 8156 spin_lock_bh(&bp->ntp_fltr_lock); 8157 hlist_del_rcu(&fltr->hash); 8158 bp->ntp_fltr_count--; 8159 spin_unlock_bh(&bp->ntp_fltr_lock); 8160 synchronize_rcu(); 8161 clear_bit(fltr->sw_id, bp->ntp_fltr_bmap); 8162 kfree(fltr); 8163 } 8164 } 8165 } 8166 if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event)) 8167 netdev_info(bp->dev, "Receive PF driver unload event!"); 8168 } 8169 8170 #else 8171 8172 static void bnxt_cfg_ntp_filters(struct bnxt *bp) 8173 { 8174 } 8175 8176 #endif /* CONFIG_RFS_ACCEL */ 8177 8178 static void bnxt_udp_tunnel_add(struct net_device *dev, 8179 struct udp_tunnel_info *ti) 8180 { 8181 struct bnxt *bp = netdev_priv(dev); 8182 8183 if (ti->sa_family != AF_INET6 && ti->sa_family != AF_INET) 8184 return; 8185 8186 if (!netif_running(dev)) 8187 return; 8188 8189 switch (ti->type) { 8190 case UDP_TUNNEL_TYPE_VXLAN: 8191 if (bp->vxlan_port_cnt && bp->vxlan_port != ti->port) 8192 return; 8193 8194 bp->vxlan_port_cnt++; 8195 if (bp->vxlan_port_cnt == 1) { 8196 bp->vxlan_port = ti->port; 8197 set_bit(BNXT_VXLAN_ADD_PORT_SP_EVENT, &bp->sp_event); 8198 bnxt_queue_sp_work(bp); 8199 } 8200 break; 8201 case UDP_TUNNEL_TYPE_GENEVE: 8202 if (bp->nge_port_cnt && bp->nge_port != ti->port) 8203 return; 8204 8205 bp->nge_port_cnt++; 8206 if (bp->nge_port_cnt == 1) { 8207 bp->nge_port = ti->port; 8208 set_bit(BNXT_GENEVE_ADD_PORT_SP_EVENT, &bp->sp_event); 8209 } 8210 break; 8211 default: 8212 return; 8213 } 8214 8215 bnxt_queue_sp_work(bp); 8216 } 8217 8218 static void bnxt_udp_tunnel_del(struct net_device *dev, 8219 struct udp_tunnel_info *ti) 8220 { 8221 struct bnxt *bp = netdev_priv(dev); 8222 8223 if (ti->sa_family != AF_INET6 && ti->sa_family != AF_INET) 8224 return; 8225 8226 if (!netif_running(dev)) 8227 return; 8228 8229 switch (ti->type) { 8230 case UDP_TUNNEL_TYPE_VXLAN: 8231 if (!bp->vxlan_port_cnt || bp->vxlan_port != ti->port) 8232 return; 8233 bp->vxlan_port_cnt--; 8234 8235 if (bp->vxlan_port_cnt != 0) 8236 return; 8237 8238 set_bit(BNXT_VXLAN_DEL_PORT_SP_EVENT, &bp->sp_event); 8239 break; 8240 case UDP_TUNNEL_TYPE_GENEVE: 8241 if (!bp->nge_port_cnt || bp->nge_port != ti->port) 8242 return; 8243 bp->nge_port_cnt--; 8244 8245 if (bp->nge_port_cnt != 0) 8246 return; 8247 8248 set_bit(BNXT_GENEVE_DEL_PORT_SP_EVENT, &bp->sp_event); 8249 break; 8250 default: 8251 return; 8252 } 8253 8254 bnxt_queue_sp_work(bp); 8255 } 8256 8257 static int bnxt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 8258 struct net_device *dev, u32 filter_mask, 8259 int nlflags) 8260 { 8261 struct bnxt *bp = netdev_priv(dev); 8262 8263 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bp->br_mode, 0, 0, 8264 nlflags, filter_mask, NULL); 8265 } 8266 8267 static int bnxt_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh, 8268 u16 flags) 8269 { 8270 struct bnxt *bp = netdev_priv(dev); 8271 struct nlattr *attr, *br_spec; 8272 int rem, rc = 0; 8273 8274 if (bp->hwrm_spec_code < 0x10708 || !BNXT_SINGLE_PF(bp)) 8275 return -EOPNOTSUPP; 8276 8277 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 8278 if (!br_spec) 8279 return -EINVAL; 8280 8281 nla_for_each_nested(attr, br_spec, rem) { 8282 u16 mode; 8283 8284 if (nla_type(attr) != IFLA_BRIDGE_MODE) 8285 continue; 8286 8287 if (nla_len(attr) < sizeof(mode)) 8288 return -EINVAL; 8289 8290 mode = nla_get_u16(attr); 8291 if (mode == bp->br_mode) 8292 break; 8293 8294 rc = bnxt_hwrm_set_br_mode(bp, mode); 8295 if (!rc) 8296 bp->br_mode = mode; 8297 break; 8298 } 8299 return rc; 8300 } 8301 8302 static int bnxt_get_phys_port_name(struct net_device *dev, char *buf, 8303 size_t len) 8304 { 8305 struct bnxt *bp = netdev_priv(dev); 8306 int rc; 8307 8308 /* The PF and it's VF-reps only support the switchdev framework */ 8309 if (!BNXT_PF(bp)) 8310 return -EOPNOTSUPP; 8311 8312 rc = snprintf(buf, len, "p%d", bp->pf.port_id); 8313 8314 if (rc >= len) 8315 return -EOPNOTSUPP; 8316 return 0; 8317 } 8318 8319 int bnxt_port_attr_get(struct bnxt *bp, struct switchdev_attr *attr) 8320 { 8321 if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV) 8322 return -EOPNOTSUPP; 8323 8324 /* The PF and it's VF-reps only support the switchdev framework */ 8325 if (!BNXT_PF(bp)) 8326 return -EOPNOTSUPP; 8327 8328 switch (attr->id) { 8329 case SWITCHDEV_ATTR_ID_PORT_PARENT_ID: 8330 attr->u.ppid.id_len = sizeof(bp->switch_id); 8331 memcpy(attr->u.ppid.id, bp->switch_id, attr->u.ppid.id_len); 8332 break; 8333 default: 8334 return -EOPNOTSUPP; 8335 } 8336 return 0; 8337 } 8338 8339 static int bnxt_swdev_port_attr_get(struct net_device *dev, 8340 struct switchdev_attr *attr) 8341 { 8342 return bnxt_port_attr_get(netdev_priv(dev), attr); 8343 } 8344 8345 static const struct switchdev_ops bnxt_switchdev_ops = { 8346 .switchdev_port_attr_get = bnxt_swdev_port_attr_get 8347 }; 8348 8349 static const struct net_device_ops bnxt_netdev_ops = { 8350 .ndo_open = bnxt_open, 8351 .ndo_start_xmit = bnxt_start_xmit, 8352 .ndo_stop = bnxt_close, 8353 .ndo_get_stats64 = bnxt_get_stats64, 8354 .ndo_set_rx_mode = bnxt_set_rx_mode, 8355 .ndo_do_ioctl = bnxt_ioctl, 8356 .ndo_validate_addr = eth_validate_addr, 8357 .ndo_set_mac_address = bnxt_change_mac_addr, 8358 .ndo_change_mtu = bnxt_change_mtu, 8359 .ndo_fix_features = bnxt_fix_features, 8360 .ndo_set_features = bnxt_set_features, 8361 .ndo_tx_timeout = bnxt_tx_timeout, 8362 #ifdef CONFIG_BNXT_SRIOV 8363 .ndo_get_vf_config = bnxt_get_vf_config, 8364 .ndo_set_vf_mac = bnxt_set_vf_mac, 8365 .ndo_set_vf_vlan = bnxt_set_vf_vlan, 8366 .ndo_set_vf_rate = bnxt_set_vf_bw, 8367 .ndo_set_vf_link_state = bnxt_set_vf_link_state, 8368 .ndo_set_vf_spoofchk = bnxt_set_vf_spoofchk, 8369 .ndo_set_vf_trust = bnxt_set_vf_trust, 8370 #endif 8371 #ifdef CONFIG_NET_POLL_CONTROLLER 8372 .ndo_poll_controller = bnxt_poll_controller, 8373 #endif 8374 .ndo_setup_tc = bnxt_setup_tc, 8375 #ifdef CONFIG_RFS_ACCEL 8376 .ndo_rx_flow_steer = bnxt_rx_flow_steer, 8377 #endif 8378 .ndo_udp_tunnel_add = bnxt_udp_tunnel_add, 8379 .ndo_udp_tunnel_del = bnxt_udp_tunnel_del, 8380 .ndo_bpf = bnxt_xdp, 8381 .ndo_bridge_getlink = bnxt_bridge_getlink, 8382 .ndo_bridge_setlink = bnxt_bridge_setlink, 8383 .ndo_get_phys_port_name = bnxt_get_phys_port_name 8384 }; 8385 8386 static void bnxt_remove_one(struct pci_dev *pdev) 8387 { 8388 struct net_device *dev = pci_get_drvdata(pdev); 8389 struct bnxt *bp = netdev_priv(dev); 8390 8391 if (BNXT_PF(bp)) { 8392 bnxt_sriov_disable(bp); 8393 bnxt_dl_unregister(bp); 8394 } 8395 8396 pci_disable_pcie_error_reporting(pdev); 8397 unregister_netdev(dev); 8398 bnxt_shutdown_tc(bp); 8399 bnxt_cancel_sp_work(bp); 8400 bp->sp_event = 0; 8401 8402 bnxt_clear_int_mode(bp); 8403 bnxt_hwrm_func_drv_unrgtr(bp); 8404 bnxt_free_hwrm_resources(bp); 8405 bnxt_free_hwrm_short_cmd_req(bp); 8406 bnxt_ethtool_free(bp); 8407 bnxt_dcb_free(bp); 8408 kfree(bp->edev); 8409 bp->edev = NULL; 8410 bnxt_cleanup_pci(bp); 8411 free_netdev(dev); 8412 } 8413 8414 static int bnxt_probe_phy(struct bnxt *bp) 8415 { 8416 int rc = 0; 8417 struct bnxt_link_info *link_info = &bp->link_info; 8418 8419 rc = bnxt_hwrm_phy_qcaps(bp); 8420 if (rc) { 8421 netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n", 8422 rc); 8423 return rc; 8424 } 8425 mutex_init(&bp->link_lock); 8426 8427 rc = bnxt_update_link(bp, false); 8428 if (rc) { 8429 netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n", 8430 rc); 8431 return rc; 8432 } 8433 8434 /* Older firmware does not have supported_auto_speeds, so assume 8435 * that all supported speeds can be autonegotiated. 8436 */ 8437 if (link_info->auto_link_speeds && !link_info->support_auto_speeds) 8438 link_info->support_auto_speeds = link_info->support_speeds; 8439 8440 /*initialize the ethool setting copy with NVM settings */ 8441 if (BNXT_AUTO_MODE(link_info->auto_mode)) { 8442 link_info->autoneg = BNXT_AUTONEG_SPEED; 8443 if (bp->hwrm_spec_code >= 0x10201) { 8444 if (link_info->auto_pause_setting & 8445 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE) 8446 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL; 8447 } else { 8448 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL; 8449 } 8450 link_info->advertising = link_info->auto_link_speeds; 8451 } else { 8452 link_info->req_link_speed = link_info->force_link_speed; 8453 link_info->req_duplex = link_info->duplex_setting; 8454 } 8455 if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) 8456 link_info->req_flow_ctrl = 8457 link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH; 8458 else 8459 link_info->req_flow_ctrl = link_info->force_pause_setting; 8460 return rc; 8461 } 8462 8463 static int bnxt_get_max_irq(struct pci_dev *pdev) 8464 { 8465 u16 ctrl; 8466 8467 if (!pdev->msix_cap) 8468 return 1; 8469 8470 pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl); 8471 return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1; 8472 } 8473 8474 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, 8475 int *max_cp) 8476 { 8477 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 8478 int max_ring_grps = 0; 8479 8480 *max_tx = hw_resc->max_tx_rings; 8481 *max_rx = hw_resc->max_rx_rings; 8482 *max_cp = min_t(int, hw_resc->max_irqs, hw_resc->max_cp_rings); 8483 *max_cp = min_t(int, *max_cp, hw_resc->max_stat_ctxs); 8484 max_ring_grps = hw_resc->max_hw_ring_grps; 8485 if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) { 8486 *max_cp -= 1; 8487 *max_rx -= 2; 8488 } 8489 if (bp->flags & BNXT_FLAG_AGG_RINGS) 8490 *max_rx >>= 1; 8491 *max_rx = min_t(int, *max_rx, max_ring_grps); 8492 } 8493 8494 int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared) 8495 { 8496 int rx, tx, cp; 8497 8498 _bnxt_get_max_rings(bp, &rx, &tx, &cp); 8499 *max_rx = rx; 8500 *max_tx = tx; 8501 if (!rx || !tx || !cp) 8502 return -ENOMEM; 8503 8504 return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared); 8505 } 8506 8507 static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx, 8508 bool shared) 8509 { 8510 int rc; 8511 8512 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared); 8513 if (rc && (bp->flags & BNXT_FLAG_AGG_RINGS)) { 8514 /* Not enough rings, try disabling agg rings. */ 8515 bp->flags &= ~BNXT_FLAG_AGG_RINGS; 8516 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared); 8517 if (rc) { 8518 /* set BNXT_FLAG_AGG_RINGS back for consistency */ 8519 bp->flags |= BNXT_FLAG_AGG_RINGS; 8520 return rc; 8521 } 8522 bp->flags |= BNXT_FLAG_NO_AGG_RINGS; 8523 bp->dev->hw_features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); 8524 bp->dev->features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); 8525 bnxt_set_ring_params(bp); 8526 } 8527 8528 if (bp->flags & BNXT_FLAG_ROCE_CAP) { 8529 int max_cp, max_stat, max_irq; 8530 8531 /* Reserve minimum resources for RoCE */ 8532 max_cp = bnxt_get_max_func_cp_rings(bp); 8533 max_stat = bnxt_get_max_func_stat_ctxs(bp); 8534 max_irq = bnxt_get_max_func_irqs(bp); 8535 if (max_cp <= BNXT_MIN_ROCE_CP_RINGS || 8536 max_irq <= BNXT_MIN_ROCE_CP_RINGS || 8537 max_stat <= BNXT_MIN_ROCE_STAT_CTXS) 8538 return 0; 8539 8540 max_cp -= BNXT_MIN_ROCE_CP_RINGS; 8541 max_irq -= BNXT_MIN_ROCE_CP_RINGS; 8542 max_stat -= BNXT_MIN_ROCE_STAT_CTXS; 8543 max_cp = min_t(int, max_cp, max_irq); 8544 max_cp = min_t(int, max_cp, max_stat); 8545 rc = bnxt_trim_rings(bp, max_rx, max_tx, max_cp, shared); 8546 if (rc) 8547 rc = 0; 8548 } 8549 return rc; 8550 } 8551 8552 /* In initial default shared ring setting, each shared ring must have a 8553 * RX/TX ring pair. 8554 */ 8555 static void bnxt_trim_dflt_sh_rings(struct bnxt *bp) 8556 { 8557 bp->cp_nr_rings = min_t(int, bp->tx_nr_rings_per_tc, bp->rx_nr_rings); 8558 bp->rx_nr_rings = bp->cp_nr_rings; 8559 bp->tx_nr_rings_per_tc = bp->cp_nr_rings; 8560 bp->tx_nr_rings = bp->tx_nr_rings_per_tc; 8561 } 8562 8563 static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh) 8564 { 8565 int dflt_rings, max_rx_rings, max_tx_rings, rc; 8566 8567 if (!bnxt_can_reserve_rings(bp)) 8568 return 0; 8569 8570 if (sh) 8571 bp->flags |= BNXT_FLAG_SHARED_RINGS; 8572 dflt_rings = netif_get_num_default_rss_queues(); 8573 /* Reduce default rings on multi-port cards so that total default 8574 * rings do not exceed CPU count. 8575 */ 8576 if (bp->port_count > 1) { 8577 int max_rings = 8578 max_t(int, num_online_cpus() / bp->port_count, 1); 8579 8580 dflt_rings = min_t(int, dflt_rings, max_rings); 8581 } 8582 rc = bnxt_get_dflt_rings(bp, &max_rx_rings, &max_tx_rings, sh); 8583 if (rc) 8584 return rc; 8585 bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings); 8586 bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings); 8587 if (sh) 8588 bnxt_trim_dflt_sh_rings(bp); 8589 else 8590 bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings; 8591 bp->tx_nr_rings = bp->tx_nr_rings_per_tc; 8592 8593 rc = __bnxt_reserve_rings(bp); 8594 if (rc) 8595 netdev_warn(bp->dev, "Unable to reserve tx rings\n"); 8596 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 8597 if (sh) 8598 bnxt_trim_dflt_sh_rings(bp); 8599 8600 /* Rings may have been trimmed, re-reserve the trimmed rings. */ 8601 if (bnxt_need_reserve_rings(bp)) { 8602 rc = __bnxt_reserve_rings(bp); 8603 if (rc) 8604 netdev_warn(bp->dev, "2nd rings reservation failed.\n"); 8605 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 8606 } 8607 bp->num_stat_ctxs = bp->cp_nr_rings; 8608 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 8609 bp->rx_nr_rings++; 8610 bp->cp_nr_rings++; 8611 } 8612 return rc; 8613 } 8614 8615 static int bnxt_init_dflt_ring_mode(struct bnxt *bp) 8616 { 8617 int rc; 8618 8619 if (bp->tx_nr_rings) 8620 return 0; 8621 8622 rc = bnxt_set_dflt_rings(bp, true); 8623 if (rc) { 8624 netdev_err(bp->dev, "Not enough rings available.\n"); 8625 return rc; 8626 } 8627 rc = bnxt_init_int_mode(bp); 8628 if (rc) 8629 return rc; 8630 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 8631 if (bnxt_rfs_supported(bp) && bnxt_rfs_capable(bp)) { 8632 bp->flags |= BNXT_FLAG_RFS; 8633 bp->dev->features |= NETIF_F_NTUPLE; 8634 } 8635 return 0; 8636 } 8637 8638 int bnxt_restore_pf_fw_resources(struct bnxt *bp) 8639 { 8640 int rc; 8641 8642 ASSERT_RTNL(); 8643 bnxt_hwrm_func_qcaps(bp); 8644 8645 if (netif_running(bp->dev)) 8646 __bnxt_close_nic(bp, true, false); 8647 8648 bnxt_ulp_irq_stop(bp); 8649 bnxt_clear_int_mode(bp); 8650 rc = bnxt_init_int_mode(bp); 8651 bnxt_ulp_irq_restart(bp, rc); 8652 8653 if (netif_running(bp->dev)) { 8654 if (rc) 8655 dev_close(bp->dev); 8656 else 8657 rc = bnxt_open_nic(bp, true, false); 8658 } 8659 8660 return rc; 8661 } 8662 8663 static int bnxt_init_mac_addr(struct bnxt *bp) 8664 { 8665 int rc = 0; 8666 8667 if (BNXT_PF(bp)) { 8668 memcpy(bp->dev->dev_addr, bp->pf.mac_addr, ETH_ALEN); 8669 } else { 8670 #ifdef CONFIG_BNXT_SRIOV 8671 struct bnxt_vf_info *vf = &bp->vf; 8672 8673 if (is_valid_ether_addr(vf->mac_addr)) { 8674 /* overwrite netdev dev_addr with admin VF MAC */ 8675 memcpy(bp->dev->dev_addr, vf->mac_addr, ETH_ALEN); 8676 } else { 8677 eth_hw_addr_random(bp->dev); 8678 } 8679 rc = bnxt_approve_mac(bp, bp->dev->dev_addr); 8680 #endif 8681 } 8682 return rc; 8683 } 8684 8685 static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) 8686 { 8687 static int version_printed; 8688 struct net_device *dev; 8689 struct bnxt *bp; 8690 int rc, max_irqs; 8691 8692 if (pci_is_bridge(pdev)) 8693 return -ENODEV; 8694 8695 if (version_printed++ == 0) 8696 pr_info("%s", version); 8697 8698 max_irqs = bnxt_get_max_irq(pdev); 8699 dev = alloc_etherdev_mq(sizeof(*bp), max_irqs); 8700 if (!dev) 8701 return -ENOMEM; 8702 8703 bp = netdev_priv(dev); 8704 8705 if (bnxt_vf_pciid(ent->driver_data)) 8706 bp->flags |= BNXT_FLAG_VF; 8707 8708 if (pdev->msix_cap) 8709 bp->flags |= BNXT_FLAG_MSIX_CAP; 8710 8711 rc = bnxt_init_board(pdev, dev); 8712 if (rc < 0) 8713 goto init_err_free; 8714 8715 dev->netdev_ops = &bnxt_netdev_ops; 8716 dev->watchdog_timeo = BNXT_TX_TIMEOUT; 8717 dev->ethtool_ops = &bnxt_ethtool_ops; 8718 SWITCHDEV_SET_OPS(dev, &bnxt_switchdev_ops); 8719 pci_set_drvdata(pdev, dev); 8720 8721 rc = bnxt_alloc_hwrm_resources(bp); 8722 if (rc) 8723 goto init_err_pci_clean; 8724 8725 mutex_init(&bp->hwrm_cmd_lock); 8726 rc = bnxt_hwrm_ver_get(bp); 8727 if (rc) 8728 goto init_err_pci_clean; 8729 8730 if (bp->flags & BNXT_FLAG_SHORT_CMD) { 8731 rc = bnxt_alloc_hwrm_short_cmd_req(bp); 8732 if (rc) 8733 goto init_err_pci_clean; 8734 } 8735 8736 rc = bnxt_hwrm_func_reset(bp); 8737 if (rc) 8738 goto init_err_pci_clean; 8739 8740 bnxt_hwrm_fw_set_time(bp); 8741 8742 dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG | 8743 NETIF_F_TSO | NETIF_F_TSO6 | 8744 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE | 8745 NETIF_F_GSO_IPXIP4 | 8746 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM | 8747 NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH | 8748 NETIF_F_RXCSUM | NETIF_F_GRO; 8749 8750 if (!BNXT_CHIP_TYPE_NITRO_A0(bp)) 8751 dev->hw_features |= NETIF_F_LRO; 8752 8753 dev->hw_enc_features = 8754 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG | 8755 NETIF_F_TSO | NETIF_F_TSO6 | 8756 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE | 8757 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM | 8758 NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL; 8759 dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM | 8760 NETIF_F_GSO_GRE_CSUM; 8761 dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA; 8762 dev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX | 8763 NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX; 8764 if (!BNXT_CHIP_TYPE_NITRO_A0(bp)) 8765 dev->hw_features |= NETIF_F_GRO_HW; 8766 dev->features |= dev->hw_features | NETIF_F_HIGHDMA; 8767 if (dev->features & NETIF_F_GRO_HW) 8768 dev->features &= ~NETIF_F_LRO; 8769 dev->priv_flags |= IFF_UNICAST_FLT; 8770 8771 #ifdef CONFIG_BNXT_SRIOV 8772 init_waitqueue_head(&bp->sriov_cfg_wait); 8773 mutex_init(&bp->sriov_lock); 8774 #endif 8775 bp->gro_func = bnxt_gro_func_5730x; 8776 if (BNXT_CHIP_P4_PLUS(bp)) 8777 bp->gro_func = bnxt_gro_func_5731x; 8778 else 8779 bp->flags |= BNXT_FLAG_DOUBLE_DB; 8780 8781 rc = bnxt_hwrm_func_drv_rgtr(bp); 8782 if (rc) 8783 goto init_err_pci_clean; 8784 8785 rc = bnxt_hwrm_func_rgtr_async_events(bp, NULL, 0); 8786 if (rc) 8787 goto init_err_pci_clean; 8788 8789 bp->ulp_probe = bnxt_ulp_probe; 8790 8791 /* Get the MAX capabilities for this function */ 8792 rc = bnxt_hwrm_func_qcaps(bp); 8793 if (rc) { 8794 netdev_err(bp->dev, "hwrm query capability failure rc: %x\n", 8795 rc); 8796 rc = -1; 8797 goto init_err_pci_clean; 8798 } 8799 rc = bnxt_init_mac_addr(bp); 8800 if (rc) { 8801 dev_err(&pdev->dev, "Unable to initialize mac address.\n"); 8802 rc = -EADDRNOTAVAIL; 8803 goto init_err_pci_clean; 8804 } 8805 rc = bnxt_hwrm_queue_qportcfg(bp); 8806 if (rc) { 8807 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %x\n", 8808 rc); 8809 rc = -1; 8810 goto init_err_pci_clean; 8811 } 8812 8813 bnxt_hwrm_func_qcfg(bp); 8814 bnxt_hwrm_port_led_qcaps(bp); 8815 bnxt_ethtool_init(bp); 8816 bnxt_dcb_init(bp); 8817 8818 /* MTU range: 60 - FW defined max */ 8819 dev->min_mtu = ETH_ZLEN; 8820 dev->max_mtu = bp->max_mtu; 8821 8822 rc = bnxt_probe_phy(bp); 8823 if (rc) 8824 goto init_err_pci_clean; 8825 8826 bnxt_set_rx_skb_mode(bp, false); 8827 bnxt_set_tpa_flags(bp); 8828 bnxt_set_ring_params(bp); 8829 bnxt_set_max_func_irqs(bp, max_irqs); 8830 rc = bnxt_set_dflt_rings(bp, true); 8831 if (rc) { 8832 netdev_err(bp->dev, "Not enough rings available.\n"); 8833 rc = -ENOMEM; 8834 goto init_err_pci_clean; 8835 } 8836 8837 /* Default RSS hash cfg. */ 8838 bp->rss_hash_cfg = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 | 8839 VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 | 8840 VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 | 8841 VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6; 8842 if (BNXT_CHIP_P4_PLUS(bp) && bp->hwrm_spec_code >= 0x10501) { 8843 bp->flags |= BNXT_FLAG_UDP_RSS_CAP; 8844 bp->rss_hash_cfg |= VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 | 8845 VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6; 8846 } 8847 8848 bnxt_hwrm_vnic_qcaps(bp); 8849 if (bnxt_rfs_supported(bp)) { 8850 dev->hw_features |= NETIF_F_NTUPLE; 8851 if (bnxt_rfs_capable(bp)) { 8852 bp->flags |= BNXT_FLAG_RFS; 8853 dev->features |= NETIF_F_NTUPLE; 8854 } 8855 } 8856 8857 if (dev->hw_features & NETIF_F_HW_VLAN_CTAG_RX) 8858 bp->flags |= BNXT_FLAG_STRIP_VLAN; 8859 8860 rc = bnxt_init_int_mode(bp); 8861 if (rc) 8862 goto init_err_pci_clean; 8863 8864 /* No TC has been set yet and rings may have been trimmed due to 8865 * limited MSIX, so we re-initialize the TX rings per TC. 8866 */ 8867 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 8868 8869 bnxt_get_wol_settings(bp); 8870 if (bp->flags & BNXT_FLAG_WOL_CAP) 8871 device_set_wakeup_enable(&pdev->dev, bp->wol); 8872 else 8873 device_set_wakeup_capable(&pdev->dev, false); 8874 8875 bnxt_hwrm_set_cache_line_size(bp, cache_line_size()); 8876 8877 if (BNXT_PF(bp)) { 8878 if (!bnxt_pf_wq) { 8879 bnxt_pf_wq = 8880 create_singlethread_workqueue("bnxt_pf_wq"); 8881 if (!bnxt_pf_wq) { 8882 dev_err(&pdev->dev, "Unable to create workqueue.\n"); 8883 goto init_err_pci_clean; 8884 } 8885 } 8886 bnxt_init_tc(bp); 8887 } 8888 8889 rc = register_netdev(dev); 8890 if (rc) 8891 goto init_err_cleanup_tc; 8892 8893 if (BNXT_PF(bp)) 8894 bnxt_dl_register(bp); 8895 8896 netdev_info(dev, "%s found at mem %lx, node addr %pM\n", 8897 board_info[ent->driver_data].name, 8898 (long)pci_resource_start(pdev, 0), dev->dev_addr); 8899 pcie_print_link_status(pdev); 8900 8901 return 0; 8902 8903 init_err_cleanup_tc: 8904 bnxt_shutdown_tc(bp); 8905 bnxt_clear_int_mode(bp); 8906 8907 init_err_pci_clean: 8908 bnxt_cleanup_pci(bp); 8909 8910 init_err_free: 8911 free_netdev(dev); 8912 return rc; 8913 } 8914 8915 static void bnxt_shutdown(struct pci_dev *pdev) 8916 { 8917 struct net_device *dev = pci_get_drvdata(pdev); 8918 struct bnxt *bp; 8919 8920 if (!dev) 8921 return; 8922 8923 rtnl_lock(); 8924 bp = netdev_priv(dev); 8925 if (!bp) 8926 goto shutdown_exit; 8927 8928 if (netif_running(dev)) 8929 dev_close(dev); 8930 8931 bnxt_ulp_shutdown(bp); 8932 8933 if (system_state == SYSTEM_POWER_OFF) { 8934 bnxt_clear_int_mode(bp); 8935 pci_wake_from_d3(pdev, bp->wol); 8936 pci_set_power_state(pdev, PCI_D3hot); 8937 } 8938 8939 shutdown_exit: 8940 rtnl_unlock(); 8941 } 8942 8943 #ifdef CONFIG_PM_SLEEP 8944 static int bnxt_suspend(struct device *device) 8945 { 8946 struct pci_dev *pdev = to_pci_dev(device); 8947 struct net_device *dev = pci_get_drvdata(pdev); 8948 struct bnxt *bp = netdev_priv(dev); 8949 int rc = 0; 8950 8951 rtnl_lock(); 8952 if (netif_running(dev)) { 8953 netif_device_detach(dev); 8954 rc = bnxt_close(dev); 8955 } 8956 bnxt_hwrm_func_drv_unrgtr(bp); 8957 rtnl_unlock(); 8958 return rc; 8959 } 8960 8961 static int bnxt_resume(struct device *device) 8962 { 8963 struct pci_dev *pdev = to_pci_dev(device); 8964 struct net_device *dev = pci_get_drvdata(pdev); 8965 struct bnxt *bp = netdev_priv(dev); 8966 int rc = 0; 8967 8968 rtnl_lock(); 8969 if (bnxt_hwrm_ver_get(bp) || bnxt_hwrm_func_drv_rgtr(bp)) { 8970 rc = -ENODEV; 8971 goto resume_exit; 8972 } 8973 rc = bnxt_hwrm_func_reset(bp); 8974 if (rc) { 8975 rc = -EBUSY; 8976 goto resume_exit; 8977 } 8978 bnxt_get_wol_settings(bp); 8979 if (netif_running(dev)) { 8980 rc = bnxt_open(dev); 8981 if (!rc) 8982 netif_device_attach(dev); 8983 } 8984 8985 resume_exit: 8986 rtnl_unlock(); 8987 return rc; 8988 } 8989 8990 static SIMPLE_DEV_PM_OPS(bnxt_pm_ops, bnxt_suspend, bnxt_resume); 8991 #define BNXT_PM_OPS (&bnxt_pm_ops) 8992 8993 #else 8994 8995 #define BNXT_PM_OPS NULL 8996 8997 #endif /* CONFIG_PM_SLEEP */ 8998 8999 /** 9000 * bnxt_io_error_detected - called when PCI error is detected 9001 * @pdev: Pointer to PCI device 9002 * @state: The current pci connection state 9003 * 9004 * This function is called after a PCI bus error affecting 9005 * this device has been detected. 9006 */ 9007 static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev, 9008 pci_channel_state_t state) 9009 { 9010 struct net_device *netdev = pci_get_drvdata(pdev); 9011 struct bnxt *bp = netdev_priv(netdev); 9012 9013 netdev_info(netdev, "PCI I/O error detected\n"); 9014 9015 rtnl_lock(); 9016 netif_device_detach(netdev); 9017 9018 bnxt_ulp_stop(bp); 9019 9020 if (state == pci_channel_io_perm_failure) { 9021 rtnl_unlock(); 9022 return PCI_ERS_RESULT_DISCONNECT; 9023 } 9024 9025 if (netif_running(netdev)) 9026 bnxt_close(netdev); 9027 9028 pci_disable_device(pdev); 9029 rtnl_unlock(); 9030 9031 /* Request a slot slot reset. */ 9032 return PCI_ERS_RESULT_NEED_RESET; 9033 } 9034 9035 /** 9036 * bnxt_io_slot_reset - called after the pci bus has been reset. 9037 * @pdev: Pointer to PCI device 9038 * 9039 * Restart the card from scratch, as if from a cold-boot. 9040 * At this point, the card has exprienced a hard reset, 9041 * followed by fixups by BIOS, and has its config space 9042 * set up identically to what it was at cold boot. 9043 */ 9044 static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev) 9045 { 9046 struct net_device *netdev = pci_get_drvdata(pdev); 9047 struct bnxt *bp = netdev_priv(netdev); 9048 int err = 0; 9049 pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT; 9050 9051 netdev_info(bp->dev, "PCI Slot Reset\n"); 9052 9053 rtnl_lock(); 9054 9055 if (pci_enable_device(pdev)) { 9056 dev_err(&pdev->dev, 9057 "Cannot re-enable PCI device after reset.\n"); 9058 } else { 9059 pci_set_master(pdev); 9060 9061 err = bnxt_hwrm_func_reset(bp); 9062 if (!err && netif_running(netdev)) 9063 err = bnxt_open(netdev); 9064 9065 if (!err) { 9066 result = PCI_ERS_RESULT_RECOVERED; 9067 bnxt_ulp_start(bp); 9068 } 9069 } 9070 9071 if (result != PCI_ERS_RESULT_RECOVERED && netif_running(netdev)) 9072 dev_close(netdev); 9073 9074 rtnl_unlock(); 9075 9076 err = pci_cleanup_aer_uncorrect_error_status(pdev); 9077 if (err) { 9078 dev_err(&pdev->dev, 9079 "pci_cleanup_aer_uncorrect_error_status failed 0x%0x\n", 9080 err); /* non-fatal, continue */ 9081 } 9082 9083 return PCI_ERS_RESULT_RECOVERED; 9084 } 9085 9086 /** 9087 * bnxt_io_resume - called when traffic can start flowing again. 9088 * @pdev: Pointer to PCI device 9089 * 9090 * This callback is called when the error recovery driver tells 9091 * us that its OK to resume normal operation. 9092 */ 9093 static void bnxt_io_resume(struct pci_dev *pdev) 9094 { 9095 struct net_device *netdev = pci_get_drvdata(pdev); 9096 9097 rtnl_lock(); 9098 9099 netif_device_attach(netdev); 9100 9101 rtnl_unlock(); 9102 } 9103 9104 static const struct pci_error_handlers bnxt_err_handler = { 9105 .error_detected = bnxt_io_error_detected, 9106 .slot_reset = bnxt_io_slot_reset, 9107 .resume = bnxt_io_resume 9108 }; 9109 9110 static struct pci_driver bnxt_pci_driver = { 9111 .name = DRV_MODULE_NAME, 9112 .id_table = bnxt_pci_tbl, 9113 .probe = bnxt_init_one, 9114 .remove = bnxt_remove_one, 9115 .shutdown = bnxt_shutdown, 9116 .driver.pm = BNXT_PM_OPS, 9117 .err_handler = &bnxt_err_handler, 9118 #if defined(CONFIG_BNXT_SRIOV) 9119 .sriov_configure = bnxt_sriov_configure, 9120 #endif 9121 }; 9122 9123 static int __init bnxt_init(void) 9124 { 9125 bnxt_debug_init(); 9126 return pci_register_driver(&bnxt_pci_driver); 9127 } 9128 9129 static void __exit bnxt_exit(void) 9130 { 9131 pci_unregister_driver(&bnxt_pci_driver); 9132 if (bnxt_pf_wq) 9133 destroy_workqueue(bnxt_pf_wq); 9134 bnxt_debug_exit(); 9135 } 9136 9137 module_init(bnxt_init); 9138 module_exit(bnxt_exit); 9139