1 /* Broadcom NetXtreme-C/E network driver. 2 * 3 * Copyright (c) 2014-2016 Broadcom Corporation 4 * Copyright (c) 2016-2018 Broadcom Limited 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License as published by 8 * the Free Software Foundation. 9 */ 10 11 #include <linux/module.h> 12 13 #include <linux/stringify.h> 14 #include <linux/kernel.h> 15 #include <linux/timer.h> 16 #include <linux/errno.h> 17 #include <linux/ioport.h> 18 #include <linux/slab.h> 19 #include <linux/vmalloc.h> 20 #include <linux/interrupt.h> 21 #include <linux/pci.h> 22 #include <linux/netdevice.h> 23 #include <linux/etherdevice.h> 24 #include <linux/skbuff.h> 25 #include <linux/dma-mapping.h> 26 #include <linux/bitops.h> 27 #include <linux/io.h> 28 #include <linux/irq.h> 29 #include <linux/delay.h> 30 #include <asm/byteorder.h> 31 #include <asm/page.h> 32 #include <linux/time.h> 33 #include <linux/mii.h> 34 #include <linux/if.h> 35 #include <linux/if_vlan.h> 36 #include <linux/if_bridge.h> 37 #include <linux/rtc.h> 38 #include <linux/bpf.h> 39 #include <net/ip.h> 40 #include <net/tcp.h> 41 #include <net/udp.h> 42 #include <net/checksum.h> 43 #include <net/ip6_checksum.h> 44 #include <net/udp_tunnel.h> 45 #include <linux/workqueue.h> 46 #include <linux/prefetch.h> 47 #include <linux/cache.h> 48 #include <linux/log2.h> 49 #include <linux/aer.h> 50 #include <linux/bitmap.h> 51 #include <linux/cpu_rmap.h> 52 #include <linux/cpumask.h> 53 #include <net/pkt_cls.h> 54 55 #include "bnxt_hsi.h" 56 #include "bnxt.h" 57 #include "bnxt_ulp.h" 58 #include "bnxt_sriov.h" 59 #include "bnxt_ethtool.h" 60 #include "bnxt_dcb.h" 61 #include "bnxt_xdp.h" 62 #include "bnxt_vfr.h" 63 #include "bnxt_tc.h" 64 #include "bnxt_devlink.h" 65 #include "bnxt_debugfs.h" 66 67 #define BNXT_TX_TIMEOUT (5 * HZ) 68 69 static const char version[] = 70 "Broadcom NetXtreme-C/E driver " DRV_MODULE_NAME " v" DRV_MODULE_VERSION "\n"; 71 72 MODULE_LICENSE("GPL"); 73 MODULE_DESCRIPTION("Broadcom BCM573xx network driver"); 74 MODULE_VERSION(DRV_MODULE_VERSION); 75 76 #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN) 77 #define BNXT_RX_DMA_OFFSET NET_SKB_PAD 78 #define BNXT_RX_COPY_THRESH 256 79 80 #define BNXT_TX_PUSH_THRESH 164 81 82 enum board_idx { 83 BCM57301, 84 BCM57302, 85 BCM57304, 86 BCM57417_NPAR, 87 BCM58700, 88 BCM57311, 89 BCM57312, 90 BCM57402, 91 BCM57404, 92 BCM57406, 93 BCM57402_NPAR, 94 BCM57407, 95 BCM57412, 96 BCM57414, 97 BCM57416, 98 BCM57417, 99 BCM57412_NPAR, 100 BCM57314, 101 BCM57417_SFP, 102 BCM57416_SFP, 103 BCM57404_NPAR, 104 BCM57406_NPAR, 105 BCM57407_SFP, 106 BCM57407_NPAR, 107 BCM57414_NPAR, 108 BCM57416_NPAR, 109 BCM57452, 110 BCM57454, 111 BCM5745x_NPAR, 112 BCM58802, 113 BCM58804, 114 BCM58808, 115 NETXTREME_E_VF, 116 NETXTREME_C_VF, 117 NETXTREME_S_VF, 118 }; 119 120 /* indexed by enum above */ 121 static const struct { 122 char *name; 123 } board_info[] = { 124 [BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" }, 125 [BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" }, 126 [BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" }, 127 [BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" }, 128 [BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" }, 129 [BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" }, 130 [BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" }, 131 [BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" }, 132 [BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" }, 133 [BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" }, 134 [BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" }, 135 [BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" }, 136 [BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" }, 137 [BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" }, 138 [BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" }, 139 [BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" }, 140 [BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" }, 141 [BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" }, 142 [BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" }, 143 [BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" }, 144 [BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" }, 145 [BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" }, 146 [BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" }, 147 [BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" }, 148 [BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" }, 149 [BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" }, 150 [BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" }, 151 [BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" }, 152 [BCM5745x_NPAR] = { "Broadcom BCM5745x NetXtreme-E Ethernet Partition" }, 153 [BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" }, 154 [BCM58804] = { "Broadcom BCM58804 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" }, 155 [BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" }, 156 [NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" }, 157 [NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" }, 158 [NETXTREME_S_VF] = { "Broadcom NetXtreme-S Ethernet Virtual Function" }, 159 }; 160 161 static const struct pci_device_id bnxt_pci_tbl[] = { 162 { PCI_VDEVICE(BROADCOM, 0x1604), .driver_data = BCM5745x_NPAR }, 163 { PCI_VDEVICE(BROADCOM, 0x1605), .driver_data = BCM5745x_NPAR }, 164 { PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 }, 165 { PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR }, 166 { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 }, 167 { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 }, 168 { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 }, 169 { PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR }, 170 { PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 }, 171 { PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 }, 172 { PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 }, 173 { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 }, 174 { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 }, 175 { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 }, 176 { PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR }, 177 { PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 }, 178 { PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 }, 179 { PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 }, 180 { PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 }, 181 { PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 }, 182 { PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR }, 183 { PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 }, 184 { PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP }, 185 { PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP }, 186 { PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR }, 187 { PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR }, 188 { PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP }, 189 { PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR }, 190 { PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR }, 191 { PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR }, 192 { PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR }, 193 { PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR }, 194 { PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR }, 195 { PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 }, 196 { PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 }, 197 { PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 }, 198 { PCI_VDEVICE(BROADCOM, 0xd804), .driver_data = BCM58804 }, 199 #ifdef CONFIG_BNXT_SRIOV 200 { PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF }, 201 { PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF }, 202 { PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF }, 203 { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF }, 204 { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF }, 205 { PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF }, 206 { PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF }, 207 { PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF }, 208 { PCI_VDEVICE(BROADCOM, 0xd800), .driver_data = NETXTREME_S_VF }, 209 #endif 210 { 0 } 211 }; 212 213 MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl); 214 215 static const u16 bnxt_vf_req_snif[] = { 216 HWRM_FUNC_CFG, 217 HWRM_FUNC_VF_CFG, 218 HWRM_PORT_PHY_QCFG, 219 HWRM_CFA_L2_FILTER_ALLOC, 220 }; 221 222 static const u16 bnxt_async_events_arr[] = { 223 ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE, 224 ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD, 225 ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED, 226 ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE, 227 ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE, 228 }; 229 230 static struct workqueue_struct *bnxt_pf_wq; 231 232 static bool bnxt_vf_pciid(enum board_idx idx) 233 { 234 return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF || 235 idx == NETXTREME_S_VF); 236 } 237 238 #define DB_CP_REARM_FLAGS (DB_KEY_CP | DB_IDX_VALID) 239 #define DB_CP_FLAGS (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS) 240 #define DB_CP_IRQ_DIS_FLAGS (DB_KEY_CP | DB_IRQ_DIS) 241 242 #define BNXT_CP_DB_REARM(db, raw_cons) \ 243 writel(DB_CP_REARM_FLAGS | RING_CMP(raw_cons), db) 244 245 #define BNXT_CP_DB(db, raw_cons) \ 246 writel(DB_CP_FLAGS | RING_CMP(raw_cons), db) 247 248 #define BNXT_CP_DB_IRQ_DIS(db) \ 249 writel(DB_CP_IRQ_DIS_FLAGS, db) 250 251 const u16 bnxt_lhint_arr[] = { 252 TX_BD_FLAGS_LHINT_512_AND_SMALLER, 253 TX_BD_FLAGS_LHINT_512_TO_1023, 254 TX_BD_FLAGS_LHINT_1024_TO_2047, 255 TX_BD_FLAGS_LHINT_1024_TO_2047, 256 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 257 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 258 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 259 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 260 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 261 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 262 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 263 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 264 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 265 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 266 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 267 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 268 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 269 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 270 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 271 }; 272 273 static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb) 274 { 275 struct metadata_dst *md_dst = skb_metadata_dst(skb); 276 277 if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX) 278 return 0; 279 280 return md_dst->u.port_info.port_id; 281 } 282 283 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev) 284 { 285 struct bnxt *bp = netdev_priv(dev); 286 struct tx_bd *txbd; 287 struct tx_bd_ext *txbd1; 288 struct netdev_queue *txq; 289 int i; 290 dma_addr_t mapping; 291 unsigned int length, pad = 0; 292 u32 len, free_size, vlan_tag_flags, cfa_action, flags; 293 u16 prod, last_frag; 294 struct pci_dev *pdev = bp->pdev; 295 struct bnxt_tx_ring_info *txr; 296 struct bnxt_sw_tx_bd *tx_buf; 297 298 i = skb_get_queue_mapping(skb); 299 if (unlikely(i >= bp->tx_nr_rings)) { 300 dev_kfree_skb_any(skb); 301 return NETDEV_TX_OK; 302 } 303 304 txq = netdev_get_tx_queue(dev, i); 305 txr = &bp->tx_ring[bp->tx_ring_map[i]]; 306 prod = txr->tx_prod; 307 308 free_size = bnxt_tx_avail(bp, txr); 309 if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) { 310 netif_tx_stop_queue(txq); 311 return NETDEV_TX_BUSY; 312 } 313 314 length = skb->len; 315 len = skb_headlen(skb); 316 last_frag = skb_shinfo(skb)->nr_frags; 317 318 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 319 320 txbd->tx_bd_opaque = prod; 321 322 tx_buf = &txr->tx_buf_ring[prod]; 323 tx_buf->skb = skb; 324 tx_buf->nr_frags = last_frag; 325 326 vlan_tag_flags = 0; 327 cfa_action = bnxt_xmit_get_cfa_action(skb); 328 if (skb_vlan_tag_present(skb)) { 329 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN | 330 skb_vlan_tag_get(skb); 331 /* Currently supports 8021Q, 8021AD vlan offloads 332 * QINQ1, QINQ2, QINQ3 vlan headers are deprecated 333 */ 334 if (skb->vlan_proto == htons(ETH_P_8021Q)) 335 vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT; 336 } 337 338 if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh) { 339 struct tx_push_buffer *tx_push_buf = txr->tx_push; 340 struct tx_push_bd *tx_push = &tx_push_buf->push_bd; 341 struct tx_bd_ext *tx_push1 = &tx_push->txbd2; 342 void *pdata = tx_push_buf->data; 343 u64 *end; 344 int j, push_len; 345 346 /* Set COAL_NOW to be ready quickly for the next push */ 347 tx_push->tx_bd_len_flags_type = 348 cpu_to_le32((length << TX_BD_LEN_SHIFT) | 349 TX_BD_TYPE_LONG_TX_BD | 350 TX_BD_FLAGS_LHINT_512_AND_SMALLER | 351 TX_BD_FLAGS_COAL_NOW | 352 TX_BD_FLAGS_PACKET_END | 353 (2 << TX_BD_FLAGS_BD_CNT_SHIFT)); 354 355 if (skb->ip_summed == CHECKSUM_PARTIAL) 356 tx_push1->tx_bd_hsize_lflags = 357 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM); 358 else 359 tx_push1->tx_bd_hsize_lflags = 0; 360 361 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags); 362 tx_push1->tx_bd_cfa_action = 363 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT); 364 365 end = pdata + length; 366 end = PTR_ALIGN(end, 8) - 1; 367 *end = 0; 368 369 skb_copy_from_linear_data(skb, pdata, len); 370 pdata += len; 371 for (j = 0; j < last_frag; j++) { 372 skb_frag_t *frag = &skb_shinfo(skb)->frags[j]; 373 void *fptr; 374 375 fptr = skb_frag_address_safe(frag); 376 if (!fptr) 377 goto normal_tx; 378 379 memcpy(pdata, fptr, skb_frag_size(frag)); 380 pdata += skb_frag_size(frag); 381 } 382 383 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type; 384 txbd->tx_bd_haddr = txr->data_mapping; 385 prod = NEXT_TX(prod); 386 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 387 memcpy(txbd, tx_push1, sizeof(*txbd)); 388 prod = NEXT_TX(prod); 389 tx_push->doorbell = 390 cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | prod); 391 txr->tx_prod = prod; 392 393 tx_buf->is_push = 1; 394 netdev_tx_sent_queue(txq, skb->len); 395 wmb(); /* Sync is_push and byte queue before pushing data */ 396 397 push_len = (length + sizeof(*tx_push) + 7) / 8; 398 if (push_len > 16) { 399 __iowrite64_copy(txr->tx_doorbell, tx_push_buf, 16); 400 __iowrite32_copy(txr->tx_doorbell + 4, tx_push_buf + 1, 401 (push_len - 16) << 1); 402 } else { 403 __iowrite64_copy(txr->tx_doorbell, tx_push_buf, 404 push_len); 405 } 406 407 goto tx_done; 408 } 409 410 normal_tx: 411 if (length < BNXT_MIN_PKT_SIZE) { 412 pad = BNXT_MIN_PKT_SIZE - length; 413 if (skb_pad(skb, pad)) { 414 /* SKB already freed. */ 415 tx_buf->skb = NULL; 416 return NETDEV_TX_OK; 417 } 418 length = BNXT_MIN_PKT_SIZE; 419 } 420 421 mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE); 422 423 if (unlikely(dma_mapping_error(&pdev->dev, mapping))) { 424 dev_kfree_skb_any(skb); 425 tx_buf->skb = NULL; 426 return NETDEV_TX_OK; 427 } 428 429 dma_unmap_addr_set(tx_buf, mapping, mapping); 430 flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD | 431 ((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT); 432 433 txbd->tx_bd_haddr = cpu_to_le64(mapping); 434 435 prod = NEXT_TX(prod); 436 txbd1 = (struct tx_bd_ext *) 437 &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 438 439 txbd1->tx_bd_hsize_lflags = 0; 440 if (skb_is_gso(skb)) { 441 u32 hdr_len; 442 443 if (skb->encapsulation) 444 hdr_len = skb_inner_network_offset(skb) + 445 skb_inner_network_header_len(skb) + 446 inner_tcp_hdrlen(skb); 447 else 448 hdr_len = skb_transport_offset(skb) + 449 tcp_hdrlen(skb); 450 451 txbd1->tx_bd_hsize_lflags = cpu_to_le32(TX_BD_FLAGS_LSO | 452 TX_BD_FLAGS_T_IPID | 453 (hdr_len << (TX_BD_HSIZE_SHIFT - 1))); 454 length = skb_shinfo(skb)->gso_size; 455 txbd1->tx_bd_mss = cpu_to_le32(length); 456 length += hdr_len; 457 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { 458 txbd1->tx_bd_hsize_lflags = 459 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM); 460 txbd1->tx_bd_mss = 0; 461 } 462 463 length >>= 9; 464 flags |= bnxt_lhint_arr[length]; 465 txbd->tx_bd_len_flags_type = cpu_to_le32(flags); 466 467 txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags); 468 txbd1->tx_bd_cfa_action = 469 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT); 470 for (i = 0; i < last_frag; i++) { 471 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 472 473 prod = NEXT_TX(prod); 474 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 475 476 len = skb_frag_size(frag); 477 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len, 478 DMA_TO_DEVICE); 479 480 if (unlikely(dma_mapping_error(&pdev->dev, mapping))) 481 goto tx_dma_error; 482 483 tx_buf = &txr->tx_buf_ring[prod]; 484 dma_unmap_addr_set(tx_buf, mapping, mapping); 485 486 txbd->tx_bd_haddr = cpu_to_le64(mapping); 487 488 flags = len << TX_BD_LEN_SHIFT; 489 txbd->tx_bd_len_flags_type = cpu_to_le32(flags); 490 } 491 492 flags &= ~TX_BD_LEN; 493 txbd->tx_bd_len_flags_type = 494 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags | 495 TX_BD_FLAGS_PACKET_END); 496 497 netdev_tx_sent_queue(txq, skb->len); 498 499 /* Sync BD data before updating doorbell */ 500 wmb(); 501 502 prod = NEXT_TX(prod); 503 txr->tx_prod = prod; 504 505 if (!skb->xmit_more || netif_xmit_stopped(txq)) 506 bnxt_db_write(bp, txr->tx_doorbell, DB_KEY_TX | prod); 507 508 tx_done: 509 510 mmiowb(); 511 512 if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) { 513 if (skb->xmit_more && !tx_buf->is_push) 514 bnxt_db_write(bp, txr->tx_doorbell, DB_KEY_TX | prod); 515 516 netif_tx_stop_queue(txq); 517 518 /* netif_tx_stop_queue() must be done before checking 519 * tx index in bnxt_tx_avail() below, because in 520 * bnxt_tx_int(), we update tx index before checking for 521 * netif_tx_queue_stopped(). 522 */ 523 smp_mb(); 524 if (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh) 525 netif_tx_wake_queue(txq); 526 } 527 return NETDEV_TX_OK; 528 529 tx_dma_error: 530 last_frag = i; 531 532 /* start back at beginning and unmap skb */ 533 prod = txr->tx_prod; 534 tx_buf = &txr->tx_buf_ring[prod]; 535 tx_buf->skb = NULL; 536 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping), 537 skb_headlen(skb), PCI_DMA_TODEVICE); 538 prod = NEXT_TX(prod); 539 540 /* unmap remaining mapped pages */ 541 for (i = 0; i < last_frag; i++) { 542 prod = NEXT_TX(prod); 543 tx_buf = &txr->tx_buf_ring[prod]; 544 dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping), 545 skb_frag_size(&skb_shinfo(skb)->frags[i]), 546 PCI_DMA_TODEVICE); 547 } 548 549 dev_kfree_skb_any(skb); 550 return NETDEV_TX_OK; 551 } 552 553 static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts) 554 { 555 struct bnxt_tx_ring_info *txr = bnapi->tx_ring; 556 struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index); 557 u16 cons = txr->tx_cons; 558 struct pci_dev *pdev = bp->pdev; 559 int i; 560 unsigned int tx_bytes = 0; 561 562 for (i = 0; i < nr_pkts; i++) { 563 struct bnxt_sw_tx_bd *tx_buf; 564 struct sk_buff *skb; 565 int j, last; 566 567 tx_buf = &txr->tx_buf_ring[cons]; 568 cons = NEXT_TX(cons); 569 skb = tx_buf->skb; 570 tx_buf->skb = NULL; 571 572 if (tx_buf->is_push) { 573 tx_buf->is_push = 0; 574 goto next_tx_int; 575 } 576 577 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping), 578 skb_headlen(skb), PCI_DMA_TODEVICE); 579 last = tx_buf->nr_frags; 580 581 for (j = 0; j < last; j++) { 582 cons = NEXT_TX(cons); 583 tx_buf = &txr->tx_buf_ring[cons]; 584 dma_unmap_page( 585 &pdev->dev, 586 dma_unmap_addr(tx_buf, mapping), 587 skb_frag_size(&skb_shinfo(skb)->frags[j]), 588 PCI_DMA_TODEVICE); 589 } 590 591 next_tx_int: 592 cons = NEXT_TX(cons); 593 594 tx_bytes += skb->len; 595 dev_kfree_skb_any(skb); 596 } 597 598 netdev_tx_completed_queue(txq, nr_pkts, tx_bytes); 599 txr->tx_cons = cons; 600 601 /* Need to make the tx_cons update visible to bnxt_start_xmit() 602 * before checking for netif_tx_queue_stopped(). Without the 603 * memory barrier, there is a small possibility that bnxt_start_xmit() 604 * will miss it and cause the queue to be stopped forever. 605 */ 606 smp_mb(); 607 608 if (unlikely(netif_tx_queue_stopped(txq)) && 609 (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)) { 610 __netif_tx_lock(txq, smp_processor_id()); 611 if (netif_tx_queue_stopped(txq) && 612 bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh && 613 txr->dev_state != BNXT_DEV_STATE_CLOSING) 614 netif_tx_wake_queue(txq); 615 __netif_tx_unlock(txq); 616 } 617 } 618 619 static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping, 620 gfp_t gfp) 621 { 622 struct device *dev = &bp->pdev->dev; 623 struct page *page; 624 625 page = alloc_page(gfp); 626 if (!page) 627 return NULL; 628 629 *mapping = dma_map_page_attrs(dev, page, 0, PAGE_SIZE, bp->rx_dir, 630 DMA_ATTR_WEAK_ORDERING); 631 if (dma_mapping_error(dev, *mapping)) { 632 __free_page(page); 633 return NULL; 634 } 635 *mapping += bp->rx_dma_offset; 636 return page; 637 } 638 639 static inline u8 *__bnxt_alloc_rx_data(struct bnxt *bp, dma_addr_t *mapping, 640 gfp_t gfp) 641 { 642 u8 *data; 643 struct pci_dev *pdev = bp->pdev; 644 645 data = kmalloc(bp->rx_buf_size, gfp); 646 if (!data) 647 return NULL; 648 649 *mapping = dma_map_single_attrs(&pdev->dev, data + bp->rx_dma_offset, 650 bp->rx_buf_use_size, bp->rx_dir, 651 DMA_ATTR_WEAK_ORDERING); 652 653 if (dma_mapping_error(&pdev->dev, *mapping)) { 654 kfree(data); 655 data = NULL; 656 } 657 return data; 658 } 659 660 int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, 661 u16 prod, gfp_t gfp) 662 { 663 struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 664 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod]; 665 dma_addr_t mapping; 666 667 if (BNXT_RX_PAGE_MODE(bp)) { 668 struct page *page = __bnxt_alloc_rx_page(bp, &mapping, gfp); 669 670 if (!page) 671 return -ENOMEM; 672 673 rx_buf->data = page; 674 rx_buf->data_ptr = page_address(page) + bp->rx_offset; 675 } else { 676 u8 *data = __bnxt_alloc_rx_data(bp, &mapping, gfp); 677 678 if (!data) 679 return -ENOMEM; 680 681 rx_buf->data = data; 682 rx_buf->data_ptr = data + bp->rx_offset; 683 } 684 rx_buf->mapping = mapping; 685 686 rxbd->rx_bd_haddr = cpu_to_le64(mapping); 687 return 0; 688 } 689 690 void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data) 691 { 692 u16 prod = rxr->rx_prod; 693 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf; 694 struct rx_bd *cons_bd, *prod_bd; 695 696 prod_rx_buf = &rxr->rx_buf_ring[prod]; 697 cons_rx_buf = &rxr->rx_buf_ring[cons]; 698 699 prod_rx_buf->data = data; 700 prod_rx_buf->data_ptr = cons_rx_buf->data_ptr; 701 702 prod_rx_buf->mapping = cons_rx_buf->mapping; 703 704 prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 705 cons_bd = &rxr->rx_desc_ring[RX_RING(cons)][RX_IDX(cons)]; 706 707 prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr; 708 } 709 710 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx) 711 { 712 u16 next, max = rxr->rx_agg_bmap_size; 713 714 next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx); 715 if (next >= max) 716 next = find_first_zero_bit(rxr->rx_agg_bmap, max); 717 return next; 718 } 719 720 static inline int bnxt_alloc_rx_page(struct bnxt *bp, 721 struct bnxt_rx_ring_info *rxr, 722 u16 prod, gfp_t gfp) 723 { 724 struct rx_bd *rxbd = 725 &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 726 struct bnxt_sw_rx_agg_bd *rx_agg_buf; 727 struct pci_dev *pdev = bp->pdev; 728 struct page *page; 729 dma_addr_t mapping; 730 u16 sw_prod = rxr->rx_sw_agg_prod; 731 unsigned int offset = 0; 732 733 if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) { 734 page = rxr->rx_page; 735 if (!page) { 736 page = alloc_page(gfp); 737 if (!page) 738 return -ENOMEM; 739 rxr->rx_page = page; 740 rxr->rx_page_offset = 0; 741 } 742 offset = rxr->rx_page_offset; 743 rxr->rx_page_offset += BNXT_RX_PAGE_SIZE; 744 if (rxr->rx_page_offset == PAGE_SIZE) 745 rxr->rx_page = NULL; 746 else 747 get_page(page); 748 } else { 749 page = alloc_page(gfp); 750 if (!page) 751 return -ENOMEM; 752 } 753 754 mapping = dma_map_page_attrs(&pdev->dev, page, offset, 755 BNXT_RX_PAGE_SIZE, PCI_DMA_FROMDEVICE, 756 DMA_ATTR_WEAK_ORDERING); 757 if (dma_mapping_error(&pdev->dev, mapping)) { 758 __free_page(page); 759 return -EIO; 760 } 761 762 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap))) 763 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod); 764 765 __set_bit(sw_prod, rxr->rx_agg_bmap); 766 rx_agg_buf = &rxr->rx_agg_ring[sw_prod]; 767 rxr->rx_sw_agg_prod = NEXT_RX_AGG(sw_prod); 768 769 rx_agg_buf->page = page; 770 rx_agg_buf->offset = offset; 771 rx_agg_buf->mapping = mapping; 772 rxbd->rx_bd_haddr = cpu_to_le64(mapping); 773 rxbd->rx_bd_opaque = sw_prod; 774 return 0; 775 } 776 777 static void bnxt_reuse_rx_agg_bufs(struct bnxt_napi *bnapi, u16 cp_cons, 778 u32 agg_bufs) 779 { 780 struct bnxt *bp = bnapi->bp; 781 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 782 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 783 u16 prod = rxr->rx_agg_prod; 784 u16 sw_prod = rxr->rx_sw_agg_prod; 785 u32 i; 786 787 for (i = 0; i < agg_bufs; i++) { 788 u16 cons; 789 struct rx_agg_cmp *agg; 790 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf; 791 struct rx_bd *prod_bd; 792 struct page *page; 793 794 agg = (struct rx_agg_cmp *) 795 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 796 cons = agg->rx_agg_cmp_opaque; 797 __clear_bit(cons, rxr->rx_agg_bmap); 798 799 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap))) 800 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod); 801 802 __set_bit(sw_prod, rxr->rx_agg_bmap); 803 prod_rx_buf = &rxr->rx_agg_ring[sw_prod]; 804 cons_rx_buf = &rxr->rx_agg_ring[cons]; 805 806 /* It is possible for sw_prod to be equal to cons, so 807 * set cons_rx_buf->page to NULL first. 808 */ 809 page = cons_rx_buf->page; 810 cons_rx_buf->page = NULL; 811 prod_rx_buf->page = page; 812 prod_rx_buf->offset = cons_rx_buf->offset; 813 814 prod_rx_buf->mapping = cons_rx_buf->mapping; 815 816 prod_bd = &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 817 818 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping); 819 prod_bd->rx_bd_opaque = sw_prod; 820 821 prod = NEXT_RX_AGG(prod); 822 sw_prod = NEXT_RX_AGG(sw_prod); 823 cp_cons = NEXT_CMP(cp_cons); 824 } 825 rxr->rx_agg_prod = prod; 826 rxr->rx_sw_agg_prod = sw_prod; 827 } 828 829 static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp, 830 struct bnxt_rx_ring_info *rxr, 831 u16 cons, void *data, u8 *data_ptr, 832 dma_addr_t dma_addr, 833 unsigned int offset_and_len) 834 { 835 unsigned int payload = offset_and_len >> 16; 836 unsigned int len = offset_and_len & 0xffff; 837 struct skb_frag_struct *frag; 838 struct page *page = data; 839 u16 prod = rxr->rx_prod; 840 struct sk_buff *skb; 841 int off, err; 842 843 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC); 844 if (unlikely(err)) { 845 bnxt_reuse_rx_data(rxr, cons, data); 846 return NULL; 847 } 848 dma_addr -= bp->rx_dma_offset; 849 dma_unmap_page_attrs(&bp->pdev->dev, dma_addr, PAGE_SIZE, bp->rx_dir, 850 DMA_ATTR_WEAK_ORDERING); 851 852 if (unlikely(!payload)) 853 payload = eth_get_headlen(data_ptr, len); 854 855 skb = napi_alloc_skb(&rxr->bnapi->napi, payload); 856 if (!skb) { 857 __free_page(page); 858 return NULL; 859 } 860 861 off = (void *)data_ptr - page_address(page); 862 skb_add_rx_frag(skb, 0, page, off, len, PAGE_SIZE); 863 memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN, 864 payload + NET_IP_ALIGN); 865 866 frag = &skb_shinfo(skb)->frags[0]; 867 skb_frag_size_sub(frag, payload); 868 frag->page_offset += payload; 869 skb->data_len -= payload; 870 skb->tail += payload; 871 872 return skb; 873 } 874 875 static struct sk_buff *bnxt_rx_skb(struct bnxt *bp, 876 struct bnxt_rx_ring_info *rxr, u16 cons, 877 void *data, u8 *data_ptr, 878 dma_addr_t dma_addr, 879 unsigned int offset_and_len) 880 { 881 u16 prod = rxr->rx_prod; 882 struct sk_buff *skb; 883 int err; 884 885 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC); 886 if (unlikely(err)) { 887 bnxt_reuse_rx_data(rxr, cons, data); 888 return NULL; 889 } 890 891 skb = build_skb(data, 0); 892 dma_unmap_single_attrs(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size, 893 bp->rx_dir, DMA_ATTR_WEAK_ORDERING); 894 if (!skb) { 895 kfree(data); 896 return NULL; 897 } 898 899 skb_reserve(skb, bp->rx_offset); 900 skb_put(skb, offset_and_len & 0xffff); 901 return skb; 902 } 903 904 static struct sk_buff *bnxt_rx_pages(struct bnxt *bp, struct bnxt_napi *bnapi, 905 struct sk_buff *skb, u16 cp_cons, 906 u32 agg_bufs) 907 { 908 struct pci_dev *pdev = bp->pdev; 909 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 910 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 911 u16 prod = rxr->rx_agg_prod; 912 u32 i; 913 914 for (i = 0; i < agg_bufs; i++) { 915 u16 cons, frag_len; 916 struct rx_agg_cmp *agg; 917 struct bnxt_sw_rx_agg_bd *cons_rx_buf; 918 struct page *page; 919 dma_addr_t mapping; 920 921 agg = (struct rx_agg_cmp *) 922 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 923 cons = agg->rx_agg_cmp_opaque; 924 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) & 925 RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT; 926 927 cons_rx_buf = &rxr->rx_agg_ring[cons]; 928 skb_fill_page_desc(skb, i, cons_rx_buf->page, 929 cons_rx_buf->offset, frag_len); 930 __clear_bit(cons, rxr->rx_agg_bmap); 931 932 /* It is possible for bnxt_alloc_rx_page() to allocate 933 * a sw_prod index that equals the cons index, so we 934 * need to clear the cons entry now. 935 */ 936 mapping = cons_rx_buf->mapping; 937 page = cons_rx_buf->page; 938 cons_rx_buf->page = NULL; 939 940 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) { 941 struct skb_shared_info *shinfo; 942 unsigned int nr_frags; 943 944 shinfo = skb_shinfo(skb); 945 nr_frags = --shinfo->nr_frags; 946 __skb_frag_set_page(&shinfo->frags[nr_frags], NULL); 947 948 dev_kfree_skb(skb); 949 950 cons_rx_buf->page = page; 951 952 /* Update prod since possibly some pages have been 953 * allocated already. 954 */ 955 rxr->rx_agg_prod = prod; 956 bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs - i); 957 return NULL; 958 } 959 960 dma_unmap_page_attrs(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE, 961 PCI_DMA_FROMDEVICE, 962 DMA_ATTR_WEAK_ORDERING); 963 964 skb->data_len += frag_len; 965 skb->len += frag_len; 966 skb->truesize += PAGE_SIZE; 967 968 prod = NEXT_RX_AGG(prod); 969 cp_cons = NEXT_CMP(cp_cons); 970 } 971 rxr->rx_agg_prod = prod; 972 return skb; 973 } 974 975 static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 976 u8 agg_bufs, u32 *raw_cons) 977 { 978 u16 last; 979 struct rx_agg_cmp *agg; 980 981 *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs); 982 last = RING_CMP(*raw_cons); 983 agg = (struct rx_agg_cmp *) 984 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)]; 985 return RX_AGG_CMP_VALID(agg, *raw_cons); 986 } 987 988 static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data, 989 unsigned int len, 990 dma_addr_t mapping) 991 { 992 struct bnxt *bp = bnapi->bp; 993 struct pci_dev *pdev = bp->pdev; 994 struct sk_buff *skb; 995 996 skb = napi_alloc_skb(&bnapi->napi, len); 997 if (!skb) 998 return NULL; 999 1000 dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copy_thresh, 1001 bp->rx_dir); 1002 1003 memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN, 1004 len + NET_IP_ALIGN); 1005 1006 dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copy_thresh, 1007 bp->rx_dir); 1008 1009 skb_put(skb, len); 1010 return skb; 1011 } 1012 1013 static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_napi *bnapi, 1014 u32 *raw_cons, void *cmp) 1015 { 1016 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 1017 struct rx_cmp *rxcmp = cmp; 1018 u32 tmp_raw_cons = *raw_cons; 1019 u8 cmp_type, agg_bufs = 0; 1020 1021 cmp_type = RX_CMP_TYPE(rxcmp); 1022 1023 if (cmp_type == CMP_TYPE_RX_L2_CMP) { 1024 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) & 1025 RX_CMP_AGG_BUFS) >> 1026 RX_CMP_AGG_BUFS_SHIFT; 1027 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) { 1028 struct rx_tpa_end_cmp *tpa_end = cmp; 1029 1030 agg_bufs = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) & 1031 RX_TPA_END_CMP_AGG_BUFS) >> 1032 RX_TPA_END_CMP_AGG_BUFS_SHIFT; 1033 } 1034 1035 if (agg_bufs) { 1036 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons)) 1037 return -EBUSY; 1038 } 1039 *raw_cons = tmp_raw_cons; 1040 return 0; 1041 } 1042 1043 static void bnxt_queue_sp_work(struct bnxt *bp) 1044 { 1045 if (BNXT_PF(bp)) 1046 queue_work(bnxt_pf_wq, &bp->sp_task); 1047 else 1048 schedule_work(&bp->sp_task); 1049 } 1050 1051 static void bnxt_cancel_sp_work(struct bnxt *bp) 1052 { 1053 if (BNXT_PF(bp)) 1054 flush_workqueue(bnxt_pf_wq); 1055 else 1056 cancel_work_sync(&bp->sp_task); 1057 } 1058 1059 static void bnxt_sched_reset(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) 1060 { 1061 if (!rxr->bnapi->in_reset) { 1062 rxr->bnapi->in_reset = true; 1063 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event); 1064 bnxt_queue_sp_work(bp); 1065 } 1066 rxr->rx_next_cons = 0xffff; 1067 } 1068 1069 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, 1070 struct rx_tpa_start_cmp *tpa_start, 1071 struct rx_tpa_start_cmp_ext *tpa_start1) 1072 { 1073 u8 agg_id = TPA_START_AGG_ID(tpa_start); 1074 u16 cons, prod; 1075 struct bnxt_tpa_info *tpa_info; 1076 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf; 1077 struct rx_bd *prod_bd; 1078 dma_addr_t mapping; 1079 1080 cons = tpa_start->rx_tpa_start_cmp_opaque; 1081 prod = rxr->rx_prod; 1082 cons_rx_buf = &rxr->rx_buf_ring[cons]; 1083 prod_rx_buf = &rxr->rx_buf_ring[prod]; 1084 tpa_info = &rxr->rx_tpa[agg_id]; 1085 1086 if (unlikely(cons != rxr->rx_next_cons)) { 1087 bnxt_sched_reset(bp, rxr); 1088 return; 1089 } 1090 /* Store cfa_code in tpa_info to use in tpa_end 1091 * completion processing. 1092 */ 1093 tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1); 1094 prod_rx_buf->data = tpa_info->data; 1095 prod_rx_buf->data_ptr = tpa_info->data_ptr; 1096 1097 mapping = tpa_info->mapping; 1098 prod_rx_buf->mapping = mapping; 1099 1100 prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 1101 1102 prod_bd->rx_bd_haddr = cpu_to_le64(mapping); 1103 1104 tpa_info->data = cons_rx_buf->data; 1105 tpa_info->data_ptr = cons_rx_buf->data_ptr; 1106 cons_rx_buf->data = NULL; 1107 tpa_info->mapping = cons_rx_buf->mapping; 1108 1109 tpa_info->len = 1110 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >> 1111 RX_TPA_START_CMP_LEN_SHIFT; 1112 if (likely(TPA_START_HASH_VALID(tpa_start))) { 1113 u32 hash_type = TPA_START_HASH_TYPE(tpa_start); 1114 1115 tpa_info->hash_type = PKT_HASH_TYPE_L4; 1116 tpa_info->gso_type = SKB_GSO_TCPV4; 1117 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */ 1118 if (hash_type == 3) 1119 tpa_info->gso_type = SKB_GSO_TCPV6; 1120 tpa_info->rss_hash = 1121 le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash); 1122 } else { 1123 tpa_info->hash_type = PKT_HASH_TYPE_NONE; 1124 tpa_info->gso_type = 0; 1125 if (netif_msg_rx_err(bp)) 1126 netdev_warn(bp->dev, "TPA packet without valid hash\n"); 1127 } 1128 tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2); 1129 tpa_info->metadata = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata); 1130 tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info); 1131 1132 rxr->rx_prod = NEXT_RX(prod); 1133 cons = NEXT_RX(cons); 1134 rxr->rx_next_cons = NEXT_RX(cons); 1135 cons_rx_buf = &rxr->rx_buf_ring[cons]; 1136 1137 bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data); 1138 rxr->rx_prod = NEXT_RX(rxr->rx_prod); 1139 cons_rx_buf->data = NULL; 1140 } 1141 1142 static void bnxt_abort_tpa(struct bnxt *bp, struct bnxt_napi *bnapi, 1143 u16 cp_cons, u32 agg_bufs) 1144 { 1145 if (agg_bufs) 1146 bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs); 1147 } 1148 1149 static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info, 1150 int payload_off, int tcp_ts, 1151 struct sk_buff *skb) 1152 { 1153 #ifdef CONFIG_INET 1154 struct tcphdr *th; 1155 int len, nw_off; 1156 u16 outer_ip_off, inner_ip_off, inner_mac_off; 1157 u32 hdr_info = tpa_info->hdr_info; 1158 bool loopback = false; 1159 1160 inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info); 1161 inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info); 1162 outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info); 1163 1164 /* If the packet is an internal loopback packet, the offsets will 1165 * have an extra 4 bytes. 1166 */ 1167 if (inner_mac_off == 4) { 1168 loopback = true; 1169 } else if (inner_mac_off > 4) { 1170 __be16 proto = *((__be16 *)(skb->data + inner_ip_off - 1171 ETH_HLEN - 2)); 1172 1173 /* We only support inner iPv4/ipv6. If we don't see the 1174 * correct protocol ID, it must be a loopback packet where 1175 * the offsets are off by 4. 1176 */ 1177 if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6)) 1178 loopback = true; 1179 } 1180 if (loopback) { 1181 /* internal loopback packet, subtract all offsets by 4 */ 1182 inner_ip_off -= 4; 1183 inner_mac_off -= 4; 1184 outer_ip_off -= 4; 1185 } 1186 1187 nw_off = inner_ip_off - ETH_HLEN; 1188 skb_set_network_header(skb, nw_off); 1189 if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) { 1190 struct ipv6hdr *iph = ipv6_hdr(skb); 1191 1192 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr)); 1193 len = skb->len - skb_transport_offset(skb); 1194 th = tcp_hdr(skb); 1195 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0); 1196 } else { 1197 struct iphdr *iph = ip_hdr(skb); 1198 1199 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr)); 1200 len = skb->len - skb_transport_offset(skb); 1201 th = tcp_hdr(skb); 1202 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0); 1203 } 1204 1205 if (inner_mac_off) { /* tunnel */ 1206 struct udphdr *uh = NULL; 1207 __be16 proto = *((__be16 *)(skb->data + outer_ip_off - 1208 ETH_HLEN - 2)); 1209 1210 if (proto == htons(ETH_P_IP)) { 1211 struct iphdr *iph = (struct iphdr *)skb->data; 1212 1213 if (iph->protocol == IPPROTO_UDP) 1214 uh = (struct udphdr *)(iph + 1); 1215 } else { 1216 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data; 1217 1218 if (iph->nexthdr == IPPROTO_UDP) 1219 uh = (struct udphdr *)(iph + 1); 1220 } 1221 if (uh) { 1222 if (uh->check) 1223 skb_shinfo(skb)->gso_type |= 1224 SKB_GSO_UDP_TUNNEL_CSUM; 1225 else 1226 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL; 1227 } 1228 } 1229 #endif 1230 return skb; 1231 } 1232 1233 #define BNXT_IPV4_HDR_SIZE (sizeof(struct iphdr) + sizeof(struct tcphdr)) 1234 #define BNXT_IPV6_HDR_SIZE (sizeof(struct ipv6hdr) + sizeof(struct tcphdr)) 1235 1236 static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info, 1237 int payload_off, int tcp_ts, 1238 struct sk_buff *skb) 1239 { 1240 #ifdef CONFIG_INET 1241 struct tcphdr *th; 1242 int len, nw_off, tcp_opt_len = 0; 1243 1244 if (tcp_ts) 1245 tcp_opt_len = 12; 1246 1247 if (tpa_info->gso_type == SKB_GSO_TCPV4) { 1248 struct iphdr *iph; 1249 1250 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len - 1251 ETH_HLEN; 1252 skb_set_network_header(skb, nw_off); 1253 iph = ip_hdr(skb); 1254 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr)); 1255 len = skb->len - skb_transport_offset(skb); 1256 th = tcp_hdr(skb); 1257 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0); 1258 } else if (tpa_info->gso_type == SKB_GSO_TCPV6) { 1259 struct ipv6hdr *iph; 1260 1261 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len - 1262 ETH_HLEN; 1263 skb_set_network_header(skb, nw_off); 1264 iph = ipv6_hdr(skb); 1265 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr)); 1266 len = skb->len - skb_transport_offset(skb); 1267 th = tcp_hdr(skb); 1268 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0); 1269 } else { 1270 dev_kfree_skb_any(skb); 1271 return NULL; 1272 } 1273 1274 if (nw_off) { /* tunnel */ 1275 struct udphdr *uh = NULL; 1276 1277 if (skb->protocol == htons(ETH_P_IP)) { 1278 struct iphdr *iph = (struct iphdr *)skb->data; 1279 1280 if (iph->protocol == IPPROTO_UDP) 1281 uh = (struct udphdr *)(iph + 1); 1282 } else { 1283 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data; 1284 1285 if (iph->nexthdr == IPPROTO_UDP) 1286 uh = (struct udphdr *)(iph + 1); 1287 } 1288 if (uh) { 1289 if (uh->check) 1290 skb_shinfo(skb)->gso_type |= 1291 SKB_GSO_UDP_TUNNEL_CSUM; 1292 else 1293 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL; 1294 } 1295 } 1296 #endif 1297 return skb; 1298 } 1299 1300 static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp, 1301 struct bnxt_tpa_info *tpa_info, 1302 struct rx_tpa_end_cmp *tpa_end, 1303 struct rx_tpa_end_cmp_ext *tpa_end1, 1304 struct sk_buff *skb) 1305 { 1306 #ifdef CONFIG_INET 1307 int payload_off; 1308 u16 segs; 1309 1310 segs = TPA_END_TPA_SEGS(tpa_end); 1311 if (segs == 1) 1312 return skb; 1313 1314 NAPI_GRO_CB(skb)->count = segs; 1315 skb_shinfo(skb)->gso_size = 1316 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len); 1317 skb_shinfo(skb)->gso_type = tpa_info->gso_type; 1318 payload_off = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) & 1319 RX_TPA_END_CMP_PAYLOAD_OFFSET) >> 1320 RX_TPA_END_CMP_PAYLOAD_OFFSET_SHIFT; 1321 skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb); 1322 if (likely(skb)) 1323 tcp_gro_complete(skb); 1324 #endif 1325 return skb; 1326 } 1327 1328 /* Given the cfa_code of a received packet determine which 1329 * netdev (vf-rep or PF) the packet is destined to. 1330 */ 1331 static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code) 1332 { 1333 struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code); 1334 1335 /* if vf-rep dev is NULL, the must belongs to the PF */ 1336 return dev ? dev : bp->dev; 1337 } 1338 1339 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp, 1340 struct bnxt_napi *bnapi, 1341 u32 *raw_cons, 1342 struct rx_tpa_end_cmp *tpa_end, 1343 struct rx_tpa_end_cmp_ext *tpa_end1, 1344 u8 *event) 1345 { 1346 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 1347 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1348 u8 agg_id = TPA_END_AGG_ID(tpa_end); 1349 u8 *data_ptr, agg_bufs; 1350 u16 cp_cons = RING_CMP(*raw_cons); 1351 unsigned int len; 1352 struct bnxt_tpa_info *tpa_info; 1353 dma_addr_t mapping; 1354 struct sk_buff *skb; 1355 void *data; 1356 1357 if (unlikely(bnapi->in_reset)) { 1358 int rc = bnxt_discard_rx(bp, bnapi, raw_cons, tpa_end); 1359 1360 if (rc < 0) 1361 return ERR_PTR(-EBUSY); 1362 return NULL; 1363 } 1364 1365 tpa_info = &rxr->rx_tpa[agg_id]; 1366 data = tpa_info->data; 1367 data_ptr = tpa_info->data_ptr; 1368 prefetch(data_ptr); 1369 len = tpa_info->len; 1370 mapping = tpa_info->mapping; 1371 1372 agg_bufs = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) & 1373 RX_TPA_END_CMP_AGG_BUFS) >> RX_TPA_END_CMP_AGG_BUFS_SHIFT; 1374 1375 if (agg_bufs) { 1376 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons)) 1377 return ERR_PTR(-EBUSY); 1378 1379 *event |= BNXT_AGG_EVENT; 1380 cp_cons = NEXT_CMP(cp_cons); 1381 } 1382 1383 if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) { 1384 bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs); 1385 if (agg_bufs > MAX_SKB_FRAGS) 1386 netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n", 1387 agg_bufs, (int)MAX_SKB_FRAGS); 1388 return NULL; 1389 } 1390 1391 if (len <= bp->rx_copy_thresh) { 1392 skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping); 1393 if (!skb) { 1394 bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs); 1395 return NULL; 1396 } 1397 } else { 1398 u8 *new_data; 1399 dma_addr_t new_mapping; 1400 1401 new_data = __bnxt_alloc_rx_data(bp, &new_mapping, GFP_ATOMIC); 1402 if (!new_data) { 1403 bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs); 1404 return NULL; 1405 } 1406 1407 tpa_info->data = new_data; 1408 tpa_info->data_ptr = new_data + bp->rx_offset; 1409 tpa_info->mapping = new_mapping; 1410 1411 skb = build_skb(data, 0); 1412 dma_unmap_single_attrs(&bp->pdev->dev, mapping, 1413 bp->rx_buf_use_size, bp->rx_dir, 1414 DMA_ATTR_WEAK_ORDERING); 1415 1416 if (!skb) { 1417 kfree(data); 1418 bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs); 1419 return NULL; 1420 } 1421 skb_reserve(skb, bp->rx_offset); 1422 skb_put(skb, len); 1423 } 1424 1425 if (agg_bufs) { 1426 skb = bnxt_rx_pages(bp, bnapi, skb, cp_cons, agg_bufs); 1427 if (!skb) { 1428 /* Page reuse already handled by bnxt_rx_pages(). */ 1429 return NULL; 1430 } 1431 } 1432 1433 skb->protocol = 1434 eth_type_trans(skb, bnxt_get_pkt_dev(bp, tpa_info->cfa_code)); 1435 1436 if (tpa_info->hash_type != PKT_HASH_TYPE_NONE) 1437 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type); 1438 1439 if ((tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) && 1440 (skb->dev->features & NETIF_F_HW_VLAN_CTAG_RX)) { 1441 u16 vlan_proto = tpa_info->metadata >> 1442 RX_CMP_FLAGS2_METADATA_TPID_SFT; 1443 u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_TCI_MASK; 1444 1445 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag); 1446 } 1447 1448 skb_checksum_none_assert(skb); 1449 if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) { 1450 skb->ip_summed = CHECKSUM_UNNECESSARY; 1451 skb->csum_level = 1452 (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3; 1453 } 1454 1455 if (TPA_END_GRO(tpa_end)) 1456 skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb); 1457 1458 return skb; 1459 } 1460 1461 static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi, 1462 struct sk_buff *skb) 1463 { 1464 if (skb->dev != bp->dev) { 1465 /* this packet belongs to a vf-rep */ 1466 bnxt_vf_rep_rx(bp, skb); 1467 return; 1468 } 1469 skb_record_rx_queue(skb, bnapi->index); 1470 napi_gro_receive(&bnapi->napi, skb); 1471 } 1472 1473 /* returns the following: 1474 * 1 - 1 packet successfully received 1475 * 0 - successful TPA_START, packet not completed yet 1476 * -EBUSY - completion ring does not have all the agg buffers yet 1477 * -ENOMEM - packet aborted due to out of memory 1478 * -EIO - packet aborted due to hw error indicated in BD 1479 */ 1480 static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_napi *bnapi, u32 *raw_cons, 1481 u8 *event) 1482 { 1483 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 1484 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1485 struct net_device *dev = bp->dev; 1486 struct rx_cmp *rxcmp; 1487 struct rx_cmp_ext *rxcmp1; 1488 u32 tmp_raw_cons = *raw_cons; 1489 u16 cfa_code, cons, prod, cp_cons = RING_CMP(tmp_raw_cons); 1490 struct bnxt_sw_rx_bd *rx_buf; 1491 unsigned int len; 1492 u8 *data_ptr, agg_bufs, cmp_type; 1493 dma_addr_t dma_addr; 1494 struct sk_buff *skb; 1495 void *data; 1496 int rc = 0; 1497 u32 misc; 1498 1499 rxcmp = (struct rx_cmp *) 1500 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1501 1502 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons); 1503 cp_cons = RING_CMP(tmp_raw_cons); 1504 rxcmp1 = (struct rx_cmp_ext *) 1505 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1506 1507 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons)) 1508 return -EBUSY; 1509 1510 cmp_type = RX_CMP_TYPE(rxcmp); 1511 1512 prod = rxr->rx_prod; 1513 1514 if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) { 1515 bnxt_tpa_start(bp, rxr, (struct rx_tpa_start_cmp *)rxcmp, 1516 (struct rx_tpa_start_cmp_ext *)rxcmp1); 1517 1518 *event |= BNXT_RX_EVENT; 1519 goto next_rx_no_prod_no_len; 1520 1521 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) { 1522 skb = bnxt_tpa_end(bp, bnapi, &tmp_raw_cons, 1523 (struct rx_tpa_end_cmp *)rxcmp, 1524 (struct rx_tpa_end_cmp_ext *)rxcmp1, event); 1525 1526 if (IS_ERR(skb)) 1527 return -EBUSY; 1528 1529 rc = -ENOMEM; 1530 if (likely(skb)) { 1531 bnxt_deliver_skb(bp, bnapi, skb); 1532 rc = 1; 1533 } 1534 *event |= BNXT_RX_EVENT; 1535 goto next_rx_no_prod_no_len; 1536 } 1537 1538 cons = rxcmp->rx_cmp_opaque; 1539 rx_buf = &rxr->rx_buf_ring[cons]; 1540 data = rx_buf->data; 1541 data_ptr = rx_buf->data_ptr; 1542 if (unlikely(cons != rxr->rx_next_cons)) { 1543 int rc1 = bnxt_discard_rx(bp, bnapi, raw_cons, rxcmp); 1544 1545 bnxt_sched_reset(bp, rxr); 1546 return rc1; 1547 } 1548 prefetch(data_ptr); 1549 1550 misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1); 1551 agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT; 1552 1553 if (agg_bufs) { 1554 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons)) 1555 return -EBUSY; 1556 1557 cp_cons = NEXT_CMP(cp_cons); 1558 *event |= BNXT_AGG_EVENT; 1559 } 1560 *event |= BNXT_RX_EVENT; 1561 1562 rx_buf->data = NULL; 1563 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) { 1564 bnxt_reuse_rx_data(rxr, cons, data); 1565 if (agg_bufs) 1566 bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs); 1567 1568 rc = -EIO; 1569 goto next_rx; 1570 } 1571 1572 len = le32_to_cpu(rxcmp->rx_cmp_len_flags_type) >> RX_CMP_LEN_SHIFT; 1573 dma_addr = rx_buf->mapping; 1574 1575 if (bnxt_rx_xdp(bp, rxr, cons, data, &data_ptr, &len, event)) { 1576 rc = 1; 1577 goto next_rx; 1578 } 1579 1580 if (len <= bp->rx_copy_thresh) { 1581 skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr); 1582 bnxt_reuse_rx_data(rxr, cons, data); 1583 if (!skb) { 1584 rc = -ENOMEM; 1585 goto next_rx; 1586 } 1587 } else { 1588 u32 payload; 1589 1590 if (rx_buf->data_ptr == data_ptr) 1591 payload = misc & RX_CMP_PAYLOAD_OFFSET; 1592 else 1593 payload = 0; 1594 skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr, 1595 payload | len); 1596 if (!skb) { 1597 rc = -ENOMEM; 1598 goto next_rx; 1599 } 1600 } 1601 1602 if (agg_bufs) { 1603 skb = bnxt_rx_pages(bp, bnapi, skb, cp_cons, agg_bufs); 1604 if (!skb) { 1605 rc = -ENOMEM; 1606 goto next_rx; 1607 } 1608 } 1609 1610 if (RX_CMP_HASH_VALID(rxcmp)) { 1611 u32 hash_type = RX_CMP_HASH_TYPE(rxcmp); 1612 enum pkt_hash_types type = PKT_HASH_TYPE_L4; 1613 1614 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */ 1615 if (hash_type != 1 && hash_type != 3) 1616 type = PKT_HASH_TYPE_L3; 1617 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type); 1618 } 1619 1620 cfa_code = RX_CMP_CFA_CODE(rxcmp1); 1621 skb->protocol = eth_type_trans(skb, bnxt_get_pkt_dev(bp, cfa_code)); 1622 1623 if ((rxcmp1->rx_cmp_flags2 & 1624 cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) && 1625 (skb->dev->features & NETIF_F_HW_VLAN_CTAG_RX)) { 1626 u32 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data); 1627 u16 vtag = meta_data & RX_CMP_FLAGS2_METADATA_TCI_MASK; 1628 u16 vlan_proto = meta_data >> RX_CMP_FLAGS2_METADATA_TPID_SFT; 1629 1630 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag); 1631 } 1632 1633 skb_checksum_none_assert(skb); 1634 if (RX_CMP_L4_CS_OK(rxcmp1)) { 1635 if (dev->features & NETIF_F_RXCSUM) { 1636 skb->ip_summed = CHECKSUM_UNNECESSARY; 1637 skb->csum_level = RX_CMP_ENCAP(rxcmp1); 1638 } 1639 } else { 1640 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) { 1641 if (dev->features & NETIF_F_RXCSUM) 1642 cpr->rx_l4_csum_errors++; 1643 } 1644 } 1645 1646 bnxt_deliver_skb(bp, bnapi, skb); 1647 rc = 1; 1648 1649 next_rx: 1650 rxr->rx_prod = NEXT_RX(prod); 1651 rxr->rx_next_cons = NEXT_RX(cons); 1652 1653 cpr->rx_packets += 1; 1654 cpr->rx_bytes += len; 1655 1656 next_rx_no_prod_no_len: 1657 *raw_cons = tmp_raw_cons; 1658 1659 return rc; 1660 } 1661 1662 /* In netpoll mode, if we are using a combined completion ring, we need to 1663 * discard the rx packets and recycle the buffers. 1664 */ 1665 static int bnxt_force_rx_discard(struct bnxt *bp, struct bnxt_napi *bnapi, 1666 u32 *raw_cons, u8 *event) 1667 { 1668 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 1669 u32 tmp_raw_cons = *raw_cons; 1670 struct rx_cmp_ext *rxcmp1; 1671 struct rx_cmp *rxcmp; 1672 u16 cp_cons; 1673 u8 cmp_type; 1674 1675 cp_cons = RING_CMP(tmp_raw_cons); 1676 rxcmp = (struct rx_cmp *) 1677 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1678 1679 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons); 1680 cp_cons = RING_CMP(tmp_raw_cons); 1681 rxcmp1 = (struct rx_cmp_ext *) 1682 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1683 1684 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons)) 1685 return -EBUSY; 1686 1687 cmp_type = RX_CMP_TYPE(rxcmp); 1688 if (cmp_type == CMP_TYPE_RX_L2_CMP) { 1689 rxcmp1->rx_cmp_cfa_code_errors_v2 |= 1690 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR); 1691 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) { 1692 struct rx_tpa_end_cmp_ext *tpa_end1; 1693 1694 tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1; 1695 tpa_end1->rx_tpa_end_cmp_errors_v2 |= 1696 cpu_to_le32(RX_TPA_END_CMP_ERRORS); 1697 } 1698 return bnxt_rx_pkt(bp, bnapi, raw_cons, event); 1699 } 1700 1701 #define BNXT_GET_EVENT_PORT(data) \ 1702 ((data) & \ 1703 ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK) 1704 1705 static int bnxt_async_event_process(struct bnxt *bp, 1706 struct hwrm_async_event_cmpl *cmpl) 1707 { 1708 u16 event_id = le16_to_cpu(cmpl->event_id); 1709 1710 /* TODO CHIMP_FW: Define event id's for link change, error etc */ 1711 switch (event_id) { 1712 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: { 1713 u32 data1 = le32_to_cpu(cmpl->event_data1); 1714 struct bnxt_link_info *link_info = &bp->link_info; 1715 1716 if (BNXT_VF(bp)) 1717 goto async_event_process_exit; 1718 1719 /* print unsupported speed warning in forced speed mode only */ 1720 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED) && 1721 (data1 & 0x20000)) { 1722 u16 fw_speed = link_info->force_link_speed; 1723 u32 speed = bnxt_fw_to_ethtool_speed(fw_speed); 1724 1725 if (speed != SPEED_UNKNOWN) 1726 netdev_warn(bp->dev, "Link speed %d no longer supported\n", 1727 speed); 1728 } 1729 set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event); 1730 /* fall thru */ 1731 } 1732 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE: 1733 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event); 1734 break; 1735 case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD: 1736 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event); 1737 break; 1738 case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: { 1739 u32 data1 = le32_to_cpu(cmpl->event_data1); 1740 u16 port_id = BNXT_GET_EVENT_PORT(data1); 1741 1742 if (BNXT_VF(bp)) 1743 break; 1744 1745 if (bp->pf.port_id != port_id) 1746 break; 1747 1748 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event); 1749 break; 1750 } 1751 case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE: 1752 if (BNXT_PF(bp)) 1753 goto async_event_process_exit; 1754 set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event); 1755 break; 1756 default: 1757 goto async_event_process_exit; 1758 } 1759 bnxt_queue_sp_work(bp); 1760 async_event_process_exit: 1761 bnxt_ulp_async_events(bp, cmpl); 1762 return 0; 1763 } 1764 1765 static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp) 1766 { 1767 u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id; 1768 struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp; 1769 struct hwrm_fwd_req_cmpl *fwd_req_cmpl = 1770 (struct hwrm_fwd_req_cmpl *)txcmp; 1771 1772 switch (cmpl_type) { 1773 case CMPL_BASE_TYPE_HWRM_DONE: 1774 seq_id = le16_to_cpu(h_cmpl->sequence_id); 1775 if (seq_id == bp->hwrm_intr_seq_id) 1776 bp->hwrm_intr_seq_id = HWRM_SEQ_ID_INVALID; 1777 else 1778 netdev_err(bp->dev, "Invalid hwrm seq id %d\n", seq_id); 1779 break; 1780 1781 case CMPL_BASE_TYPE_HWRM_FWD_REQ: 1782 vf_id = le16_to_cpu(fwd_req_cmpl->source_id); 1783 1784 if ((vf_id < bp->pf.first_vf_id) || 1785 (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) { 1786 netdev_err(bp->dev, "Msg contains invalid VF id %x\n", 1787 vf_id); 1788 return -EINVAL; 1789 } 1790 1791 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap); 1792 set_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event); 1793 bnxt_queue_sp_work(bp); 1794 break; 1795 1796 case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT: 1797 bnxt_async_event_process(bp, 1798 (struct hwrm_async_event_cmpl *)txcmp); 1799 1800 default: 1801 break; 1802 } 1803 1804 return 0; 1805 } 1806 1807 static irqreturn_t bnxt_msix(int irq, void *dev_instance) 1808 { 1809 struct bnxt_napi *bnapi = dev_instance; 1810 struct bnxt *bp = bnapi->bp; 1811 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 1812 u32 cons = RING_CMP(cpr->cp_raw_cons); 1813 1814 cpr->event_ctr++; 1815 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]); 1816 napi_schedule(&bnapi->napi); 1817 return IRQ_HANDLED; 1818 } 1819 1820 static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr) 1821 { 1822 u32 raw_cons = cpr->cp_raw_cons; 1823 u16 cons = RING_CMP(raw_cons); 1824 struct tx_cmp *txcmp; 1825 1826 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]; 1827 1828 return TX_CMP_VALID(txcmp, raw_cons); 1829 } 1830 1831 static irqreturn_t bnxt_inta(int irq, void *dev_instance) 1832 { 1833 struct bnxt_napi *bnapi = dev_instance; 1834 struct bnxt *bp = bnapi->bp; 1835 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 1836 u32 cons = RING_CMP(cpr->cp_raw_cons); 1837 u32 int_status; 1838 1839 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]); 1840 1841 if (!bnxt_has_work(bp, cpr)) { 1842 int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS); 1843 /* return if erroneous interrupt */ 1844 if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id))) 1845 return IRQ_NONE; 1846 } 1847 1848 /* disable ring IRQ */ 1849 BNXT_CP_DB_IRQ_DIS(cpr->cp_doorbell); 1850 1851 /* Return here if interrupt is shared and is disabled. */ 1852 if (unlikely(atomic_read(&bp->intr_sem) != 0)) 1853 return IRQ_HANDLED; 1854 1855 napi_schedule(&bnapi->napi); 1856 return IRQ_HANDLED; 1857 } 1858 1859 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_napi *bnapi, int budget) 1860 { 1861 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 1862 u32 raw_cons = cpr->cp_raw_cons; 1863 u32 cons; 1864 int tx_pkts = 0; 1865 int rx_pkts = 0; 1866 u8 event = 0; 1867 struct tx_cmp *txcmp; 1868 1869 while (1) { 1870 int rc; 1871 1872 cons = RING_CMP(raw_cons); 1873 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]; 1874 1875 if (!TX_CMP_VALID(txcmp, raw_cons)) 1876 break; 1877 1878 /* The valid test of the entry must be done first before 1879 * reading any further. 1880 */ 1881 dma_rmb(); 1882 if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) { 1883 tx_pkts++; 1884 /* return full budget so NAPI will complete. */ 1885 if (unlikely(tx_pkts > bp->tx_wake_thresh)) 1886 rx_pkts = budget; 1887 } else if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) { 1888 if (likely(budget)) 1889 rc = bnxt_rx_pkt(bp, bnapi, &raw_cons, &event); 1890 else 1891 rc = bnxt_force_rx_discard(bp, bnapi, &raw_cons, 1892 &event); 1893 if (likely(rc >= 0)) 1894 rx_pkts += rc; 1895 /* Increment rx_pkts when rc is -ENOMEM to count towards 1896 * the NAPI budget. Otherwise, we may potentially loop 1897 * here forever if we consistently cannot allocate 1898 * buffers. 1899 */ 1900 else if (rc == -ENOMEM && budget) 1901 rx_pkts++; 1902 else if (rc == -EBUSY) /* partial completion */ 1903 break; 1904 } else if (unlikely((TX_CMP_TYPE(txcmp) == 1905 CMPL_BASE_TYPE_HWRM_DONE) || 1906 (TX_CMP_TYPE(txcmp) == 1907 CMPL_BASE_TYPE_HWRM_FWD_REQ) || 1908 (TX_CMP_TYPE(txcmp) == 1909 CMPL_BASE_TYPE_HWRM_ASYNC_EVENT))) { 1910 bnxt_hwrm_handler(bp, txcmp); 1911 } 1912 raw_cons = NEXT_RAW_CMP(raw_cons); 1913 1914 if (rx_pkts == budget) 1915 break; 1916 } 1917 1918 if (event & BNXT_TX_EVENT) { 1919 struct bnxt_tx_ring_info *txr = bnapi->tx_ring; 1920 void __iomem *db = txr->tx_doorbell; 1921 u16 prod = txr->tx_prod; 1922 1923 /* Sync BD data before updating doorbell */ 1924 wmb(); 1925 1926 bnxt_db_write_relaxed(bp, db, DB_KEY_TX | prod); 1927 } 1928 1929 cpr->cp_raw_cons = raw_cons; 1930 /* ACK completion ring before freeing tx ring and producing new 1931 * buffers in rx/agg rings to prevent overflowing the completion 1932 * ring. 1933 */ 1934 BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons); 1935 1936 if (tx_pkts) 1937 bnapi->tx_int(bp, bnapi, tx_pkts); 1938 1939 if (event & BNXT_RX_EVENT) { 1940 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1941 1942 bnxt_db_write(bp, rxr->rx_doorbell, DB_KEY_RX | rxr->rx_prod); 1943 if (event & BNXT_AGG_EVENT) 1944 bnxt_db_write(bp, rxr->rx_agg_doorbell, 1945 DB_KEY_RX | rxr->rx_agg_prod); 1946 } 1947 return rx_pkts; 1948 } 1949 1950 static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget) 1951 { 1952 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi); 1953 struct bnxt *bp = bnapi->bp; 1954 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 1955 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1956 struct tx_cmp *txcmp; 1957 struct rx_cmp_ext *rxcmp1; 1958 u32 cp_cons, tmp_raw_cons; 1959 u32 raw_cons = cpr->cp_raw_cons; 1960 u32 rx_pkts = 0; 1961 u8 event = 0; 1962 1963 while (1) { 1964 int rc; 1965 1966 cp_cons = RING_CMP(raw_cons); 1967 txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1968 1969 if (!TX_CMP_VALID(txcmp, raw_cons)) 1970 break; 1971 1972 if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) { 1973 tmp_raw_cons = NEXT_RAW_CMP(raw_cons); 1974 cp_cons = RING_CMP(tmp_raw_cons); 1975 rxcmp1 = (struct rx_cmp_ext *) 1976 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1977 1978 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons)) 1979 break; 1980 1981 /* force an error to recycle the buffer */ 1982 rxcmp1->rx_cmp_cfa_code_errors_v2 |= 1983 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR); 1984 1985 rc = bnxt_rx_pkt(bp, bnapi, &raw_cons, &event); 1986 if (likely(rc == -EIO) && budget) 1987 rx_pkts++; 1988 else if (rc == -EBUSY) /* partial completion */ 1989 break; 1990 } else if (unlikely(TX_CMP_TYPE(txcmp) == 1991 CMPL_BASE_TYPE_HWRM_DONE)) { 1992 bnxt_hwrm_handler(bp, txcmp); 1993 } else { 1994 netdev_err(bp->dev, 1995 "Invalid completion received on special ring\n"); 1996 } 1997 raw_cons = NEXT_RAW_CMP(raw_cons); 1998 1999 if (rx_pkts == budget) 2000 break; 2001 } 2002 2003 cpr->cp_raw_cons = raw_cons; 2004 BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons); 2005 bnxt_db_write(bp, rxr->rx_doorbell, DB_KEY_RX | rxr->rx_prod); 2006 2007 if (event & BNXT_AGG_EVENT) 2008 bnxt_db_write(bp, rxr->rx_agg_doorbell, 2009 DB_KEY_RX | rxr->rx_agg_prod); 2010 2011 if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) { 2012 napi_complete_done(napi, rx_pkts); 2013 BNXT_CP_DB_REARM(cpr->cp_doorbell, cpr->cp_raw_cons); 2014 } 2015 return rx_pkts; 2016 } 2017 2018 static int bnxt_poll(struct napi_struct *napi, int budget) 2019 { 2020 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi); 2021 struct bnxt *bp = bnapi->bp; 2022 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2023 int work_done = 0; 2024 2025 while (1) { 2026 work_done += bnxt_poll_work(bp, bnapi, budget - work_done); 2027 2028 if (work_done >= budget) 2029 break; 2030 2031 if (!bnxt_has_work(bp, cpr)) { 2032 if (napi_complete_done(napi, work_done)) 2033 BNXT_CP_DB_REARM(cpr->cp_doorbell, 2034 cpr->cp_raw_cons); 2035 break; 2036 } 2037 } 2038 if (bp->flags & BNXT_FLAG_DIM) { 2039 struct net_dim_sample dim_sample; 2040 2041 net_dim_sample(cpr->event_ctr, 2042 cpr->rx_packets, 2043 cpr->rx_bytes, 2044 &dim_sample); 2045 net_dim(&cpr->dim, dim_sample); 2046 } 2047 mmiowb(); 2048 return work_done; 2049 } 2050 2051 static void bnxt_free_tx_skbs(struct bnxt *bp) 2052 { 2053 int i, max_idx; 2054 struct pci_dev *pdev = bp->pdev; 2055 2056 if (!bp->tx_ring) 2057 return; 2058 2059 max_idx = bp->tx_nr_pages * TX_DESC_CNT; 2060 for (i = 0; i < bp->tx_nr_rings; i++) { 2061 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 2062 int j; 2063 2064 for (j = 0; j < max_idx;) { 2065 struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j]; 2066 struct sk_buff *skb = tx_buf->skb; 2067 int k, last; 2068 2069 if (!skb) { 2070 j++; 2071 continue; 2072 } 2073 2074 tx_buf->skb = NULL; 2075 2076 if (tx_buf->is_push) { 2077 dev_kfree_skb(skb); 2078 j += 2; 2079 continue; 2080 } 2081 2082 dma_unmap_single(&pdev->dev, 2083 dma_unmap_addr(tx_buf, mapping), 2084 skb_headlen(skb), 2085 PCI_DMA_TODEVICE); 2086 2087 last = tx_buf->nr_frags; 2088 j += 2; 2089 for (k = 0; k < last; k++, j++) { 2090 int ring_idx = j & bp->tx_ring_mask; 2091 skb_frag_t *frag = &skb_shinfo(skb)->frags[k]; 2092 2093 tx_buf = &txr->tx_buf_ring[ring_idx]; 2094 dma_unmap_page( 2095 &pdev->dev, 2096 dma_unmap_addr(tx_buf, mapping), 2097 skb_frag_size(frag), PCI_DMA_TODEVICE); 2098 } 2099 dev_kfree_skb(skb); 2100 } 2101 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i)); 2102 } 2103 } 2104 2105 static void bnxt_free_rx_skbs(struct bnxt *bp) 2106 { 2107 int i, max_idx, max_agg_idx; 2108 struct pci_dev *pdev = bp->pdev; 2109 2110 if (!bp->rx_ring) 2111 return; 2112 2113 max_idx = bp->rx_nr_pages * RX_DESC_CNT; 2114 max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT; 2115 for (i = 0; i < bp->rx_nr_rings; i++) { 2116 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 2117 int j; 2118 2119 if (rxr->rx_tpa) { 2120 for (j = 0; j < MAX_TPA; j++) { 2121 struct bnxt_tpa_info *tpa_info = 2122 &rxr->rx_tpa[j]; 2123 u8 *data = tpa_info->data; 2124 2125 if (!data) 2126 continue; 2127 2128 dma_unmap_single_attrs(&pdev->dev, 2129 tpa_info->mapping, 2130 bp->rx_buf_use_size, 2131 bp->rx_dir, 2132 DMA_ATTR_WEAK_ORDERING); 2133 2134 tpa_info->data = NULL; 2135 2136 kfree(data); 2137 } 2138 } 2139 2140 for (j = 0; j < max_idx; j++) { 2141 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[j]; 2142 dma_addr_t mapping = rx_buf->mapping; 2143 void *data = rx_buf->data; 2144 2145 if (!data) 2146 continue; 2147 2148 rx_buf->data = NULL; 2149 2150 if (BNXT_RX_PAGE_MODE(bp)) { 2151 mapping -= bp->rx_dma_offset; 2152 dma_unmap_page_attrs(&pdev->dev, mapping, 2153 PAGE_SIZE, bp->rx_dir, 2154 DMA_ATTR_WEAK_ORDERING); 2155 __free_page(data); 2156 } else { 2157 dma_unmap_single_attrs(&pdev->dev, mapping, 2158 bp->rx_buf_use_size, 2159 bp->rx_dir, 2160 DMA_ATTR_WEAK_ORDERING); 2161 kfree(data); 2162 } 2163 } 2164 2165 for (j = 0; j < max_agg_idx; j++) { 2166 struct bnxt_sw_rx_agg_bd *rx_agg_buf = 2167 &rxr->rx_agg_ring[j]; 2168 struct page *page = rx_agg_buf->page; 2169 2170 if (!page) 2171 continue; 2172 2173 dma_unmap_page_attrs(&pdev->dev, rx_agg_buf->mapping, 2174 BNXT_RX_PAGE_SIZE, 2175 PCI_DMA_FROMDEVICE, 2176 DMA_ATTR_WEAK_ORDERING); 2177 2178 rx_agg_buf->page = NULL; 2179 __clear_bit(j, rxr->rx_agg_bmap); 2180 2181 __free_page(page); 2182 } 2183 if (rxr->rx_page) { 2184 __free_page(rxr->rx_page); 2185 rxr->rx_page = NULL; 2186 } 2187 } 2188 } 2189 2190 static void bnxt_free_skbs(struct bnxt *bp) 2191 { 2192 bnxt_free_tx_skbs(bp); 2193 bnxt_free_rx_skbs(bp); 2194 } 2195 2196 static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_struct *ring) 2197 { 2198 struct pci_dev *pdev = bp->pdev; 2199 int i; 2200 2201 for (i = 0; i < ring->nr_pages; i++) { 2202 if (!ring->pg_arr[i]) 2203 continue; 2204 2205 dma_free_coherent(&pdev->dev, ring->page_size, 2206 ring->pg_arr[i], ring->dma_arr[i]); 2207 2208 ring->pg_arr[i] = NULL; 2209 } 2210 if (ring->pg_tbl) { 2211 dma_free_coherent(&pdev->dev, ring->nr_pages * 8, 2212 ring->pg_tbl, ring->pg_tbl_map); 2213 ring->pg_tbl = NULL; 2214 } 2215 if (ring->vmem_size && *ring->vmem) { 2216 vfree(*ring->vmem); 2217 *ring->vmem = NULL; 2218 } 2219 } 2220 2221 static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_struct *ring) 2222 { 2223 int i; 2224 struct pci_dev *pdev = bp->pdev; 2225 2226 if (ring->nr_pages > 1) { 2227 ring->pg_tbl = dma_alloc_coherent(&pdev->dev, 2228 ring->nr_pages * 8, 2229 &ring->pg_tbl_map, 2230 GFP_KERNEL); 2231 if (!ring->pg_tbl) 2232 return -ENOMEM; 2233 } 2234 2235 for (i = 0; i < ring->nr_pages; i++) { 2236 ring->pg_arr[i] = dma_alloc_coherent(&pdev->dev, 2237 ring->page_size, 2238 &ring->dma_arr[i], 2239 GFP_KERNEL); 2240 if (!ring->pg_arr[i]) 2241 return -ENOMEM; 2242 2243 if (ring->nr_pages > 1) 2244 ring->pg_tbl[i] = cpu_to_le64(ring->dma_arr[i]); 2245 } 2246 2247 if (ring->vmem_size) { 2248 *ring->vmem = vzalloc(ring->vmem_size); 2249 if (!(*ring->vmem)) 2250 return -ENOMEM; 2251 } 2252 return 0; 2253 } 2254 2255 static void bnxt_free_rx_rings(struct bnxt *bp) 2256 { 2257 int i; 2258 2259 if (!bp->rx_ring) 2260 return; 2261 2262 for (i = 0; i < bp->rx_nr_rings; i++) { 2263 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 2264 struct bnxt_ring_struct *ring; 2265 2266 if (rxr->xdp_prog) 2267 bpf_prog_put(rxr->xdp_prog); 2268 2269 if (xdp_rxq_info_is_reg(&rxr->xdp_rxq)) 2270 xdp_rxq_info_unreg(&rxr->xdp_rxq); 2271 2272 kfree(rxr->rx_tpa); 2273 rxr->rx_tpa = NULL; 2274 2275 kfree(rxr->rx_agg_bmap); 2276 rxr->rx_agg_bmap = NULL; 2277 2278 ring = &rxr->rx_ring_struct; 2279 bnxt_free_ring(bp, ring); 2280 2281 ring = &rxr->rx_agg_ring_struct; 2282 bnxt_free_ring(bp, ring); 2283 } 2284 } 2285 2286 static int bnxt_alloc_rx_rings(struct bnxt *bp) 2287 { 2288 int i, rc, agg_rings = 0, tpa_rings = 0; 2289 2290 if (!bp->rx_ring) 2291 return -ENOMEM; 2292 2293 if (bp->flags & BNXT_FLAG_AGG_RINGS) 2294 agg_rings = 1; 2295 2296 if (bp->flags & BNXT_FLAG_TPA) 2297 tpa_rings = 1; 2298 2299 for (i = 0; i < bp->rx_nr_rings; i++) { 2300 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 2301 struct bnxt_ring_struct *ring; 2302 2303 ring = &rxr->rx_ring_struct; 2304 2305 rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i); 2306 if (rc < 0) 2307 return rc; 2308 2309 rc = bnxt_alloc_ring(bp, ring); 2310 if (rc) 2311 return rc; 2312 2313 if (agg_rings) { 2314 u16 mem_size; 2315 2316 ring = &rxr->rx_agg_ring_struct; 2317 rc = bnxt_alloc_ring(bp, ring); 2318 if (rc) 2319 return rc; 2320 2321 ring->grp_idx = i; 2322 rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1; 2323 mem_size = rxr->rx_agg_bmap_size / 8; 2324 rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL); 2325 if (!rxr->rx_agg_bmap) 2326 return -ENOMEM; 2327 2328 if (tpa_rings) { 2329 rxr->rx_tpa = kcalloc(MAX_TPA, 2330 sizeof(struct bnxt_tpa_info), 2331 GFP_KERNEL); 2332 if (!rxr->rx_tpa) 2333 return -ENOMEM; 2334 } 2335 } 2336 } 2337 return 0; 2338 } 2339 2340 static void bnxt_free_tx_rings(struct bnxt *bp) 2341 { 2342 int i; 2343 struct pci_dev *pdev = bp->pdev; 2344 2345 if (!bp->tx_ring) 2346 return; 2347 2348 for (i = 0; i < bp->tx_nr_rings; i++) { 2349 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 2350 struct bnxt_ring_struct *ring; 2351 2352 if (txr->tx_push) { 2353 dma_free_coherent(&pdev->dev, bp->tx_push_size, 2354 txr->tx_push, txr->tx_push_mapping); 2355 txr->tx_push = NULL; 2356 } 2357 2358 ring = &txr->tx_ring_struct; 2359 2360 bnxt_free_ring(bp, ring); 2361 } 2362 } 2363 2364 static int bnxt_alloc_tx_rings(struct bnxt *bp) 2365 { 2366 int i, j, rc; 2367 struct pci_dev *pdev = bp->pdev; 2368 2369 bp->tx_push_size = 0; 2370 if (bp->tx_push_thresh) { 2371 int push_size; 2372 2373 push_size = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) + 2374 bp->tx_push_thresh); 2375 2376 if (push_size > 256) { 2377 push_size = 0; 2378 bp->tx_push_thresh = 0; 2379 } 2380 2381 bp->tx_push_size = push_size; 2382 } 2383 2384 for (i = 0, j = 0; i < bp->tx_nr_rings; i++) { 2385 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 2386 struct bnxt_ring_struct *ring; 2387 u8 qidx; 2388 2389 ring = &txr->tx_ring_struct; 2390 2391 rc = bnxt_alloc_ring(bp, ring); 2392 if (rc) 2393 return rc; 2394 2395 ring->grp_idx = txr->bnapi->index; 2396 if (bp->tx_push_size) { 2397 dma_addr_t mapping; 2398 2399 /* One pre-allocated DMA buffer to backup 2400 * TX push operation 2401 */ 2402 txr->tx_push = dma_alloc_coherent(&pdev->dev, 2403 bp->tx_push_size, 2404 &txr->tx_push_mapping, 2405 GFP_KERNEL); 2406 2407 if (!txr->tx_push) 2408 return -ENOMEM; 2409 2410 mapping = txr->tx_push_mapping + 2411 sizeof(struct tx_push_bd); 2412 txr->data_mapping = cpu_to_le64(mapping); 2413 2414 memset(txr->tx_push, 0, sizeof(struct tx_push_bd)); 2415 } 2416 qidx = bp->tc_to_qidx[j]; 2417 ring->queue_id = bp->q_info[qidx].queue_id; 2418 if (i < bp->tx_nr_rings_xdp) 2419 continue; 2420 if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1)) 2421 j++; 2422 } 2423 return 0; 2424 } 2425 2426 static void bnxt_free_cp_rings(struct bnxt *bp) 2427 { 2428 int i; 2429 2430 if (!bp->bnapi) 2431 return; 2432 2433 for (i = 0; i < bp->cp_nr_rings; i++) { 2434 struct bnxt_napi *bnapi = bp->bnapi[i]; 2435 struct bnxt_cp_ring_info *cpr; 2436 struct bnxt_ring_struct *ring; 2437 2438 if (!bnapi) 2439 continue; 2440 2441 cpr = &bnapi->cp_ring; 2442 ring = &cpr->cp_ring_struct; 2443 2444 bnxt_free_ring(bp, ring); 2445 } 2446 } 2447 2448 static int bnxt_alloc_cp_rings(struct bnxt *bp) 2449 { 2450 int i, rc, ulp_base_vec, ulp_msix; 2451 2452 ulp_msix = bnxt_get_ulp_msix_num(bp); 2453 ulp_base_vec = bnxt_get_ulp_msix_base(bp); 2454 for (i = 0; i < bp->cp_nr_rings; i++) { 2455 struct bnxt_napi *bnapi = bp->bnapi[i]; 2456 struct bnxt_cp_ring_info *cpr; 2457 struct bnxt_ring_struct *ring; 2458 2459 if (!bnapi) 2460 continue; 2461 2462 cpr = &bnapi->cp_ring; 2463 ring = &cpr->cp_ring_struct; 2464 2465 rc = bnxt_alloc_ring(bp, ring); 2466 if (rc) 2467 return rc; 2468 2469 if (ulp_msix && i >= ulp_base_vec) 2470 ring->map_idx = i + ulp_msix; 2471 else 2472 ring->map_idx = i; 2473 } 2474 return 0; 2475 } 2476 2477 static void bnxt_init_ring_struct(struct bnxt *bp) 2478 { 2479 int i; 2480 2481 for (i = 0; i < bp->cp_nr_rings; i++) { 2482 struct bnxt_napi *bnapi = bp->bnapi[i]; 2483 struct bnxt_cp_ring_info *cpr; 2484 struct bnxt_rx_ring_info *rxr; 2485 struct bnxt_tx_ring_info *txr; 2486 struct bnxt_ring_struct *ring; 2487 2488 if (!bnapi) 2489 continue; 2490 2491 cpr = &bnapi->cp_ring; 2492 ring = &cpr->cp_ring_struct; 2493 ring->nr_pages = bp->cp_nr_pages; 2494 ring->page_size = HW_CMPD_RING_SIZE; 2495 ring->pg_arr = (void **)cpr->cp_desc_ring; 2496 ring->dma_arr = cpr->cp_desc_mapping; 2497 ring->vmem_size = 0; 2498 2499 rxr = bnapi->rx_ring; 2500 if (!rxr) 2501 goto skip_rx; 2502 2503 ring = &rxr->rx_ring_struct; 2504 ring->nr_pages = bp->rx_nr_pages; 2505 ring->page_size = HW_RXBD_RING_SIZE; 2506 ring->pg_arr = (void **)rxr->rx_desc_ring; 2507 ring->dma_arr = rxr->rx_desc_mapping; 2508 ring->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages; 2509 ring->vmem = (void **)&rxr->rx_buf_ring; 2510 2511 ring = &rxr->rx_agg_ring_struct; 2512 ring->nr_pages = bp->rx_agg_nr_pages; 2513 ring->page_size = HW_RXBD_RING_SIZE; 2514 ring->pg_arr = (void **)rxr->rx_agg_desc_ring; 2515 ring->dma_arr = rxr->rx_agg_desc_mapping; 2516 ring->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages; 2517 ring->vmem = (void **)&rxr->rx_agg_ring; 2518 2519 skip_rx: 2520 txr = bnapi->tx_ring; 2521 if (!txr) 2522 continue; 2523 2524 ring = &txr->tx_ring_struct; 2525 ring->nr_pages = bp->tx_nr_pages; 2526 ring->page_size = HW_RXBD_RING_SIZE; 2527 ring->pg_arr = (void **)txr->tx_desc_ring; 2528 ring->dma_arr = txr->tx_desc_mapping; 2529 ring->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages; 2530 ring->vmem = (void **)&txr->tx_buf_ring; 2531 } 2532 } 2533 2534 static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type) 2535 { 2536 int i; 2537 u32 prod; 2538 struct rx_bd **rx_buf_ring; 2539 2540 rx_buf_ring = (struct rx_bd **)ring->pg_arr; 2541 for (i = 0, prod = 0; i < ring->nr_pages; i++) { 2542 int j; 2543 struct rx_bd *rxbd; 2544 2545 rxbd = rx_buf_ring[i]; 2546 if (!rxbd) 2547 continue; 2548 2549 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) { 2550 rxbd->rx_bd_len_flags_type = cpu_to_le32(type); 2551 rxbd->rx_bd_opaque = prod; 2552 } 2553 } 2554 } 2555 2556 static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr) 2557 { 2558 struct net_device *dev = bp->dev; 2559 struct bnxt_rx_ring_info *rxr; 2560 struct bnxt_ring_struct *ring; 2561 u32 prod, type; 2562 int i; 2563 2564 type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) | 2565 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP; 2566 2567 if (NET_IP_ALIGN == 2) 2568 type |= RX_BD_FLAGS_SOP; 2569 2570 rxr = &bp->rx_ring[ring_nr]; 2571 ring = &rxr->rx_ring_struct; 2572 bnxt_init_rxbd_pages(ring, type); 2573 2574 if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) { 2575 rxr->xdp_prog = bpf_prog_add(bp->xdp_prog, 1); 2576 if (IS_ERR(rxr->xdp_prog)) { 2577 int rc = PTR_ERR(rxr->xdp_prog); 2578 2579 rxr->xdp_prog = NULL; 2580 return rc; 2581 } 2582 } 2583 prod = rxr->rx_prod; 2584 for (i = 0; i < bp->rx_ring_size; i++) { 2585 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL) != 0) { 2586 netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n", 2587 ring_nr, i, bp->rx_ring_size); 2588 break; 2589 } 2590 prod = NEXT_RX(prod); 2591 } 2592 rxr->rx_prod = prod; 2593 ring->fw_ring_id = INVALID_HW_RING_ID; 2594 2595 ring = &rxr->rx_agg_ring_struct; 2596 ring->fw_ring_id = INVALID_HW_RING_ID; 2597 2598 if (!(bp->flags & BNXT_FLAG_AGG_RINGS)) 2599 return 0; 2600 2601 type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) | 2602 RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP; 2603 2604 bnxt_init_rxbd_pages(ring, type); 2605 2606 prod = rxr->rx_agg_prod; 2607 for (i = 0; i < bp->rx_agg_ring_size; i++) { 2608 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL) != 0) { 2609 netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n", 2610 ring_nr, i, bp->rx_ring_size); 2611 break; 2612 } 2613 prod = NEXT_RX_AGG(prod); 2614 } 2615 rxr->rx_agg_prod = prod; 2616 2617 if (bp->flags & BNXT_FLAG_TPA) { 2618 if (rxr->rx_tpa) { 2619 u8 *data; 2620 dma_addr_t mapping; 2621 2622 for (i = 0; i < MAX_TPA; i++) { 2623 data = __bnxt_alloc_rx_data(bp, &mapping, 2624 GFP_KERNEL); 2625 if (!data) 2626 return -ENOMEM; 2627 2628 rxr->rx_tpa[i].data = data; 2629 rxr->rx_tpa[i].data_ptr = data + bp->rx_offset; 2630 rxr->rx_tpa[i].mapping = mapping; 2631 } 2632 } else { 2633 netdev_err(bp->dev, "No resource allocated for LRO/GRO\n"); 2634 return -ENOMEM; 2635 } 2636 } 2637 2638 return 0; 2639 } 2640 2641 static void bnxt_init_cp_rings(struct bnxt *bp) 2642 { 2643 int i; 2644 2645 for (i = 0; i < bp->cp_nr_rings; i++) { 2646 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring; 2647 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 2648 2649 ring->fw_ring_id = INVALID_HW_RING_ID; 2650 cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks; 2651 cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs; 2652 } 2653 } 2654 2655 static int bnxt_init_rx_rings(struct bnxt *bp) 2656 { 2657 int i, rc = 0; 2658 2659 if (BNXT_RX_PAGE_MODE(bp)) { 2660 bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM; 2661 bp->rx_dma_offset = XDP_PACKET_HEADROOM; 2662 } else { 2663 bp->rx_offset = BNXT_RX_OFFSET; 2664 bp->rx_dma_offset = BNXT_RX_DMA_OFFSET; 2665 } 2666 2667 for (i = 0; i < bp->rx_nr_rings; i++) { 2668 rc = bnxt_init_one_rx_ring(bp, i); 2669 if (rc) 2670 break; 2671 } 2672 2673 return rc; 2674 } 2675 2676 static int bnxt_init_tx_rings(struct bnxt *bp) 2677 { 2678 u16 i; 2679 2680 bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2, 2681 MAX_SKB_FRAGS + 1); 2682 2683 for (i = 0; i < bp->tx_nr_rings; i++) { 2684 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 2685 struct bnxt_ring_struct *ring = &txr->tx_ring_struct; 2686 2687 ring->fw_ring_id = INVALID_HW_RING_ID; 2688 } 2689 2690 return 0; 2691 } 2692 2693 static void bnxt_free_ring_grps(struct bnxt *bp) 2694 { 2695 kfree(bp->grp_info); 2696 bp->grp_info = NULL; 2697 } 2698 2699 static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init) 2700 { 2701 int i; 2702 2703 if (irq_re_init) { 2704 bp->grp_info = kcalloc(bp->cp_nr_rings, 2705 sizeof(struct bnxt_ring_grp_info), 2706 GFP_KERNEL); 2707 if (!bp->grp_info) 2708 return -ENOMEM; 2709 } 2710 for (i = 0; i < bp->cp_nr_rings; i++) { 2711 if (irq_re_init) 2712 bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID; 2713 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID; 2714 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID; 2715 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID; 2716 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID; 2717 } 2718 return 0; 2719 } 2720 2721 static void bnxt_free_vnics(struct bnxt *bp) 2722 { 2723 kfree(bp->vnic_info); 2724 bp->vnic_info = NULL; 2725 bp->nr_vnics = 0; 2726 } 2727 2728 static int bnxt_alloc_vnics(struct bnxt *bp) 2729 { 2730 int num_vnics = 1; 2731 2732 #ifdef CONFIG_RFS_ACCEL 2733 if (bp->flags & BNXT_FLAG_RFS) 2734 num_vnics += bp->rx_nr_rings; 2735 #endif 2736 2737 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 2738 num_vnics++; 2739 2740 bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info), 2741 GFP_KERNEL); 2742 if (!bp->vnic_info) 2743 return -ENOMEM; 2744 2745 bp->nr_vnics = num_vnics; 2746 return 0; 2747 } 2748 2749 static void bnxt_init_vnics(struct bnxt *bp) 2750 { 2751 int i; 2752 2753 for (i = 0; i < bp->nr_vnics; i++) { 2754 struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; 2755 2756 vnic->fw_vnic_id = INVALID_HW_RING_ID; 2757 vnic->fw_rss_cos_lb_ctx[0] = INVALID_HW_RING_ID; 2758 vnic->fw_rss_cos_lb_ctx[1] = INVALID_HW_RING_ID; 2759 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID; 2760 2761 if (bp->vnic_info[i].rss_hash_key) { 2762 if (i == 0) 2763 prandom_bytes(vnic->rss_hash_key, 2764 HW_HASH_KEY_SIZE); 2765 else 2766 memcpy(vnic->rss_hash_key, 2767 bp->vnic_info[0].rss_hash_key, 2768 HW_HASH_KEY_SIZE); 2769 } 2770 } 2771 } 2772 2773 static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg) 2774 { 2775 int pages; 2776 2777 pages = ring_size / desc_per_pg; 2778 2779 if (!pages) 2780 return 1; 2781 2782 pages++; 2783 2784 while (pages & (pages - 1)) 2785 pages++; 2786 2787 return pages; 2788 } 2789 2790 void bnxt_set_tpa_flags(struct bnxt *bp) 2791 { 2792 bp->flags &= ~BNXT_FLAG_TPA; 2793 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS) 2794 return; 2795 if (bp->dev->features & NETIF_F_LRO) 2796 bp->flags |= BNXT_FLAG_LRO; 2797 else if (bp->dev->features & NETIF_F_GRO_HW) 2798 bp->flags |= BNXT_FLAG_GRO; 2799 } 2800 2801 /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must 2802 * be set on entry. 2803 */ 2804 void bnxt_set_ring_params(struct bnxt *bp) 2805 { 2806 u32 ring_size, rx_size, rx_space; 2807 u32 agg_factor = 0, agg_ring_size = 0; 2808 2809 /* 8 for CRC and VLAN */ 2810 rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8); 2811 2812 rx_space = rx_size + NET_SKB_PAD + 2813 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 2814 2815 bp->rx_copy_thresh = BNXT_RX_COPY_THRESH; 2816 ring_size = bp->rx_ring_size; 2817 bp->rx_agg_ring_size = 0; 2818 bp->rx_agg_nr_pages = 0; 2819 2820 if (bp->flags & BNXT_FLAG_TPA) 2821 agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE); 2822 2823 bp->flags &= ~BNXT_FLAG_JUMBO; 2824 if (rx_space > PAGE_SIZE && !(bp->flags & BNXT_FLAG_NO_AGG_RINGS)) { 2825 u32 jumbo_factor; 2826 2827 bp->flags |= BNXT_FLAG_JUMBO; 2828 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT; 2829 if (jumbo_factor > agg_factor) 2830 agg_factor = jumbo_factor; 2831 } 2832 agg_ring_size = ring_size * agg_factor; 2833 2834 if (agg_ring_size) { 2835 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size, 2836 RX_DESC_CNT); 2837 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) { 2838 u32 tmp = agg_ring_size; 2839 2840 bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES; 2841 agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1; 2842 netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n", 2843 tmp, agg_ring_size); 2844 } 2845 bp->rx_agg_ring_size = agg_ring_size; 2846 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1; 2847 rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN); 2848 rx_space = rx_size + NET_SKB_PAD + 2849 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 2850 } 2851 2852 bp->rx_buf_use_size = rx_size; 2853 bp->rx_buf_size = rx_space; 2854 2855 bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT); 2856 bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1; 2857 2858 ring_size = bp->tx_ring_size; 2859 bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT); 2860 bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1; 2861 2862 ring_size = bp->rx_ring_size * (2 + agg_factor) + bp->tx_ring_size; 2863 bp->cp_ring_size = ring_size; 2864 2865 bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT); 2866 if (bp->cp_nr_pages > MAX_CP_PAGES) { 2867 bp->cp_nr_pages = MAX_CP_PAGES; 2868 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1; 2869 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n", 2870 ring_size, bp->cp_ring_size); 2871 } 2872 bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT; 2873 bp->cp_ring_mask = bp->cp_bit - 1; 2874 } 2875 2876 /* Changing allocation mode of RX rings. 2877 * TODO: Update when extending xdp_rxq_info to support allocation modes. 2878 */ 2879 int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode) 2880 { 2881 if (page_mode) { 2882 if (bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU) 2883 return -EOPNOTSUPP; 2884 bp->dev->max_mtu = 2885 min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU); 2886 bp->flags &= ~BNXT_FLAG_AGG_RINGS; 2887 bp->flags |= BNXT_FLAG_NO_AGG_RINGS | BNXT_FLAG_RX_PAGE_MODE; 2888 bp->rx_dir = DMA_BIDIRECTIONAL; 2889 bp->rx_skb_func = bnxt_rx_page_skb; 2890 /* Disable LRO or GRO_HW */ 2891 netdev_update_features(bp->dev); 2892 } else { 2893 bp->dev->max_mtu = bp->max_mtu; 2894 bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE; 2895 bp->rx_dir = DMA_FROM_DEVICE; 2896 bp->rx_skb_func = bnxt_rx_skb; 2897 } 2898 return 0; 2899 } 2900 2901 static void bnxt_free_vnic_attributes(struct bnxt *bp) 2902 { 2903 int i; 2904 struct bnxt_vnic_info *vnic; 2905 struct pci_dev *pdev = bp->pdev; 2906 2907 if (!bp->vnic_info) 2908 return; 2909 2910 for (i = 0; i < bp->nr_vnics; i++) { 2911 vnic = &bp->vnic_info[i]; 2912 2913 kfree(vnic->fw_grp_ids); 2914 vnic->fw_grp_ids = NULL; 2915 2916 kfree(vnic->uc_list); 2917 vnic->uc_list = NULL; 2918 2919 if (vnic->mc_list) { 2920 dma_free_coherent(&pdev->dev, vnic->mc_list_size, 2921 vnic->mc_list, vnic->mc_list_mapping); 2922 vnic->mc_list = NULL; 2923 } 2924 2925 if (vnic->rss_table) { 2926 dma_free_coherent(&pdev->dev, PAGE_SIZE, 2927 vnic->rss_table, 2928 vnic->rss_table_dma_addr); 2929 vnic->rss_table = NULL; 2930 } 2931 2932 vnic->rss_hash_key = NULL; 2933 vnic->flags = 0; 2934 } 2935 } 2936 2937 static int bnxt_alloc_vnic_attributes(struct bnxt *bp) 2938 { 2939 int i, rc = 0, size; 2940 struct bnxt_vnic_info *vnic; 2941 struct pci_dev *pdev = bp->pdev; 2942 int max_rings; 2943 2944 for (i = 0; i < bp->nr_vnics; i++) { 2945 vnic = &bp->vnic_info[i]; 2946 2947 if (vnic->flags & BNXT_VNIC_UCAST_FLAG) { 2948 int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN; 2949 2950 if (mem_size > 0) { 2951 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL); 2952 if (!vnic->uc_list) { 2953 rc = -ENOMEM; 2954 goto out; 2955 } 2956 } 2957 } 2958 2959 if (vnic->flags & BNXT_VNIC_MCAST_FLAG) { 2960 vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN; 2961 vnic->mc_list = 2962 dma_alloc_coherent(&pdev->dev, 2963 vnic->mc_list_size, 2964 &vnic->mc_list_mapping, 2965 GFP_KERNEL); 2966 if (!vnic->mc_list) { 2967 rc = -ENOMEM; 2968 goto out; 2969 } 2970 } 2971 2972 if (vnic->flags & BNXT_VNIC_RSS_FLAG) 2973 max_rings = bp->rx_nr_rings; 2974 else 2975 max_rings = 1; 2976 2977 vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL); 2978 if (!vnic->fw_grp_ids) { 2979 rc = -ENOMEM; 2980 goto out; 2981 } 2982 2983 if ((bp->flags & BNXT_FLAG_NEW_RSS_CAP) && 2984 !(vnic->flags & BNXT_VNIC_RSS_FLAG)) 2985 continue; 2986 2987 /* Allocate rss table and hash key */ 2988 vnic->rss_table = dma_alloc_coherent(&pdev->dev, PAGE_SIZE, 2989 &vnic->rss_table_dma_addr, 2990 GFP_KERNEL); 2991 if (!vnic->rss_table) { 2992 rc = -ENOMEM; 2993 goto out; 2994 } 2995 2996 size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16)); 2997 2998 vnic->rss_hash_key = ((void *)vnic->rss_table) + size; 2999 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size; 3000 } 3001 return 0; 3002 3003 out: 3004 return rc; 3005 } 3006 3007 static void bnxt_free_hwrm_resources(struct bnxt *bp) 3008 { 3009 struct pci_dev *pdev = bp->pdev; 3010 3011 dma_free_coherent(&pdev->dev, PAGE_SIZE, bp->hwrm_cmd_resp_addr, 3012 bp->hwrm_cmd_resp_dma_addr); 3013 3014 bp->hwrm_cmd_resp_addr = NULL; 3015 if (bp->hwrm_dbg_resp_addr) { 3016 dma_free_coherent(&pdev->dev, HWRM_DBG_REG_BUF_SIZE, 3017 bp->hwrm_dbg_resp_addr, 3018 bp->hwrm_dbg_resp_dma_addr); 3019 3020 bp->hwrm_dbg_resp_addr = NULL; 3021 } 3022 } 3023 3024 static int bnxt_alloc_hwrm_resources(struct bnxt *bp) 3025 { 3026 struct pci_dev *pdev = bp->pdev; 3027 3028 bp->hwrm_cmd_resp_addr = dma_alloc_coherent(&pdev->dev, PAGE_SIZE, 3029 &bp->hwrm_cmd_resp_dma_addr, 3030 GFP_KERNEL); 3031 if (!bp->hwrm_cmd_resp_addr) 3032 return -ENOMEM; 3033 bp->hwrm_dbg_resp_addr = dma_alloc_coherent(&pdev->dev, 3034 HWRM_DBG_REG_BUF_SIZE, 3035 &bp->hwrm_dbg_resp_dma_addr, 3036 GFP_KERNEL); 3037 if (!bp->hwrm_dbg_resp_addr) 3038 netdev_warn(bp->dev, "fail to alloc debug register dma mem\n"); 3039 3040 return 0; 3041 } 3042 3043 static void bnxt_free_hwrm_short_cmd_req(struct bnxt *bp) 3044 { 3045 if (bp->hwrm_short_cmd_req_addr) { 3046 struct pci_dev *pdev = bp->pdev; 3047 3048 dma_free_coherent(&pdev->dev, BNXT_HWRM_MAX_REQ_LEN, 3049 bp->hwrm_short_cmd_req_addr, 3050 bp->hwrm_short_cmd_req_dma_addr); 3051 bp->hwrm_short_cmd_req_addr = NULL; 3052 } 3053 } 3054 3055 static int bnxt_alloc_hwrm_short_cmd_req(struct bnxt *bp) 3056 { 3057 struct pci_dev *pdev = bp->pdev; 3058 3059 bp->hwrm_short_cmd_req_addr = 3060 dma_alloc_coherent(&pdev->dev, BNXT_HWRM_MAX_REQ_LEN, 3061 &bp->hwrm_short_cmd_req_dma_addr, 3062 GFP_KERNEL); 3063 if (!bp->hwrm_short_cmd_req_addr) 3064 return -ENOMEM; 3065 3066 return 0; 3067 } 3068 3069 static void bnxt_free_stats(struct bnxt *bp) 3070 { 3071 u32 size, i; 3072 struct pci_dev *pdev = bp->pdev; 3073 3074 bp->flags &= ~BNXT_FLAG_PORT_STATS; 3075 bp->flags &= ~BNXT_FLAG_PORT_STATS_EXT; 3076 3077 if (bp->hw_rx_port_stats) { 3078 dma_free_coherent(&pdev->dev, bp->hw_port_stats_size, 3079 bp->hw_rx_port_stats, 3080 bp->hw_rx_port_stats_map); 3081 bp->hw_rx_port_stats = NULL; 3082 } 3083 3084 if (bp->hw_rx_port_stats_ext) { 3085 dma_free_coherent(&pdev->dev, sizeof(struct rx_port_stats_ext), 3086 bp->hw_rx_port_stats_ext, 3087 bp->hw_rx_port_stats_ext_map); 3088 bp->hw_rx_port_stats_ext = NULL; 3089 } 3090 3091 if (!bp->bnapi) 3092 return; 3093 3094 size = sizeof(struct ctx_hw_stats); 3095 3096 for (i = 0; i < bp->cp_nr_rings; i++) { 3097 struct bnxt_napi *bnapi = bp->bnapi[i]; 3098 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 3099 3100 if (cpr->hw_stats) { 3101 dma_free_coherent(&pdev->dev, size, cpr->hw_stats, 3102 cpr->hw_stats_map); 3103 cpr->hw_stats = NULL; 3104 } 3105 } 3106 } 3107 3108 static int bnxt_alloc_stats(struct bnxt *bp) 3109 { 3110 u32 size, i; 3111 struct pci_dev *pdev = bp->pdev; 3112 3113 size = sizeof(struct ctx_hw_stats); 3114 3115 for (i = 0; i < bp->cp_nr_rings; i++) { 3116 struct bnxt_napi *bnapi = bp->bnapi[i]; 3117 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 3118 3119 cpr->hw_stats = dma_alloc_coherent(&pdev->dev, size, 3120 &cpr->hw_stats_map, 3121 GFP_KERNEL); 3122 if (!cpr->hw_stats) 3123 return -ENOMEM; 3124 3125 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID; 3126 } 3127 3128 if (BNXT_PF(bp) && bp->chip_num != CHIP_NUM_58700) { 3129 bp->hw_port_stats_size = sizeof(struct rx_port_stats) + 3130 sizeof(struct tx_port_stats) + 1024; 3131 3132 bp->hw_rx_port_stats = 3133 dma_alloc_coherent(&pdev->dev, bp->hw_port_stats_size, 3134 &bp->hw_rx_port_stats_map, 3135 GFP_KERNEL); 3136 if (!bp->hw_rx_port_stats) 3137 return -ENOMEM; 3138 3139 bp->hw_tx_port_stats = (void *)(bp->hw_rx_port_stats + 1) + 3140 512; 3141 bp->hw_tx_port_stats_map = bp->hw_rx_port_stats_map + 3142 sizeof(struct rx_port_stats) + 512; 3143 bp->flags |= BNXT_FLAG_PORT_STATS; 3144 3145 /* Display extended statistics only if FW supports it */ 3146 if (bp->hwrm_spec_code < 0x10804 || 3147 bp->hwrm_spec_code == 0x10900) 3148 return 0; 3149 3150 bp->hw_rx_port_stats_ext = 3151 dma_zalloc_coherent(&pdev->dev, 3152 sizeof(struct rx_port_stats_ext), 3153 &bp->hw_rx_port_stats_ext_map, 3154 GFP_KERNEL); 3155 if (!bp->hw_rx_port_stats_ext) 3156 return 0; 3157 3158 bp->flags |= BNXT_FLAG_PORT_STATS_EXT; 3159 } 3160 return 0; 3161 } 3162 3163 static void bnxt_clear_ring_indices(struct bnxt *bp) 3164 { 3165 int i; 3166 3167 if (!bp->bnapi) 3168 return; 3169 3170 for (i = 0; i < bp->cp_nr_rings; i++) { 3171 struct bnxt_napi *bnapi = bp->bnapi[i]; 3172 struct bnxt_cp_ring_info *cpr; 3173 struct bnxt_rx_ring_info *rxr; 3174 struct bnxt_tx_ring_info *txr; 3175 3176 if (!bnapi) 3177 continue; 3178 3179 cpr = &bnapi->cp_ring; 3180 cpr->cp_raw_cons = 0; 3181 3182 txr = bnapi->tx_ring; 3183 if (txr) { 3184 txr->tx_prod = 0; 3185 txr->tx_cons = 0; 3186 } 3187 3188 rxr = bnapi->rx_ring; 3189 if (rxr) { 3190 rxr->rx_prod = 0; 3191 rxr->rx_agg_prod = 0; 3192 rxr->rx_sw_agg_prod = 0; 3193 rxr->rx_next_cons = 0; 3194 } 3195 } 3196 } 3197 3198 static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool irq_reinit) 3199 { 3200 #ifdef CONFIG_RFS_ACCEL 3201 int i; 3202 3203 /* Under rtnl_lock and all our NAPIs have been disabled. It's 3204 * safe to delete the hash table. 3205 */ 3206 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) { 3207 struct hlist_head *head; 3208 struct hlist_node *tmp; 3209 struct bnxt_ntuple_filter *fltr; 3210 3211 head = &bp->ntp_fltr_hash_tbl[i]; 3212 hlist_for_each_entry_safe(fltr, tmp, head, hash) { 3213 hlist_del(&fltr->hash); 3214 kfree(fltr); 3215 } 3216 } 3217 if (irq_reinit) { 3218 kfree(bp->ntp_fltr_bmap); 3219 bp->ntp_fltr_bmap = NULL; 3220 } 3221 bp->ntp_fltr_count = 0; 3222 #endif 3223 } 3224 3225 static int bnxt_alloc_ntp_fltrs(struct bnxt *bp) 3226 { 3227 #ifdef CONFIG_RFS_ACCEL 3228 int i, rc = 0; 3229 3230 if (!(bp->flags & BNXT_FLAG_RFS)) 3231 return 0; 3232 3233 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) 3234 INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]); 3235 3236 bp->ntp_fltr_count = 0; 3237 bp->ntp_fltr_bmap = kcalloc(BITS_TO_LONGS(BNXT_NTP_FLTR_MAX_FLTR), 3238 sizeof(long), 3239 GFP_KERNEL); 3240 3241 if (!bp->ntp_fltr_bmap) 3242 rc = -ENOMEM; 3243 3244 return rc; 3245 #else 3246 return 0; 3247 #endif 3248 } 3249 3250 static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init) 3251 { 3252 bnxt_free_vnic_attributes(bp); 3253 bnxt_free_tx_rings(bp); 3254 bnxt_free_rx_rings(bp); 3255 bnxt_free_cp_rings(bp); 3256 bnxt_free_ntp_fltrs(bp, irq_re_init); 3257 if (irq_re_init) { 3258 bnxt_free_stats(bp); 3259 bnxt_free_ring_grps(bp); 3260 bnxt_free_vnics(bp); 3261 kfree(bp->tx_ring_map); 3262 bp->tx_ring_map = NULL; 3263 kfree(bp->tx_ring); 3264 bp->tx_ring = NULL; 3265 kfree(bp->rx_ring); 3266 bp->rx_ring = NULL; 3267 kfree(bp->bnapi); 3268 bp->bnapi = NULL; 3269 } else { 3270 bnxt_clear_ring_indices(bp); 3271 } 3272 } 3273 3274 static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init) 3275 { 3276 int i, j, rc, size, arr_size; 3277 void *bnapi; 3278 3279 if (irq_re_init) { 3280 /* Allocate bnapi mem pointer array and mem block for 3281 * all queues 3282 */ 3283 arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) * 3284 bp->cp_nr_rings); 3285 size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi)); 3286 bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL); 3287 if (!bnapi) 3288 return -ENOMEM; 3289 3290 bp->bnapi = bnapi; 3291 bnapi += arr_size; 3292 for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) { 3293 bp->bnapi[i] = bnapi; 3294 bp->bnapi[i]->index = i; 3295 bp->bnapi[i]->bp = bp; 3296 } 3297 3298 bp->rx_ring = kcalloc(bp->rx_nr_rings, 3299 sizeof(struct bnxt_rx_ring_info), 3300 GFP_KERNEL); 3301 if (!bp->rx_ring) 3302 return -ENOMEM; 3303 3304 for (i = 0; i < bp->rx_nr_rings; i++) { 3305 bp->rx_ring[i].bnapi = bp->bnapi[i]; 3306 bp->bnapi[i]->rx_ring = &bp->rx_ring[i]; 3307 } 3308 3309 bp->tx_ring = kcalloc(bp->tx_nr_rings, 3310 sizeof(struct bnxt_tx_ring_info), 3311 GFP_KERNEL); 3312 if (!bp->tx_ring) 3313 return -ENOMEM; 3314 3315 bp->tx_ring_map = kcalloc(bp->tx_nr_rings, sizeof(u16), 3316 GFP_KERNEL); 3317 3318 if (!bp->tx_ring_map) 3319 return -ENOMEM; 3320 3321 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 3322 j = 0; 3323 else 3324 j = bp->rx_nr_rings; 3325 3326 for (i = 0; i < bp->tx_nr_rings; i++, j++) { 3327 bp->tx_ring[i].bnapi = bp->bnapi[j]; 3328 bp->bnapi[j]->tx_ring = &bp->tx_ring[i]; 3329 bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i; 3330 if (i >= bp->tx_nr_rings_xdp) { 3331 bp->tx_ring[i].txq_index = i - 3332 bp->tx_nr_rings_xdp; 3333 bp->bnapi[j]->tx_int = bnxt_tx_int; 3334 } else { 3335 bp->bnapi[j]->flags |= BNXT_NAPI_FLAG_XDP; 3336 bp->bnapi[j]->tx_int = bnxt_tx_int_xdp; 3337 } 3338 } 3339 3340 rc = bnxt_alloc_stats(bp); 3341 if (rc) 3342 goto alloc_mem_err; 3343 3344 rc = bnxt_alloc_ntp_fltrs(bp); 3345 if (rc) 3346 goto alloc_mem_err; 3347 3348 rc = bnxt_alloc_vnics(bp); 3349 if (rc) 3350 goto alloc_mem_err; 3351 } 3352 3353 bnxt_init_ring_struct(bp); 3354 3355 rc = bnxt_alloc_rx_rings(bp); 3356 if (rc) 3357 goto alloc_mem_err; 3358 3359 rc = bnxt_alloc_tx_rings(bp); 3360 if (rc) 3361 goto alloc_mem_err; 3362 3363 rc = bnxt_alloc_cp_rings(bp); 3364 if (rc) 3365 goto alloc_mem_err; 3366 3367 bp->vnic_info[0].flags |= BNXT_VNIC_RSS_FLAG | BNXT_VNIC_MCAST_FLAG | 3368 BNXT_VNIC_UCAST_FLAG; 3369 rc = bnxt_alloc_vnic_attributes(bp); 3370 if (rc) 3371 goto alloc_mem_err; 3372 return 0; 3373 3374 alloc_mem_err: 3375 bnxt_free_mem(bp, true); 3376 return rc; 3377 } 3378 3379 static void bnxt_disable_int(struct bnxt *bp) 3380 { 3381 int i; 3382 3383 if (!bp->bnapi) 3384 return; 3385 3386 for (i = 0; i < bp->cp_nr_rings; i++) { 3387 struct bnxt_napi *bnapi = bp->bnapi[i]; 3388 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 3389 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 3390 3391 if (ring->fw_ring_id != INVALID_HW_RING_ID) 3392 BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons); 3393 } 3394 } 3395 3396 static int bnxt_cp_num_to_irq_num(struct bnxt *bp, int n) 3397 { 3398 struct bnxt_napi *bnapi = bp->bnapi[n]; 3399 struct bnxt_cp_ring_info *cpr; 3400 3401 cpr = &bnapi->cp_ring; 3402 return cpr->cp_ring_struct.map_idx; 3403 } 3404 3405 static void bnxt_disable_int_sync(struct bnxt *bp) 3406 { 3407 int i; 3408 3409 atomic_inc(&bp->intr_sem); 3410 3411 bnxt_disable_int(bp); 3412 for (i = 0; i < bp->cp_nr_rings; i++) { 3413 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 3414 3415 synchronize_irq(bp->irq_tbl[map_idx].vector); 3416 } 3417 } 3418 3419 static void bnxt_enable_int(struct bnxt *bp) 3420 { 3421 int i; 3422 3423 atomic_set(&bp->intr_sem, 0); 3424 for (i = 0; i < bp->cp_nr_rings; i++) { 3425 struct bnxt_napi *bnapi = bp->bnapi[i]; 3426 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 3427 3428 BNXT_CP_DB_REARM(cpr->cp_doorbell, cpr->cp_raw_cons); 3429 } 3430 } 3431 3432 void bnxt_hwrm_cmd_hdr_init(struct bnxt *bp, void *request, u16 req_type, 3433 u16 cmpl_ring, u16 target_id) 3434 { 3435 struct input *req = request; 3436 3437 req->req_type = cpu_to_le16(req_type); 3438 req->cmpl_ring = cpu_to_le16(cmpl_ring); 3439 req->target_id = cpu_to_le16(target_id); 3440 req->resp_addr = cpu_to_le64(bp->hwrm_cmd_resp_dma_addr); 3441 } 3442 3443 static int bnxt_hwrm_do_send_msg(struct bnxt *bp, void *msg, u32 msg_len, 3444 int timeout, bool silent) 3445 { 3446 int i, intr_process, rc, tmo_count; 3447 struct input *req = msg; 3448 u32 *data = msg; 3449 __le32 *resp_len; 3450 u8 *valid; 3451 u16 cp_ring_id, len = 0; 3452 struct hwrm_err_output *resp = bp->hwrm_cmd_resp_addr; 3453 u16 max_req_len = BNXT_HWRM_MAX_REQ_LEN; 3454 struct hwrm_short_input short_input = {0}; 3455 3456 req->seq_id = cpu_to_le16(bp->hwrm_cmd_seq++); 3457 memset(resp, 0, PAGE_SIZE); 3458 cp_ring_id = le16_to_cpu(req->cmpl_ring); 3459 intr_process = (cp_ring_id == INVALID_HW_RING_ID) ? 0 : 1; 3460 3461 if (bp->flags & BNXT_FLAG_SHORT_CMD) { 3462 void *short_cmd_req = bp->hwrm_short_cmd_req_addr; 3463 3464 memcpy(short_cmd_req, req, msg_len); 3465 memset(short_cmd_req + msg_len, 0, BNXT_HWRM_MAX_REQ_LEN - 3466 msg_len); 3467 3468 short_input.req_type = req->req_type; 3469 short_input.signature = 3470 cpu_to_le16(SHORT_REQ_SIGNATURE_SHORT_CMD); 3471 short_input.size = cpu_to_le16(msg_len); 3472 short_input.req_addr = 3473 cpu_to_le64(bp->hwrm_short_cmd_req_dma_addr); 3474 3475 data = (u32 *)&short_input; 3476 msg_len = sizeof(short_input); 3477 3478 /* Sync memory write before updating doorbell */ 3479 wmb(); 3480 3481 max_req_len = BNXT_HWRM_SHORT_REQ_LEN; 3482 } 3483 3484 /* Write request msg to hwrm channel */ 3485 __iowrite32_copy(bp->bar0, data, msg_len / 4); 3486 3487 for (i = msg_len; i < max_req_len; i += 4) 3488 writel(0, bp->bar0 + i); 3489 3490 /* currently supports only one outstanding message */ 3491 if (intr_process) 3492 bp->hwrm_intr_seq_id = le16_to_cpu(req->seq_id); 3493 3494 /* Ring channel doorbell */ 3495 writel(1, bp->bar0 + 0x100); 3496 3497 if (!timeout) 3498 timeout = DFLT_HWRM_CMD_TIMEOUT; 3499 /* convert timeout to usec */ 3500 timeout *= 1000; 3501 3502 i = 0; 3503 /* Short timeout for the first few iterations: 3504 * number of loops = number of loops for short timeout + 3505 * number of loops for standard timeout. 3506 */ 3507 tmo_count = HWRM_SHORT_TIMEOUT_COUNTER; 3508 timeout = timeout - HWRM_SHORT_MIN_TIMEOUT * HWRM_SHORT_TIMEOUT_COUNTER; 3509 tmo_count += DIV_ROUND_UP(timeout, HWRM_MIN_TIMEOUT); 3510 resp_len = bp->hwrm_cmd_resp_addr + HWRM_RESP_LEN_OFFSET; 3511 if (intr_process) { 3512 /* Wait until hwrm response cmpl interrupt is processed */ 3513 while (bp->hwrm_intr_seq_id != HWRM_SEQ_ID_INVALID && 3514 i++ < tmo_count) { 3515 /* on first few passes, just barely sleep */ 3516 if (i < HWRM_SHORT_TIMEOUT_COUNTER) 3517 usleep_range(HWRM_SHORT_MIN_TIMEOUT, 3518 HWRM_SHORT_MAX_TIMEOUT); 3519 else 3520 usleep_range(HWRM_MIN_TIMEOUT, 3521 HWRM_MAX_TIMEOUT); 3522 } 3523 3524 if (bp->hwrm_intr_seq_id != HWRM_SEQ_ID_INVALID) { 3525 netdev_err(bp->dev, "Resp cmpl intr err msg: 0x%x\n", 3526 le16_to_cpu(req->req_type)); 3527 return -1; 3528 } 3529 len = (le32_to_cpu(*resp_len) & HWRM_RESP_LEN_MASK) >> 3530 HWRM_RESP_LEN_SFT; 3531 valid = bp->hwrm_cmd_resp_addr + len - 1; 3532 } else { 3533 int j; 3534 3535 /* Check if response len is updated */ 3536 for (i = 0; i < tmo_count; i++) { 3537 len = (le32_to_cpu(*resp_len) & HWRM_RESP_LEN_MASK) >> 3538 HWRM_RESP_LEN_SFT; 3539 if (len) 3540 break; 3541 /* on first few passes, just barely sleep */ 3542 if (i < DFLT_HWRM_CMD_TIMEOUT) 3543 usleep_range(HWRM_SHORT_MIN_TIMEOUT, 3544 HWRM_SHORT_MAX_TIMEOUT); 3545 else 3546 usleep_range(HWRM_MIN_TIMEOUT, 3547 HWRM_MAX_TIMEOUT); 3548 } 3549 3550 if (i >= tmo_count) { 3551 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d\n", 3552 HWRM_TOTAL_TIMEOUT(i), 3553 le16_to_cpu(req->req_type), 3554 le16_to_cpu(req->seq_id), len); 3555 return -1; 3556 } 3557 3558 /* Last byte of resp contains valid bit */ 3559 valid = bp->hwrm_cmd_resp_addr + len - 1; 3560 for (j = 0; j < HWRM_VALID_BIT_DELAY_USEC; j++) { 3561 /* make sure we read from updated DMA memory */ 3562 dma_rmb(); 3563 if (*valid) 3564 break; 3565 udelay(1); 3566 } 3567 3568 if (j >= HWRM_VALID_BIT_DELAY_USEC) { 3569 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d v:%d\n", 3570 HWRM_TOTAL_TIMEOUT(i), 3571 le16_to_cpu(req->req_type), 3572 le16_to_cpu(req->seq_id), len, *valid); 3573 return -1; 3574 } 3575 } 3576 3577 /* Zero valid bit for compatibility. Valid bit in an older spec 3578 * may become a new field in a newer spec. We must make sure that 3579 * a new field not implemented by old spec will read zero. 3580 */ 3581 *valid = 0; 3582 rc = le16_to_cpu(resp->error_code); 3583 if (rc && !silent) 3584 netdev_err(bp->dev, "hwrm req_type 0x%x seq id 0x%x error 0x%x\n", 3585 le16_to_cpu(resp->req_type), 3586 le16_to_cpu(resp->seq_id), rc); 3587 return rc; 3588 } 3589 3590 int _hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout) 3591 { 3592 return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, false); 3593 } 3594 3595 int _hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len, 3596 int timeout) 3597 { 3598 return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true); 3599 } 3600 3601 int hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout) 3602 { 3603 int rc; 3604 3605 mutex_lock(&bp->hwrm_cmd_lock); 3606 rc = _hwrm_send_message(bp, msg, msg_len, timeout); 3607 mutex_unlock(&bp->hwrm_cmd_lock); 3608 return rc; 3609 } 3610 3611 int hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len, 3612 int timeout) 3613 { 3614 int rc; 3615 3616 mutex_lock(&bp->hwrm_cmd_lock); 3617 rc = bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true); 3618 mutex_unlock(&bp->hwrm_cmd_lock); 3619 return rc; 3620 } 3621 3622 int bnxt_hwrm_func_rgtr_async_events(struct bnxt *bp, unsigned long *bmap, 3623 int bmap_size) 3624 { 3625 struct hwrm_func_drv_rgtr_input req = {0}; 3626 DECLARE_BITMAP(async_events_bmap, 256); 3627 u32 *events = (u32 *)async_events_bmap; 3628 int i; 3629 3630 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1); 3631 3632 req.enables = 3633 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD); 3634 3635 memset(async_events_bmap, 0, sizeof(async_events_bmap)); 3636 for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++) 3637 __set_bit(bnxt_async_events_arr[i], async_events_bmap); 3638 3639 if (bmap && bmap_size) { 3640 for (i = 0; i < bmap_size; i++) { 3641 if (test_bit(i, bmap)) 3642 __set_bit(i, async_events_bmap); 3643 } 3644 } 3645 3646 for (i = 0; i < 8; i++) 3647 req.async_event_fwd[i] |= cpu_to_le32(events[i]); 3648 3649 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3650 } 3651 3652 static int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp) 3653 { 3654 struct hwrm_func_drv_rgtr_input req = {0}; 3655 3656 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1); 3657 3658 req.enables = 3659 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE | 3660 FUNC_DRV_RGTR_REQ_ENABLES_VER); 3661 3662 req.os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX); 3663 req.flags = cpu_to_le32(FUNC_DRV_RGTR_REQ_FLAGS_16BIT_VER_MODE); 3664 req.ver_maj_8b = DRV_VER_MAJ; 3665 req.ver_min_8b = DRV_VER_MIN; 3666 req.ver_upd_8b = DRV_VER_UPD; 3667 req.ver_maj = cpu_to_le16(DRV_VER_MAJ); 3668 req.ver_min = cpu_to_le16(DRV_VER_MIN); 3669 req.ver_upd = cpu_to_le16(DRV_VER_UPD); 3670 3671 if (BNXT_PF(bp)) { 3672 u32 data[8]; 3673 int i; 3674 3675 memset(data, 0, sizeof(data)); 3676 for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) { 3677 u16 cmd = bnxt_vf_req_snif[i]; 3678 unsigned int bit, idx; 3679 3680 idx = cmd / 32; 3681 bit = cmd % 32; 3682 data[idx] |= 1 << bit; 3683 } 3684 3685 for (i = 0; i < 8; i++) 3686 req.vf_req_fwd[i] = cpu_to_le32(data[i]); 3687 3688 req.enables |= 3689 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD); 3690 } 3691 3692 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3693 } 3694 3695 static int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp) 3696 { 3697 struct hwrm_func_drv_unrgtr_input req = {0}; 3698 3699 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_UNRGTR, -1, -1); 3700 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3701 } 3702 3703 static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type) 3704 { 3705 u32 rc = 0; 3706 struct hwrm_tunnel_dst_port_free_input req = {0}; 3707 3708 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_FREE, -1, -1); 3709 req.tunnel_type = tunnel_type; 3710 3711 switch (tunnel_type) { 3712 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN: 3713 req.tunnel_dst_port_id = bp->vxlan_fw_dst_port_id; 3714 break; 3715 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE: 3716 req.tunnel_dst_port_id = bp->nge_fw_dst_port_id; 3717 break; 3718 default: 3719 break; 3720 } 3721 3722 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3723 if (rc) 3724 netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n", 3725 rc); 3726 return rc; 3727 } 3728 3729 static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port, 3730 u8 tunnel_type) 3731 { 3732 u32 rc = 0; 3733 struct hwrm_tunnel_dst_port_alloc_input req = {0}; 3734 struct hwrm_tunnel_dst_port_alloc_output *resp = bp->hwrm_cmd_resp_addr; 3735 3736 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_ALLOC, -1, -1); 3737 3738 req.tunnel_type = tunnel_type; 3739 req.tunnel_dst_port_val = port; 3740 3741 mutex_lock(&bp->hwrm_cmd_lock); 3742 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3743 if (rc) { 3744 netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n", 3745 rc); 3746 goto err_out; 3747 } 3748 3749 switch (tunnel_type) { 3750 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN: 3751 bp->vxlan_fw_dst_port_id = resp->tunnel_dst_port_id; 3752 break; 3753 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE: 3754 bp->nge_fw_dst_port_id = resp->tunnel_dst_port_id; 3755 break; 3756 default: 3757 break; 3758 } 3759 3760 err_out: 3761 mutex_unlock(&bp->hwrm_cmd_lock); 3762 return rc; 3763 } 3764 3765 static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id) 3766 { 3767 struct hwrm_cfa_l2_set_rx_mask_input req = {0}; 3768 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 3769 3770 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_SET_RX_MASK, -1, -1); 3771 req.vnic_id = cpu_to_le32(vnic->fw_vnic_id); 3772 3773 req.num_mc_entries = cpu_to_le32(vnic->mc_list_count); 3774 req.mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping); 3775 req.mask = cpu_to_le32(vnic->rx_mask); 3776 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3777 } 3778 3779 #ifdef CONFIG_RFS_ACCEL 3780 static int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp, 3781 struct bnxt_ntuple_filter *fltr) 3782 { 3783 struct hwrm_cfa_ntuple_filter_free_input req = {0}; 3784 3785 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_FREE, -1, -1); 3786 req.ntuple_filter_id = fltr->filter_id; 3787 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3788 } 3789 3790 #define BNXT_NTP_FLTR_FLAGS \ 3791 (CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID | \ 3792 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE | \ 3793 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR | \ 3794 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE | \ 3795 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR | \ 3796 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK | \ 3797 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR | \ 3798 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK | \ 3799 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL | \ 3800 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT | \ 3801 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK | \ 3802 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT | \ 3803 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK | \ 3804 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID) 3805 3806 #define BNXT_NTP_TUNNEL_FLTR_FLAG \ 3807 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE 3808 3809 static int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp, 3810 struct bnxt_ntuple_filter *fltr) 3811 { 3812 int rc = 0; 3813 struct hwrm_cfa_ntuple_filter_alloc_input req = {0}; 3814 struct hwrm_cfa_ntuple_filter_alloc_output *resp = 3815 bp->hwrm_cmd_resp_addr; 3816 struct flow_keys *keys = &fltr->fkeys; 3817 struct bnxt_vnic_info *vnic = &bp->vnic_info[fltr->rxq + 1]; 3818 3819 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_ALLOC, -1, -1); 3820 req.l2_filter_id = bp->vnic_info[0].fw_l2_filter_id[fltr->l2_fltr_idx]; 3821 3822 req.enables = cpu_to_le32(BNXT_NTP_FLTR_FLAGS); 3823 3824 req.ethertype = htons(ETH_P_IP); 3825 memcpy(req.src_macaddr, fltr->src_mac_addr, ETH_ALEN); 3826 req.ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4; 3827 req.ip_protocol = keys->basic.ip_proto; 3828 3829 if (keys->basic.n_proto == htons(ETH_P_IPV6)) { 3830 int i; 3831 3832 req.ethertype = htons(ETH_P_IPV6); 3833 req.ip_addr_type = 3834 CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6; 3835 *(struct in6_addr *)&req.src_ipaddr[0] = 3836 keys->addrs.v6addrs.src; 3837 *(struct in6_addr *)&req.dst_ipaddr[0] = 3838 keys->addrs.v6addrs.dst; 3839 for (i = 0; i < 4; i++) { 3840 req.src_ipaddr_mask[i] = cpu_to_be32(0xffffffff); 3841 req.dst_ipaddr_mask[i] = cpu_to_be32(0xffffffff); 3842 } 3843 } else { 3844 req.src_ipaddr[0] = keys->addrs.v4addrs.src; 3845 req.src_ipaddr_mask[0] = cpu_to_be32(0xffffffff); 3846 req.dst_ipaddr[0] = keys->addrs.v4addrs.dst; 3847 req.dst_ipaddr_mask[0] = cpu_to_be32(0xffffffff); 3848 } 3849 if (keys->control.flags & FLOW_DIS_ENCAPSULATION) { 3850 req.enables |= cpu_to_le32(BNXT_NTP_TUNNEL_FLTR_FLAG); 3851 req.tunnel_type = 3852 CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL; 3853 } 3854 3855 req.src_port = keys->ports.src; 3856 req.src_port_mask = cpu_to_be16(0xffff); 3857 req.dst_port = keys->ports.dst; 3858 req.dst_port_mask = cpu_to_be16(0xffff); 3859 3860 req.dst_id = cpu_to_le16(vnic->fw_vnic_id); 3861 mutex_lock(&bp->hwrm_cmd_lock); 3862 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3863 if (!rc) 3864 fltr->filter_id = resp->ntuple_filter_id; 3865 mutex_unlock(&bp->hwrm_cmd_lock); 3866 return rc; 3867 } 3868 #endif 3869 3870 static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx, 3871 u8 *mac_addr) 3872 { 3873 u32 rc = 0; 3874 struct hwrm_cfa_l2_filter_alloc_input req = {0}; 3875 struct hwrm_cfa_l2_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr; 3876 3877 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_ALLOC, -1, -1); 3878 req.flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX); 3879 if (!BNXT_CHIP_TYPE_NITRO_A0(bp)) 3880 req.flags |= 3881 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST); 3882 req.dst_id = cpu_to_le16(bp->vnic_info[vnic_id].fw_vnic_id); 3883 req.enables = 3884 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR | 3885 CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID | 3886 CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK); 3887 memcpy(req.l2_addr, mac_addr, ETH_ALEN); 3888 req.l2_addr_mask[0] = 0xff; 3889 req.l2_addr_mask[1] = 0xff; 3890 req.l2_addr_mask[2] = 0xff; 3891 req.l2_addr_mask[3] = 0xff; 3892 req.l2_addr_mask[4] = 0xff; 3893 req.l2_addr_mask[5] = 0xff; 3894 3895 mutex_lock(&bp->hwrm_cmd_lock); 3896 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3897 if (!rc) 3898 bp->vnic_info[vnic_id].fw_l2_filter_id[idx] = 3899 resp->l2_filter_id; 3900 mutex_unlock(&bp->hwrm_cmd_lock); 3901 return rc; 3902 } 3903 3904 static int bnxt_hwrm_clear_vnic_filter(struct bnxt *bp) 3905 { 3906 u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */ 3907 int rc = 0; 3908 3909 /* Any associated ntuple filters will also be cleared by firmware. */ 3910 mutex_lock(&bp->hwrm_cmd_lock); 3911 for (i = 0; i < num_of_vnics; i++) { 3912 struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; 3913 3914 for (j = 0; j < vnic->uc_filter_count; j++) { 3915 struct hwrm_cfa_l2_filter_free_input req = {0}; 3916 3917 bnxt_hwrm_cmd_hdr_init(bp, &req, 3918 HWRM_CFA_L2_FILTER_FREE, -1, -1); 3919 3920 req.l2_filter_id = vnic->fw_l2_filter_id[j]; 3921 3922 rc = _hwrm_send_message(bp, &req, sizeof(req), 3923 HWRM_CMD_TIMEOUT); 3924 } 3925 vnic->uc_filter_count = 0; 3926 } 3927 mutex_unlock(&bp->hwrm_cmd_lock); 3928 3929 return rc; 3930 } 3931 3932 static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags) 3933 { 3934 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 3935 struct hwrm_vnic_tpa_cfg_input req = {0}; 3936 3937 if (vnic->fw_vnic_id == INVALID_HW_RING_ID) 3938 return 0; 3939 3940 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_TPA_CFG, -1, -1); 3941 3942 if (tpa_flags) { 3943 u16 mss = bp->dev->mtu - 40; 3944 u32 nsegs, n, segs = 0, flags; 3945 3946 flags = VNIC_TPA_CFG_REQ_FLAGS_TPA | 3947 VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA | 3948 VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE | 3949 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN | 3950 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ; 3951 if (tpa_flags & BNXT_FLAG_GRO) 3952 flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO; 3953 3954 req.flags = cpu_to_le32(flags); 3955 3956 req.enables = 3957 cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS | 3958 VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS | 3959 VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN); 3960 3961 /* Number of segs are log2 units, and first packet is not 3962 * included as part of this units. 3963 */ 3964 if (mss <= BNXT_RX_PAGE_SIZE) { 3965 n = BNXT_RX_PAGE_SIZE / mss; 3966 nsegs = (MAX_SKB_FRAGS - 1) * n; 3967 } else { 3968 n = mss / BNXT_RX_PAGE_SIZE; 3969 if (mss & (BNXT_RX_PAGE_SIZE - 1)) 3970 n++; 3971 nsegs = (MAX_SKB_FRAGS - n) / n; 3972 } 3973 3974 segs = ilog2(nsegs); 3975 req.max_agg_segs = cpu_to_le16(segs); 3976 req.max_aggs = cpu_to_le16(VNIC_TPA_CFG_REQ_MAX_AGGS_MAX); 3977 3978 req.min_agg_len = cpu_to_le32(512); 3979 } 3980 req.vnic_id = cpu_to_le16(vnic->fw_vnic_id); 3981 3982 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3983 } 3984 3985 static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss) 3986 { 3987 u32 i, j, max_rings; 3988 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 3989 struct hwrm_vnic_rss_cfg_input req = {0}; 3990 3991 if (vnic->fw_rss_cos_lb_ctx[0] == INVALID_HW_RING_ID) 3992 return 0; 3993 3994 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1); 3995 if (set_rss) { 3996 req.hash_type = cpu_to_le32(bp->rss_hash_cfg); 3997 if (vnic->flags & BNXT_VNIC_RSS_FLAG) { 3998 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 3999 max_rings = bp->rx_nr_rings - 1; 4000 else 4001 max_rings = bp->rx_nr_rings; 4002 } else { 4003 max_rings = 1; 4004 } 4005 4006 /* Fill the RSS indirection table with ring group ids */ 4007 for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++, j++) { 4008 if (j == max_rings) 4009 j = 0; 4010 vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]); 4011 } 4012 4013 req.ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr); 4014 req.hash_key_tbl_addr = 4015 cpu_to_le64(vnic->rss_hash_key_dma_addr); 4016 } 4017 req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]); 4018 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4019 } 4020 4021 static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id) 4022 { 4023 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 4024 struct hwrm_vnic_plcmodes_cfg_input req = {0}; 4025 4026 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_PLCMODES_CFG, -1, -1); 4027 req.flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT | 4028 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 | 4029 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6); 4030 req.enables = 4031 cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID | 4032 VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID); 4033 /* thresholds not implemented in firmware yet */ 4034 req.jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh); 4035 req.hds_threshold = cpu_to_le16(bp->rx_copy_thresh); 4036 req.vnic_id = cpu_to_le32(vnic->fw_vnic_id); 4037 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4038 } 4039 4040 static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id, 4041 u16 ctx_idx) 4042 { 4043 struct hwrm_vnic_rss_cos_lb_ctx_free_input req = {0}; 4044 4045 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_FREE, -1, -1); 4046 req.rss_cos_lb_ctx_id = 4047 cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx]); 4048 4049 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4050 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID; 4051 } 4052 4053 static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp) 4054 { 4055 int i, j; 4056 4057 for (i = 0; i < bp->nr_vnics; i++) { 4058 struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; 4059 4060 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) { 4061 if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID) 4062 bnxt_hwrm_vnic_ctx_free_one(bp, i, j); 4063 } 4064 } 4065 bp->rsscos_nr_ctxs = 0; 4066 } 4067 4068 static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id, u16 ctx_idx) 4069 { 4070 int rc; 4071 struct hwrm_vnic_rss_cos_lb_ctx_alloc_input req = {0}; 4072 struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp = 4073 bp->hwrm_cmd_resp_addr; 4074 4075 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC, -1, 4076 -1); 4077 4078 mutex_lock(&bp->hwrm_cmd_lock); 4079 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4080 if (!rc) 4081 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = 4082 le16_to_cpu(resp->rss_cos_lb_ctx_id); 4083 mutex_unlock(&bp->hwrm_cmd_lock); 4084 4085 return rc; 4086 } 4087 4088 static u32 bnxt_get_roce_vnic_mode(struct bnxt *bp) 4089 { 4090 if (bp->flags & BNXT_FLAG_ROCE_MIRROR_CAP) 4091 return VNIC_CFG_REQ_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE; 4092 return VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE; 4093 } 4094 4095 int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id) 4096 { 4097 unsigned int ring = 0, grp_idx; 4098 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 4099 struct hwrm_vnic_cfg_input req = {0}; 4100 u16 def_vlan = 0; 4101 4102 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_CFG, -1, -1); 4103 4104 req.enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP); 4105 /* Only RSS support for now TBD: COS & LB */ 4106 if (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID) { 4107 req.rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]); 4108 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE | 4109 VNIC_CFG_REQ_ENABLES_MRU); 4110 } else if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) { 4111 req.rss_rule = 4112 cpu_to_le16(bp->vnic_info[0].fw_rss_cos_lb_ctx[0]); 4113 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE | 4114 VNIC_CFG_REQ_ENABLES_MRU); 4115 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE); 4116 } else { 4117 req.rss_rule = cpu_to_le16(0xffff); 4118 } 4119 4120 if (BNXT_CHIP_TYPE_NITRO_A0(bp) && 4121 (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID)) { 4122 req.cos_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[1]); 4123 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_COS_RULE); 4124 } else { 4125 req.cos_rule = cpu_to_le16(0xffff); 4126 } 4127 4128 if (vnic->flags & BNXT_VNIC_RSS_FLAG) 4129 ring = 0; 4130 else if (vnic->flags & BNXT_VNIC_RFS_FLAG) 4131 ring = vnic_id - 1; 4132 else if ((vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp)) 4133 ring = bp->rx_nr_rings - 1; 4134 4135 grp_idx = bp->rx_ring[ring].bnapi->index; 4136 req.vnic_id = cpu_to_le16(vnic->fw_vnic_id); 4137 req.dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id); 4138 4139 req.lb_rule = cpu_to_le16(0xffff); 4140 req.mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + ETH_FCS_LEN + 4141 VLAN_HLEN); 4142 4143 #ifdef CONFIG_BNXT_SRIOV 4144 if (BNXT_VF(bp)) 4145 def_vlan = bp->vf.vlan; 4146 #endif 4147 if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan) 4148 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE); 4149 if (!vnic_id && bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP)) 4150 req.flags |= cpu_to_le32(bnxt_get_roce_vnic_mode(bp)); 4151 4152 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4153 } 4154 4155 static int bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id) 4156 { 4157 u32 rc = 0; 4158 4159 if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) { 4160 struct hwrm_vnic_free_input req = {0}; 4161 4162 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_FREE, -1, -1); 4163 req.vnic_id = 4164 cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id); 4165 4166 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4167 if (rc) 4168 return rc; 4169 bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID; 4170 } 4171 return rc; 4172 } 4173 4174 static void bnxt_hwrm_vnic_free(struct bnxt *bp) 4175 { 4176 u16 i; 4177 4178 for (i = 0; i < bp->nr_vnics; i++) 4179 bnxt_hwrm_vnic_free_one(bp, i); 4180 } 4181 4182 static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id, 4183 unsigned int start_rx_ring_idx, 4184 unsigned int nr_rings) 4185 { 4186 int rc = 0; 4187 unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings; 4188 struct hwrm_vnic_alloc_input req = {0}; 4189 struct hwrm_vnic_alloc_output *resp = bp->hwrm_cmd_resp_addr; 4190 4191 /* map ring groups to this vnic */ 4192 for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) { 4193 grp_idx = bp->rx_ring[i].bnapi->index; 4194 if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) { 4195 netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n", 4196 j, nr_rings); 4197 break; 4198 } 4199 bp->vnic_info[vnic_id].fw_grp_ids[j] = 4200 bp->grp_info[grp_idx].fw_grp_id; 4201 } 4202 4203 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[0] = INVALID_HW_RING_ID; 4204 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[1] = INVALID_HW_RING_ID; 4205 if (vnic_id == 0) 4206 req.flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT); 4207 4208 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_ALLOC, -1, -1); 4209 4210 mutex_lock(&bp->hwrm_cmd_lock); 4211 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4212 if (!rc) 4213 bp->vnic_info[vnic_id].fw_vnic_id = le32_to_cpu(resp->vnic_id); 4214 mutex_unlock(&bp->hwrm_cmd_lock); 4215 return rc; 4216 } 4217 4218 static int bnxt_hwrm_vnic_qcaps(struct bnxt *bp) 4219 { 4220 struct hwrm_vnic_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 4221 struct hwrm_vnic_qcaps_input req = {0}; 4222 int rc; 4223 4224 if (bp->hwrm_spec_code < 0x10600) 4225 return 0; 4226 4227 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_QCAPS, -1, -1); 4228 mutex_lock(&bp->hwrm_cmd_lock); 4229 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4230 if (!rc) { 4231 u32 flags = le32_to_cpu(resp->flags); 4232 4233 if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP) 4234 bp->flags |= BNXT_FLAG_NEW_RSS_CAP; 4235 if (flags & 4236 VNIC_QCAPS_RESP_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_CAP) 4237 bp->flags |= BNXT_FLAG_ROCE_MIRROR_CAP; 4238 } 4239 mutex_unlock(&bp->hwrm_cmd_lock); 4240 return rc; 4241 } 4242 4243 static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp) 4244 { 4245 u16 i; 4246 u32 rc = 0; 4247 4248 mutex_lock(&bp->hwrm_cmd_lock); 4249 for (i = 0; i < bp->rx_nr_rings; i++) { 4250 struct hwrm_ring_grp_alloc_input req = {0}; 4251 struct hwrm_ring_grp_alloc_output *resp = 4252 bp->hwrm_cmd_resp_addr; 4253 unsigned int grp_idx = bp->rx_ring[i].bnapi->index; 4254 4255 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_ALLOC, -1, -1); 4256 4257 req.cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id); 4258 req.rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id); 4259 req.ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id); 4260 req.sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx); 4261 4262 rc = _hwrm_send_message(bp, &req, sizeof(req), 4263 HWRM_CMD_TIMEOUT); 4264 if (rc) 4265 break; 4266 4267 bp->grp_info[grp_idx].fw_grp_id = 4268 le32_to_cpu(resp->ring_group_id); 4269 } 4270 mutex_unlock(&bp->hwrm_cmd_lock); 4271 return rc; 4272 } 4273 4274 static int bnxt_hwrm_ring_grp_free(struct bnxt *bp) 4275 { 4276 u16 i; 4277 u32 rc = 0; 4278 struct hwrm_ring_grp_free_input req = {0}; 4279 4280 if (!bp->grp_info) 4281 return 0; 4282 4283 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_FREE, -1, -1); 4284 4285 mutex_lock(&bp->hwrm_cmd_lock); 4286 for (i = 0; i < bp->cp_nr_rings; i++) { 4287 if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID) 4288 continue; 4289 req.ring_group_id = 4290 cpu_to_le32(bp->grp_info[i].fw_grp_id); 4291 4292 rc = _hwrm_send_message(bp, &req, sizeof(req), 4293 HWRM_CMD_TIMEOUT); 4294 if (rc) 4295 break; 4296 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID; 4297 } 4298 mutex_unlock(&bp->hwrm_cmd_lock); 4299 return rc; 4300 } 4301 4302 static int hwrm_ring_alloc_send_msg(struct bnxt *bp, 4303 struct bnxt_ring_struct *ring, 4304 u32 ring_type, u32 map_index) 4305 { 4306 int rc = 0, err = 0; 4307 struct hwrm_ring_alloc_input req = {0}; 4308 struct hwrm_ring_alloc_output *resp = bp->hwrm_cmd_resp_addr; 4309 struct bnxt_ring_grp_info *grp_info; 4310 u16 ring_id; 4311 4312 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_ALLOC, -1, -1); 4313 4314 req.enables = 0; 4315 if (ring->nr_pages > 1) { 4316 req.page_tbl_addr = cpu_to_le64(ring->pg_tbl_map); 4317 /* Page size is in log2 units */ 4318 req.page_size = BNXT_PAGE_SHIFT; 4319 req.page_tbl_depth = 1; 4320 } else { 4321 req.page_tbl_addr = cpu_to_le64(ring->dma_arr[0]); 4322 } 4323 req.fbo = 0; 4324 /* Association of ring index with doorbell index and MSIX number */ 4325 req.logical_id = cpu_to_le16(map_index); 4326 4327 switch (ring_type) { 4328 case HWRM_RING_ALLOC_TX: 4329 req.ring_type = RING_ALLOC_REQ_RING_TYPE_TX; 4330 /* Association of transmit ring with completion ring */ 4331 grp_info = &bp->grp_info[ring->grp_idx]; 4332 req.cmpl_ring_id = cpu_to_le16(grp_info->cp_fw_ring_id); 4333 req.length = cpu_to_le32(bp->tx_ring_mask + 1); 4334 req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx); 4335 req.queue_id = cpu_to_le16(ring->queue_id); 4336 break; 4337 case HWRM_RING_ALLOC_RX: 4338 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX; 4339 req.length = cpu_to_le32(bp->rx_ring_mask + 1); 4340 break; 4341 case HWRM_RING_ALLOC_AGG: 4342 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX; 4343 req.length = cpu_to_le32(bp->rx_agg_ring_mask + 1); 4344 break; 4345 case HWRM_RING_ALLOC_CMPL: 4346 req.ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL; 4347 req.length = cpu_to_le32(bp->cp_ring_mask + 1); 4348 if (bp->flags & BNXT_FLAG_USING_MSIX) 4349 req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX; 4350 break; 4351 default: 4352 netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n", 4353 ring_type); 4354 return -1; 4355 } 4356 4357 mutex_lock(&bp->hwrm_cmd_lock); 4358 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4359 err = le16_to_cpu(resp->error_code); 4360 ring_id = le16_to_cpu(resp->ring_id); 4361 mutex_unlock(&bp->hwrm_cmd_lock); 4362 4363 if (rc || err) { 4364 netdev_err(bp->dev, "hwrm_ring_alloc type %d failed. rc:%x err:%x\n", 4365 ring_type, rc, err); 4366 return -EIO; 4367 } 4368 ring->fw_ring_id = ring_id; 4369 return rc; 4370 } 4371 4372 static int bnxt_hwrm_set_async_event_cr(struct bnxt *bp, int idx) 4373 { 4374 int rc; 4375 4376 if (BNXT_PF(bp)) { 4377 struct hwrm_func_cfg_input req = {0}; 4378 4379 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1); 4380 req.fid = cpu_to_le16(0xffff); 4381 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR); 4382 req.async_event_cr = cpu_to_le16(idx); 4383 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4384 } else { 4385 struct hwrm_func_vf_cfg_input req = {0}; 4386 4387 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_VF_CFG, -1, -1); 4388 req.enables = 4389 cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR); 4390 req.async_event_cr = cpu_to_le16(idx); 4391 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4392 } 4393 return rc; 4394 } 4395 4396 static int bnxt_hwrm_ring_alloc(struct bnxt *bp) 4397 { 4398 int i, rc = 0; 4399 4400 for (i = 0; i < bp->cp_nr_rings; i++) { 4401 struct bnxt_napi *bnapi = bp->bnapi[i]; 4402 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 4403 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 4404 u32 map_idx = ring->map_idx; 4405 4406 cpr->cp_doorbell = bp->bar1 + map_idx * 0x80; 4407 rc = hwrm_ring_alloc_send_msg(bp, ring, HWRM_RING_ALLOC_CMPL, 4408 map_idx); 4409 if (rc) 4410 goto err_out; 4411 BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons); 4412 bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id; 4413 4414 if (!i) { 4415 rc = bnxt_hwrm_set_async_event_cr(bp, ring->fw_ring_id); 4416 if (rc) 4417 netdev_warn(bp->dev, "Failed to set async event completion ring.\n"); 4418 } 4419 } 4420 4421 for (i = 0; i < bp->tx_nr_rings; i++) { 4422 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 4423 struct bnxt_ring_struct *ring = &txr->tx_ring_struct; 4424 u32 map_idx = i; 4425 4426 rc = hwrm_ring_alloc_send_msg(bp, ring, HWRM_RING_ALLOC_TX, 4427 map_idx); 4428 if (rc) 4429 goto err_out; 4430 txr->tx_doorbell = bp->bar1 + map_idx * 0x80; 4431 } 4432 4433 for (i = 0; i < bp->rx_nr_rings; i++) { 4434 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 4435 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct; 4436 u32 map_idx = rxr->bnapi->index; 4437 4438 rc = hwrm_ring_alloc_send_msg(bp, ring, HWRM_RING_ALLOC_RX, 4439 map_idx); 4440 if (rc) 4441 goto err_out; 4442 rxr->rx_doorbell = bp->bar1 + map_idx * 0x80; 4443 writel(DB_KEY_RX | rxr->rx_prod, rxr->rx_doorbell); 4444 bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id; 4445 } 4446 4447 if (bp->flags & BNXT_FLAG_AGG_RINGS) { 4448 for (i = 0; i < bp->rx_nr_rings; i++) { 4449 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 4450 struct bnxt_ring_struct *ring = 4451 &rxr->rx_agg_ring_struct; 4452 u32 grp_idx = ring->grp_idx; 4453 u32 map_idx = grp_idx + bp->rx_nr_rings; 4454 4455 rc = hwrm_ring_alloc_send_msg(bp, ring, 4456 HWRM_RING_ALLOC_AGG, 4457 map_idx); 4458 if (rc) 4459 goto err_out; 4460 4461 rxr->rx_agg_doorbell = bp->bar1 + map_idx * 0x80; 4462 writel(DB_KEY_RX | rxr->rx_agg_prod, 4463 rxr->rx_agg_doorbell); 4464 bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id; 4465 } 4466 } 4467 err_out: 4468 return rc; 4469 } 4470 4471 static int hwrm_ring_free_send_msg(struct bnxt *bp, 4472 struct bnxt_ring_struct *ring, 4473 u32 ring_type, int cmpl_ring_id) 4474 { 4475 int rc; 4476 struct hwrm_ring_free_input req = {0}; 4477 struct hwrm_ring_free_output *resp = bp->hwrm_cmd_resp_addr; 4478 u16 error_code; 4479 4480 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_FREE, cmpl_ring_id, -1); 4481 req.ring_type = ring_type; 4482 req.ring_id = cpu_to_le16(ring->fw_ring_id); 4483 4484 mutex_lock(&bp->hwrm_cmd_lock); 4485 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4486 error_code = le16_to_cpu(resp->error_code); 4487 mutex_unlock(&bp->hwrm_cmd_lock); 4488 4489 if (rc || error_code) { 4490 netdev_err(bp->dev, "hwrm_ring_free type %d failed. rc:%x err:%x\n", 4491 ring_type, rc, error_code); 4492 return -EIO; 4493 } 4494 return 0; 4495 } 4496 4497 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path) 4498 { 4499 int i; 4500 4501 if (!bp->bnapi) 4502 return; 4503 4504 for (i = 0; i < bp->tx_nr_rings; i++) { 4505 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 4506 struct bnxt_ring_struct *ring = &txr->tx_ring_struct; 4507 u32 grp_idx = txr->bnapi->index; 4508 u32 cmpl_ring_id = bp->grp_info[grp_idx].cp_fw_ring_id; 4509 4510 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 4511 hwrm_ring_free_send_msg(bp, ring, 4512 RING_FREE_REQ_RING_TYPE_TX, 4513 close_path ? cmpl_ring_id : 4514 INVALID_HW_RING_ID); 4515 ring->fw_ring_id = INVALID_HW_RING_ID; 4516 } 4517 } 4518 4519 for (i = 0; i < bp->rx_nr_rings; i++) { 4520 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 4521 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct; 4522 u32 grp_idx = rxr->bnapi->index; 4523 u32 cmpl_ring_id = bp->grp_info[grp_idx].cp_fw_ring_id; 4524 4525 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 4526 hwrm_ring_free_send_msg(bp, ring, 4527 RING_FREE_REQ_RING_TYPE_RX, 4528 close_path ? cmpl_ring_id : 4529 INVALID_HW_RING_ID); 4530 ring->fw_ring_id = INVALID_HW_RING_ID; 4531 bp->grp_info[grp_idx].rx_fw_ring_id = 4532 INVALID_HW_RING_ID; 4533 } 4534 } 4535 4536 for (i = 0; i < bp->rx_nr_rings; i++) { 4537 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 4538 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct; 4539 u32 grp_idx = rxr->bnapi->index; 4540 u32 cmpl_ring_id = bp->grp_info[grp_idx].cp_fw_ring_id; 4541 4542 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 4543 hwrm_ring_free_send_msg(bp, ring, 4544 RING_FREE_REQ_RING_TYPE_RX, 4545 close_path ? cmpl_ring_id : 4546 INVALID_HW_RING_ID); 4547 ring->fw_ring_id = INVALID_HW_RING_ID; 4548 bp->grp_info[grp_idx].agg_fw_ring_id = 4549 INVALID_HW_RING_ID; 4550 } 4551 } 4552 4553 /* The completion rings are about to be freed. After that the 4554 * IRQ doorbell will not work anymore. So we need to disable 4555 * IRQ here. 4556 */ 4557 bnxt_disable_int_sync(bp); 4558 4559 for (i = 0; i < bp->cp_nr_rings; i++) { 4560 struct bnxt_napi *bnapi = bp->bnapi[i]; 4561 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 4562 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 4563 4564 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 4565 hwrm_ring_free_send_msg(bp, ring, 4566 RING_FREE_REQ_RING_TYPE_L2_CMPL, 4567 INVALID_HW_RING_ID); 4568 ring->fw_ring_id = INVALID_HW_RING_ID; 4569 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID; 4570 } 4571 } 4572 } 4573 4574 static int bnxt_hwrm_get_rings(struct bnxt *bp) 4575 { 4576 struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 4577 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 4578 struct hwrm_func_qcfg_input req = {0}; 4579 int rc; 4580 4581 if (bp->hwrm_spec_code < 0x10601) 4582 return 0; 4583 4584 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1); 4585 req.fid = cpu_to_le16(0xffff); 4586 mutex_lock(&bp->hwrm_cmd_lock); 4587 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4588 if (rc) { 4589 mutex_unlock(&bp->hwrm_cmd_lock); 4590 return -EIO; 4591 } 4592 4593 hw_resc->resv_tx_rings = le16_to_cpu(resp->alloc_tx_rings); 4594 if (bp->flags & BNXT_FLAG_NEW_RM) { 4595 u16 cp, stats; 4596 4597 hw_resc->resv_rx_rings = le16_to_cpu(resp->alloc_rx_rings); 4598 hw_resc->resv_hw_ring_grps = 4599 le32_to_cpu(resp->alloc_hw_ring_grps); 4600 hw_resc->resv_vnics = le16_to_cpu(resp->alloc_vnics); 4601 cp = le16_to_cpu(resp->alloc_cmpl_rings); 4602 stats = le16_to_cpu(resp->alloc_stat_ctx); 4603 cp = min_t(u16, cp, stats); 4604 hw_resc->resv_cp_rings = cp; 4605 } 4606 mutex_unlock(&bp->hwrm_cmd_lock); 4607 return 0; 4608 } 4609 4610 /* Caller must hold bp->hwrm_cmd_lock */ 4611 int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings) 4612 { 4613 struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 4614 struct hwrm_func_qcfg_input req = {0}; 4615 int rc; 4616 4617 if (bp->hwrm_spec_code < 0x10601) 4618 return 0; 4619 4620 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1); 4621 req.fid = cpu_to_le16(fid); 4622 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4623 if (!rc) 4624 *tx_rings = le16_to_cpu(resp->alloc_tx_rings); 4625 4626 return rc; 4627 } 4628 4629 static void 4630 __bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, struct hwrm_func_cfg_input *req, 4631 int tx_rings, int rx_rings, int ring_grps, 4632 int cp_rings, int vnics) 4633 { 4634 u32 enables = 0; 4635 4636 bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_CFG, -1, -1); 4637 req->fid = cpu_to_le16(0xffff); 4638 enables |= tx_rings ? FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS : 0; 4639 req->num_tx_rings = cpu_to_le16(tx_rings); 4640 if (bp->flags & BNXT_FLAG_NEW_RM) { 4641 enables |= rx_rings ? FUNC_CFG_REQ_ENABLES_NUM_RX_RINGS : 0; 4642 enables |= cp_rings ? FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS | 4643 FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0; 4644 enables |= ring_grps ? 4645 FUNC_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0; 4646 enables |= vnics ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0; 4647 4648 req->num_rx_rings = cpu_to_le16(rx_rings); 4649 req->num_hw_ring_grps = cpu_to_le16(ring_grps); 4650 req->num_cmpl_rings = cpu_to_le16(cp_rings); 4651 req->num_stat_ctxs = req->num_cmpl_rings; 4652 req->num_vnics = cpu_to_le16(vnics); 4653 } 4654 req->enables = cpu_to_le32(enables); 4655 } 4656 4657 static void 4658 __bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, 4659 struct hwrm_func_vf_cfg_input *req, int tx_rings, 4660 int rx_rings, int ring_grps, int cp_rings, 4661 int vnics) 4662 { 4663 u32 enables = 0; 4664 4665 bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_VF_CFG, -1, -1); 4666 enables |= tx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_TX_RINGS : 0; 4667 enables |= rx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_RX_RINGS : 0; 4668 enables |= cp_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS | 4669 FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0; 4670 enables |= ring_grps ? FUNC_VF_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0; 4671 enables |= vnics ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0; 4672 4673 req->num_tx_rings = cpu_to_le16(tx_rings); 4674 req->num_rx_rings = cpu_to_le16(rx_rings); 4675 req->num_hw_ring_grps = cpu_to_le16(ring_grps); 4676 req->num_cmpl_rings = cpu_to_le16(cp_rings); 4677 req->num_stat_ctxs = req->num_cmpl_rings; 4678 req->num_vnics = cpu_to_le16(vnics); 4679 4680 req->enables = cpu_to_le32(enables); 4681 } 4682 4683 static int 4684 bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 4685 int ring_grps, int cp_rings, int vnics) 4686 { 4687 struct hwrm_func_cfg_input req = {0}; 4688 int rc; 4689 4690 __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps, 4691 cp_rings, vnics); 4692 if (!req.enables) 4693 return 0; 4694 4695 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4696 if (rc) 4697 return -ENOMEM; 4698 4699 if (bp->hwrm_spec_code < 0x10601) 4700 bp->hw_resc.resv_tx_rings = tx_rings; 4701 4702 rc = bnxt_hwrm_get_rings(bp); 4703 return rc; 4704 } 4705 4706 static int 4707 bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 4708 int ring_grps, int cp_rings, int vnics) 4709 { 4710 struct hwrm_func_vf_cfg_input req = {0}; 4711 int rc; 4712 4713 if (!(bp->flags & BNXT_FLAG_NEW_RM)) { 4714 bp->hw_resc.resv_tx_rings = tx_rings; 4715 return 0; 4716 } 4717 4718 __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps, 4719 cp_rings, vnics); 4720 req.enables |= cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS | 4721 FUNC_VF_CFG_REQ_ENABLES_NUM_L2_CTXS); 4722 req.num_rsscos_ctxs = cpu_to_le16(BNXT_VF_MAX_RSS_CTX); 4723 req.num_l2_ctxs = cpu_to_le16(BNXT_VF_MAX_L2_CTX); 4724 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4725 if (rc) 4726 return -ENOMEM; 4727 4728 rc = bnxt_hwrm_get_rings(bp); 4729 return rc; 4730 } 4731 4732 static int bnxt_hwrm_reserve_rings(struct bnxt *bp, int tx, int rx, int grp, 4733 int cp, int vnic) 4734 { 4735 if (BNXT_PF(bp)) 4736 return bnxt_hwrm_reserve_pf_rings(bp, tx, rx, grp, cp, vnic); 4737 else 4738 return bnxt_hwrm_reserve_vf_rings(bp, tx, rx, grp, cp, vnic); 4739 } 4740 4741 static int bnxt_cp_rings_in_use(struct bnxt *bp) 4742 { 4743 int cp = bp->cp_nr_rings; 4744 int ulp_msix, ulp_base; 4745 4746 ulp_msix = bnxt_get_ulp_msix_num(bp); 4747 if (ulp_msix) { 4748 ulp_base = bnxt_get_ulp_msix_base(bp); 4749 cp += ulp_msix; 4750 if ((ulp_base + ulp_msix) > cp) 4751 cp = ulp_base + ulp_msix; 4752 } 4753 return cp; 4754 } 4755 4756 static bool bnxt_need_reserve_rings(struct bnxt *bp) 4757 { 4758 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 4759 int cp = bnxt_cp_rings_in_use(bp); 4760 int rx = bp->rx_nr_rings; 4761 int vnic = 1, grp = rx; 4762 4763 if (bp->hwrm_spec_code < 0x10601) 4764 return false; 4765 4766 if (hw_resc->resv_tx_rings != bp->tx_nr_rings) 4767 return true; 4768 4769 if (bp->flags & BNXT_FLAG_RFS) 4770 vnic = rx + 1; 4771 if (bp->flags & BNXT_FLAG_AGG_RINGS) 4772 rx <<= 1; 4773 if ((bp->flags & BNXT_FLAG_NEW_RM) && 4774 (hw_resc->resv_rx_rings != rx || hw_resc->resv_cp_rings != cp || 4775 hw_resc->resv_hw_ring_grps != grp || hw_resc->resv_vnics != vnic)) 4776 return true; 4777 return false; 4778 } 4779 4780 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max, 4781 bool shared); 4782 4783 static int __bnxt_reserve_rings(struct bnxt *bp) 4784 { 4785 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 4786 int cp = bnxt_cp_rings_in_use(bp); 4787 int tx = bp->tx_nr_rings; 4788 int rx = bp->rx_nr_rings; 4789 int grp, rx_rings, rc; 4790 bool sh = false; 4791 int vnic = 1; 4792 4793 if (!bnxt_need_reserve_rings(bp)) 4794 return 0; 4795 4796 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 4797 sh = true; 4798 if (bp->flags & BNXT_FLAG_RFS) 4799 vnic = rx + 1; 4800 if (bp->flags & BNXT_FLAG_AGG_RINGS) 4801 rx <<= 1; 4802 grp = bp->rx_nr_rings; 4803 4804 rc = bnxt_hwrm_reserve_rings(bp, tx, rx, grp, cp, vnic); 4805 if (rc) 4806 return rc; 4807 4808 tx = hw_resc->resv_tx_rings; 4809 if (bp->flags & BNXT_FLAG_NEW_RM) { 4810 rx = hw_resc->resv_rx_rings; 4811 cp = hw_resc->resv_cp_rings; 4812 grp = hw_resc->resv_hw_ring_grps; 4813 vnic = hw_resc->resv_vnics; 4814 } 4815 4816 rx_rings = rx; 4817 if (bp->flags & BNXT_FLAG_AGG_RINGS) { 4818 if (rx >= 2) { 4819 rx_rings = rx >> 1; 4820 } else { 4821 if (netif_running(bp->dev)) 4822 return -ENOMEM; 4823 4824 bp->flags &= ~BNXT_FLAG_AGG_RINGS; 4825 bp->flags |= BNXT_FLAG_NO_AGG_RINGS; 4826 bp->dev->hw_features &= ~NETIF_F_LRO; 4827 bp->dev->features &= ~NETIF_F_LRO; 4828 bnxt_set_ring_params(bp); 4829 } 4830 } 4831 rx_rings = min_t(int, rx_rings, grp); 4832 rc = bnxt_trim_rings(bp, &rx_rings, &tx, cp, sh); 4833 if (bp->flags & BNXT_FLAG_AGG_RINGS) 4834 rx = rx_rings << 1; 4835 cp = sh ? max_t(int, tx, rx_rings) : tx + rx_rings; 4836 bp->tx_nr_rings = tx; 4837 bp->rx_nr_rings = rx_rings; 4838 bp->cp_nr_rings = cp; 4839 4840 if (!tx || !rx || !cp || !grp || !vnic) 4841 return -ENOMEM; 4842 4843 return rc; 4844 } 4845 4846 static int bnxt_hwrm_check_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 4847 int ring_grps, int cp_rings, int vnics) 4848 { 4849 struct hwrm_func_vf_cfg_input req = {0}; 4850 u32 flags; 4851 int rc; 4852 4853 if (!(bp->flags & BNXT_FLAG_NEW_RM)) 4854 return 0; 4855 4856 __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps, 4857 cp_rings, vnics); 4858 flags = FUNC_VF_CFG_REQ_FLAGS_TX_ASSETS_TEST | 4859 FUNC_VF_CFG_REQ_FLAGS_RX_ASSETS_TEST | 4860 FUNC_VF_CFG_REQ_FLAGS_CMPL_ASSETS_TEST | 4861 FUNC_VF_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST | 4862 FUNC_VF_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST | 4863 FUNC_VF_CFG_REQ_FLAGS_VNIC_ASSETS_TEST; 4864 4865 req.flags = cpu_to_le32(flags); 4866 rc = hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4867 if (rc) 4868 return -ENOMEM; 4869 return 0; 4870 } 4871 4872 static int bnxt_hwrm_check_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 4873 int ring_grps, int cp_rings, int vnics) 4874 { 4875 struct hwrm_func_cfg_input req = {0}; 4876 u32 flags; 4877 int rc; 4878 4879 __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps, 4880 cp_rings, vnics); 4881 flags = FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST; 4882 if (bp->flags & BNXT_FLAG_NEW_RM) 4883 flags |= FUNC_CFG_REQ_FLAGS_RX_ASSETS_TEST | 4884 FUNC_CFG_REQ_FLAGS_CMPL_ASSETS_TEST | 4885 FUNC_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST | 4886 FUNC_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST | 4887 FUNC_CFG_REQ_FLAGS_VNIC_ASSETS_TEST; 4888 4889 req.flags = cpu_to_le32(flags); 4890 rc = hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4891 if (rc) 4892 return -ENOMEM; 4893 return 0; 4894 } 4895 4896 static int bnxt_hwrm_check_rings(struct bnxt *bp, int tx_rings, int rx_rings, 4897 int ring_grps, int cp_rings, int vnics) 4898 { 4899 if (bp->hwrm_spec_code < 0x10801) 4900 return 0; 4901 4902 if (BNXT_PF(bp)) 4903 return bnxt_hwrm_check_pf_rings(bp, tx_rings, rx_rings, 4904 ring_grps, cp_rings, vnics); 4905 4906 return bnxt_hwrm_check_vf_rings(bp, tx_rings, rx_rings, ring_grps, 4907 cp_rings, vnics); 4908 } 4909 4910 static void bnxt_hwrm_set_coal_params(struct bnxt_coal *hw_coal, 4911 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req) 4912 { 4913 u16 val, tmr, max, flags; 4914 4915 max = hw_coal->bufs_per_record * 128; 4916 if (hw_coal->budget) 4917 max = hw_coal->bufs_per_record * hw_coal->budget; 4918 4919 val = clamp_t(u16, hw_coal->coal_bufs, 1, max); 4920 req->num_cmpl_aggr_int = cpu_to_le16(val); 4921 4922 /* This is a 6-bit value and must not be 0, or we'll get non stop IRQ */ 4923 val = min_t(u16, val, 63); 4924 req->num_cmpl_dma_aggr = cpu_to_le16(val); 4925 4926 /* This is a 6-bit value and must not be 0, or we'll get non stop IRQ */ 4927 val = clamp_t(u16, hw_coal->coal_bufs_irq, 1, 63); 4928 req->num_cmpl_dma_aggr_during_int = cpu_to_le16(val); 4929 4930 tmr = BNXT_USEC_TO_COAL_TIMER(hw_coal->coal_ticks); 4931 tmr = max_t(u16, tmr, 1); 4932 req->int_lat_tmr_max = cpu_to_le16(tmr); 4933 4934 /* min timer set to 1/2 of interrupt timer */ 4935 val = tmr / 2; 4936 req->int_lat_tmr_min = cpu_to_le16(val); 4937 4938 /* buf timer set to 1/4 of interrupt timer */ 4939 val = max_t(u16, tmr / 4, 1); 4940 req->cmpl_aggr_dma_tmr = cpu_to_le16(val); 4941 4942 tmr = BNXT_USEC_TO_COAL_TIMER(hw_coal->coal_ticks_irq); 4943 tmr = max_t(u16, tmr, 1); 4944 req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(tmr); 4945 4946 flags = RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET; 4947 if (hw_coal->idle_thresh && hw_coal->coal_ticks < hw_coal->idle_thresh) 4948 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE; 4949 req->flags = cpu_to_le16(flags); 4950 } 4951 4952 int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi) 4953 { 4954 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0}; 4955 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 4956 struct bnxt_coal coal; 4957 unsigned int grp_idx; 4958 4959 /* Tick values in micro seconds. 4960 * 1 coal_buf x bufs_per_record = 1 completion record. 4961 */ 4962 memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal)); 4963 4964 coal.coal_ticks = cpr->rx_ring_coal.coal_ticks; 4965 coal.coal_bufs = cpr->rx_ring_coal.coal_bufs; 4966 4967 if (!bnapi->rx_ring) 4968 return -ENODEV; 4969 4970 bnxt_hwrm_cmd_hdr_init(bp, &req_rx, 4971 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1); 4972 4973 bnxt_hwrm_set_coal_params(&coal, &req_rx); 4974 4975 grp_idx = bnapi->index; 4976 req_rx.ring_id = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id); 4977 4978 return hwrm_send_message(bp, &req_rx, sizeof(req_rx), 4979 HWRM_CMD_TIMEOUT); 4980 } 4981 4982 int bnxt_hwrm_set_coal(struct bnxt *bp) 4983 { 4984 int i, rc = 0; 4985 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0}, 4986 req_tx = {0}, *req; 4987 4988 bnxt_hwrm_cmd_hdr_init(bp, &req_rx, 4989 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1); 4990 bnxt_hwrm_cmd_hdr_init(bp, &req_tx, 4991 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1); 4992 4993 bnxt_hwrm_set_coal_params(&bp->rx_coal, &req_rx); 4994 bnxt_hwrm_set_coal_params(&bp->tx_coal, &req_tx); 4995 4996 mutex_lock(&bp->hwrm_cmd_lock); 4997 for (i = 0; i < bp->cp_nr_rings; i++) { 4998 struct bnxt_napi *bnapi = bp->bnapi[i]; 4999 5000 req = &req_rx; 5001 if (!bnapi->rx_ring) 5002 req = &req_tx; 5003 req->ring_id = cpu_to_le16(bp->grp_info[i].cp_fw_ring_id); 5004 5005 rc = _hwrm_send_message(bp, req, sizeof(*req), 5006 HWRM_CMD_TIMEOUT); 5007 if (rc) 5008 break; 5009 } 5010 mutex_unlock(&bp->hwrm_cmd_lock); 5011 return rc; 5012 } 5013 5014 static int bnxt_hwrm_stat_ctx_free(struct bnxt *bp) 5015 { 5016 int rc = 0, i; 5017 struct hwrm_stat_ctx_free_input req = {0}; 5018 5019 if (!bp->bnapi) 5020 return 0; 5021 5022 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 5023 return 0; 5024 5025 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_FREE, -1, -1); 5026 5027 mutex_lock(&bp->hwrm_cmd_lock); 5028 for (i = 0; i < bp->cp_nr_rings; i++) { 5029 struct bnxt_napi *bnapi = bp->bnapi[i]; 5030 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 5031 5032 if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) { 5033 req.stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id); 5034 5035 rc = _hwrm_send_message(bp, &req, sizeof(req), 5036 HWRM_CMD_TIMEOUT); 5037 if (rc) 5038 break; 5039 5040 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID; 5041 } 5042 } 5043 mutex_unlock(&bp->hwrm_cmd_lock); 5044 return rc; 5045 } 5046 5047 static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp) 5048 { 5049 int rc = 0, i; 5050 struct hwrm_stat_ctx_alloc_input req = {0}; 5051 struct hwrm_stat_ctx_alloc_output *resp = bp->hwrm_cmd_resp_addr; 5052 5053 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 5054 return 0; 5055 5056 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_ALLOC, -1, -1); 5057 5058 req.update_period_ms = cpu_to_le32(bp->stats_coal_ticks / 1000); 5059 5060 mutex_lock(&bp->hwrm_cmd_lock); 5061 for (i = 0; i < bp->cp_nr_rings; i++) { 5062 struct bnxt_napi *bnapi = bp->bnapi[i]; 5063 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 5064 5065 req.stats_dma_addr = cpu_to_le64(cpr->hw_stats_map); 5066 5067 rc = _hwrm_send_message(bp, &req, sizeof(req), 5068 HWRM_CMD_TIMEOUT); 5069 if (rc) 5070 break; 5071 5072 cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id); 5073 5074 bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id; 5075 } 5076 mutex_unlock(&bp->hwrm_cmd_lock); 5077 return rc; 5078 } 5079 5080 static int bnxt_hwrm_func_qcfg(struct bnxt *bp) 5081 { 5082 struct hwrm_func_qcfg_input req = {0}; 5083 struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 5084 u16 flags; 5085 int rc; 5086 5087 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1); 5088 req.fid = cpu_to_le16(0xffff); 5089 mutex_lock(&bp->hwrm_cmd_lock); 5090 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5091 if (rc) 5092 goto func_qcfg_exit; 5093 5094 #ifdef CONFIG_BNXT_SRIOV 5095 if (BNXT_VF(bp)) { 5096 struct bnxt_vf_info *vf = &bp->vf; 5097 5098 vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK; 5099 } 5100 #endif 5101 flags = le16_to_cpu(resp->flags); 5102 if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED | 5103 FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED)) { 5104 bp->flags |= BNXT_FLAG_FW_LLDP_AGENT; 5105 if (flags & FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED) 5106 bp->flags |= BNXT_FLAG_FW_DCBX_AGENT; 5107 } 5108 if (BNXT_PF(bp) && (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST)) 5109 bp->flags |= BNXT_FLAG_MULTI_HOST; 5110 5111 switch (resp->port_partition_type) { 5112 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0: 5113 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5: 5114 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0: 5115 bp->port_partition_type = resp->port_partition_type; 5116 break; 5117 } 5118 if (bp->hwrm_spec_code < 0x10707 || 5119 resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEB) 5120 bp->br_mode = BRIDGE_MODE_VEB; 5121 else if (resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEPA) 5122 bp->br_mode = BRIDGE_MODE_VEPA; 5123 else 5124 bp->br_mode = BRIDGE_MODE_UNDEF; 5125 5126 bp->max_mtu = le16_to_cpu(resp->max_mtu_configured); 5127 if (!bp->max_mtu) 5128 bp->max_mtu = BNXT_MAX_MTU; 5129 5130 func_qcfg_exit: 5131 mutex_unlock(&bp->hwrm_cmd_lock); 5132 return rc; 5133 } 5134 5135 int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp, bool all) 5136 { 5137 struct hwrm_func_resource_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 5138 struct hwrm_func_resource_qcaps_input req = {0}; 5139 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 5140 int rc; 5141 5142 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESOURCE_QCAPS, -1, -1); 5143 req.fid = cpu_to_le16(0xffff); 5144 5145 mutex_lock(&bp->hwrm_cmd_lock); 5146 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5147 if (rc) { 5148 rc = -EIO; 5149 goto hwrm_func_resc_qcaps_exit; 5150 } 5151 5152 hw_resc->max_tx_sch_inputs = le16_to_cpu(resp->max_tx_scheduler_inputs); 5153 if (!all) 5154 goto hwrm_func_resc_qcaps_exit; 5155 5156 hw_resc->min_rsscos_ctxs = le16_to_cpu(resp->min_rsscos_ctx); 5157 hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx); 5158 hw_resc->min_cp_rings = le16_to_cpu(resp->min_cmpl_rings); 5159 hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings); 5160 hw_resc->min_tx_rings = le16_to_cpu(resp->min_tx_rings); 5161 hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings); 5162 hw_resc->min_rx_rings = le16_to_cpu(resp->min_rx_rings); 5163 hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings); 5164 hw_resc->min_hw_ring_grps = le16_to_cpu(resp->min_hw_ring_grps); 5165 hw_resc->max_hw_ring_grps = le16_to_cpu(resp->max_hw_ring_grps); 5166 hw_resc->min_l2_ctxs = le16_to_cpu(resp->min_l2_ctxs); 5167 hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs); 5168 hw_resc->min_vnics = le16_to_cpu(resp->min_vnics); 5169 hw_resc->max_vnics = le16_to_cpu(resp->max_vnics); 5170 hw_resc->min_stat_ctxs = le16_to_cpu(resp->min_stat_ctx); 5171 hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx); 5172 5173 if (BNXT_PF(bp)) { 5174 struct bnxt_pf_info *pf = &bp->pf; 5175 5176 pf->vf_resv_strategy = 5177 le16_to_cpu(resp->vf_reservation_strategy); 5178 if (pf->vf_resv_strategy > BNXT_VF_RESV_STRATEGY_MINIMAL) 5179 pf->vf_resv_strategy = BNXT_VF_RESV_STRATEGY_MAXIMAL; 5180 } 5181 hwrm_func_resc_qcaps_exit: 5182 mutex_unlock(&bp->hwrm_cmd_lock); 5183 return rc; 5184 } 5185 5186 static int __bnxt_hwrm_func_qcaps(struct bnxt *bp) 5187 { 5188 int rc = 0; 5189 struct hwrm_func_qcaps_input req = {0}; 5190 struct hwrm_func_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 5191 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 5192 u32 flags; 5193 5194 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCAPS, -1, -1); 5195 req.fid = cpu_to_le16(0xffff); 5196 5197 mutex_lock(&bp->hwrm_cmd_lock); 5198 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5199 if (rc) 5200 goto hwrm_func_qcaps_exit; 5201 5202 flags = le32_to_cpu(resp->flags); 5203 if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED) 5204 bp->flags |= BNXT_FLAG_ROCEV1_CAP; 5205 if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED) 5206 bp->flags |= BNXT_FLAG_ROCEV2_CAP; 5207 5208 bp->tx_push_thresh = 0; 5209 if (flags & FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED) 5210 bp->tx_push_thresh = BNXT_TX_PUSH_THRESH; 5211 5212 hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx); 5213 hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings); 5214 hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings); 5215 hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings); 5216 hw_resc->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps); 5217 if (!hw_resc->max_hw_ring_grps) 5218 hw_resc->max_hw_ring_grps = hw_resc->max_tx_rings; 5219 hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs); 5220 hw_resc->max_vnics = le16_to_cpu(resp->max_vnics); 5221 hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx); 5222 5223 if (BNXT_PF(bp)) { 5224 struct bnxt_pf_info *pf = &bp->pf; 5225 5226 pf->fw_fid = le16_to_cpu(resp->fid); 5227 pf->port_id = le16_to_cpu(resp->port_id); 5228 bp->dev->dev_port = pf->port_id; 5229 memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN); 5230 pf->first_vf_id = le16_to_cpu(resp->first_vf_id); 5231 pf->max_vfs = le16_to_cpu(resp->max_vfs); 5232 pf->max_encap_records = le32_to_cpu(resp->max_encap_records); 5233 pf->max_decap_records = le32_to_cpu(resp->max_decap_records); 5234 pf->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows); 5235 pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows); 5236 pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows); 5237 pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows); 5238 if (flags & FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED) 5239 bp->flags |= BNXT_FLAG_WOL_CAP; 5240 } else { 5241 #ifdef CONFIG_BNXT_SRIOV 5242 struct bnxt_vf_info *vf = &bp->vf; 5243 5244 vf->fw_fid = le16_to_cpu(resp->fid); 5245 memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN); 5246 #endif 5247 } 5248 5249 hwrm_func_qcaps_exit: 5250 mutex_unlock(&bp->hwrm_cmd_lock); 5251 return rc; 5252 } 5253 5254 static int bnxt_hwrm_func_qcaps(struct bnxt *bp) 5255 { 5256 int rc; 5257 5258 rc = __bnxt_hwrm_func_qcaps(bp); 5259 if (rc) 5260 return rc; 5261 if (bp->hwrm_spec_code >= 0x10803) { 5262 rc = bnxt_hwrm_func_resc_qcaps(bp, true); 5263 if (!rc) 5264 bp->flags |= BNXT_FLAG_NEW_RM; 5265 } 5266 return 0; 5267 } 5268 5269 static int bnxt_hwrm_func_reset(struct bnxt *bp) 5270 { 5271 struct hwrm_func_reset_input req = {0}; 5272 5273 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESET, -1, -1); 5274 req.enables = 0; 5275 5276 return hwrm_send_message(bp, &req, sizeof(req), HWRM_RESET_TIMEOUT); 5277 } 5278 5279 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp) 5280 { 5281 int rc = 0; 5282 struct hwrm_queue_qportcfg_input req = {0}; 5283 struct hwrm_queue_qportcfg_output *resp = bp->hwrm_cmd_resp_addr; 5284 u8 i, *qptr; 5285 5286 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_QUEUE_QPORTCFG, -1, -1); 5287 5288 mutex_lock(&bp->hwrm_cmd_lock); 5289 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5290 if (rc) 5291 goto qportcfg_exit; 5292 5293 if (!resp->max_configurable_queues) { 5294 rc = -EINVAL; 5295 goto qportcfg_exit; 5296 } 5297 bp->max_tc = resp->max_configurable_queues; 5298 bp->max_lltc = resp->max_configurable_lossless_queues; 5299 if (bp->max_tc > BNXT_MAX_QUEUE) 5300 bp->max_tc = BNXT_MAX_QUEUE; 5301 5302 if (resp->queue_cfg_info & QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG) 5303 bp->max_tc = 1; 5304 5305 if (bp->max_lltc > bp->max_tc) 5306 bp->max_lltc = bp->max_tc; 5307 5308 qptr = &resp->queue_id0; 5309 for (i = 0; i < bp->max_tc; i++) { 5310 bp->q_info[i].queue_id = *qptr++; 5311 bp->q_info[i].queue_profile = *qptr++; 5312 bp->tc_to_qidx[i] = i; 5313 } 5314 5315 qportcfg_exit: 5316 mutex_unlock(&bp->hwrm_cmd_lock); 5317 return rc; 5318 } 5319 5320 static int bnxt_hwrm_ver_get(struct bnxt *bp) 5321 { 5322 int rc; 5323 struct hwrm_ver_get_input req = {0}; 5324 struct hwrm_ver_get_output *resp = bp->hwrm_cmd_resp_addr; 5325 u32 dev_caps_cfg; 5326 5327 bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN; 5328 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VER_GET, -1, -1); 5329 req.hwrm_intf_maj = HWRM_VERSION_MAJOR; 5330 req.hwrm_intf_min = HWRM_VERSION_MINOR; 5331 req.hwrm_intf_upd = HWRM_VERSION_UPDATE; 5332 mutex_lock(&bp->hwrm_cmd_lock); 5333 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5334 if (rc) 5335 goto hwrm_ver_get_exit; 5336 5337 memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output)); 5338 5339 bp->hwrm_spec_code = resp->hwrm_intf_maj_8b << 16 | 5340 resp->hwrm_intf_min_8b << 8 | 5341 resp->hwrm_intf_upd_8b; 5342 if (resp->hwrm_intf_maj_8b < 1) { 5343 netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n", 5344 resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b, 5345 resp->hwrm_intf_upd_8b); 5346 netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n"); 5347 } 5348 snprintf(bp->fw_ver_str, BC_HWRM_STR_LEN, "%d.%d.%d.%d", 5349 resp->hwrm_fw_maj_8b, resp->hwrm_fw_min_8b, 5350 resp->hwrm_fw_bld_8b, resp->hwrm_fw_rsvd_8b); 5351 5352 bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout); 5353 if (!bp->hwrm_cmd_timeout) 5354 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT; 5355 5356 if (resp->hwrm_intf_maj_8b >= 1) 5357 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len); 5358 5359 bp->chip_num = le16_to_cpu(resp->chip_num); 5360 if (bp->chip_num == CHIP_NUM_58700 && !resp->chip_rev && 5361 !resp->chip_metal) 5362 bp->flags |= BNXT_FLAG_CHIP_NITRO_A0; 5363 5364 dev_caps_cfg = le32_to_cpu(resp->dev_caps_cfg); 5365 if ((dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) && 5366 (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED)) 5367 bp->flags |= BNXT_FLAG_SHORT_CMD; 5368 5369 hwrm_ver_get_exit: 5370 mutex_unlock(&bp->hwrm_cmd_lock); 5371 return rc; 5372 } 5373 5374 int bnxt_hwrm_fw_set_time(struct bnxt *bp) 5375 { 5376 struct hwrm_fw_set_time_input req = {0}; 5377 struct tm tm; 5378 time64_t now = ktime_get_real_seconds(); 5379 5380 if ((BNXT_VF(bp) && bp->hwrm_spec_code < 0x10901) || 5381 bp->hwrm_spec_code < 0x10400) 5382 return -EOPNOTSUPP; 5383 5384 time64_to_tm(now, 0, &tm); 5385 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FW_SET_TIME, -1, -1); 5386 req.year = cpu_to_le16(1900 + tm.tm_year); 5387 req.month = 1 + tm.tm_mon; 5388 req.day = tm.tm_mday; 5389 req.hour = tm.tm_hour; 5390 req.minute = tm.tm_min; 5391 req.second = tm.tm_sec; 5392 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5393 } 5394 5395 static int bnxt_hwrm_port_qstats(struct bnxt *bp) 5396 { 5397 int rc; 5398 struct bnxt_pf_info *pf = &bp->pf; 5399 struct hwrm_port_qstats_input req = {0}; 5400 5401 if (!(bp->flags & BNXT_FLAG_PORT_STATS)) 5402 return 0; 5403 5404 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS, -1, -1); 5405 req.port_id = cpu_to_le16(pf->port_id); 5406 req.tx_stat_host_addr = cpu_to_le64(bp->hw_tx_port_stats_map); 5407 req.rx_stat_host_addr = cpu_to_le64(bp->hw_rx_port_stats_map); 5408 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5409 return rc; 5410 } 5411 5412 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp) 5413 { 5414 struct hwrm_port_qstats_ext_input req = {0}; 5415 struct bnxt_pf_info *pf = &bp->pf; 5416 5417 if (!(bp->flags & BNXT_FLAG_PORT_STATS_EXT)) 5418 return 0; 5419 5420 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS_EXT, -1, -1); 5421 req.port_id = cpu_to_le16(pf->port_id); 5422 req.rx_stat_size = cpu_to_le16(sizeof(struct rx_port_stats_ext)); 5423 req.rx_stat_host_addr = cpu_to_le64(bp->hw_rx_port_stats_ext_map); 5424 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5425 } 5426 5427 static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp) 5428 { 5429 if (bp->vxlan_port_cnt) { 5430 bnxt_hwrm_tunnel_dst_port_free( 5431 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN); 5432 } 5433 bp->vxlan_port_cnt = 0; 5434 if (bp->nge_port_cnt) { 5435 bnxt_hwrm_tunnel_dst_port_free( 5436 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE); 5437 } 5438 bp->nge_port_cnt = 0; 5439 } 5440 5441 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa) 5442 { 5443 int rc, i; 5444 u32 tpa_flags = 0; 5445 5446 if (set_tpa) 5447 tpa_flags = bp->flags & BNXT_FLAG_TPA; 5448 for (i = 0; i < bp->nr_vnics; i++) { 5449 rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags); 5450 if (rc) { 5451 netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n", 5452 i, rc); 5453 return rc; 5454 } 5455 } 5456 return 0; 5457 } 5458 5459 static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp) 5460 { 5461 int i; 5462 5463 for (i = 0; i < bp->nr_vnics; i++) 5464 bnxt_hwrm_vnic_set_rss(bp, i, false); 5465 } 5466 5467 static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path, 5468 bool irq_re_init) 5469 { 5470 if (bp->vnic_info) { 5471 bnxt_hwrm_clear_vnic_filter(bp); 5472 /* clear all RSS setting before free vnic ctx */ 5473 bnxt_hwrm_clear_vnic_rss(bp); 5474 bnxt_hwrm_vnic_ctx_free(bp); 5475 /* before free the vnic, undo the vnic tpa settings */ 5476 if (bp->flags & BNXT_FLAG_TPA) 5477 bnxt_set_tpa(bp, false); 5478 bnxt_hwrm_vnic_free(bp); 5479 } 5480 bnxt_hwrm_ring_free(bp, close_path); 5481 bnxt_hwrm_ring_grp_free(bp); 5482 if (irq_re_init) { 5483 bnxt_hwrm_stat_ctx_free(bp); 5484 bnxt_hwrm_free_tunnel_ports(bp); 5485 } 5486 } 5487 5488 static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode) 5489 { 5490 struct hwrm_func_cfg_input req = {0}; 5491 int rc; 5492 5493 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1); 5494 req.fid = cpu_to_le16(0xffff); 5495 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_EVB_MODE); 5496 if (br_mode == BRIDGE_MODE_VEB) 5497 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEB; 5498 else if (br_mode == BRIDGE_MODE_VEPA) 5499 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEPA; 5500 else 5501 return -EINVAL; 5502 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5503 if (rc) 5504 rc = -EIO; 5505 return rc; 5506 } 5507 5508 static int bnxt_hwrm_set_cache_line_size(struct bnxt *bp, int size) 5509 { 5510 struct hwrm_func_cfg_input req = {0}; 5511 int rc; 5512 5513 if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10803) 5514 return 0; 5515 5516 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1); 5517 req.fid = cpu_to_le16(0xffff); 5518 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_CACHE_LINESIZE); 5519 req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_64; 5520 if (size == 128) 5521 req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_128; 5522 5523 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5524 if (rc) 5525 rc = -EIO; 5526 return rc; 5527 } 5528 5529 static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id) 5530 { 5531 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5532 int rc; 5533 5534 if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) 5535 goto skip_rss_ctx; 5536 5537 /* allocate context for vnic */ 5538 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 0); 5539 if (rc) { 5540 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n", 5541 vnic_id, rc); 5542 goto vnic_setup_err; 5543 } 5544 bp->rsscos_nr_ctxs++; 5545 5546 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 5547 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 1); 5548 if (rc) { 5549 netdev_err(bp->dev, "hwrm vnic %d cos ctx alloc failure rc: %x\n", 5550 vnic_id, rc); 5551 goto vnic_setup_err; 5552 } 5553 bp->rsscos_nr_ctxs++; 5554 } 5555 5556 skip_rss_ctx: 5557 /* configure default vnic, ring grp */ 5558 rc = bnxt_hwrm_vnic_cfg(bp, vnic_id); 5559 if (rc) { 5560 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n", 5561 vnic_id, rc); 5562 goto vnic_setup_err; 5563 } 5564 5565 /* Enable RSS hashing on vnic */ 5566 rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true); 5567 if (rc) { 5568 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n", 5569 vnic_id, rc); 5570 goto vnic_setup_err; 5571 } 5572 5573 if (bp->flags & BNXT_FLAG_AGG_RINGS) { 5574 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id); 5575 if (rc) { 5576 netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n", 5577 vnic_id, rc); 5578 } 5579 } 5580 5581 vnic_setup_err: 5582 return rc; 5583 } 5584 5585 static int bnxt_alloc_rfs_vnics(struct bnxt *bp) 5586 { 5587 #ifdef CONFIG_RFS_ACCEL 5588 int i, rc = 0; 5589 5590 for (i = 0; i < bp->rx_nr_rings; i++) { 5591 struct bnxt_vnic_info *vnic; 5592 u16 vnic_id = i + 1; 5593 u16 ring_id = i; 5594 5595 if (vnic_id >= bp->nr_vnics) 5596 break; 5597 5598 vnic = &bp->vnic_info[vnic_id]; 5599 vnic->flags |= BNXT_VNIC_RFS_FLAG; 5600 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP) 5601 vnic->flags |= BNXT_VNIC_RFS_NEW_RSS_FLAG; 5602 rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, ring_id, 1); 5603 if (rc) { 5604 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n", 5605 vnic_id, rc); 5606 break; 5607 } 5608 rc = bnxt_setup_vnic(bp, vnic_id); 5609 if (rc) 5610 break; 5611 } 5612 return rc; 5613 #else 5614 return 0; 5615 #endif 5616 } 5617 5618 /* Allow PF and VF with default VLAN to be in promiscuous mode */ 5619 static bool bnxt_promisc_ok(struct bnxt *bp) 5620 { 5621 #ifdef CONFIG_BNXT_SRIOV 5622 if (BNXT_VF(bp) && !bp->vf.vlan) 5623 return false; 5624 #endif 5625 return true; 5626 } 5627 5628 static int bnxt_setup_nitroa0_vnic(struct bnxt *bp) 5629 { 5630 unsigned int rc = 0; 5631 5632 rc = bnxt_hwrm_vnic_alloc(bp, 1, bp->rx_nr_rings - 1, 1); 5633 if (rc) { 5634 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n", 5635 rc); 5636 return rc; 5637 } 5638 5639 rc = bnxt_hwrm_vnic_cfg(bp, 1); 5640 if (rc) { 5641 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n", 5642 rc); 5643 return rc; 5644 } 5645 return rc; 5646 } 5647 5648 static int bnxt_cfg_rx_mode(struct bnxt *); 5649 static bool bnxt_mc_list_updated(struct bnxt *, u32 *); 5650 5651 static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init) 5652 { 5653 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 5654 int rc = 0; 5655 unsigned int rx_nr_rings = bp->rx_nr_rings; 5656 5657 if (irq_re_init) { 5658 rc = bnxt_hwrm_stat_ctx_alloc(bp); 5659 if (rc) { 5660 netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n", 5661 rc); 5662 goto err_out; 5663 } 5664 } 5665 5666 rc = bnxt_hwrm_ring_alloc(bp); 5667 if (rc) { 5668 netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc); 5669 goto err_out; 5670 } 5671 5672 rc = bnxt_hwrm_ring_grp_alloc(bp); 5673 if (rc) { 5674 netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc); 5675 goto err_out; 5676 } 5677 5678 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 5679 rx_nr_rings--; 5680 5681 /* default vnic 0 */ 5682 rc = bnxt_hwrm_vnic_alloc(bp, 0, 0, rx_nr_rings); 5683 if (rc) { 5684 netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc); 5685 goto err_out; 5686 } 5687 5688 rc = bnxt_setup_vnic(bp, 0); 5689 if (rc) 5690 goto err_out; 5691 5692 if (bp->flags & BNXT_FLAG_RFS) { 5693 rc = bnxt_alloc_rfs_vnics(bp); 5694 if (rc) 5695 goto err_out; 5696 } 5697 5698 if (bp->flags & BNXT_FLAG_TPA) { 5699 rc = bnxt_set_tpa(bp, true); 5700 if (rc) 5701 goto err_out; 5702 } 5703 5704 if (BNXT_VF(bp)) 5705 bnxt_update_vf_mac(bp); 5706 5707 /* Filter for default vnic 0 */ 5708 rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr); 5709 if (rc) { 5710 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc); 5711 goto err_out; 5712 } 5713 vnic->uc_filter_count = 1; 5714 5715 vnic->rx_mask = 0; 5716 if (bp->dev->flags & IFF_BROADCAST) 5717 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST; 5718 5719 if ((bp->dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp)) 5720 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 5721 5722 if (bp->dev->flags & IFF_ALLMULTI) { 5723 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 5724 vnic->mc_list_count = 0; 5725 } else { 5726 u32 mask = 0; 5727 5728 bnxt_mc_list_updated(bp, &mask); 5729 vnic->rx_mask |= mask; 5730 } 5731 5732 rc = bnxt_cfg_rx_mode(bp); 5733 if (rc) 5734 goto err_out; 5735 5736 rc = bnxt_hwrm_set_coal(bp); 5737 if (rc) 5738 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n", 5739 rc); 5740 5741 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 5742 rc = bnxt_setup_nitroa0_vnic(bp); 5743 if (rc) 5744 netdev_err(bp->dev, "Special vnic setup failure for NS2 A0 rc: %x\n", 5745 rc); 5746 } 5747 5748 if (BNXT_VF(bp)) { 5749 bnxt_hwrm_func_qcfg(bp); 5750 netdev_update_features(bp->dev); 5751 } 5752 5753 return 0; 5754 5755 err_out: 5756 bnxt_hwrm_resource_free(bp, 0, true); 5757 5758 return rc; 5759 } 5760 5761 static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init) 5762 { 5763 bnxt_hwrm_resource_free(bp, 1, irq_re_init); 5764 return 0; 5765 } 5766 5767 static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init) 5768 { 5769 bnxt_init_cp_rings(bp); 5770 bnxt_init_rx_rings(bp); 5771 bnxt_init_tx_rings(bp); 5772 bnxt_init_ring_grps(bp, irq_re_init); 5773 bnxt_init_vnics(bp); 5774 5775 return bnxt_init_chip(bp, irq_re_init); 5776 } 5777 5778 static int bnxt_set_real_num_queues(struct bnxt *bp) 5779 { 5780 int rc; 5781 struct net_device *dev = bp->dev; 5782 5783 rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings - 5784 bp->tx_nr_rings_xdp); 5785 if (rc) 5786 return rc; 5787 5788 rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings); 5789 if (rc) 5790 return rc; 5791 5792 #ifdef CONFIG_RFS_ACCEL 5793 if (bp->flags & BNXT_FLAG_RFS) 5794 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings); 5795 #endif 5796 5797 return rc; 5798 } 5799 5800 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max, 5801 bool shared) 5802 { 5803 int _rx = *rx, _tx = *tx; 5804 5805 if (shared) { 5806 *rx = min_t(int, _rx, max); 5807 *tx = min_t(int, _tx, max); 5808 } else { 5809 if (max < 2) 5810 return -ENOMEM; 5811 5812 while (_rx + _tx > max) { 5813 if (_rx > _tx && _rx > 1) 5814 _rx--; 5815 else if (_tx > 1) 5816 _tx--; 5817 } 5818 *rx = _rx; 5819 *tx = _tx; 5820 } 5821 return 0; 5822 } 5823 5824 static void bnxt_setup_msix(struct bnxt *bp) 5825 { 5826 const int len = sizeof(bp->irq_tbl[0].name); 5827 struct net_device *dev = bp->dev; 5828 int tcs, i; 5829 5830 tcs = netdev_get_num_tc(dev); 5831 if (tcs > 1) { 5832 int i, off, count; 5833 5834 for (i = 0; i < tcs; i++) { 5835 count = bp->tx_nr_rings_per_tc; 5836 off = i * count; 5837 netdev_set_tc_queue(dev, i, count, off); 5838 } 5839 } 5840 5841 for (i = 0; i < bp->cp_nr_rings; i++) { 5842 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 5843 char *attr; 5844 5845 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 5846 attr = "TxRx"; 5847 else if (i < bp->rx_nr_rings) 5848 attr = "rx"; 5849 else 5850 attr = "tx"; 5851 5852 snprintf(bp->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name, 5853 attr, i); 5854 bp->irq_tbl[map_idx].handler = bnxt_msix; 5855 } 5856 } 5857 5858 static void bnxt_setup_inta(struct bnxt *bp) 5859 { 5860 const int len = sizeof(bp->irq_tbl[0].name); 5861 5862 if (netdev_get_num_tc(bp->dev)) 5863 netdev_reset_tc(bp->dev); 5864 5865 snprintf(bp->irq_tbl[0].name, len, "%s-%s-%d", bp->dev->name, "TxRx", 5866 0); 5867 bp->irq_tbl[0].handler = bnxt_inta; 5868 } 5869 5870 static int bnxt_setup_int_mode(struct bnxt *bp) 5871 { 5872 int rc; 5873 5874 if (bp->flags & BNXT_FLAG_USING_MSIX) 5875 bnxt_setup_msix(bp); 5876 else 5877 bnxt_setup_inta(bp); 5878 5879 rc = bnxt_set_real_num_queues(bp); 5880 return rc; 5881 } 5882 5883 #ifdef CONFIG_RFS_ACCEL 5884 static unsigned int bnxt_get_max_func_rss_ctxs(struct bnxt *bp) 5885 { 5886 return bp->hw_resc.max_rsscos_ctxs; 5887 } 5888 5889 static unsigned int bnxt_get_max_func_vnics(struct bnxt *bp) 5890 { 5891 return bp->hw_resc.max_vnics; 5892 } 5893 #endif 5894 5895 unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp) 5896 { 5897 return bp->hw_resc.max_stat_ctxs; 5898 } 5899 5900 void bnxt_set_max_func_stat_ctxs(struct bnxt *bp, unsigned int max) 5901 { 5902 bp->hw_resc.max_stat_ctxs = max; 5903 } 5904 5905 unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp) 5906 { 5907 return bp->hw_resc.max_cp_rings; 5908 } 5909 5910 void bnxt_set_max_func_cp_rings(struct bnxt *bp, unsigned int max) 5911 { 5912 bp->hw_resc.max_cp_rings = max; 5913 } 5914 5915 unsigned int bnxt_get_max_func_irqs(struct bnxt *bp) 5916 { 5917 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 5918 5919 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_cp_rings); 5920 } 5921 5922 static void bnxt_set_max_func_irqs(struct bnxt *bp, unsigned int max_irqs) 5923 { 5924 bp->hw_resc.max_irqs = max_irqs; 5925 } 5926 5927 int bnxt_get_avail_msix(struct bnxt *bp, int num) 5928 { 5929 int max_cp = bnxt_get_max_func_cp_rings(bp); 5930 int max_irq = bnxt_get_max_func_irqs(bp); 5931 int total_req = bp->cp_nr_rings + num; 5932 int max_idx, avail_msix; 5933 5934 max_idx = min_t(int, bp->total_irqs, max_cp); 5935 avail_msix = max_idx - bp->cp_nr_rings; 5936 if (!(bp->flags & BNXT_FLAG_NEW_RM) || avail_msix >= num) 5937 return avail_msix; 5938 5939 if (max_irq < total_req) { 5940 num = max_irq - bp->cp_nr_rings; 5941 if (num <= 0) 5942 return 0; 5943 } 5944 return num; 5945 } 5946 5947 static int bnxt_get_num_msix(struct bnxt *bp) 5948 { 5949 if (!(bp->flags & BNXT_FLAG_NEW_RM)) 5950 return bnxt_get_max_func_irqs(bp); 5951 5952 return bnxt_cp_rings_in_use(bp); 5953 } 5954 5955 static int bnxt_init_msix(struct bnxt *bp) 5956 { 5957 int i, total_vecs, max, rc = 0, min = 1, ulp_msix; 5958 struct msix_entry *msix_ent; 5959 5960 total_vecs = bnxt_get_num_msix(bp); 5961 max = bnxt_get_max_func_irqs(bp); 5962 if (total_vecs > max) 5963 total_vecs = max; 5964 5965 if (!total_vecs) 5966 return 0; 5967 5968 msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL); 5969 if (!msix_ent) 5970 return -ENOMEM; 5971 5972 for (i = 0; i < total_vecs; i++) { 5973 msix_ent[i].entry = i; 5974 msix_ent[i].vector = 0; 5975 } 5976 5977 if (!(bp->flags & BNXT_FLAG_SHARED_RINGS)) 5978 min = 2; 5979 5980 total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs); 5981 ulp_msix = bnxt_get_ulp_msix_num(bp); 5982 if (total_vecs < 0 || total_vecs < ulp_msix) { 5983 rc = -ENODEV; 5984 goto msix_setup_exit; 5985 } 5986 5987 bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL); 5988 if (bp->irq_tbl) { 5989 for (i = 0; i < total_vecs; i++) 5990 bp->irq_tbl[i].vector = msix_ent[i].vector; 5991 5992 bp->total_irqs = total_vecs; 5993 /* Trim rings based upon num of vectors allocated */ 5994 rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings, 5995 total_vecs - ulp_msix, min == 1); 5996 if (rc) 5997 goto msix_setup_exit; 5998 5999 bp->cp_nr_rings = (min == 1) ? 6000 max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) : 6001 bp->tx_nr_rings + bp->rx_nr_rings; 6002 6003 } else { 6004 rc = -ENOMEM; 6005 goto msix_setup_exit; 6006 } 6007 bp->flags |= BNXT_FLAG_USING_MSIX; 6008 kfree(msix_ent); 6009 return 0; 6010 6011 msix_setup_exit: 6012 netdev_err(bp->dev, "bnxt_init_msix err: %x\n", rc); 6013 kfree(bp->irq_tbl); 6014 bp->irq_tbl = NULL; 6015 pci_disable_msix(bp->pdev); 6016 kfree(msix_ent); 6017 return rc; 6018 } 6019 6020 static int bnxt_init_inta(struct bnxt *bp) 6021 { 6022 bp->irq_tbl = kcalloc(1, sizeof(struct bnxt_irq), GFP_KERNEL); 6023 if (!bp->irq_tbl) 6024 return -ENOMEM; 6025 6026 bp->total_irqs = 1; 6027 bp->rx_nr_rings = 1; 6028 bp->tx_nr_rings = 1; 6029 bp->cp_nr_rings = 1; 6030 bp->flags |= BNXT_FLAG_SHARED_RINGS; 6031 bp->irq_tbl[0].vector = bp->pdev->irq; 6032 return 0; 6033 } 6034 6035 static int bnxt_init_int_mode(struct bnxt *bp) 6036 { 6037 int rc = 0; 6038 6039 if (bp->flags & BNXT_FLAG_MSIX_CAP) 6040 rc = bnxt_init_msix(bp); 6041 6042 if (!(bp->flags & BNXT_FLAG_USING_MSIX) && BNXT_PF(bp)) { 6043 /* fallback to INTA */ 6044 rc = bnxt_init_inta(bp); 6045 } 6046 return rc; 6047 } 6048 6049 static void bnxt_clear_int_mode(struct bnxt *bp) 6050 { 6051 if (bp->flags & BNXT_FLAG_USING_MSIX) 6052 pci_disable_msix(bp->pdev); 6053 6054 kfree(bp->irq_tbl); 6055 bp->irq_tbl = NULL; 6056 bp->flags &= ~BNXT_FLAG_USING_MSIX; 6057 } 6058 6059 int bnxt_reserve_rings(struct bnxt *bp) 6060 { 6061 int tcs = netdev_get_num_tc(bp->dev); 6062 int rc; 6063 6064 if (!bnxt_need_reserve_rings(bp)) 6065 return 0; 6066 6067 rc = __bnxt_reserve_rings(bp); 6068 if (rc) { 6069 netdev_err(bp->dev, "ring reservation failure rc: %d\n", rc); 6070 return rc; 6071 } 6072 if ((bp->flags & BNXT_FLAG_NEW_RM) && 6073 (bnxt_get_num_msix(bp) != bp->total_irqs)) { 6074 bnxt_ulp_irq_stop(bp); 6075 bnxt_clear_int_mode(bp); 6076 rc = bnxt_init_int_mode(bp); 6077 bnxt_ulp_irq_restart(bp, rc); 6078 if (rc) 6079 return rc; 6080 } 6081 if (tcs && (bp->tx_nr_rings_per_tc * tcs != bp->tx_nr_rings)) { 6082 netdev_err(bp->dev, "tx ring reservation failure\n"); 6083 netdev_reset_tc(bp->dev); 6084 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 6085 return -ENOMEM; 6086 } 6087 bp->num_stat_ctxs = bp->cp_nr_rings; 6088 return 0; 6089 } 6090 6091 static void bnxt_free_irq(struct bnxt *bp) 6092 { 6093 struct bnxt_irq *irq; 6094 int i; 6095 6096 #ifdef CONFIG_RFS_ACCEL 6097 free_irq_cpu_rmap(bp->dev->rx_cpu_rmap); 6098 bp->dev->rx_cpu_rmap = NULL; 6099 #endif 6100 if (!bp->irq_tbl || !bp->bnapi) 6101 return; 6102 6103 for (i = 0; i < bp->cp_nr_rings; i++) { 6104 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 6105 6106 irq = &bp->irq_tbl[map_idx]; 6107 if (irq->requested) { 6108 if (irq->have_cpumask) { 6109 irq_set_affinity_hint(irq->vector, NULL); 6110 free_cpumask_var(irq->cpu_mask); 6111 irq->have_cpumask = 0; 6112 } 6113 free_irq(irq->vector, bp->bnapi[i]); 6114 } 6115 6116 irq->requested = 0; 6117 } 6118 } 6119 6120 static int bnxt_request_irq(struct bnxt *bp) 6121 { 6122 int i, j, rc = 0; 6123 unsigned long flags = 0; 6124 #ifdef CONFIG_RFS_ACCEL 6125 struct cpu_rmap *rmap; 6126 #endif 6127 6128 rc = bnxt_setup_int_mode(bp); 6129 if (rc) { 6130 netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n", 6131 rc); 6132 return rc; 6133 } 6134 #ifdef CONFIG_RFS_ACCEL 6135 rmap = bp->dev->rx_cpu_rmap; 6136 #endif 6137 if (!(bp->flags & BNXT_FLAG_USING_MSIX)) 6138 flags = IRQF_SHARED; 6139 6140 for (i = 0, j = 0; i < bp->cp_nr_rings; i++) { 6141 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 6142 struct bnxt_irq *irq = &bp->irq_tbl[map_idx]; 6143 6144 #ifdef CONFIG_RFS_ACCEL 6145 if (rmap && bp->bnapi[i]->rx_ring) { 6146 rc = irq_cpu_rmap_add(rmap, irq->vector); 6147 if (rc) 6148 netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n", 6149 j); 6150 j++; 6151 } 6152 #endif 6153 rc = request_irq(irq->vector, irq->handler, flags, irq->name, 6154 bp->bnapi[i]); 6155 if (rc) 6156 break; 6157 6158 irq->requested = 1; 6159 6160 if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) { 6161 int numa_node = dev_to_node(&bp->pdev->dev); 6162 6163 irq->have_cpumask = 1; 6164 cpumask_set_cpu(cpumask_local_spread(i, numa_node), 6165 irq->cpu_mask); 6166 rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask); 6167 if (rc) { 6168 netdev_warn(bp->dev, 6169 "Set affinity failed, IRQ = %d\n", 6170 irq->vector); 6171 break; 6172 } 6173 } 6174 } 6175 return rc; 6176 } 6177 6178 static void bnxt_del_napi(struct bnxt *bp) 6179 { 6180 int i; 6181 6182 if (!bp->bnapi) 6183 return; 6184 6185 for (i = 0; i < bp->cp_nr_rings; i++) { 6186 struct bnxt_napi *bnapi = bp->bnapi[i]; 6187 6188 napi_hash_del(&bnapi->napi); 6189 netif_napi_del(&bnapi->napi); 6190 } 6191 /* We called napi_hash_del() before netif_napi_del(), we need 6192 * to respect an RCU grace period before freeing napi structures. 6193 */ 6194 synchronize_net(); 6195 } 6196 6197 static void bnxt_init_napi(struct bnxt *bp) 6198 { 6199 int i; 6200 unsigned int cp_nr_rings = bp->cp_nr_rings; 6201 struct bnxt_napi *bnapi; 6202 6203 if (bp->flags & BNXT_FLAG_USING_MSIX) { 6204 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 6205 cp_nr_rings--; 6206 for (i = 0; i < cp_nr_rings; i++) { 6207 bnapi = bp->bnapi[i]; 6208 netif_napi_add(bp->dev, &bnapi->napi, 6209 bnxt_poll, 64); 6210 } 6211 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 6212 bnapi = bp->bnapi[cp_nr_rings]; 6213 netif_napi_add(bp->dev, &bnapi->napi, 6214 bnxt_poll_nitroa0, 64); 6215 } 6216 } else { 6217 bnapi = bp->bnapi[0]; 6218 netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll, 64); 6219 } 6220 } 6221 6222 static void bnxt_disable_napi(struct bnxt *bp) 6223 { 6224 int i; 6225 6226 if (!bp->bnapi) 6227 return; 6228 6229 for (i = 0; i < bp->cp_nr_rings; i++) { 6230 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring; 6231 6232 if (bp->bnapi[i]->rx_ring) 6233 cancel_work_sync(&cpr->dim.work); 6234 6235 napi_disable(&bp->bnapi[i]->napi); 6236 } 6237 } 6238 6239 static void bnxt_enable_napi(struct bnxt *bp) 6240 { 6241 int i; 6242 6243 for (i = 0; i < bp->cp_nr_rings; i++) { 6244 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring; 6245 bp->bnapi[i]->in_reset = false; 6246 6247 if (bp->bnapi[i]->rx_ring) { 6248 INIT_WORK(&cpr->dim.work, bnxt_dim_work); 6249 cpr->dim.mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE; 6250 } 6251 napi_enable(&bp->bnapi[i]->napi); 6252 } 6253 } 6254 6255 void bnxt_tx_disable(struct bnxt *bp) 6256 { 6257 int i; 6258 struct bnxt_tx_ring_info *txr; 6259 6260 if (bp->tx_ring) { 6261 for (i = 0; i < bp->tx_nr_rings; i++) { 6262 txr = &bp->tx_ring[i]; 6263 txr->dev_state = BNXT_DEV_STATE_CLOSING; 6264 } 6265 } 6266 /* Stop all TX queues */ 6267 netif_tx_disable(bp->dev); 6268 netif_carrier_off(bp->dev); 6269 } 6270 6271 void bnxt_tx_enable(struct bnxt *bp) 6272 { 6273 int i; 6274 struct bnxt_tx_ring_info *txr; 6275 6276 for (i = 0; i < bp->tx_nr_rings; i++) { 6277 txr = &bp->tx_ring[i]; 6278 txr->dev_state = 0; 6279 } 6280 netif_tx_wake_all_queues(bp->dev); 6281 if (bp->link_info.link_up) 6282 netif_carrier_on(bp->dev); 6283 } 6284 6285 static void bnxt_report_link(struct bnxt *bp) 6286 { 6287 if (bp->link_info.link_up) { 6288 const char *duplex; 6289 const char *flow_ctrl; 6290 u32 speed; 6291 u16 fec; 6292 6293 netif_carrier_on(bp->dev); 6294 if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL) 6295 duplex = "full"; 6296 else 6297 duplex = "half"; 6298 if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH) 6299 flow_ctrl = "ON - receive & transmit"; 6300 else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX) 6301 flow_ctrl = "ON - transmit"; 6302 else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX) 6303 flow_ctrl = "ON - receive"; 6304 else 6305 flow_ctrl = "none"; 6306 speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed); 6307 netdev_info(bp->dev, "NIC Link is Up, %u Mbps %s duplex, Flow control: %s\n", 6308 speed, duplex, flow_ctrl); 6309 if (bp->flags & BNXT_FLAG_EEE_CAP) 6310 netdev_info(bp->dev, "EEE is %s\n", 6311 bp->eee.eee_active ? "active" : 6312 "not active"); 6313 fec = bp->link_info.fec_cfg; 6314 if (!(fec & PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED)) 6315 netdev_info(bp->dev, "FEC autoneg %s encodings: %s\n", 6316 (fec & BNXT_FEC_AUTONEG) ? "on" : "off", 6317 (fec & BNXT_FEC_ENC_BASE_R) ? "BaseR" : 6318 (fec & BNXT_FEC_ENC_RS) ? "RS" : "None"); 6319 } else { 6320 netif_carrier_off(bp->dev); 6321 netdev_err(bp->dev, "NIC Link is Down\n"); 6322 } 6323 } 6324 6325 static int bnxt_hwrm_phy_qcaps(struct bnxt *bp) 6326 { 6327 int rc = 0; 6328 struct hwrm_port_phy_qcaps_input req = {0}; 6329 struct hwrm_port_phy_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 6330 struct bnxt_link_info *link_info = &bp->link_info; 6331 6332 if (bp->hwrm_spec_code < 0x10201) 6333 return 0; 6334 6335 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCAPS, -1, -1); 6336 6337 mutex_lock(&bp->hwrm_cmd_lock); 6338 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6339 if (rc) 6340 goto hwrm_phy_qcaps_exit; 6341 6342 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) { 6343 struct ethtool_eee *eee = &bp->eee; 6344 u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode); 6345 6346 bp->flags |= BNXT_FLAG_EEE_CAP; 6347 eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0); 6348 bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) & 6349 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK; 6350 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) & 6351 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK; 6352 } 6353 if (resp->supported_speeds_auto_mode) 6354 link_info->support_auto_speeds = 6355 le16_to_cpu(resp->supported_speeds_auto_mode); 6356 6357 bp->port_count = resp->port_cnt; 6358 6359 hwrm_phy_qcaps_exit: 6360 mutex_unlock(&bp->hwrm_cmd_lock); 6361 return rc; 6362 } 6363 6364 static int bnxt_update_link(struct bnxt *bp, bool chng_link_state) 6365 { 6366 int rc = 0; 6367 struct bnxt_link_info *link_info = &bp->link_info; 6368 struct hwrm_port_phy_qcfg_input req = {0}; 6369 struct hwrm_port_phy_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 6370 u8 link_up = link_info->link_up; 6371 u16 diff; 6372 6373 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCFG, -1, -1); 6374 6375 mutex_lock(&bp->hwrm_cmd_lock); 6376 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6377 if (rc) { 6378 mutex_unlock(&bp->hwrm_cmd_lock); 6379 return rc; 6380 } 6381 6382 memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp)); 6383 link_info->phy_link_status = resp->link; 6384 link_info->duplex = resp->duplex_cfg; 6385 if (bp->hwrm_spec_code >= 0x10800) 6386 link_info->duplex = resp->duplex_state; 6387 link_info->pause = resp->pause; 6388 link_info->auto_mode = resp->auto_mode; 6389 link_info->auto_pause_setting = resp->auto_pause; 6390 link_info->lp_pause = resp->link_partner_adv_pause; 6391 link_info->force_pause_setting = resp->force_pause; 6392 link_info->duplex_setting = resp->duplex_cfg; 6393 if (link_info->phy_link_status == BNXT_LINK_LINK) 6394 link_info->link_speed = le16_to_cpu(resp->link_speed); 6395 else 6396 link_info->link_speed = 0; 6397 link_info->force_link_speed = le16_to_cpu(resp->force_link_speed); 6398 link_info->support_speeds = le16_to_cpu(resp->support_speeds); 6399 link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask); 6400 link_info->lp_auto_link_speeds = 6401 le16_to_cpu(resp->link_partner_adv_speeds); 6402 link_info->preemphasis = le32_to_cpu(resp->preemphasis); 6403 link_info->phy_ver[0] = resp->phy_maj; 6404 link_info->phy_ver[1] = resp->phy_min; 6405 link_info->phy_ver[2] = resp->phy_bld; 6406 link_info->media_type = resp->media_type; 6407 link_info->phy_type = resp->phy_type; 6408 link_info->transceiver = resp->xcvr_pkg_type; 6409 link_info->phy_addr = resp->eee_config_phy_addr & 6410 PORT_PHY_QCFG_RESP_PHY_ADDR_MASK; 6411 link_info->module_status = resp->module_status; 6412 6413 if (bp->flags & BNXT_FLAG_EEE_CAP) { 6414 struct ethtool_eee *eee = &bp->eee; 6415 u16 fw_speeds; 6416 6417 eee->eee_active = 0; 6418 if (resp->eee_config_phy_addr & 6419 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) { 6420 eee->eee_active = 1; 6421 fw_speeds = le16_to_cpu( 6422 resp->link_partner_adv_eee_link_speed_mask); 6423 eee->lp_advertised = 6424 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0); 6425 } 6426 6427 /* Pull initial EEE config */ 6428 if (!chng_link_state) { 6429 if (resp->eee_config_phy_addr & 6430 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED) 6431 eee->eee_enabled = 1; 6432 6433 fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask); 6434 eee->advertised = 6435 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0); 6436 6437 if (resp->eee_config_phy_addr & 6438 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) { 6439 __le32 tmr; 6440 6441 eee->tx_lpi_enabled = 1; 6442 tmr = resp->xcvr_identifier_type_tx_lpi_timer; 6443 eee->tx_lpi_timer = le32_to_cpu(tmr) & 6444 PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK; 6445 } 6446 } 6447 } 6448 6449 link_info->fec_cfg = PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED; 6450 if (bp->hwrm_spec_code >= 0x10504) 6451 link_info->fec_cfg = le16_to_cpu(resp->fec_cfg); 6452 6453 /* TODO: need to add more logic to report VF link */ 6454 if (chng_link_state) { 6455 if (link_info->phy_link_status == BNXT_LINK_LINK) 6456 link_info->link_up = 1; 6457 else 6458 link_info->link_up = 0; 6459 if (link_up != link_info->link_up) 6460 bnxt_report_link(bp); 6461 } else { 6462 /* alwasy link down if not require to update link state */ 6463 link_info->link_up = 0; 6464 } 6465 mutex_unlock(&bp->hwrm_cmd_lock); 6466 6467 if (!BNXT_SINGLE_PF(bp)) 6468 return 0; 6469 6470 diff = link_info->support_auto_speeds ^ link_info->advertising; 6471 if ((link_info->support_auto_speeds | diff) != 6472 link_info->support_auto_speeds) { 6473 /* An advertised speed is no longer supported, so we need to 6474 * update the advertisement settings. Caller holds RTNL 6475 * so we can modify link settings. 6476 */ 6477 link_info->advertising = link_info->support_auto_speeds; 6478 if (link_info->autoneg & BNXT_AUTONEG_SPEED) 6479 bnxt_hwrm_set_link_setting(bp, true, false); 6480 } 6481 return 0; 6482 } 6483 6484 static void bnxt_get_port_module_status(struct bnxt *bp) 6485 { 6486 struct bnxt_link_info *link_info = &bp->link_info; 6487 struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp; 6488 u8 module_status; 6489 6490 if (bnxt_update_link(bp, true)) 6491 return; 6492 6493 module_status = link_info->module_status; 6494 switch (module_status) { 6495 case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX: 6496 case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN: 6497 case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG: 6498 netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n", 6499 bp->pf.port_id); 6500 if (bp->hwrm_spec_code >= 0x10201) { 6501 netdev_warn(bp->dev, "Module part number %s\n", 6502 resp->phy_vendor_partnumber); 6503 } 6504 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX) 6505 netdev_warn(bp->dev, "TX is disabled\n"); 6506 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN) 6507 netdev_warn(bp->dev, "SFP+ module is shutdown\n"); 6508 } 6509 } 6510 6511 static void 6512 bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req) 6513 { 6514 if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) { 6515 if (bp->hwrm_spec_code >= 0x10201) 6516 req->auto_pause = 6517 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE; 6518 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX) 6519 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX; 6520 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX) 6521 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX; 6522 req->enables |= 6523 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE); 6524 } else { 6525 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX) 6526 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX; 6527 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX) 6528 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX; 6529 req->enables |= 6530 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE); 6531 if (bp->hwrm_spec_code >= 0x10201) { 6532 req->auto_pause = req->force_pause; 6533 req->enables |= cpu_to_le32( 6534 PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE); 6535 } 6536 } 6537 } 6538 6539 static void bnxt_hwrm_set_link_common(struct bnxt *bp, 6540 struct hwrm_port_phy_cfg_input *req) 6541 { 6542 u8 autoneg = bp->link_info.autoneg; 6543 u16 fw_link_speed = bp->link_info.req_link_speed; 6544 u16 advertising = bp->link_info.advertising; 6545 6546 if (autoneg & BNXT_AUTONEG_SPEED) { 6547 req->auto_mode |= 6548 PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK; 6549 6550 req->enables |= cpu_to_le32( 6551 PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK); 6552 req->auto_link_speed_mask = cpu_to_le16(advertising); 6553 6554 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE); 6555 req->flags |= 6556 cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG); 6557 } else { 6558 req->force_link_speed = cpu_to_le16(fw_link_speed); 6559 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE); 6560 } 6561 6562 /* tell chimp that the setting takes effect immediately */ 6563 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY); 6564 } 6565 6566 int bnxt_hwrm_set_pause(struct bnxt *bp) 6567 { 6568 struct hwrm_port_phy_cfg_input req = {0}; 6569 int rc; 6570 6571 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1); 6572 bnxt_hwrm_set_pause_common(bp, &req); 6573 6574 if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) || 6575 bp->link_info.force_link_chng) 6576 bnxt_hwrm_set_link_common(bp, &req); 6577 6578 mutex_lock(&bp->hwrm_cmd_lock); 6579 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6580 if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) { 6581 /* since changing of pause setting doesn't trigger any link 6582 * change event, the driver needs to update the current pause 6583 * result upon successfully return of the phy_cfg command 6584 */ 6585 bp->link_info.pause = 6586 bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl; 6587 bp->link_info.auto_pause_setting = 0; 6588 if (!bp->link_info.force_link_chng) 6589 bnxt_report_link(bp); 6590 } 6591 bp->link_info.force_link_chng = false; 6592 mutex_unlock(&bp->hwrm_cmd_lock); 6593 return rc; 6594 } 6595 6596 static void bnxt_hwrm_set_eee(struct bnxt *bp, 6597 struct hwrm_port_phy_cfg_input *req) 6598 { 6599 struct ethtool_eee *eee = &bp->eee; 6600 6601 if (eee->eee_enabled) { 6602 u16 eee_speeds; 6603 u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE; 6604 6605 if (eee->tx_lpi_enabled) 6606 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE; 6607 else 6608 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE; 6609 6610 req->flags |= cpu_to_le32(flags); 6611 eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised); 6612 req->eee_link_speed_mask = cpu_to_le16(eee_speeds); 6613 req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer); 6614 } else { 6615 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE); 6616 } 6617 } 6618 6619 int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee) 6620 { 6621 struct hwrm_port_phy_cfg_input req = {0}; 6622 6623 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1); 6624 if (set_pause) 6625 bnxt_hwrm_set_pause_common(bp, &req); 6626 6627 bnxt_hwrm_set_link_common(bp, &req); 6628 6629 if (set_eee) 6630 bnxt_hwrm_set_eee(bp, &req); 6631 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6632 } 6633 6634 static int bnxt_hwrm_shutdown_link(struct bnxt *bp) 6635 { 6636 struct hwrm_port_phy_cfg_input req = {0}; 6637 6638 if (!BNXT_SINGLE_PF(bp)) 6639 return 0; 6640 6641 if (pci_num_vf(bp->pdev)) 6642 return 0; 6643 6644 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1); 6645 req.flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN); 6646 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6647 } 6648 6649 static int bnxt_hwrm_port_led_qcaps(struct bnxt *bp) 6650 { 6651 struct hwrm_port_led_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 6652 struct hwrm_port_led_qcaps_input req = {0}; 6653 struct bnxt_pf_info *pf = &bp->pf; 6654 int rc; 6655 6656 if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10601) 6657 return 0; 6658 6659 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_LED_QCAPS, -1, -1); 6660 req.port_id = cpu_to_le16(pf->port_id); 6661 mutex_lock(&bp->hwrm_cmd_lock); 6662 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6663 if (rc) { 6664 mutex_unlock(&bp->hwrm_cmd_lock); 6665 return rc; 6666 } 6667 if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) { 6668 int i; 6669 6670 bp->num_leds = resp->num_leds; 6671 memcpy(bp->leds, &resp->led0_id, sizeof(bp->leds[0]) * 6672 bp->num_leds); 6673 for (i = 0; i < bp->num_leds; i++) { 6674 struct bnxt_led_info *led = &bp->leds[i]; 6675 __le16 caps = led->led_state_caps; 6676 6677 if (!led->led_group_id || 6678 !BNXT_LED_ALT_BLINK_CAP(caps)) { 6679 bp->num_leds = 0; 6680 break; 6681 } 6682 } 6683 } 6684 mutex_unlock(&bp->hwrm_cmd_lock); 6685 return 0; 6686 } 6687 6688 int bnxt_hwrm_alloc_wol_fltr(struct bnxt *bp) 6689 { 6690 struct hwrm_wol_filter_alloc_input req = {0}; 6691 struct hwrm_wol_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr; 6692 int rc; 6693 6694 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_ALLOC, -1, -1); 6695 req.port_id = cpu_to_le16(bp->pf.port_id); 6696 req.wol_type = WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT; 6697 req.enables = cpu_to_le32(WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS); 6698 memcpy(req.mac_address, bp->dev->dev_addr, ETH_ALEN); 6699 mutex_lock(&bp->hwrm_cmd_lock); 6700 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6701 if (!rc) 6702 bp->wol_filter_id = resp->wol_filter_id; 6703 mutex_unlock(&bp->hwrm_cmd_lock); 6704 return rc; 6705 } 6706 6707 int bnxt_hwrm_free_wol_fltr(struct bnxt *bp) 6708 { 6709 struct hwrm_wol_filter_free_input req = {0}; 6710 int rc; 6711 6712 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_FREE, -1, -1); 6713 req.port_id = cpu_to_le16(bp->pf.port_id); 6714 req.enables = cpu_to_le32(WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID); 6715 req.wol_filter_id = bp->wol_filter_id; 6716 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6717 return rc; 6718 } 6719 6720 static u16 bnxt_hwrm_get_wol_fltrs(struct bnxt *bp, u16 handle) 6721 { 6722 struct hwrm_wol_filter_qcfg_input req = {0}; 6723 struct hwrm_wol_filter_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 6724 u16 next_handle = 0; 6725 int rc; 6726 6727 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_QCFG, -1, -1); 6728 req.port_id = cpu_to_le16(bp->pf.port_id); 6729 req.handle = cpu_to_le16(handle); 6730 mutex_lock(&bp->hwrm_cmd_lock); 6731 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6732 if (!rc) { 6733 next_handle = le16_to_cpu(resp->next_handle); 6734 if (next_handle != 0) { 6735 if (resp->wol_type == 6736 WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT) { 6737 bp->wol = 1; 6738 bp->wol_filter_id = resp->wol_filter_id; 6739 } 6740 } 6741 } 6742 mutex_unlock(&bp->hwrm_cmd_lock); 6743 return next_handle; 6744 } 6745 6746 static void bnxt_get_wol_settings(struct bnxt *bp) 6747 { 6748 u16 handle = 0; 6749 6750 if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_WOL_CAP)) 6751 return; 6752 6753 do { 6754 handle = bnxt_hwrm_get_wol_fltrs(bp, handle); 6755 } while (handle && handle != 0xffff); 6756 } 6757 6758 static bool bnxt_eee_config_ok(struct bnxt *bp) 6759 { 6760 struct ethtool_eee *eee = &bp->eee; 6761 struct bnxt_link_info *link_info = &bp->link_info; 6762 6763 if (!(bp->flags & BNXT_FLAG_EEE_CAP)) 6764 return true; 6765 6766 if (eee->eee_enabled) { 6767 u32 advertising = 6768 _bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0); 6769 6770 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) { 6771 eee->eee_enabled = 0; 6772 return false; 6773 } 6774 if (eee->advertised & ~advertising) { 6775 eee->advertised = advertising & eee->supported; 6776 return false; 6777 } 6778 } 6779 return true; 6780 } 6781 6782 static int bnxt_update_phy_setting(struct bnxt *bp) 6783 { 6784 int rc; 6785 bool update_link = false; 6786 bool update_pause = false; 6787 bool update_eee = false; 6788 struct bnxt_link_info *link_info = &bp->link_info; 6789 6790 rc = bnxt_update_link(bp, true); 6791 if (rc) { 6792 netdev_err(bp->dev, "failed to update link (rc: %x)\n", 6793 rc); 6794 return rc; 6795 } 6796 if (!BNXT_SINGLE_PF(bp)) 6797 return 0; 6798 6799 if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) && 6800 (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) != 6801 link_info->req_flow_ctrl) 6802 update_pause = true; 6803 if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) && 6804 link_info->force_pause_setting != link_info->req_flow_ctrl) 6805 update_pause = true; 6806 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) { 6807 if (BNXT_AUTO_MODE(link_info->auto_mode)) 6808 update_link = true; 6809 if (link_info->req_link_speed != link_info->force_link_speed) 6810 update_link = true; 6811 if (link_info->req_duplex != link_info->duplex_setting) 6812 update_link = true; 6813 } else { 6814 if (link_info->auto_mode == BNXT_LINK_AUTO_NONE) 6815 update_link = true; 6816 if (link_info->advertising != link_info->auto_link_speeds) 6817 update_link = true; 6818 } 6819 6820 /* The last close may have shutdown the link, so need to call 6821 * PHY_CFG to bring it back up. 6822 */ 6823 if (!netif_carrier_ok(bp->dev)) 6824 update_link = true; 6825 6826 if (!bnxt_eee_config_ok(bp)) 6827 update_eee = true; 6828 6829 if (update_link) 6830 rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee); 6831 else if (update_pause) 6832 rc = bnxt_hwrm_set_pause(bp); 6833 if (rc) { 6834 netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n", 6835 rc); 6836 return rc; 6837 } 6838 6839 return rc; 6840 } 6841 6842 /* Common routine to pre-map certain register block to different GRC window. 6843 * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows 6844 * in PF and 3 windows in VF that can be customized to map in different 6845 * register blocks. 6846 */ 6847 static void bnxt_preset_reg_win(struct bnxt *bp) 6848 { 6849 if (BNXT_PF(bp)) { 6850 /* CAG registers map to GRC window #4 */ 6851 writel(BNXT_CAG_REG_BASE, 6852 bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12); 6853 } 6854 } 6855 6856 static int bnxt_init_dflt_ring_mode(struct bnxt *bp); 6857 6858 static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) 6859 { 6860 int rc = 0; 6861 6862 bnxt_preset_reg_win(bp); 6863 netif_carrier_off(bp->dev); 6864 if (irq_re_init) { 6865 /* Reserve rings now if none were reserved at driver probe. */ 6866 rc = bnxt_init_dflt_ring_mode(bp); 6867 if (rc) { 6868 netdev_err(bp->dev, "Failed to reserve default rings at open\n"); 6869 return rc; 6870 } 6871 rc = bnxt_reserve_rings(bp); 6872 if (rc) 6873 return rc; 6874 } 6875 if ((bp->flags & BNXT_FLAG_RFS) && 6876 !(bp->flags & BNXT_FLAG_USING_MSIX)) { 6877 /* disable RFS if falling back to INTA */ 6878 bp->dev->hw_features &= ~NETIF_F_NTUPLE; 6879 bp->flags &= ~BNXT_FLAG_RFS; 6880 } 6881 6882 rc = bnxt_alloc_mem(bp, irq_re_init); 6883 if (rc) { 6884 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc); 6885 goto open_err_free_mem; 6886 } 6887 6888 if (irq_re_init) { 6889 bnxt_init_napi(bp); 6890 rc = bnxt_request_irq(bp); 6891 if (rc) { 6892 netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc); 6893 goto open_err_irq; 6894 } 6895 } 6896 6897 bnxt_enable_napi(bp); 6898 bnxt_debug_dev_init(bp); 6899 6900 rc = bnxt_init_nic(bp, irq_re_init); 6901 if (rc) { 6902 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc); 6903 goto open_err; 6904 } 6905 6906 if (link_re_init) { 6907 mutex_lock(&bp->link_lock); 6908 rc = bnxt_update_phy_setting(bp); 6909 mutex_unlock(&bp->link_lock); 6910 if (rc) 6911 netdev_warn(bp->dev, "failed to update phy settings\n"); 6912 } 6913 6914 if (irq_re_init) 6915 udp_tunnel_get_rx_info(bp->dev); 6916 6917 set_bit(BNXT_STATE_OPEN, &bp->state); 6918 bnxt_enable_int(bp); 6919 /* Enable TX queues */ 6920 bnxt_tx_enable(bp); 6921 mod_timer(&bp->timer, jiffies + bp->current_interval); 6922 /* Poll link status and check for SFP+ module status */ 6923 bnxt_get_port_module_status(bp); 6924 6925 /* VF-reps may need to be re-opened after the PF is re-opened */ 6926 if (BNXT_PF(bp)) 6927 bnxt_vf_reps_open(bp); 6928 return 0; 6929 6930 open_err: 6931 bnxt_debug_dev_exit(bp); 6932 bnxt_disable_napi(bp); 6933 6934 open_err_irq: 6935 bnxt_del_napi(bp); 6936 6937 open_err_free_mem: 6938 bnxt_free_skbs(bp); 6939 bnxt_free_irq(bp); 6940 bnxt_free_mem(bp, true); 6941 return rc; 6942 } 6943 6944 /* rtnl_lock held */ 6945 int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) 6946 { 6947 int rc = 0; 6948 6949 rc = __bnxt_open_nic(bp, irq_re_init, link_re_init); 6950 if (rc) { 6951 netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc); 6952 dev_close(bp->dev); 6953 } 6954 return rc; 6955 } 6956 6957 /* rtnl_lock held, open the NIC half way by allocating all resources, but 6958 * NAPI, IRQ, and TX are not enabled. This is mainly used for offline 6959 * self tests. 6960 */ 6961 int bnxt_half_open_nic(struct bnxt *bp) 6962 { 6963 int rc = 0; 6964 6965 rc = bnxt_alloc_mem(bp, false); 6966 if (rc) { 6967 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc); 6968 goto half_open_err; 6969 } 6970 rc = bnxt_init_nic(bp, false); 6971 if (rc) { 6972 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc); 6973 goto half_open_err; 6974 } 6975 return 0; 6976 6977 half_open_err: 6978 bnxt_free_skbs(bp); 6979 bnxt_free_mem(bp, false); 6980 dev_close(bp->dev); 6981 return rc; 6982 } 6983 6984 /* rtnl_lock held, this call can only be made after a previous successful 6985 * call to bnxt_half_open_nic(). 6986 */ 6987 void bnxt_half_close_nic(struct bnxt *bp) 6988 { 6989 bnxt_hwrm_resource_free(bp, false, false); 6990 bnxt_free_skbs(bp); 6991 bnxt_free_mem(bp, false); 6992 } 6993 6994 static int bnxt_open(struct net_device *dev) 6995 { 6996 struct bnxt *bp = netdev_priv(dev); 6997 6998 return __bnxt_open_nic(bp, true, true); 6999 } 7000 7001 static bool bnxt_drv_busy(struct bnxt *bp) 7002 { 7003 return (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state) || 7004 test_bit(BNXT_STATE_READ_STATS, &bp->state)); 7005 } 7006 7007 static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init, 7008 bool link_re_init) 7009 { 7010 /* Close the VF-reps before closing PF */ 7011 if (BNXT_PF(bp)) 7012 bnxt_vf_reps_close(bp); 7013 7014 /* Change device state to avoid TX queue wake up's */ 7015 bnxt_tx_disable(bp); 7016 7017 clear_bit(BNXT_STATE_OPEN, &bp->state); 7018 smp_mb__after_atomic(); 7019 while (bnxt_drv_busy(bp)) 7020 msleep(20); 7021 7022 /* Flush rings and and disable interrupts */ 7023 bnxt_shutdown_nic(bp, irq_re_init); 7024 7025 /* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */ 7026 7027 bnxt_debug_dev_exit(bp); 7028 bnxt_disable_napi(bp); 7029 del_timer_sync(&bp->timer); 7030 bnxt_free_skbs(bp); 7031 7032 if (irq_re_init) { 7033 bnxt_free_irq(bp); 7034 bnxt_del_napi(bp); 7035 } 7036 bnxt_free_mem(bp, irq_re_init); 7037 } 7038 7039 int bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) 7040 { 7041 int rc = 0; 7042 7043 #ifdef CONFIG_BNXT_SRIOV 7044 if (bp->sriov_cfg) { 7045 rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait, 7046 !bp->sriov_cfg, 7047 BNXT_SRIOV_CFG_WAIT_TMO); 7048 if (rc) 7049 netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete!\n"); 7050 } 7051 #endif 7052 __bnxt_close_nic(bp, irq_re_init, link_re_init); 7053 return rc; 7054 } 7055 7056 static int bnxt_close(struct net_device *dev) 7057 { 7058 struct bnxt *bp = netdev_priv(dev); 7059 7060 bnxt_close_nic(bp, true, true); 7061 bnxt_hwrm_shutdown_link(bp); 7062 return 0; 7063 } 7064 7065 /* rtnl_lock held */ 7066 static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) 7067 { 7068 switch (cmd) { 7069 case SIOCGMIIPHY: 7070 /* fallthru */ 7071 case SIOCGMIIREG: { 7072 if (!netif_running(dev)) 7073 return -EAGAIN; 7074 7075 return 0; 7076 } 7077 7078 case SIOCSMIIREG: 7079 if (!netif_running(dev)) 7080 return -EAGAIN; 7081 7082 return 0; 7083 7084 default: 7085 /* do nothing */ 7086 break; 7087 } 7088 return -EOPNOTSUPP; 7089 } 7090 7091 static void 7092 bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) 7093 { 7094 u32 i; 7095 struct bnxt *bp = netdev_priv(dev); 7096 7097 set_bit(BNXT_STATE_READ_STATS, &bp->state); 7098 /* Make sure bnxt_close_nic() sees that we are reading stats before 7099 * we check the BNXT_STATE_OPEN flag. 7100 */ 7101 smp_mb__after_atomic(); 7102 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 7103 clear_bit(BNXT_STATE_READ_STATS, &bp->state); 7104 return; 7105 } 7106 7107 /* TODO check if we need to synchronize with bnxt_close path */ 7108 for (i = 0; i < bp->cp_nr_rings; i++) { 7109 struct bnxt_napi *bnapi = bp->bnapi[i]; 7110 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 7111 struct ctx_hw_stats *hw_stats = cpr->hw_stats; 7112 7113 stats->rx_packets += le64_to_cpu(hw_stats->rx_ucast_pkts); 7114 stats->rx_packets += le64_to_cpu(hw_stats->rx_mcast_pkts); 7115 stats->rx_packets += le64_to_cpu(hw_stats->rx_bcast_pkts); 7116 7117 stats->tx_packets += le64_to_cpu(hw_stats->tx_ucast_pkts); 7118 stats->tx_packets += le64_to_cpu(hw_stats->tx_mcast_pkts); 7119 stats->tx_packets += le64_to_cpu(hw_stats->tx_bcast_pkts); 7120 7121 stats->rx_bytes += le64_to_cpu(hw_stats->rx_ucast_bytes); 7122 stats->rx_bytes += le64_to_cpu(hw_stats->rx_mcast_bytes); 7123 stats->rx_bytes += le64_to_cpu(hw_stats->rx_bcast_bytes); 7124 7125 stats->tx_bytes += le64_to_cpu(hw_stats->tx_ucast_bytes); 7126 stats->tx_bytes += le64_to_cpu(hw_stats->tx_mcast_bytes); 7127 stats->tx_bytes += le64_to_cpu(hw_stats->tx_bcast_bytes); 7128 7129 stats->rx_missed_errors += 7130 le64_to_cpu(hw_stats->rx_discard_pkts); 7131 7132 stats->multicast += le64_to_cpu(hw_stats->rx_mcast_pkts); 7133 7134 stats->tx_dropped += le64_to_cpu(hw_stats->tx_drop_pkts); 7135 } 7136 7137 if (bp->flags & BNXT_FLAG_PORT_STATS) { 7138 struct rx_port_stats *rx = bp->hw_rx_port_stats; 7139 struct tx_port_stats *tx = bp->hw_tx_port_stats; 7140 7141 stats->rx_crc_errors = le64_to_cpu(rx->rx_fcs_err_frames); 7142 stats->rx_frame_errors = le64_to_cpu(rx->rx_align_err_frames); 7143 stats->rx_length_errors = le64_to_cpu(rx->rx_undrsz_frames) + 7144 le64_to_cpu(rx->rx_ovrsz_frames) + 7145 le64_to_cpu(rx->rx_runt_frames); 7146 stats->rx_errors = le64_to_cpu(rx->rx_false_carrier_frames) + 7147 le64_to_cpu(rx->rx_jbr_frames); 7148 stats->collisions = le64_to_cpu(tx->tx_total_collisions); 7149 stats->tx_fifo_errors = le64_to_cpu(tx->tx_fifo_underruns); 7150 stats->tx_errors = le64_to_cpu(tx->tx_err); 7151 } 7152 clear_bit(BNXT_STATE_READ_STATS, &bp->state); 7153 } 7154 7155 static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask) 7156 { 7157 struct net_device *dev = bp->dev; 7158 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 7159 struct netdev_hw_addr *ha; 7160 u8 *haddr; 7161 int mc_count = 0; 7162 bool update = false; 7163 int off = 0; 7164 7165 netdev_for_each_mc_addr(ha, dev) { 7166 if (mc_count >= BNXT_MAX_MC_ADDRS) { 7167 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 7168 vnic->mc_list_count = 0; 7169 return false; 7170 } 7171 haddr = ha->addr; 7172 if (!ether_addr_equal(haddr, vnic->mc_list + off)) { 7173 memcpy(vnic->mc_list + off, haddr, ETH_ALEN); 7174 update = true; 7175 } 7176 off += ETH_ALEN; 7177 mc_count++; 7178 } 7179 if (mc_count) 7180 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST; 7181 7182 if (mc_count != vnic->mc_list_count) { 7183 vnic->mc_list_count = mc_count; 7184 update = true; 7185 } 7186 return update; 7187 } 7188 7189 static bool bnxt_uc_list_updated(struct bnxt *bp) 7190 { 7191 struct net_device *dev = bp->dev; 7192 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 7193 struct netdev_hw_addr *ha; 7194 int off = 0; 7195 7196 if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1)) 7197 return true; 7198 7199 netdev_for_each_uc_addr(ha, dev) { 7200 if (!ether_addr_equal(ha->addr, vnic->uc_list + off)) 7201 return true; 7202 7203 off += ETH_ALEN; 7204 } 7205 return false; 7206 } 7207 7208 static void bnxt_set_rx_mode(struct net_device *dev) 7209 { 7210 struct bnxt *bp = netdev_priv(dev); 7211 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 7212 u32 mask = vnic->rx_mask; 7213 bool mc_update = false; 7214 bool uc_update; 7215 7216 if (!netif_running(dev)) 7217 return; 7218 7219 mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS | 7220 CFA_L2_SET_RX_MASK_REQ_MASK_MCAST | 7221 CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST | 7222 CFA_L2_SET_RX_MASK_REQ_MASK_BCAST); 7223 7224 if ((dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp)) 7225 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 7226 7227 uc_update = bnxt_uc_list_updated(bp); 7228 7229 if (dev->flags & IFF_BROADCAST) 7230 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST; 7231 if (dev->flags & IFF_ALLMULTI) { 7232 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 7233 vnic->mc_list_count = 0; 7234 } else { 7235 mc_update = bnxt_mc_list_updated(bp, &mask); 7236 } 7237 7238 if (mask != vnic->rx_mask || uc_update || mc_update) { 7239 vnic->rx_mask = mask; 7240 7241 set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event); 7242 bnxt_queue_sp_work(bp); 7243 } 7244 } 7245 7246 static int bnxt_cfg_rx_mode(struct bnxt *bp) 7247 { 7248 struct net_device *dev = bp->dev; 7249 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 7250 struct netdev_hw_addr *ha; 7251 int i, off = 0, rc; 7252 bool uc_update; 7253 7254 netif_addr_lock_bh(dev); 7255 uc_update = bnxt_uc_list_updated(bp); 7256 netif_addr_unlock_bh(dev); 7257 7258 if (!uc_update) 7259 goto skip_uc; 7260 7261 mutex_lock(&bp->hwrm_cmd_lock); 7262 for (i = 1; i < vnic->uc_filter_count; i++) { 7263 struct hwrm_cfa_l2_filter_free_input req = {0}; 7264 7265 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_FREE, -1, 7266 -1); 7267 7268 req.l2_filter_id = vnic->fw_l2_filter_id[i]; 7269 7270 rc = _hwrm_send_message(bp, &req, sizeof(req), 7271 HWRM_CMD_TIMEOUT); 7272 } 7273 mutex_unlock(&bp->hwrm_cmd_lock); 7274 7275 vnic->uc_filter_count = 1; 7276 7277 netif_addr_lock_bh(dev); 7278 if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) { 7279 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 7280 } else { 7281 netdev_for_each_uc_addr(ha, dev) { 7282 memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN); 7283 off += ETH_ALEN; 7284 vnic->uc_filter_count++; 7285 } 7286 } 7287 netif_addr_unlock_bh(dev); 7288 7289 for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) { 7290 rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off); 7291 if (rc) { 7292 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", 7293 rc); 7294 vnic->uc_filter_count = i; 7295 return rc; 7296 } 7297 } 7298 7299 skip_uc: 7300 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0); 7301 if (rc) 7302 netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %x\n", 7303 rc); 7304 7305 return rc; 7306 } 7307 7308 static bool bnxt_can_reserve_rings(struct bnxt *bp) 7309 { 7310 #ifdef CONFIG_BNXT_SRIOV 7311 if ((bp->flags & BNXT_FLAG_NEW_RM) && BNXT_VF(bp)) { 7312 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 7313 7314 /* No minimum rings were provisioned by the PF. Don't 7315 * reserve rings by default when device is down. 7316 */ 7317 if (hw_resc->min_tx_rings || hw_resc->resv_tx_rings) 7318 return true; 7319 7320 if (!netif_running(bp->dev)) 7321 return false; 7322 } 7323 #endif 7324 return true; 7325 } 7326 7327 /* If the chip and firmware supports RFS */ 7328 static bool bnxt_rfs_supported(struct bnxt *bp) 7329 { 7330 if (BNXT_PF(bp) && !BNXT_CHIP_TYPE_NITRO_A0(bp)) 7331 return true; 7332 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP) 7333 return true; 7334 return false; 7335 } 7336 7337 /* If runtime conditions support RFS */ 7338 static bool bnxt_rfs_capable(struct bnxt *bp) 7339 { 7340 #ifdef CONFIG_RFS_ACCEL 7341 int vnics, max_vnics, max_rss_ctxs; 7342 7343 if (!(bp->flags & BNXT_FLAG_MSIX_CAP) || !bnxt_can_reserve_rings(bp)) 7344 return false; 7345 7346 vnics = 1 + bp->rx_nr_rings; 7347 max_vnics = bnxt_get_max_func_vnics(bp); 7348 max_rss_ctxs = bnxt_get_max_func_rss_ctxs(bp); 7349 7350 /* RSS contexts not a limiting factor */ 7351 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP) 7352 max_rss_ctxs = max_vnics; 7353 if (vnics > max_vnics || vnics > max_rss_ctxs) { 7354 if (bp->rx_nr_rings > 1) 7355 netdev_warn(bp->dev, 7356 "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n", 7357 min(max_rss_ctxs - 1, max_vnics - 1)); 7358 return false; 7359 } 7360 7361 if (!(bp->flags & BNXT_FLAG_NEW_RM)) 7362 return true; 7363 7364 if (vnics == bp->hw_resc.resv_vnics) 7365 return true; 7366 7367 bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, vnics); 7368 if (vnics <= bp->hw_resc.resv_vnics) 7369 return true; 7370 7371 netdev_warn(bp->dev, "Unable to reserve resources to support NTUPLE filters.\n"); 7372 bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 1); 7373 return false; 7374 #else 7375 return false; 7376 #endif 7377 } 7378 7379 static netdev_features_t bnxt_fix_features(struct net_device *dev, 7380 netdev_features_t features) 7381 { 7382 struct bnxt *bp = netdev_priv(dev); 7383 7384 if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp)) 7385 features &= ~NETIF_F_NTUPLE; 7386 7387 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS) 7388 features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); 7389 7390 if (!(features & NETIF_F_GRO)) 7391 features &= ~NETIF_F_GRO_HW; 7392 7393 if (features & NETIF_F_GRO_HW) 7394 features &= ~NETIF_F_LRO; 7395 7396 /* Both CTAG and STAG VLAN accelaration on the RX side have to be 7397 * turned on or off together. 7398 */ 7399 if ((features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)) != 7400 (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)) { 7401 if (dev->features & NETIF_F_HW_VLAN_CTAG_RX) 7402 features &= ~(NETIF_F_HW_VLAN_CTAG_RX | 7403 NETIF_F_HW_VLAN_STAG_RX); 7404 else 7405 features |= NETIF_F_HW_VLAN_CTAG_RX | 7406 NETIF_F_HW_VLAN_STAG_RX; 7407 } 7408 #ifdef CONFIG_BNXT_SRIOV 7409 if (BNXT_VF(bp)) { 7410 if (bp->vf.vlan) { 7411 features &= ~(NETIF_F_HW_VLAN_CTAG_RX | 7412 NETIF_F_HW_VLAN_STAG_RX); 7413 } 7414 } 7415 #endif 7416 return features; 7417 } 7418 7419 static int bnxt_set_features(struct net_device *dev, netdev_features_t features) 7420 { 7421 struct bnxt *bp = netdev_priv(dev); 7422 u32 flags = bp->flags; 7423 u32 changes; 7424 int rc = 0; 7425 bool re_init = false; 7426 bool update_tpa = false; 7427 7428 flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS; 7429 if (features & NETIF_F_GRO_HW) 7430 flags |= BNXT_FLAG_GRO; 7431 else if (features & NETIF_F_LRO) 7432 flags |= BNXT_FLAG_LRO; 7433 7434 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS) 7435 flags &= ~BNXT_FLAG_TPA; 7436 7437 if (features & NETIF_F_HW_VLAN_CTAG_RX) 7438 flags |= BNXT_FLAG_STRIP_VLAN; 7439 7440 if (features & NETIF_F_NTUPLE) 7441 flags |= BNXT_FLAG_RFS; 7442 7443 changes = flags ^ bp->flags; 7444 if (changes & BNXT_FLAG_TPA) { 7445 update_tpa = true; 7446 if ((bp->flags & BNXT_FLAG_TPA) == 0 || 7447 (flags & BNXT_FLAG_TPA) == 0) 7448 re_init = true; 7449 } 7450 7451 if (changes & ~BNXT_FLAG_TPA) 7452 re_init = true; 7453 7454 if (flags != bp->flags) { 7455 u32 old_flags = bp->flags; 7456 7457 bp->flags = flags; 7458 7459 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 7460 if (update_tpa) 7461 bnxt_set_ring_params(bp); 7462 return rc; 7463 } 7464 7465 if (re_init) { 7466 bnxt_close_nic(bp, false, false); 7467 if (update_tpa) 7468 bnxt_set_ring_params(bp); 7469 7470 return bnxt_open_nic(bp, false, false); 7471 } 7472 if (update_tpa) { 7473 rc = bnxt_set_tpa(bp, 7474 (flags & BNXT_FLAG_TPA) ? 7475 true : false); 7476 if (rc) 7477 bp->flags = old_flags; 7478 } 7479 } 7480 return rc; 7481 } 7482 7483 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi) 7484 { 7485 struct bnxt_tx_ring_info *txr = bnapi->tx_ring; 7486 int i = bnapi->index; 7487 7488 if (!txr) 7489 return; 7490 7491 netdev_info(bnapi->bp->dev, "[%d]: tx{fw_ring: %d prod: %x cons: %x}\n", 7492 i, txr->tx_ring_struct.fw_ring_id, txr->tx_prod, 7493 txr->tx_cons); 7494 } 7495 7496 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi) 7497 { 7498 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 7499 int i = bnapi->index; 7500 7501 if (!rxr) 7502 return; 7503 7504 netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n", 7505 i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod, 7506 rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod, 7507 rxr->rx_sw_agg_prod); 7508 } 7509 7510 static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi) 7511 { 7512 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 7513 int i = bnapi->index; 7514 7515 netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n", 7516 i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons); 7517 } 7518 7519 static void bnxt_dbg_dump_states(struct bnxt *bp) 7520 { 7521 int i; 7522 struct bnxt_napi *bnapi; 7523 7524 for (i = 0; i < bp->cp_nr_rings; i++) { 7525 bnapi = bp->bnapi[i]; 7526 if (netif_msg_drv(bp)) { 7527 bnxt_dump_tx_sw_state(bnapi); 7528 bnxt_dump_rx_sw_state(bnapi); 7529 bnxt_dump_cp_sw_state(bnapi); 7530 } 7531 } 7532 } 7533 7534 static void bnxt_reset_task(struct bnxt *bp, bool silent) 7535 { 7536 if (!silent) 7537 bnxt_dbg_dump_states(bp); 7538 if (netif_running(bp->dev)) { 7539 int rc; 7540 7541 if (!silent) 7542 bnxt_ulp_stop(bp); 7543 bnxt_close_nic(bp, false, false); 7544 rc = bnxt_open_nic(bp, false, false); 7545 if (!silent && !rc) 7546 bnxt_ulp_start(bp); 7547 } 7548 } 7549 7550 static void bnxt_tx_timeout(struct net_device *dev) 7551 { 7552 struct bnxt *bp = netdev_priv(dev); 7553 7554 netdev_err(bp->dev, "TX timeout detected, starting reset task!\n"); 7555 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event); 7556 bnxt_queue_sp_work(bp); 7557 } 7558 7559 #ifdef CONFIG_NET_POLL_CONTROLLER 7560 static void bnxt_poll_controller(struct net_device *dev) 7561 { 7562 struct bnxt *bp = netdev_priv(dev); 7563 int i; 7564 7565 /* Only process tx rings/combined rings in netpoll mode. */ 7566 for (i = 0; i < bp->tx_nr_rings; i++) { 7567 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 7568 7569 napi_schedule(&txr->bnapi->napi); 7570 } 7571 } 7572 #endif 7573 7574 static void bnxt_timer(struct timer_list *t) 7575 { 7576 struct bnxt *bp = from_timer(bp, t, timer); 7577 struct net_device *dev = bp->dev; 7578 7579 if (!netif_running(dev)) 7580 return; 7581 7582 if (atomic_read(&bp->intr_sem) != 0) 7583 goto bnxt_restart_timer; 7584 7585 if (bp->link_info.link_up && (bp->flags & BNXT_FLAG_PORT_STATS) && 7586 bp->stats_coal_ticks) { 7587 set_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event); 7588 bnxt_queue_sp_work(bp); 7589 } 7590 7591 if (bnxt_tc_flower_enabled(bp)) { 7592 set_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event); 7593 bnxt_queue_sp_work(bp); 7594 } 7595 bnxt_restart_timer: 7596 mod_timer(&bp->timer, jiffies + bp->current_interval); 7597 } 7598 7599 static void bnxt_rtnl_lock_sp(struct bnxt *bp) 7600 { 7601 /* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK 7602 * set. If the device is being closed, bnxt_close() may be holding 7603 * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear. So we 7604 * must clear BNXT_STATE_IN_SP_TASK before holding rtnl(). 7605 */ 7606 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 7607 rtnl_lock(); 7608 } 7609 7610 static void bnxt_rtnl_unlock_sp(struct bnxt *bp) 7611 { 7612 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 7613 rtnl_unlock(); 7614 } 7615 7616 /* Only called from bnxt_sp_task() */ 7617 static void bnxt_reset(struct bnxt *bp, bool silent) 7618 { 7619 bnxt_rtnl_lock_sp(bp); 7620 if (test_bit(BNXT_STATE_OPEN, &bp->state)) 7621 bnxt_reset_task(bp, silent); 7622 bnxt_rtnl_unlock_sp(bp); 7623 } 7624 7625 static void bnxt_cfg_ntp_filters(struct bnxt *); 7626 7627 static void bnxt_sp_task(struct work_struct *work) 7628 { 7629 struct bnxt *bp = container_of(work, struct bnxt, sp_task); 7630 7631 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 7632 smp_mb__after_atomic(); 7633 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 7634 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 7635 return; 7636 } 7637 7638 if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event)) 7639 bnxt_cfg_rx_mode(bp); 7640 7641 if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event)) 7642 bnxt_cfg_ntp_filters(bp); 7643 if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event)) 7644 bnxt_hwrm_exec_fwd_req(bp); 7645 if (test_and_clear_bit(BNXT_VXLAN_ADD_PORT_SP_EVENT, &bp->sp_event)) { 7646 bnxt_hwrm_tunnel_dst_port_alloc( 7647 bp, bp->vxlan_port, 7648 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN); 7649 } 7650 if (test_and_clear_bit(BNXT_VXLAN_DEL_PORT_SP_EVENT, &bp->sp_event)) { 7651 bnxt_hwrm_tunnel_dst_port_free( 7652 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN); 7653 } 7654 if (test_and_clear_bit(BNXT_GENEVE_ADD_PORT_SP_EVENT, &bp->sp_event)) { 7655 bnxt_hwrm_tunnel_dst_port_alloc( 7656 bp, bp->nge_port, 7657 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE); 7658 } 7659 if (test_and_clear_bit(BNXT_GENEVE_DEL_PORT_SP_EVENT, &bp->sp_event)) { 7660 bnxt_hwrm_tunnel_dst_port_free( 7661 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE); 7662 } 7663 if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event)) { 7664 bnxt_hwrm_port_qstats(bp); 7665 bnxt_hwrm_port_qstats_ext(bp); 7666 } 7667 7668 if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) { 7669 int rc; 7670 7671 mutex_lock(&bp->link_lock); 7672 if (test_and_clear_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, 7673 &bp->sp_event)) 7674 bnxt_hwrm_phy_qcaps(bp); 7675 7676 rc = bnxt_update_link(bp, true); 7677 mutex_unlock(&bp->link_lock); 7678 if (rc) 7679 netdev_err(bp->dev, "SP task can't update link (rc: %x)\n", 7680 rc); 7681 } 7682 if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event)) { 7683 mutex_lock(&bp->link_lock); 7684 bnxt_get_port_module_status(bp); 7685 mutex_unlock(&bp->link_lock); 7686 } 7687 7688 if (test_and_clear_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event)) 7689 bnxt_tc_flow_stats_work(bp); 7690 7691 /* These functions below will clear BNXT_STATE_IN_SP_TASK. They 7692 * must be the last functions to be called before exiting. 7693 */ 7694 if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event)) 7695 bnxt_reset(bp, false); 7696 7697 if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event)) 7698 bnxt_reset(bp, true); 7699 7700 smp_mb__before_atomic(); 7701 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 7702 } 7703 7704 /* Under rtnl_lock */ 7705 int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs, 7706 int tx_xdp) 7707 { 7708 int max_rx, max_tx, tx_sets = 1; 7709 int tx_rings_needed; 7710 int rx_rings = rx; 7711 int cp, vnics, rc; 7712 7713 if (tcs) 7714 tx_sets = tcs; 7715 7716 rc = bnxt_get_max_rings(bp, &max_rx, &max_tx, sh); 7717 if (rc) 7718 return rc; 7719 7720 if (max_rx < rx) 7721 return -ENOMEM; 7722 7723 tx_rings_needed = tx * tx_sets + tx_xdp; 7724 if (max_tx < tx_rings_needed) 7725 return -ENOMEM; 7726 7727 vnics = 1; 7728 if (bp->flags & BNXT_FLAG_RFS) 7729 vnics += rx_rings; 7730 7731 if (bp->flags & BNXT_FLAG_AGG_RINGS) 7732 rx_rings <<= 1; 7733 cp = sh ? max_t(int, tx_rings_needed, rx) : tx_rings_needed + rx; 7734 if (bp->flags & BNXT_FLAG_NEW_RM) 7735 cp += bnxt_get_ulp_msix_num(bp); 7736 return bnxt_hwrm_check_rings(bp, tx_rings_needed, rx_rings, rx, cp, 7737 vnics); 7738 } 7739 7740 static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev) 7741 { 7742 if (bp->bar2) { 7743 pci_iounmap(pdev, bp->bar2); 7744 bp->bar2 = NULL; 7745 } 7746 7747 if (bp->bar1) { 7748 pci_iounmap(pdev, bp->bar1); 7749 bp->bar1 = NULL; 7750 } 7751 7752 if (bp->bar0) { 7753 pci_iounmap(pdev, bp->bar0); 7754 bp->bar0 = NULL; 7755 } 7756 } 7757 7758 static void bnxt_cleanup_pci(struct bnxt *bp) 7759 { 7760 bnxt_unmap_bars(bp, bp->pdev); 7761 pci_release_regions(bp->pdev); 7762 pci_disable_device(bp->pdev); 7763 } 7764 7765 static void bnxt_init_dflt_coal(struct bnxt *bp) 7766 { 7767 struct bnxt_coal *coal; 7768 7769 /* Tick values in micro seconds. 7770 * 1 coal_buf x bufs_per_record = 1 completion record. 7771 */ 7772 coal = &bp->rx_coal; 7773 coal->coal_ticks = 14; 7774 coal->coal_bufs = 30; 7775 coal->coal_ticks_irq = 1; 7776 coal->coal_bufs_irq = 2; 7777 coal->idle_thresh = 50; 7778 coal->bufs_per_record = 2; 7779 coal->budget = 64; /* NAPI budget */ 7780 7781 coal = &bp->tx_coal; 7782 coal->coal_ticks = 28; 7783 coal->coal_bufs = 30; 7784 coal->coal_ticks_irq = 2; 7785 coal->coal_bufs_irq = 2; 7786 coal->bufs_per_record = 1; 7787 7788 bp->stats_coal_ticks = BNXT_DEF_STATS_COAL_TICKS; 7789 } 7790 7791 static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev) 7792 { 7793 int rc; 7794 struct bnxt *bp = netdev_priv(dev); 7795 7796 SET_NETDEV_DEV(dev, &pdev->dev); 7797 7798 /* enable device (incl. PCI PM wakeup), and bus-mastering */ 7799 rc = pci_enable_device(pdev); 7800 if (rc) { 7801 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n"); 7802 goto init_err; 7803 } 7804 7805 if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) { 7806 dev_err(&pdev->dev, 7807 "Cannot find PCI device base address, aborting\n"); 7808 rc = -ENODEV; 7809 goto init_err_disable; 7810 } 7811 7812 rc = pci_request_regions(pdev, DRV_MODULE_NAME); 7813 if (rc) { 7814 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n"); 7815 goto init_err_disable; 7816 } 7817 7818 if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 && 7819 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) { 7820 dev_err(&pdev->dev, "System does not support DMA, aborting\n"); 7821 goto init_err_disable; 7822 } 7823 7824 pci_set_master(pdev); 7825 7826 bp->dev = dev; 7827 bp->pdev = pdev; 7828 7829 bp->bar0 = pci_ioremap_bar(pdev, 0); 7830 if (!bp->bar0) { 7831 dev_err(&pdev->dev, "Cannot map device registers, aborting\n"); 7832 rc = -ENOMEM; 7833 goto init_err_release; 7834 } 7835 7836 bp->bar1 = pci_ioremap_bar(pdev, 2); 7837 if (!bp->bar1) { 7838 dev_err(&pdev->dev, "Cannot map doorbell registers, aborting\n"); 7839 rc = -ENOMEM; 7840 goto init_err_release; 7841 } 7842 7843 bp->bar2 = pci_ioremap_bar(pdev, 4); 7844 if (!bp->bar2) { 7845 dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n"); 7846 rc = -ENOMEM; 7847 goto init_err_release; 7848 } 7849 7850 pci_enable_pcie_error_reporting(pdev); 7851 7852 INIT_WORK(&bp->sp_task, bnxt_sp_task); 7853 7854 spin_lock_init(&bp->ntp_fltr_lock); 7855 7856 bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE; 7857 bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE; 7858 7859 bnxt_init_dflt_coal(bp); 7860 7861 timer_setup(&bp->timer, bnxt_timer, 0); 7862 bp->current_interval = BNXT_TIMER_INTERVAL; 7863 7864 clear_bit(BNXT_STATE_OPEN, &bp->state); 7865 return 0; 7866 7867 init_err_release: 7868 bnxt_unmap_bars(bp, pdev); 7869 pci_release_regions(pdev); 7870 7871 init_err_disable: 7872 pci_disable_device(pdev); 7873 7874 init_err: 7875 return rc; 7876 } 7877 7878 /* rtnl_lock held */ 7879 static int bnxt_change_mac_addr(struct net_device *dev, void *p) 7880 { 7881 struct sockaddr *addr = p; 7882 struct bnxt *bp = netdev_priv(dev); 7883 int rc = 0; 7884 7885 if (!is_valid_ether_addr(addr->sa_data)) 7886 return -EADDRNOTAVAIL; 7887 7888 if (ether_addr_equal(addr->sa_data, dev->dev_addr)) 7889 return 0; 7890 7891 rc = bnxt_approve_mac(bp, addr->sa_data); 7892 if (rc) 7893 return rc; 7894 7895 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len); 7896 if (netif_running(dev)) { 7897 bnxt_close_nic(bp, false, false); 7898 rc = bnxt_open_nic(bp, false, false); 7899 } 7900 7901 return rc; 7902 } 7903 7904 /* rtnl_lock held */ 7905 static int bnxt_change_mtu(struct net_device *dev, int new_mtu) 7906 { 7907 struct bnxt *bp = netdev_priv(dev); 7908 7909 if (netif_running(dev)) 7910 bnxt_close_nic(bp, false, false); 7911 7912 dev->mtu = new_mtu; 7913 bnxt_set_ring_params(bp); 7914 7915 if (netif_running(dev)) 7916 return bnxt_open_nic(bp, false, false); 7917 7918 return 0; 7919 } 7920 7921 int bnxt_setup_mq_tc(struct net_device *dev, u8 tc) 7922 { 7923 struct bnxt *bp = netdev_priv(dev); 7924 bool sh = false; 7925 int rc; 7926 7927 if (tc > bp->max_tc) { 7928 netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n", 7929 tc, bp->max_tc); 7930 return -EINVAL; 7931 } 7932 7933 if (netdev_get_num_tc(dev) == tc) 7934 return 0; 7935 7936 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 7937 sh = true; 7938 7939 rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings, 7940 sh, tc, bp->tx_nr_rings_xdp); 7941 if (rc) 7942 return rc; 7943 7944 /* Needs to close the device and do hw resource re-allocations */ 7945 if (netif_running(bp->dev)) 7946 bnxt_close_nic(bp, true, false); 7947 7948 if (tc) { 7949 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc; 7950 netdev_set_num_tc(dev, tc); 7951 } else { 7952 bp->tx_nr_rings = bp->tx_nr_rings_per_tc; 7953 netdev_reset_tc(dev); 7954 } 7955 bp->tx_nr_rings += bp->tx_nr_rings_xdp; 7956 bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) : 7957 bp->tx_nr_rings + bp->rx_nr_rings; 7958 bp->num_stat_ctxs = bp->cp_nr_rings; 7959 7960 if (netif_running(bp->dev)) 7961 return bnxt_open_nic(bp, true, false); 7962 7963 return 0; 7964 } 7965 7966 static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 7967 void *cb_priv) 7968 { 7969 struct bnxt *bp = cb_priv; 7970 7971 if (!bnxt_tc_flower_enabled(bp) || 7972 !tc_cls_can_offload_and_chain0(bp->dev, type_data)) 7973 return -EOPNOTSUPP; 7974 7975 switch (type) { 7976 case TC_SETUP_CLSFLOWER: 7977 return bnxt_tc_setup_flower(bp, bp->pf.fw_fid, type_data); 7978 default: 7979 return -EOPNOTSUPP; 7980 } 7981 } 7982 7983 static int bnxt_setup_tc_block(struct net_device *dev, 7984 struct tc_block_offload *f) 7985 { 7986 struct bnxt *bp = netdev_priv(dev); 7987 7988 if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS) 7989 return -EOPNOTSUPP; 7990 7991 switch (f->command) { 7992 case TC_BLOCK_BIND: 7993 return tcf_block_cb_register(f->block, bnxt_setup_tc_block_cb, 7994 bp, bp); 7995 case TC_BLOCK_UNBIND: 7996 tcf_block_cb_unregister(f->block, bnxt_setup_tc_block_cb, bp); 7997 return 0; 7998 default: 7999 return -EOPNOTSUPP; 8000 } 8001 } 8002 8003 static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type, 8004 void *type_data) 8005 { 8006 switch (type) { 8007 case TC_SETUP_BLOCK: 8008 return bnxt_setup_tc_block(dev, type_data); 8009 case TC_SETUP_QDISC_MQPRIO: { 8010 struct tc_mqprio_qopt *mqprio = type_data; 8011 8012 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 8013 8014 return bnxt_setup_mq_tc(dev, mqprio->num_tc); 8015 } 8016 default: 8017 return -EOPNOTSUPP; 8018 } 8019 } 8020 8021 #ifdef CONFIG_RFS_ACCEL 8022 static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1, 8023 struct bnxt_ntuple_filter *f2) 8024 { 8025 struct flow_keys *keys1 = &f1->fkeys; 8026 struct flow_keys *keys2 = &f2->fkeys; 8027 8028 if (keys1->addrs.v4addrs.src == keys2->addrs.v4addrs.src && 8029 keys1->addrs.v4addrs.dst == keys2->addrs.v4addrs.dst && 8030 keys1->ports.ports == keys2->ports.ports && 8031 keys1->basic.ip_proto == keys2->basic.ip_proto && 8032 keys1->basic.n_proto == keys2->basic.n_proto && 8033 keys1->control.flags == keys2->control.flags && 8034 ether_addr_equal(f1->src_mac_addr, f2->src_mac_addr) && 8035 ether_addr_equal(f1->dst_mac_addr, f2->dst_mac_addr)) 8036 return true; 8037 8038 return false; 8039 } 8040 8041 static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb, 8042 u16 rxq_index, u32 flow_id) 8043 { 8044 struct bnxt *bp = netdev_priv(dev); 8045 struct bnxt_ntuple_filter *fltr, *new_fltr; 8046 struct flow_keys *fkeys; 8047 struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb); 8048 int rc = 0, idx, bit_id, l2_idx = 0; 8049 struct hlist_head *head; 8050 8051 if (!ether_addr_equal(dev->dev_addr, eth->h_dest)) { 8052 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 8053 int off = 0, j; 8054 8055 netif_addr_lock_bh(dev); 8056 for (j = 0; j < vnic->uc_filter_count; j++, off += ETH_ALEN) { 8057 if (ether_addr_equal(eth->h_dest, 8058 vnic->uc_list + off)) { 8059 l2_idx = j + 1; 8060 break; 8061 } 8062 } 8063 netif_addr_unlock_bh(dev); 8064 if (!l2_idx) 8065 return -EINVAL; 8066 } 8067 new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC); 8068 if (!new_fltr) 8069 return -ENOMEM; 8070 8071 fkeys = &new_fltr->fkeys; 8072 if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) { 8073 rc = -EPROTONOSUPPORT; 8074 goto err_free; 8075 } 8076 8077 if ((fkeys->basic.n_proto != htons(ETH_P_IP) && 8078 fkeys->basic.n_proto != htons(ETH_P_IPV6)) || 8079 ((fkeys->basic.ip_proto != IPPROTO_TCP) && 8080 (fkeys->basic.ip_proto != IPPROTO_UDP))) { 8081 rc = -EPROTONOSUPPORT; 8082 goto err_free; 8083 } 8084 if (fkeys->basic.n_proto == htons(ETH_P_IPV6) && 8085 bp->hwrm_spec_code < 0x10601) { 8086 rc = -EPROTONOSUPPORT; 8087 goto err_free; 8088 } 8089 if ((fkeys->control.flags & FLOW_DIS_ENCAPSULATION) && 8090 bp->hwrm_spec_code < 0x10601) { 8091 rc = -EPROTONOSUPPORT; 8092 goto err_free; 8093 } 8094 8095 memcpy(new_fltr->dst_mac_addr, eth->h_dest, ETH_ALEN); 8096 memcpy(new_fltr->src_mac_addr, eth->h_source, ETH_ALEN); 8097 8098 idx = skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK; 8099 head = &bp->ntp_fltr_hash_tbl[idx]; 8100 rcu_read_lock(); 8101 hlist_for_each_entry_rcu(fltr, head, hash) { 8102 if (bnxt_fltr_match(fltr, new_fltr)) { 8103 rcu_read_unlock(); 8104 rc = 0; 8105 goto err_free; 8106 } 8107 } 8108 rcu_read_unlock(); 8109 8110 spin_lock_bh(&bp->ntp_fltr_lock); 8111 bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap, 8112 BNXT_NTP_FLTR_MAX_FLTR, 0); 8113 if (bit_id < 0) { 8114 spin_unlock_bh(&bp->ntp_fltr_lock); 8115 rc = -ENOMEM; 8116 goto err_free; 8117 } 8118 8119 new_fltr->sw_id = (u16)bit_id; 8120 new_fltr->flow_id = flow_id; 8121 new_fltr->l2_fltr_idx = l2_idx; 8122 new_fltr->rxq = rxq_index; 8123 hlist_add_head_rcu(&new_fltr->hash, head); 8124 bp->ntp_fltr_count++; 8125 spin_unlock_bh(&bp->ntp_fltr_lock); 8126 8127 set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event); 8128 bnxt_queue_sp_work(bp); 8129 8130 return new_fltr->sw_id; 8131 8132 err_free: 8133 kfree(new_fltr); 8134 return rc; 8135 } 8136 8137 static void bnxt_cfg_ntp_filters(struct bnxt *bp) 8138 { 8139 int i; 8140 8141 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) { 8142 struct hlist_head *head; 8143 struct hlist_node *tmp; 8144 struct bnxt_ntuple_filter *fltr; 8145 int rc; 8146 8147 head = &bp->ntp_fltr_hash_tbl[i]; 8148 hlist_for_each_entry_safe(fltr, tmp, head, hash) { 8149 bool del = false; 8150 8151 if (test_bit(BNXT_FLTR_VALID, &fltr->state)) { 8152 if (rps_may_expire_flow(bp->dev, fltr->rxq, 8153 fltr->flow_id, 8154 fltr->sw_id)) { 8155 bnxt_hwrm_cfa_ntuple_filter_free(bp, 8156 fltr); 8157 del = true; 8158 } 8159 } else { 8160 rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp, 8161 fltr); 8162 if (rc) 8163 del = true; 8164 else 8165 set_bit(BNXT_FLTR_VALID, &fltr->state); 8166 } 8167 8168 if (del) { 8169 spin_lock_bh(&bp->ntp_fltr_lock); 8170 hlist_del_rcu(&fltr->hash); 8171 bp->ntp_fltr_count--; 8172 spin_unlock_bh(&bp->ntp_fltr_lock); 8173 synchronize_rcu(); 8174 clear_bit(fltr->sw_id, bp->ntp_fltr_bmap); 8175 kfree(fltr); 8176 } 8177 } 8178 } 8179 if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event)) 8180 netdev_info(bp->dev, "Receive PF driver unload event!"); 8181 } 8182 8183 #else 8184 8185 static void bnxt_cfg_ntp_filters(struct bnxt *bp) 8186 { 8187 } 8188 8189 #endif /* CONFIG_RFS_ACCEL */ 8190 8191 static void bnxt_udp_tunnel_add(struct net_device *dev, 8192 struct udp_tunnel_info *ti) 8193 { 8194 struct bnxt *bp = netdev_priv(dev); 8195 8196 if (ti->sa_family != AF_INET6 && ti->sa_family != AF_INET) 8197 return; 8198 8199 if (!netif_running(dev)) 8200 return; 8201 8202 switch (ti->type) { 8203 case UDP_TUNNEL_TYPE_VXLAN: 8204 if (bp->vxlan_port_cnt && bp->vxlan_port != ti->port) 8205 return; 8206 8207 bp->vxlan_port_cnt++; 8208 if (bp->vxlan_port_cnt == 1) { 8209 bp->vxlan_port = ti->port; 8210 set_bit(BNXT_VXLAN_ADD_PORT_SP_EVENT, &bp->sp_event); 8211 bnxt_queue_sp_work(bp); 8212 } 8213 break; 8214 case UDP_TUNNEL_TYPE_GENEVE: 8215 if (bp->nge_port_cnt && bp->nge_port != ti->port) 8216 return; 8217 8218 bp->nge_port_cnt++; 8219 if (bp->nge_port_cnt == 1) { 8220 bp->nge_port = ti->port; 8221 set_bit(BNXT_GENEVE_ADD_PORT_SP_EVENT, &bp->sp_event); 8222 } 8223 break; 8224 default: 8225 return; 8226 } 8227 8228 bnxt_queue_sp_work(bp); 8229 } 8230 8231 static void bnxt_udp_tunnel_del(struct net_device *dev, 8232 struct udp_tunnel_info *ti) 8233 { 8234 struct bnxt *bp = netdev_priv(dev); 8235 8236 if (ti->sa_family != AF_INET6 && ti->sa_family != AF_INET) 8237 return; 8238 8239 if (!netif_running(dev)) 8240 return; 8241 8242 switch (ti->type) { 8243 case UDP_TUNNEL_TYPE_VXLAN: 8244 if (!bp->vxlan_port_cnt || bp->vxlan_port != ti->port) 8245 return; 8246 bp->vxlan_port_cnt--; 8247 8248 if (bp->vxlan_port_cnt != 0) 8249 return; 8250 8251 set_bit(BNXT_VXLAN_DEL_PORT_SP_EVENT, &bp->sp_event); 8252 break; 8253 case UDP_TUNNEL_TYPE_GENEVE: 8254 if (!bp->nge_port_cnt || bp->nge_port != ti->port) 8255 return; 8256 bp->nge_port_cnt--; 8257 8258 if (bp->nge_port_cnt != 0) 8259 return; 8260 8261 set_bit(BNXT_GENEVE_DEL_PORT_SP_EVENT, &bp->sp_event); 8262 break; 8263 default: 8264 return; 8265 } 8266 8267 bnxt_queue_sp_work(bp); 8268 } 8269 8270 static int bnxt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 8271 struct net_device *dev, u32 filter_mask, 8272 int nlflags) 8273 { 8274 struct bnxt *bp = netdev_priv(dev); 8275 8276 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bp->br_mode, 0, 0, 8277 nlflags, filter_mask, NULL); 8278 } 8279 8280 static int bnxt_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh, 8281 u16 flags) 8282 { 8283 struct bnxt *bp = netdev_priv(dev); 8284 struct nlattr *attr, *br_spec; 8285 int rem, rc = 0; 8286 8287 if (bp->hwrm_spec_code < 0x10708 || !BNXT_SINGLE_PF(bp)) 8288 return -EOPNOTSUPP; 8289 8290 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 8291 if (!br_spec) 8292 return -EINVAL; 8293 8294 nla_for_each_nested(attr, br_spec, rem) { 8295 u16 mode; 8296 8297 if (nla_type(attr) != IFLA_BRIDGE_MODE) 8298 continue; 8299 8300 if (nla_len(attr) < sizeof(mode)) 8301 return -EINVAL; 8302 8303 mode = nla_get_u16(attr); 8304 if (mode == bp->br_mode) 8305 break; 8306 8307 rc = bnxt_hwrm_set_br_mode(bp, mode); 8308 if (!rc) 8309 bp->br_mode = mode; 8310 break; 8311 } 8312 return rc; 8313 } 8314 8315 static int bnxt_get_phys_port_name(struct net_device *dev, char *buf, 8316 size_t len) 8317 { 8318 struct bnxt *bp = netdev_priv(dev); 8319 int rc; 8320 8321 /* The PF and it's VF-reps only support the switchdev framework */ 8322 if (!BNXT_PF(bp)) 8323 return -EOPNOTSUPP; 8324 8325 rc = snprintf(buf, len, "p%d", bp->pf.port_id); 8326 8327 if (rc >= len) 8328 return -EOPNOTSUPP; 8329 return 0; 8330 } 8331 8332 int bnxt_port_attr_get(struct bnxt *bp, struct switchdev_attr *attr) 8333 { 8334 if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV) 8335 return -EOPNOTSUPP; 8336 8337 /* The PF and it's VF-reps only support the switchdev framework */ 8338 if (!BNXT_PF(bp)) 8339 return -EOPNOTSUPP; 8340 8341 switch (attr->id) { 8342 case SWITCHDEV_ATTR_ID_PORT_PARENT_ID: 8343 attr->u.ppid.id_len = sizeof(bp->switch_id); 8344 memcpy(attr->u.ppid.id, bp->switch_id, attr->u.ppid.id_len); 8345 break; 8346 default: 8347 return -EOPNOTSUPP; 8348 } 8349 return 0; 8350 } 8351 8352 static int bnxt_swdev_port_attr_get(struct net_device *dev, 8353 struct switchdev_attr *attr) 8354 { 8355 return bnxt_port_attr_get(netdev_priv(dev), attr); 8356 } 8357 8358 static const struct switchdev_ops bnxt_switchdev_ops = { 8359 .switchdev_port_attr_get = bnxt_swdev_port_attr_get 8360 }; 8361 8362 static const struct net_device_ops bnxt_netdev_ops = { 8363 .ndo_open = bnxt_open, 8364 .ndo_start_xmit = bnxt_start_xmit, 8365 .ndo_stop = bnxt_close, 8366 .ndo_get_stats64 = bnxt_get_stats64, 8367 .ndo_set_rx_mode = bnxt_set_rx_mode, 8368 .ndo_do_ioctl = bnxt_ioctl, 8369 .ndo_validate_addr = eth_validate_addr, 8370 .ndo_set_mac_address = bnxt_change_mac_addr, 8371 .ndo_change_mtu = bnxt_change_mtu, 8372 .ndo_fix_features = bnxt_fix_features, 8373 .ndo_set_features = bnxt_set_features, 8374 .ndo_tx_timeout = bnxt_tx_timeout, 8375 #ifdef CONFIG_BNXT_SRIOV 8376 .ndo_get_vf_config = bnxt_get_vf_config, 8377 .ndo_set_vf_mac = bnxt_set_vf_mac, 8378 .ndo_set_vf_vlan = bnxt_set_vf_vlan, 8379 .ndo_set_vf_rate = bnxt_set_vf_bw, 8380 .ndo_set_vf_link_state = bnxt_set_vf_link_state, 8381 .ndo_set_vf_spoofchk = bnxt_set_vf_spoofchk, 8382 .ndo_set_vf_trust = bnxt_set_vf_trust, 8383 #endif 8384 #ifdef CONFIG_NET_POLL_CONTROLLER 8385 .ndo_poll_controller = bnxt_poll_controller, 8386 #endif 8387 .ndo_setup_tc = bnxt_setup_tc, 8388 #ifdef CONFIG_RFS_ACCEL 8389 .ndo_rx_flow_steer = bnxt_rx_flow_steer, 8390 #endif 8391 .ndo_udp_tunnel_add = bnxt_udp_tunnel_add, 8392 .ndo_udp_tunnel_del = bnxt_udp_tunnel_del, 8393 .ndo_bpf = bnxt_xdp, 8394 .ndo_bridge_getlink = bnxt_bridge_getlink, 8395 .ndo_bridge_setlink = bnxt_bridge_setlink, 8396 .ndo_get_phys_port_name = bnxt_get_phys_port_name 8397 }; 8398 8399 static void bnxt_remove_one(struct pci_dev *pdev) 8400 { 8401 struct net_device *dev = pci_get_drvdata(pdev); 8402 struct bnxt *bp = netdev_priv(dev); 8403 8404 if (BNXT_PF(bp)) { 8405 bnxt_sriov_disable(bp); 8406 bnxt_dl_unregister(bp); 8407 } 8408 8409 pci_disable_pcie_error_reporting(pdev); 8410 unregister_netdev(dev); 8411 bnxt_shutdown_tc(bp); 8412 bnxt_cancel_sp_work(bp); 8413 bp->sp_event = 0; 8414 8415 bnxt_clear_int_mode(bp); 8416 bnxt_hwrm_func_drv_unrgtr(bp); 8417 bnxt_free_hwrm_resources(bp); 8418 bnxt_free_hwrm_short_cmd_req(bp); 8419 bnxt_ethtool_free(bp); 8420 bnxt_dcb_free(bp); 8421 kfree(bp->edev); 8422 bp->edev = NULL; 8423 bnxt_cleanup_pci(bp); 8424 free_netdev(dev); 8425 } 8426 8427 static int bnxt_probe_phy(struct bnxt *bp) 8428 { 8429 int rc = 0; 8430 struct bnxt_link_info *link_info = &bp->link_info; 8431 8432 rc = bnxt_hwrm_phy_qcaps(bp); 8433 if (rc) { 8434 netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n", 8435 rc); 8436 return rc; 8437 } 8438 mutex_init(&bp->link_lock); 8439 8440 rc = bnxt_update_link(bp, false); 8441 if (rc) { 8442 netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n", 8443 rc); 8444 return rc; 8445 } 8446 8447 /* Older firmware does not have supported_auto_speeds, so assume 8448 * that all supported speeds can be autonegotiated. 8449 */ 8450 if (link_info->auto_link_speeds && !link_info->support_auto_speeds) 8451 link_info->support_auto_speeds = link_info->support_speeds; 8452 8453 /*initialize the ethool setting copy with NVM settings */ 8454 if (BNXT_AUTO_MODE(link_info->auto_mode)) { 8455 link_info->autoneg = BNXT_AUTONEG_SPEED; 8456 if (bp->hwrm_spec_code >= 0x10201) { 8457 if (link_info->auto_pause_setting & 8458 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE) 8459 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL; 8460 } else { 8461 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL; 8462 } 8463 link_info->advertising = link_info->auto_link_speeds; 8464 } else { 8465 link_info->req_link_speed = link_info->force_link_speed; 8466 link_info->req_duplex = link_info->duplex_setting; 8467 } 8468 if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) 8469 link_info->req_flow_ctrl = 8470 link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH; 8471 else 8472 link_info->req_flow_ctrl = link_info->force_pause_setting; 8473 return rc; 8474 } 8475 8476 static int bnxt_get_max_irq(struct pci_dev *pdev) 8477 { 8478 u16 ctrl; 8479 8480 if (!pdev->msix_cap) 8481 return 1; 8482 8483 pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl); 8484 return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1; 8485 } 8486 8487 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, 8488 int *max_cp) 8489 { 8490 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 8491 int max_ring_grps = 0; 8492 8493 *max_tx = hw_resc->max_tx_rings; 8494 *max_rx = hw_resc->max_rx_rings; 8495 *max_cp = min_t(int, hw_resc->max_irqs, hw_resc->max_cp_rings); 8496 *max_cp = min_t(int, *max_cp, hw_resc->max_stat_ctxs); 8497 max_ring_grps = hw_resc->max_hw_ring_grps; 8498 if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) { 8499 *max_cp -= 1; 8500 *max_rx -= 2; 8501 } 8502 if (bp->flags & BNXT_FLAG_AGG_RINGS) 8503 *max_rx >>= 1; 8504 *max_rx = min_t(int, *max_rx, max_ring_grps); 8505 } 8506 8507 int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared) 8508 { 8509 int rx, tx, cp; 8510 8511 _bnxt_get_max_rings(bp, &rx, &tx, &cp); 8512 *max_rx = rx; 8513 *max_tx = tx; 8514 if (!rx || !tx || !cp) 8515 return -ENOMEM; 8516 8517 return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared); 8518 } 8519 8520 static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx, 8521 bool shared) 8522 { 8523 int rc; 8524 8525 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared); 8526 if (rc && (bp->flags & BNXT_FLAG_AGG_RINGS)) { 8527 /* Not enough rings, try disabling agg rings. */ 8528 bp->flags &= ~BNXT_FLAG_AGG_RINGS; 8529 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared); 8530 if (rc) { 8531 /* set BNXT_FLAG_AGG_RINGS back for consistency */ 8532 bp->flags |= BNXT_FLAG_AGG_RINGS; 8533 return rc; 8534 } 8535 bp->flags |= BNXT_FLAG_NO_AGG_RINGS; 8536 bp->dev->hw_features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); 8537 bp->dev->features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); 8538 bnxt_set_ring_params(bp); 8539 } 8540 8541 if (bp->flags & BNXT_FLAG_ROCE_CAP) { 8542 int max_cp, max_stat, max_irq; 8543 8544 /* Reserve minimum resources for RoCE */ 8545 max_cp = bnxt_get_max_func_cp_rings(bp); 8546 max_stat = bnxt_get_max_func_stat_ctxs(bp); 8547 max_irq = bnxt_get_max_func_irqs(bp); 8548 if (max_cp <= BNXT_MIN_ROCE_CP_RINGS || 8549 max_irq <= BNXT_MIN_ROCE_CP_RINGS || 8550 max_stat <= BNXT_MIN_ROCE_STAT_CTXS) 8551 return 0; 8552 8553 max_cp -= BNXT_MIN_ROCE_CP_RINGS; 8554 max_irq -= BNXT_MIN_ROCE_CP_RINGS; 8555 max_stat -= BNXT_MIN_ROCE_STAT_CTXS; 8556 max_cp = min_t(int, max_cp, max_irq); 8557 max_cp = min_t(int, max_cp, max_stat); 8558 rc = bnxt_trim_rings(bp, max_rx, max_tx, max_cp, shared); 8559 if (rc) 8560 rc = 0; 8561 } 8562 return rc; 8563 } 8564 8565 /* In initial default shared ring setting, each shared ring must have a 8566 * RX/TX ring pair. 8567 */ 8568 static void bnxt_trim_dflt_sh_rings(struct bnxt *bp) 8569 { 8570 bp->cp_nr_rings = min_t(int, bp->tx_nr_rings_per_tc, bp->rx_nr_rings); 8571 bp->rx_nr_rings = bp->cp_nr_rings; 8572 bp->tx_nr_rings_per_tc = bp->cp_nr_rings; 8573 bp->tx_nr_rings = bp->tx_nr_rings_per_tc; 8574 } 8575 8576 static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh) 8577 { 8578 int dflt_rings, max_rx_rings, max_tx_rings, rc; 8579 8580 if (!bnxt_can_reserve_rings(bp)) 8581 return 0; 8582 8583 if (sh) 8584 bp->flags |= BNXT_FLAG_SHARED_RINGS; 8585 dflt_rings = netif_get_num_default_rss_queues(); 8586 /* Reduce default rings on multi-port cards so that total default 8587 * rings do not exceed CPU count. 8588 */ 8589 if (bp->port_count > 1) { 8590 int max_rings = 8591 max_t(int, num_online_cpus() / bp->port_count, 1); 8592 8593 dflt_rings = min_t(int, dflt_rings, max_rings); 8594 } 8595 rc = bnxt_get_dflt_rings(bp, &max_rx_rings, &max_tx_rings, sh); 8596 if (rc) 8597 return rc; 8598 bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings); 8599 bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings); 8600 if (sh) 8601 bnxt_trim_dflt_sh_rings(bp); 8602 else 8603 bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings; 8604 bp->tx_nr_rings = bp->tx_nr_rings_per_tc; 8605 8606 rc = __bnxt_reserve_rings(bp); 8607 if (rc) 8608 netdev_warn(bp->dev, "Unable to reserve tx rings\n"); 8609 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 8610 if (sh) 8611 bnxt_trim_dflt_sh_rings(bp); 8612 8613 /* Rings may have been trimmed, re-reserve the trimmed rings. */ 8614 if (bnxt_need_reserve_rings(bp)) { 8615 rc = __bnxt_reserve_rings(bp); 8616 if (rc) 8617 netdev_warn(bp->dev, "2nd rings reservation failed.\n"); 8618 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 8619 } 8620 bp->num_stat_ctxs = bp->cp_nr_rings; 8621 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 8622 bp->rx_nr_rings++; 8623 bp->cp_nr_rings++; 8624 } 8625 return rc; 8626 } 8627 8628 static int bnxt_init_dflt_ring_mode(struct bnxt *bp) 8629 { 8630 int rc; 8631 8632 if (bp->tx_nr_rings) 8633 return 0; 8634 8635 rc = bnxt_set_dflt_rings(bp, true); 8636 if (rc) { 8637 netdev_err(bp->dev, "Not enough rings available.\n"); 8638 return rc; 8639 } 8640 rc = bnxt_init_int_mode(bp); 8641 if (rc) 8642 return rc; 8643 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 8644 if (bnxt_rfs_supported(bp) && bnxt_rfs_capable(bp)) { 8645 bp->flags |= BNXT_FLAG_RFS; 8646 bp->dev->features |= NETIF_F_NTUPLE; 8647 } 8648 return 0; 8649 } 8650 8651 int bnxt_restore_pf_fw_resources(struct bnxt *bp) 8652 { 8653 int rc; 8654 8655 ASSERT_RTNL(); 8656 bnxt_hwrm_func_qcaps(bp); 8657 8658 if (netif_running(bp->dev)) 8659 __bnxt_close_nic(bp, true, false); 8660 8661 bnxt_ulp_irq_stop(bp); 8662 bnxt_clear_int_mode(bp); 8663 rc = bnxt_init_int_mode(bp); 8664 bnxt_ulp_irq_restart(bp, rc); 8665 8666 if (netif_running(bp->dev)) { 8667 if (rc) 8668 dev_close(bp->dev); 8669 else 8670 rc = bnxt_open_nic(bp, true, false); 8671 } 8672 8673 return rc; 8674 } 8675 8676 static int bnxt_init_mac_addr(struct bnxt *bp) 8677 { 8678 int rc = 0; 8679 8680 if (BNXT_PF(bp)) { 8681 memcpy(bp->dev->dev_addr, bp->pf.mac_addr, ETH_ALEN); 8682 } else { 8683 #ifdef CONFIG_BNXT_SRIOV 8684 struct bnxt_vf_info *vf = &bp->vf; 8685 8686 if (is_valid_ether_addr(vf->mac_addr)) { 8687 /* overwrite netdev dev_addr with admin VF MAC */ 8688 memcpy(bp->dev->dev_addr, vf->mac_addr, ETH_ALEN); 8689 } else { 8690 eth_hw_addr_random(bp->dev); 8691 } 8692 rc = bnxt_approve_mac(bp, bp->dev->dev_addr); 8693 #endif 8694 } 8695 return rc; 8696 } 8697 8698 static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) 8699 { 8700 static int version_printed; 8701 struct net_device *dev; 8702 struct bnxt *bp; 8703 int rc, max_irqs; 8704 8705 if (pci_is_bridge(pdev)) 8706 return -ENODEV; 8707 8708 if (version_printed++ == 0) 8709 pr_info("%s", version); 8710 8711 max_irqs = bnxt_get_max_irq(pdev); 8712 dev = alloc_etherdev_mq(sizeof(*bp), max_irqs); 8713 if (!dev) 8714 return -ENOMEM; 8715 8716 bp = netdev_priv(dev); 8717 8718 if (bnxt_vf_pciid(ent->driver_data)) 8719 bp->flags |= BNXT_FLAG_VF; 8720 8721 if (pdev->msix_cap) 8722 bp->flags |= BNXT_FLAG_MSIX_CAP; 8723 8724 rc = bnxt_init_board(pdev, dev); 8725 if (rc < 0) 8726 goto init_err_free; 8727 8728 dev->netdev_ops = &bnxt_netdev_ops; 8729 dev->watchdog_timeo = BNXT_TX_TIMEOUT; 8730 dev->ethtool_ops = &bnxt_ethtool_ops; 8731 SWITCHDEV_SET_OPS(dev, &bnxt_switchdev_ops); 8732 pci_set_drvdata(pdev, dev); 8733 8734 rc = bnxt_alloc_hwrm_resources(bp); 8735 if (rc) 8736 goto init_err_pci_clean; 8737 8738 mutex_init(&bp->hwrm_cmd_lock); 8739 rc = bnxt_hwrm_ver_get(bp); 8740 if (rc) 8741 goto init_err_pci_clean; 8742 8743 if (bp->flags & BNXT_FLAG_SHORT_CMD) { 8744 rc = bnxt_alloc_hwrm_short_cmd_req(bp); 8745 if (rc) 8746 goto init_err_pci_clean; 8747 } 8748 8749 rc = bnxt_hwrm_func_reset(bp); 8750 if (rc) 8751 goto init_err_pci_clean; 8752 8753 bnxt_hwrm_fw_set_time(bp); 8754 8755 dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG | 8756 NETIF_F_TSO | NETIF_F_TSO6 | 8757 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE | 8758 NETIF_F_GSO_IPXIP4 | 8759 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM | 8760 NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH | 8761 NETIF_F_RXCSUM | NETIF_F_GRO; 8762 8763 if (!BNXT_CHIP_TYPE_NITRO_A0(bp)) 8764 dev->hw_features |= NETIF_F_LRO; 8765 8766 dev->hw_enc_features = 8767 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG | 8768 NETIF_F_TSO | NETIF_F_TSO6 | 8769 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE | 8770 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM | 8771 NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL; 8772 dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM | 8773 NETIF_F_GSO_GRE_CSUM; 8774 dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA; 8775 dev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX | 8776 NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX; 8777 if (!BNXT_CHIP_TYPE_NITRO_A0(bp)) 8778 dev->hw_features |= NETIF_F_GRO_HW; 8779 dev->features |= dev->hw_features | NETIF_F_HIGHDMA; 8780 if (dev->features & NETIF_F_GRO_HW) 8781 dev->features &= ~NETIF_F_LRO; 8782 dev->priv_flags |= IFF_UNICAST_FLT; 8783 8784 #ifdef CONFIG_BNXT_SRIOV 8785 init_waitqueue_head(&bp->sriov_cfg_wait); 8786 mutex_init(&bp->sriov_lock); 8787 #endif 8788 bp->gro_func = bnxt_gro_func_5730x; 8789 if (BNXT_CHIP_P4_PLUS(bp)) 8790 bp->gro_func = bnxt_gro_func_5731x; 8791 else 8792 bp->flags |= BNXT_FLAG_DOUBLE_DB; 8793 8794 rc = bnxt_hwrm_func_drv_rgtr(bp); 8795 if (rc) 8796 goto init_err_pci_clean; 8797 8798 rc = bnxt_hwrm_func_rgtr_async_events(bp, NULL, 0); 8799 if (rc) 8800 goto init_err_pci_clean; 8801 8802 bp->ulp_probe = bnxt_ulp_probe; 8803 8804 /* Get the MAX capabilities for this function */ 8805 rc = bnxt_hwrm_func_qcaps(bp); 8806 if (rc) { 8807 netdev_err(bp->dev, "hwrm query capability failure rc: %x\n", 8808 rc); 8809 rc = -1; 8810 goto init_err_pci_clean; 8811 } 8812 rc = bnxt_init_mac_addr(bp); 8813 if (rc) { 8814 dev_err(&pdev->dev, "Unable to initialize mac address.\n"); 8815 rc = -EADDRNOTAVAIL; 8816 goto init_err_pci_clean; 8817 } 8818 rc = bnxt_hwrm_queue_qportcfg(bp); 8819 if (rc) { 8820 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %x\n", 8821 rc); 8822 rc = -1; 8823 goto init_err_pci_clean; 8824 } 8825 8826 bnxt_hwrm_func_qcfg(bp); 8827 bnxt_hwrm_port_led_qcaps(bp); 8828 bnxt_ethtool_init(bp); 8829 bnxt_dcb_init(bp); 8830 8831 /* MTU range: 60 - FW defined max */ 8832 dev->min_mtu = ETH_ZLEN; 8833 dev->max_mtu = bp->max_mtu; 8834 8835 rc = bnxt_probe_phy(bp); 8836 if (rc) 8837 goto init_err_pci_clean; 8838 8839 bnxt_set_rx_skb_mode(bp, false); 8840 bnxt_set_tpa_flags(bp); 8841 bnxt_set_ring_params(bp); 8842 bnxt_set_max_func_irqs(bp, max_irqs); 8843 rc = bnxt_set_dflt_rings(bp, true); 8844 if (rc) { 8845 netdev_err(bp->dev, "Not enough rings available.\n"); 8846 rc = -ENOMEM; 8847 goto init_err_pci_clean; 8848 } 8849 8850 /* Default RSS hash cfg. */ 8851 bp->rss_hash_cfg = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 | 8852 VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 | 8853 VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 | 8854 VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6; 8855 if (BNXT_CHIP_P4_PLUS(bp) && bp->hwrm_spec_code >= 0x10501) { 8856 bp->flags |= BNXT_FLAG_UDP_RSS_CAP; 8857 bp->rss_hash_cfg |= VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 | 8858 VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6; 8859 } 8860 8861 bnxt_hwrm_vnic_qcaps(bp); 8862 if (bnxt_rfs_supported(bp)) { 8863 dev->hw_features |= NETIF_F_NTUPLE; 8864 if (bnxt_rfs_capable(bp)) { 8865 bp->flags |= BNXT_FLAG_RFS; 8866 dev->features |= NETIF_F_NTUPLE; 8867 } 8868 } 8869 8870 if (dev->hw_features & NETIF_F_HW_VLAN_CTAG_RX) 8871 bp->flags |= BNXT_FLAG_STRIP_VLAN; 8872 8873 rc = bnxt_init_int_mode(bp); 8874 if (rc) 8875 goto init_err_pci_clean; 8876 8877 /* No TC has been set yet and rings may have been trimmed due to 8878 * limited MSIX, so we re-initialize the TX rings per TC. 8879 */ 8880 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 8881 8882 bnxt_get_wol_settings(bp); 8883 if (bp->flags & BNXT_FLAG_WOL_CAP) 8884 device_set_wakeup_enable(&pdev->dev, bp->wol); 8885 else 8886 device_set_wakeup_capable(&pdev->dev, false); 8887 8888 bnxt_hwrm_set_cache_line_size(bp, cache_line_size()); 8889 8890 if (BNXT_PF(bp)) { 8891 if (!bnxt_pf_wq) { 8892 bnxt_pf_wq = 8893 create_singlethread_workqueue("bnxt_pf_wq"); 8894 if (!bnxt_pf_wq) { 8895 dev_err(&pdev->dev, "Unable to create workqueue.\n"); 8896 goto init_err_pci_clean; 8897 } 8898 } 8899 bnxt_init_tc(bp); 8900 } 8901 8902 rc = register_netdev(dev); 8903 if (rc) 8904 goto init_err_cleanup_tc; 8905 8906 if (BNXT_PF(bp)) 8907 bnxt_dl_register(bp); 8908 8909 netdev_info(dev, "%s found at mem %lx, node addr %pM\n", 8910 board_info[ent->driver_data].name, 8911 (long)pci_resource_start(pdev, 0), dev->dev_addr); 8912 pcie_print_link_status(pdev); 8913 8914 return 0; 8915 8916 init_err_cleanup_tc: 8917 bnxt_shutdown_tc(bp); 8918 bnxt_clear_int_mode(bp); 8919 8920 init_err_pci_clean: 8921 bnxt_cleanup_pci(bp); 8922 8923 init_err_free: 8924 free_netdev(dev); 8925 return rc; 8926 } 8927 8928 static void bnxt_shutdown(struct pci_dev *pdev) 8929 { 8930 struct net_device *dev = pci_get_drvdata(pdev); 8931 struct bnxt *bp; 8932 8933 if (!dev) 8934 return; 8935 8936 rtnl_lock(); 8937 bp = netdev_priv(dev); 8938 if (!bp) 8939 goto shutdown_exit; 8940 8941 if (netif_running(dev)) 8942 dev_close(dev); 8943 8944 bnxt_ulp_shutdown(bp); 8945 8946 if (system_state == SYSTEM_POWER_OFF) { 8947 bnxt_clear_int_mode(bp); 8948 pci_wake_from_d3(pdev, bp->wol); 8949 pci_set_power_state(pdev, PCI_D3hot); 8950 } 8951 8952 shutdown_exit: 8953 rtnl_unlock(); 8954 } 8955 8956 #ifdef CONFIG_PM_SLEEP 8957 static int bnxt_suspend(struct device *device) 8958 { 8959 struct pci_dev *pdev = to_pci_dev(device); 8960 struct net_device *dev = pci_get_drvdata(pdev); 8961 struct bnxt *bp = netdev_priv(dev); 8962 int rc = 0; 8963 8964 rtnl_lock(); 8965 if (netif_running(dev)) { 8966 netif_device_detach(dev); 8967 rc = bnxt_close(dev); 8968 } 8969 bnxt_hwrm_func_drv_unrgtr(bp); 8970 rtnl_unlock(); 8971 return rc; 8972 } 8973 8974 static int bnxt_resume(struct device *device) 8975 { 8976 struct pci_dev *pdev = to_pci_dev(device); 8977 struct net_device *dev = pci_get_drvdata(pdev); 8978 struct bnxt *bp = netdev_priv(dev); 8979 int rc = 0; 8980 8981 rtnl_lock(); 8982 if (bnxt_hwrm_ver_get(bp) || bnxt_hwrm_func_drv_rgtr(bp)) { 8983 rc = -ENODEV; 8984 goto resume_exit; 8985 } 8986 rc = bnxt_hwrm_func_reset(bp); 8987 if (rc) { 8988 rc = -EBUSY; 8989 goto resume_exit; 8990 } 8991 bnxt_get_wol_settings(bp); 8992 if (netif_running(dev)) { 8993 rc = bnxt_open(dev); 8994 if (!rc) 8995 netif_device_attach(dev); 8996 } 8997 8998 resume_exit: 8999 rtnl_unlock(); 9000 return rc; 9001 } 9002 9003 static SIMPLE_DEV_PM_OPS(bnxt_pm_ops, bnxt_suspend, bnxt_resume); 9004 #define BNXT_PM_OPS (&bnxt_pm_ops) 9005 9006 #else 9007 9008 #define BNXT_PM_OPS NULL 9009 9010 #endif /* CONFIG_PM_SLEEP */ 9011 9012 /** 9013 * bnxt_io_error_detected - called when PCI error is detected 9014 * @pdev: Pointer to PCI device 9015 * @state: The current pci connection state 9016 * 9017 * This function is called after a PCI bus error affecting 9018 * this device has been detected. 9019 */ 9020 static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev, 9021 pci_channel_state_t state) 9022 { 9023 struct net_device *netdev = pci_get_drvdata(pdev); 9024 struct bnxt *bp = netdev_priv(netdev); 9025 9026 netdev_info(netdev, "PCI I/O error detected\n"); 9027 9028 rtnl_lock(); 9029 netif_device_detach(netdev); 9030 9031 bnxt_ulp_stop(bp); 9032 9033 if (state == pci_channel_io_perm_failure) { 9034 rtnl_unlock(); 9035 return PCI_ERS_RESULT_DISCONNECT; 9036 } 9037 9038 if (netif_running(netdev)) 9039 bnxt_close(netdev); 9040 9041 pci_disable_device(pdev); 9042 rtnl_unlock(); 9043 9044 /* Request a slot slot reset. */ 9045 return PCI_ERS_RESULT_NEED_RESET; 9046 } 9047 9048 /** 9049 * bnxt_io_slot_reset - called after the pci bus has been reset. 9050 * @pdev: Pointer to PCI device 9051 * 9052 * Restart the card from scratch, as if from a cold-boot. 9053 * At this point, the card has exprienced a hard reset, 9054 * followed by fixups by BIOS, and has its config space 9055 * set up identically to what it was at cold boot. 9056 */ 9057 static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev) 9058 { 9059 struct net_device *netdev = pci_get_drvdata(pdev); 9060 struct bnxt *bp = netdev_priv(netdev); 9061 int err = 0; 9062 pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT; 9063 9064 netdev_info(bp->dev, "PCI Slot Reset\n"); 9065 9066 rtnl_lock(); 9067 9068 if (pci_enable_device(pdev)) { 9069 dev_err(&pdev->dev, 9070 "Cannot re-enable PCI device after reset.\n"); 9071 } else { 9072 pci_set_master(pdev); 9073 9074 err = bnxt_hwrm_func_reset(bp); 9075 if (!err && netif_running(netdev)) 9076 err = bnxt_open(netdev); 9077 9078 if (!err) { 9079 result = PCI_ERS_RESULT_RECOVERED; 9080 bnxt_ulp_start(bp); 9081 } 9082 } 9083 9084 if (result != PCI_ERS_RESULT_RECOVERED && netif_running(netdev)) 9085 dev_close(netdev); 9086 9087 rtnl_unlock(); 9088 9089 err = pci_cleanup_aer_uncorrect_error_status(pdev); 9090 if (err) { 9091 dev_err(&pdev->dev, 9092 "pci_cleanup_aer_uncorrect_error_status failed 0x%0x\n", 9093 err); /* non-fatal, continue */ 9094 } 9095 9096 return PCI_ERS_RESULT_RECOVERED; 9097 } 9098 9099 /** 9100 * bnxt_io_resume - called when traffic can start flowing again. 9101 * @pdev: Pointer to PCI device 9102 * 9103 * This callback is called when the error recovery driver tells 9104 * us that its OK to resume normal operation. 9105 */ 9106 static void bnxt_io_resume(struct pci_dev *pdev) 9107 { 9108 struct net_device *netdev = pci_get_drvdata(pdev); 9109 9110 rtnl_lock(); 9111 9112 netif_device_attach(netdev); 9113 9114 rtnl_unlock(); 9115 } 9116 9117 static const struct pci_error_handlers bnxt_err_handler = { 9118 .error_detected = bnxt_io_error_detected, 9119 .slot_reset = bnxt_io_slot_reset, 9120 .resume = bnxt_io_resume 9121 }; 9122 9123 static struct pci_driver bnxt_pci_driver = { 9124 .name = DRV_MODULE_NAME, 9125 .id_table = bnxt_pci_tbl, 9126 .probe = bnxt_init_one, 9127 .remove = bnxt_remove_one, 9128 .shutdown = bnxt_shutdown, 9129 .driver.pm = BNXT_PM_OPS, 9130 .err_handler = &bnxt_err_handler, 9131 #if defined(CONFIG_BNXT_SRIOV) 9132 .sriov_configure = bnxt_sriov_configure, 9133 #endif 9134 }; 9135 9136 static int __init bnxt_init(void) 9137 { 9138 bnxt_debug_init(); 9139 return pci_register_driver(&bnxt_pci_driver); 9140 } 9141 9142 static void __exit bnxt_exit(void) 9143 { 9144 pci_unregister_driver(&bnxt_pci_driver); 9145 if (bnxt_pf_wq) 9146 destroy_workqueue(bnxt_pf_wq); 9147 bnxt_debug_exit(); 9148 } 9149 9150 module_init(bnxt_init); 9151 module_exit(bnxt_exit); 9152