1 /* Broadcom NetXtreme-C/E network driver. 2 * 3 * Copyright (c) 2014-2016 Broadcom Corporation 4 * Copyright (c) 2016-2018 Broadcom Limited 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License as published by 8 * the Free Software Foundation. 9 */ 10 11 #include <linux/module.h> 12 13 #include <linux/stringify.h> 14 #include <linux/kernel.h> 15 #include <linux/timer.h> 16 #include <linux/errno.h> 17 #include <linux/ioport.h> 18 #include <linux/slab.h> 19 #include <linux/vmalloc.h> 20 #include <linux/interrupt.h> 21 #include <linux/pci.h> 22 #include <linux/netdevice.h> 23 #include <linux/etherdevice.h> 24 #include <linux/skbuff.h> 25 #include <linux/dma-mapping.h> 26 #include <linux/bitops.h> 27 #include <linux/io.h> 28 #include <linux/irq.h> 29 #include <linux/delay.h> 30 #include <asm/byteorder.h> 31 #include <asm/page.h> 32 #include <linux/time.h> 33 #include <linux/mii.h> 34 #include <linux/if.h> 35 #include <linux/if_vlan.h> 36 #include <linux/if_bridge.h> 37 #include <linux/rtc.h> 38 #include <linux/bpf.h> 39 #include <net/ip.h> 40 #include <net/tcp.h> 41 #include <net/udp.h> 42 #include <net/checksum.h> 43 #include <net/ip6_checksum.h> 44 #include <net/udp_tunnel.h> 45 #include <linux/workqueue.h> 46 #include <linux/prefetch.h> 47 #include <linux/cache.h> 48 #include <linux/log2.h> 49 #include <linux/aer.h> 50 #include <linux/bitmap.h> 51 #include <linux/cpu_rmap.h> 52 #include <linux/cpumask.h> 53 #include <net/pkt_cls.h> 54 #include <linux/hwmon.h> 55 #include <linux/hwmon-sysfs.h> 56 57 #include "bnxt_hsi.h" 58 #include "bnxt.h" 59 #include "bnxt_ulp.h" 60 #include "bnxt_sriov.h" 61 #include "bnxt_ethtool.h" 62 #include "bnxt_dcb.h" 63 #include "bnxt_xdp.h" 64 #include "bnxt_vfr.h" 65 #include "bnxt_tc.h" 66 #include "bnxt_devlink.h" 67 #include "bnxt_debugfs.h" 68 69 #define BNXT_TX_TIMEOUT (5 * HZ) 70 71 static const char version[] = 72 "Broadcom NetXtreme-C/E driver " DRV_MODULE_NAME " v" DRV_MODULE_VERSION "\n"; 73 74 MODULE_LICENSE("GPL"); 75 MODULE_DESCRIPTION("Broadcom BCM573xx network driver"); 76 MODULE_VERSION(DRV_MODULE_VERSION); 77 78 #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN) 79 #define BNXT_RX_DMA_OFFSET NET_SKB_PAD 80 #define BNXT_RX_COPY_THRESH 256 81 82 #define BNXT_TX_PUSH_THRESH 164 83 84 enum board_idx { 85 BCM57301, 86 BCM57302, 87 BCM57304, 88 BCM57417_NPAR, 89 BCM58700, 90 BCM57311, 91 BCM57312, 92 BCM57402, 93 BCM57404, 94 BCM57406, 95 BCM57402_NPAR, 96 BCM57407, 97 BCM57412, 98 BCM57414, 99 BCM57416, 100 BCM57417, 101 BCM57412_NPAR, 102 BCM57314, 103 BCM57417_SFP, 104 BCM57416_SFP, 105 BCM57404_NPAR, 106 BCM57406_NPAR, 107 BCM57407_SFP, 108 BCM57407_NPAR, 109 BCM57414_NPAR, 110 BCM57416_NPAR, 111 BCM57452, 112 BCM57454, 113 BCM5745x_NPAR, 114 BCM58802, 115 BCM58804, 116 BCM58808, 117 NETXTREME_E_VF, 118 NETXTREME_C_VF, 119 NETXTREME_S_VF, 120 }; 121 122 /* indexed by enum above */ 123 static const struct { 124 char *name; 125 } board_info[] = { 126 [BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" }, 127 [BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" }, 128 [BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" }, 129 [BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" }, 130 [BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" }, 131 [BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" }, 132 [BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" }, 133 [BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" }, 134 [BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" }, 135 [BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" }, 136 [BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" }, 137 [BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" }, 138 [BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" }, 139 [BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" }, 140 [BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" }, 141 [BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" }, 142 [BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" }, 143 [BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" }, 144 [BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" }, 145 [BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" }, 146 [BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" }, 147 [BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" }, 148 [BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" }, 149 [BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" }, 150 [BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" }, 151 [BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" }, 152 [BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" }, 153 [BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" }, 154 [BCM5745x_NPAR] = { "Broadcom BCM5745x NetXtreme-E Ethernet Partition" }, 155 [BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" }, 156 [BCM58804] = { "Broadcom BCM58804 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" }, 157 [BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" }, 158 [NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" }, 159 [NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" }, 160 [NETXTREME_S_VF] = { "Broadcom NetXtreme-S Ethernet Virtual Function" }, 161 }; 162 163 static const struct pci_device_id bnxt_pci_tbl[] = { 164 { PCI_VDEVICE(BROADCOM, 0x1604), .driver_data = BCM5745x_NPAR }, 165 { PCI_VDEVICE(BROADCOM, 0x1605), .driver_data = BCM5745x_NPAR }, 166 { PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 }, 167 { PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR }, 168 { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 }, 169 { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 }, 170 { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 }, 171 { PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR }, 172 { PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 }, 173 { PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 }, 174 { PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 }, 175 { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 }, 176 { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 }, 177 { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 }, 178 { PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR }, 179 { PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 }, 180 { PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 }, 181 { PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 }, 182 { PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 }, 183 { PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 }, 184 { PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR }, 185 { PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 }, 186 { PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP }, 187 { PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP }, 188 { PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR }, 189 { PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR }, 190 { PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP }, 191 { PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR }, 192 { PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR }, 193 { PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR }, 194 { PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR }, 195 { PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR }, 196 { PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR }, 197 { PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 }, 198 { PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 }, 199 { PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 }, 200 { PCI_VDEVICE(BROADCOM, 0xd804), .driver_data = BCM58804 }, 201 #ifdef CONFIG_BNXT_SRIOV 202 { PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF }, 203 { PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF }, 204 { PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF }, 205 { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF }, 206 { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF }, 207 { PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF }, 208 { PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF }, 209 { PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF }, 210 { PCI_VDEVICE(BROADCOM, 0xd800), .driver_data = NETXTREME_S_VF }, 211 #endif 212 { 0 } 213 }; 214 215 MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl); 216 217 static const u16 bnxt_vf_req_snif[] = { 218 HWRM_FUNC_CFG, 219 HWRM_FUNC_VF_CFG, 220 HWRM_PORT_PHY_QCFG, 221 HWRM_CFA_L2_FILTER_ALLOC, 222 }; 223 224 static const u16 bnxt_async_events_arr[] = { 225 ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE, 226 ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD, 227 ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED, 228 ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE, 229 ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE, 230 }; 231 232 static struct workqueue_struct *bnxt_pf_wq; 233 234 static bool bnxt_vf_pciid(enum board_idx idx) 235 { 236 return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF || 237 idx == NETXTREME_S_VF); 238 } 239 240 #define DB_CP_REARM_FLAGS (DB_KEY_CP | DB_IDX_VALID) 241 #define DB_CP_FLAGS (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS) 242 #define DB_CP_IRQ_DIS_FLAGS (DB_KEY_CP | DB_IRQ_DIS) 243 244 #define BNXT_CP_DB_REARM(db, raw_cons) \ 245 writel(DB_CP_REARM_FLAGS | RING_CMP(raw_cons), db) 246 247 #define BNXT_CP_DB(db, raw_cons) \ 248 writel(DB_CP_FLAGS | RING_CMP(raw_cons), db) 249 250 #define BNXT_CP_DB_IRQ_DIS(db) \ 251 writel(DB_CP_IRQ_DIS_FLAGS, db) 252 253 const u16 bnxt_lhint_arr[] = { 254 TX_BD_FLAGS_LHINT_512_AND_SMALLER, 255 TX_BD_FLAGS_LHINT_512_TO_1023, 256 TX_BD_FLAGS_LHINT_1024_TO_2047, 257 TX_BD_FLAGS_LHINT_1024_TO_2047, 258 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 259 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 260 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 261 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 262 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 263 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 264 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 265 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 266 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 267 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 268 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 269 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 270 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 271 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 272 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 273 }; 274 275 static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb) 276 { 277 struct metadata_dst *md_dst = skb_metadata_dst(skb); 278 279 if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX) 280 return 0; 281 282 return md_dst->u.port_info.port_id; 283 } 284 285 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev) 286 { 287 struct bnxt *bp = netdev_priv(dev); 288 struct tx_bd *txbd; 289 struct tx_bd_ext *txbd1; 290 struct netdev_queue *txq; 291 int i; 292 dma_addr_t mapping; 293 unsigned int length, pad = 0; 294 u32 len, free_size, vlan_tag_flags, cfa_action, flags; 295 u16 prod, last_frag; 296 struct pci_dev *pdev = bp->pdev; 297 struct bnxt_tx_ring_info *txr; 298 struct bnxt_sw_tx_bd *tx_buf; 299 300 i = skb_get_queue_mapping(skb); 301 if (unlikely(i >= bp->tx_nr_rings)) { 302 dev_kfree_skb_any(skb); 303 return NETDEV_TX_OK; 304 } 305 306 txq = netdev_get_tx_queue(dev, i); 307 txr = &bp->tx_ring[bp->tx_ring_map[i]]; 308 prod = txr->tx_prod; 309 310 free_size = bnxt_tx_avail(bp, txr); 311 if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) { 312 netif_tx_stop_queue(txq); 313 return NETDEV_TX_BUSY; 314 } 315 316 length = skb->len; 317 len = skb_headlen(skb); 318 last_frag = skb_shinfo(skb)->nr_frags; 319 320 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 321 322 txbd->tx_bd_opaque = prod; 323 324 tx_buf = &txr->tx_buf_ring[prod]; 325 tx_buf->skb = skb; 326 tx_buf->nr_frags = last_frag; 327 328 vlan_tag_flags = 0; 329 cfa_action = bnxt_xmit_get_cfa_action(skb); 330 if (skb_vlan_tag_present(skb)) { 331 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN | 332 skb_vlan_tag_get(skb); 333 /* Currently supports 8021Q, 8021AD vlan offloads 334 * QINQ1, QINQ2, QINQ3 vlan headers are deprecated 335 */ 336 if (skb->vlan_proto == htons(ETH_P_8021Q)) 337 vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT; 338 } 339 340 if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh) { 341 struct tx_push_buffer *tx_push_buf = txr->tx_push; 342 struct tx_push_bd *tx_push = &tx_push_buf->push_bd; 343 struct tx_bd_ext *tx_push1 = &tx_push->txbd2; 344 void *pdata = tx_push_buf->data; 345 u64 *end; 346 int j, push_len; 347 348 /* Set COAL_NOW to be ready quickly for the next push */ 349 tx_push->tx_bd_len_flags_type = 350 cpu_to_le32((length << TX_BD_LEN_SHIFT) | 351 TX_BD_TYPE_LONG_TX_BD | 352 TX_BD_FLAGS_LHINT_512_AND_SMALLER | 353 TX_BD_FLAGS_COAL_NOW | 354 TX_BD_FLAGS_PACKET_END | 355 (2 << TX_BD_FLAGS_BD_CNT_SHIFT)); 356 357 if (skb->ip_summed == CHECKSUM_PARTIAL) 358 tx_push1->tx_bd_hsize_lflags = 359 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM); 360 else 361 tx_push1->tx_bd_hsize_lflags = 0; 362 363 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags); 364 tx_push1->tx_bd_cfa_action = 365 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT); 366 367 end = pdata + length; 368 end = PTR_ALIGN(end, 8) - 1; 369 *end = 0; 370 371 skb_copy_from_linear_data(skb, pdata, len); 372 pdata += len; 373 for (j = 0; j < last_frag; j++) { 374 skb_frag_t *frag = &skb_shinfo(skb)->frags[j]; 375 void *fptr; 376 377 fptr = skb_frag_address_safe(frag); 378 if (!fptr) 379 goto normal_tx; 380 381 memcpy(pdata, fptr, skb_frag_size(frag)); 382 pdata += skb_frag_size(frag); 383 } 384 385 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type; 386 txbd->tx_bd_haddr = txr->data_mapping; 387 prod = NEXT_TX(prod); 388 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 389 memcpy(txbd, tx_push1, sizeof(*txbd)); 390 prod = NEXT_TX(prod); 391 tx_push->doorbell = 392 cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | prod); 393 txr->tx_prod = prod; 394 395 tx_buf->is_push = 1; 396 netdev_tx_sent_queue(txq, skb->len); 397 wmb(); /* Sync is_push and byte queue before pushing data */ 398 399 push_len = (length + sizeof(*tx_push) + 7) / 8; 400 if (push_len > 16) { 401 __iowrite64_copy(txr->tx_doorbell, tx_push_buf, 16); 402 __iowrite32_copy(txr->tx_doorbell + 4, tx_push_buf + 1, 403 (push_len - 16) << 1); 404 } else { 405 __iowrite64_copy(txr->tx_doorbell, tx_push_buf, 406 push_len); 407 } 408 409 goto tx_done; 410 } 411 412 normal_tx: 413 if (length < BNXT_MIN_PKT_SIZE) { 414 pad = BNXT_MIN_PKT_SIZE - length; 415 if (skb_pad(skb, pad)) { 416 /* SKB already freed. */ 417 tx_buf->skb = NULL; 418 return NETDEV_TX_OK; 419 } 420 length = BNXT_MIN_PKT_SIZE; 421 } 422 423 mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE); 424 425 if (unlikely(dma_mapping_error(&pdev->dev, mapping))) { 426 dev_kfree_skb_any(skb); 427 tx_buf->skb = NULL; 428 return NETDEV_TX_OK; 429 } 430 431 dma_unmap_addr_set(tx_buf, mapping, mapping); 432 flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD | 433 ((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT); 434 435 txbd->tx_bd_haddr = cpu_to_le64(mapping); 436 437 prod = NEXT_TX(prod); 438 txbd1 = (struct tx_bd_ext *) 439 &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 440 441 txbd1->tx_bd_hsize_lflags = 0; 442 if (skb_is_gso(skb)) { 443 u32 hdr_len; 444 445 if (skb->encapsulation) 446 hdr_len = skb_inner_network_offset(skb) + 447 skb_inner_network_header_len(skb) + 448 inner_tcp_hdrlen(skb); 449 else 450 hdr_len = skb_transport_offset(skb) + 451 tcp_hdrlen(skb); 452 453 txbd1->tx_bd_hsize_lflags = cpu_to_le32(TX_BD_FLAGS_LSO | 454 TX_BD_FLAGS_T_IPID | 455 (hdr_len << (TX_BD_HSIZE_SHIFT - 1))); 456 length = skb_shinfo(skb)->gso_size; 457 txbd1->tx_bd_mss = cpu_to_le32(length); 458 length += hdr_len; 459 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { 460 txbd1->tx_bd_hsize_lflags = 461 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM); 462 txbd1->tx_bd_mss = 0; 463 } 464 465 length >>= 9; 466 flags |= bnxt_lhint_arr[length]; 467 txbd->tx_bd_len_flags_type = cpu_to_le32(flags); 468 469 txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags); 470 txbd1->tx_bd_cfa_action = 471 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT); 472 for (i = 0; i < last_frag; i++) { 473 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 474 475 prod = NEXT_TX(prod); 476 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 477 478 len = skb_frag_size(frag); 479 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len, 480 DMA_TO_DEVICE); 481 482 if (unlikely(dma_mapping_error(&pdev->dev, mapping))) 483 goto tx_dma_error; 484 485 tx_buf = &txr->tx_buf_ring[prod]; 486 dma_unmap_addr_set(tx_buf, mapping, mapping); 487 488 txbd->tx_bd_haddr = cpu_to_le64(mapping); 489 490 flags = len << TX_BD_LEN_SHIFT; 491 txbd->tx_bd_len_flags_type = cpu_to_le32(flags); 492 } 493 494 flags &= ~TX_BD_LEN; 495 txbd->tx_bd_len_flags_type = 496 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags | 497 TX_BD_FLAGS_PACKET_END); 498 499 netdev_tx_sent_queue(txq, skb->len); 500 501 /* Sync BD data before updating doorbell */ 502 wmb(); 503 504 prod = NEXT_TX(prod); 505 txr->tx_prod = prod; 506 507 if (!skb->xmit_more || netif_xmit_stopped(txq)) 508 bnxt_db_write(bp, txr->tx_doorbell, DB_KEY_TX | prod); 509 510 tx_done: 511 512 mmiowb(); 513 514 if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) { 515 if (skb->xmit_more && !tx_buf->is_push) 516 bnxt_db_write(bp, txr->tx_doorbell, DB_KEY_TX | prod); 517 518 netif_tx_stop_queue(txq); 519 520 /* netif_tx_stop_queue() must be done before checking 521 * tx index in bnxt_tx_avail() below, because in 522 * bnxt_tx_int(), we update tx index before checking for 523 * netif_tx_queue_stopped(). 524 */ 525 smp_mb(); 526 if (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh) 527 netif_tx_wake_queue(txq); 528 } 529 return NETDEV_TX_OK; 530 531 tx_dma_error: 532 last_frag = i; 533 534 /* start back at beginning and unmap skb */ 535 prod = txr->tx_prod; 536 tx_buf = &txr->tx_buf_ring[prod]; 537 tx_buf->skb = NULL; 538 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping), 539 skb_headlen(skb), PCI_DMA_TODEVICE); 540 prod = NEXT_TX(prod); 541 542 /* unmap remaining mapped pages */ 543 for (i = 0; i < last_frag; i++) { 544 prod = NEXT_TX(prod); 545 tx_buf = &txr->tx_buf_ring[prod]; 546 dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping), 547 skb_frag_size(&skb_shinfo(skb)->frags[i]), 548 PCI_DMA_TODEVICE); 549 } 550 551 dev_kfree_skb_any(skb); 552 return NETDEV_TX_OK; 553 } 554 555 static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts) 556 { 557 struct bnxt_tx_ring_info *txr = bnapi->tx_ring; 558 struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index); 559 u16 cons = txr->tx_cons; 560 struct pci_dev *pdev = bp->pdev; 561 int i; 562 unsigned int tx_bytes = 0; 563 564 for (i = 0; i < nr_pkts; i++) { 565 struct bnxt_sw_tx_bd *tx_buf; 566 struct sk_buff *skb; 567 int j, last; 568 569 tx_buf = &txr->tx_buf_ring[cons]; 570 cons = NEXT_TX(cons); 571 skb = tx_buf->skb; 572 tx_buf->skb = NULL; 573 574 if (tx_buf->is_push) { 575 tx_buf->is_push = 0; 576 goto next_tx_int; 577 } 578 579 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping), 580 skb_headlen(skb), PCI_DMA_TODEVICE); 581 last = tx_buf->nr_frags; 582 583 for (j = 0; j < last; j++) { 584 cons = NEXT_TX(cons); 585 tx_buf = &txr->tx_buf_ring[cons]; 586 dma_unmap_page( 587 &pdev->dev, 588 dma_unmap_addr(tx_buf, mapping), 589 skb_frag_size(&skb_shinfo(skb)->frags[j]), 590 PCI_DMA_TODEVICE); 591 } 592 593 next_tx_int: 594 cons = NEXT_TX(cons); 595 596 tx_bytes += skb->len; 597 dev_kfree_skb_any(skb); 598 } 599 600 netdev_tx_completed_queue(txq, nr_pkts, tx_bytes); 601 txr->tx_cons = cons; 602 603 /* Need to make the tx_cons update visible to bnxt_start_xmit() 604 * before checking for netif_tx_queue_stopped(). Without the 605 * memory barrier, there is a small possibility that bnxt_start_xmit() 606 * will miss it and cause the queue to be stopped forever. 607 */ 608 smp_mb(); 609 610 if (unlikely(netif_tx_queue_stopped(txq)) && 611 (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)) { 612 __netif_tx_lock(txq, smp_processor_id()); 613 if (netif_tx_queue_stopped(txq) && 614 bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh && 615 txr->dev_state != BNXT_DEV_STATE_CLOSING) 616 netif_tx_wake_queue(txq); 617 __netif_tx_unlock(txq); 618 } 619 } 620 621 static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping, 622 gfp_t gfp) 623 { 624 struct device *dev = &bp->pdev->dev; 625 struct page *page; 626 627 page = alloc_page(gfp); 628 if (!page) 629 return NULL; 630 631 *mapping = dma_map_page_attrs(dev, page, 0, PAGE_SIZE, bp->rx_dir, 632 DMA_ATTR_WEAK_ORDERING); 633 if (dma_mapping_error(dev, *mapping)) { 634 __free_page(page); 635 return NULL; 636 } 637 *mapping += bp->rx_dma_offset; 638 return page; 639 } 640 641 static inline u8 *__bnxt_alloc_rx_data(struct bnxt *bp, dma_addr_t *mapping, 642 gfp_t gfp) 643 { 644 u8 *data; 645 struct pci_dev *pdev = bp->pdev; 646 647 data = kmalloc(bp->rx_buf_size, gfp); 648 if (!data) 649 return NULL; 650 651 *mapping = dma_map_single_attrs(&pdev->dev, data + bp->rx_dma_offset, 652 bp->rx_buf_use_size, bp->rx_dir, 653 DMA_ATTR_WEAK_ORDERING); 654 655 if (dma_mapping_error(&pdev->dev, *mapping)) { 656 kfree(data); 657 data = NULL; 658 } 659 return data; 660 } 661 662 int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, 663 u16 prod, gfp_t gfp) 664 { 665 struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 666 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod]; 667 dma_addr_t mapping; 668 669 if (BNXT_RX_PAGE_MODE(bp)) { 670 struct page *page = __bnxt_alloc_rx_page(bp, &mapping, gfp); 671 672 if (!page) 673 return -ENOMEM; 674 675 rx_buf->data = page; 676 rx_buf->data_ptr = page_address(page) + bp->rx_offset; 677 } else { 678 u8 *data = __bnxt_alloc_rx_data(bp, &mapping, gfp); 679 680 if (!data) 681 return -ENOMEM; 682 683 rx_buf->data = data; 684 rx_buf->data_ptr = data + bp->rx_offset; 685 } 686 rx_buf->mapping = mapping; 687 688 rxbd->rx_bd_haddr = cpu_to_le64(mapping); 689 return 0; 690 } 691 692 void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data) 693 { 694 u16 prod = rxr->rx_prod; 695 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf; 696 struct rx_bd *cons_bd, *prod_bd; 697 698 prod_rx_buf = &rxr->rx_buf_ring[prod]; 699 cons_rx_buf = &rxr->rx_buf_ring[cons]; 700 701 prod_rx_buf->data = data; 702 prod_rx_buf->data_ptr = cons_rx_buf->data_ptr; 703 704 prod_rx_buf->mapping = cons_rx_buf->mapping; 705 706 prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 707 cons_bd = &rxr->rx_desc_ring[RX_RING(cons)][RX_IDX(cons)]; 708 709 prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr; 710 } 711 712 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx) 713 { 714 u16 next, max = rxr->rx_agg_bmap_size; 715 716 next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx); 717 if (next >= max) 718 next = find_first_zero_bit(rxr->rx_agg_bmap, max); 719 return next; 720 } 721 722 static inline int bnxt_alloc_rx_page(struct bnxt *bp, 723 struct bnxt_rx_ring_info *rxr, 724 u16 prod, gfp_t gfp) 725 { 726 struct rx_bd *rxbd = 727 &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 728 struct bnxt_sw_rx_agg_bd *rx_agg_buf; 729 struct pci_dev *pdev = bp->pdev; 730 struct page *page; 731 dma_addr_t mapping; 732 u16 sw_prod = rxr->rx_sw_agg_prod; 733 unsigned int offset = 0; 734 735 if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) { 736 page = rxr->rx_page; 737 if (!page) { 738 page = alloc_page(gfp); 739 if (!page) 740 return -ENOMEM; 741 rxr->rx_page = page; 742 rxr->rx_page_offset = 0; 743 } 744 offset = rxr->rx_page_offset; 745 rxr->rx_page_offset += BNXT_RX_PAGE_SIZE; 746 if (rxr->rx_page_offset == PAGE_SIZE) 747 rxr->rx_page = NULL; 748 else 749 get_page(page); 750 } else { 751 page = alloc_page(gfp); 752 if (!page) 753 return -ENOMEM; 754 } 755 756 mapping = dma_map_page_attrs(&pdev->dev, page, offset, 757 BNXT_RX_PAGE_SIZE, PCI_DMA_FROMDEVICE, 758 DMA_ATTR_WEAK_ORDERING); 759 if (dma_mapping_error(&pdev->dev, mapping)) { 760 __free_page(page); 761 return -EIO; 762 } 763 764 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap))) 765 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod); 766 767 __set_bit(sw_prod, rxr->rx_agg_bmap); 768 rx_agg_buf = &rxr->rx_agg_ring[sw_prod]; 769 rxr->rx_sw_agg_prod = NEXT_RX_AGG(sw_prod); 770 771 rx_agg_buf->page = page; 772 rx_agg_buf->offset = offset; 773 rx_agg_buf->mapping = mapping; 774 rxbd->rx_bd_haddr = cpu_to_le64(mapping); 775 rxbd->rx_bd_opaque = sw_prod; 776 return 0; 777 } 778 779 static void bnxt_reuse_rx_agg_bufs(struct bnxt_napi *bnapi, u16 cp_cons, 780 u32 agg_bufs) 781 { 782 struct bnxt *bp = bnapi->bp; 783 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 784 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 785 u16 prod = rxr->rx_agg_prod; 786 u16 sw_prod = rxr->rx_sw_agg_prod; 787 u32 i; 788 789 for (i = 0; i < agg_bufs; i++) { 790 u16 cons; 791 struct rx_agg_cmp *agg; 792 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf; 793 struct rx_bd *prod_bd; 794 struct page *page; 795 796 agg = (struct rx_agg_cmp *) 797 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 798 cons = agg->rx_agg_cmp_opaque; 799 __clear_bit(cons, rxr->rx_agg_bmap); 800 801 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap))) 802 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod); 803 804 __set_bit(sw_prod, rxr->rx_agg_bmap); 805 prod_rx_buf = &rxr->rx_agg_ring[sw_prod]; 806 cons_rx_buf = &rxr->rx_agg_ring[cons]; 807 808 /* It is possible for sw_prod to be equal to cons, so 809 * set cons_rx_buf->page to NULL first. 810 */ 811 page = cons_rx_buf->page; 812 cons_rx_buf->page = NULL; 813 prod_rx_buf->page = page; 814 prod_rx_buf->offset = cons_rx_buf->offset; 815 816 prod_rx_buf->mapping = cons_rx_buf->mapping; 817 818 prod_bd = &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 819 820 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping); 821 prod_bd->rx_bd_opaque = sw_prod; 822 823 prod = NEXT_RX_AGG(prod); 824 sw_prod = NEXT_RX_AGG(sw_prod); 825 cp_cons = NEXT_CMP(cp_cons); 826 } 827 rxr->rx_agg_prod = prod; 828 rxr->rx_sw_agg_prod = sw_prod; 829 } 830 831 static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp, 832 struct bnxt_rx_ring_info *rxr, 833 u16 cons, void *data, u8 *data_ptr, 834 dma_addr_t dma_addr, 835 unsigned int offset_and_len) 836 { 837 unsigned int payload = offset_and_len >> 16; 838 unsigned int len = offset_and_len & 0xffff; 839 struct skb_frag_struct *frag; 840 struct page *page = data; 841 u16 prod = rxr->rx_prod; 842 struct sk_buff *skb; 843 int off, err; 844 845 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC); 846 if (unlikely(err)) { 847 bnxt_reuse_rx_data(rxr, cons, data); 848 return NULL; 849 } 850 dma_addr -= bp->rx_dma_offset; 851 dma_unmap_page_attrs(&bp->pdev->dev, dma_addr, PAGE_SIZE, bp->rx_dir, 852 DMA_ATTR_WEAK_ORDERING); 853 854 if (unlikely(!payload)) 855 payload = eth_get_headlen(data_ptr, len); 856 857 skb = napi_alloc_skb(&rxr->bnapi->napi, payload); 858 if (!skb) { 859 __free_page(page); 860 return NULL; 861 } 862 863 off = (void *)data_ptr - page_address(page); 864 skb_add_rx_frag(skb, 0, page, off, len, PAGE_SIZE); 865 memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN, 866 payload + NET_IP_ALIGN); 867 868 frag = &skb_shinfo(skb)->frags[0]; 869 skb_frag_size_sub(frag, payload); 870 frag->page_offset += payload; 871 skb->data_len -= payload; 872 skb->tail += payload; 873 874 return skb; 875 } 876 877 static struct sk_buff *bnxt_rx_skb(struct bnxt *bp, 878 struct bnxt_rx_ring_info *rxr, u16 cons, 879 void *data, u8 *data_ptr, 880 dma_addr_t dma_addr, 881 unsigned int offset_and_len) 882 { 883 u16 prod = rxr->rx_prod; 884 struct sk_buff *skb; 885 int err; 886 887 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC); 888 if (unlikely(err)) { 889 bnxt_reuse_rx_data(rxr, cons, data); 890 return NULL; 891 } 892 893 skb = build_skb(data, 0); 894 dma_unmap_single_attrs(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size, 895 bp->rx_dir, DMA_ATTR_WEAK_ORDERING); 896 if (!skb) { 897 kfree(data); 898 return NULL; 899 } 900 901 skb_reserve(skb, bp->rx_offset); 902 skb_put(skb, offset_and_len & 0xffff); 903 return skb; 904 } 905 906 static struct sk_buff *bnxt_rx_pages(struct bnxt *bp, struct bnxt_napi *bnapi, 907 struct sk_buff *skb, u16 cp_cons, 908 u32 agg_bufs) 909 { 910 struct pci_dev *pdev = bp->pdev; 911 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 912 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 913 u16 prod = rxr->rx_agg_prod; 914 u32 i; 915 916 for (i = 0; i < agg_bufs; i++) { 917 u16 cons, frag_len; 918 struct rx_agg_cmp *agg; 919 struct bnxt_sw_rx_agg_bd *cons_rx_buf; 920 struct page *page; 921 dma_addr_t mapping; 922 923 agg = (struct rx_agg_cmp *) 924 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 925 cons = agg->rx_agg_cmp_opaque; 926 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) & 927 RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT; 928 929 cons_rx_buf = &rxr->rx_agg_ring[cons]; 930 skb_fill_page_desc(skb, i, cons_rx_buf->page, 931 cons_rx_buf->offset, frag_len); 932 __clear_bit(cons, rxr->rx_agg_bmap); 933 934 /* It is possible for bnxt_alloc_rx_page() to allocate 935 * a sw_prod index that equals the cons index, so we 936 * need to clear the cons entry now. 937 */ 938 mapping = cons_rx_buf->mapping; 939 page = cons_rx_buf->page; 940 cons_rx_buf->page = NULL; 941 942 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) { 943 struct skb_shared_info *shinfo; 944 unsigned int nr_frags; 945 946 shinfo = skb_shinfo(skb); 947 nr_frags = --shinfo->nr_frags; 948 __skb_frag_set_page(&shinfo->frags[nr_frags], NULL); 949 950 dev_kfree_skb(skb); 951 952 cons_rx_buf->page = page; 953 954 /* Update prod since possibly some pages have been 955 * allocated already. 956 */ 957 rxr->rx_agg_prod = prod; 958 bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs - i); 959 return NULL; 960 } 961 962 dma_unmap_page_attrs(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE, 963 PCI_DMA_FROMDEVICE, 964 DMA_ATTR_WEAK_ORDERING); 965 966 skb->data_len += frag_len; 967 skb->len += frag_len; 968 skb->truesize += PAGE_SIZE; 969 970 prod = NEXT_RX_AGG(prod); 971 cp_cons = NEXT_CMP(cp_cons); 972 } 973 rxr->rx_agg_prod = prod; 974 return skb; 975 } 976 977 static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 978 u8 agg_bufs, u32 *raw_cons) 979 { 980 u16 last; 981 struct rx_agg_cmp *agg; 982 983 *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs); 984 last = RING_CMP(*raw_cons); 985 agg = (struct rx_agg_cmp *) 986 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)]; 987 return RX_AGG_CMP_VALID(agg, *raw_cons); 988 } 989 990 static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data, 991 unsigned int len, 992 dma_addr_t mapping) 993 { 994 struct bnxt *bp = bnapi->bp; 995 struct pci_dev *pdev = bp->pdev; 996 struct sk_buff *skb; 997 998 skb = napi_alloc_skb(&bnapi->napi, len); 999 if (!skb) 1000 return NULL; 1001 1002 dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copy_thresh, 1003 bp->rx_dir); 1004 1005 memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN, 1006 len + NET_IP_ALIGN); 1007 1008 dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copy_thresh, 1009 bp->rx_dir); 1010 1011 skb_put(skb, len); 1012 return skb; 1013 } 1014 1015 static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_napi *bnapi, 1016 u32 *raw_cons, void *cmp) 1017 { 1018 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 1019 struct rx_cmp *rxcmp = cmp; 1020 u32 tmp_raw_cons = *raw_cons; 1021 u8 cmp_type, agg_bufs = 0; 1022 1023 cmp_type = RX_CMP_TYPE(rxcmp); 1024 1025 if (cmp_type == CMP_TYPE_RX_L2_CMP) { 1026 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) & 1027 RX_CMP_AGG_BUFS) >> 1028 RX_CMP_AGG_BUFS_SHIFT; 1029 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) { 1030 struct rx_tpa_end_cmp *tpa_end = cmp; 1031 1032 agg_bufs = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) & 1033 RX_TPA_END_CMP_AGG_BUFS) >> 1034 RX_TPA_END_CMP_AGG_BUFS_SHIFT; 1035 } 1036 1037 if (agg_bufs) { 1038 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons)) 1039 return -EBUSY; 1040 } 1041 *raw_cons = tmp_raw_cons; 1042 return 0; 1043 } 1044 1045 static void bnxt_queue_sp_work(struct bnxt *bp) 1046 { 1047 if (BNXT_PF(bp)) 1048 queue_work(bnxt_pf_wq, &bp->sp_task); 1049 else 1050 schedule_work(&bp->sp_task); 1051 } 1052 1053 static void bnxt_cancel_sp_work(struct bnxt *bp) 1054 { 1055 if (BNXT_PF(bp)) 1056 flush_workqueue(bnxt_pf_wq); 1057 else 1058 cancel_work_sync(&bp->sp_task); 1059 } 1060 1061 static void bnxt_sched_reset(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) 1062 { 1063 if (!rxr->bnapi->in_reset) { 1064 rxr->bnapi->in_reset = true; 1065 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event); 1066 bnxt_queue_sp_work(bp); 1067 } 1068 rxr->rx_next_cons = 0xffff; 1069 } 1070 1071 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, 1072 struct rx_tpa_start_cmp *tpa_start, 1073 struct rx_tpa_start_cmp_ext *tpa_start1) 1074 { 1075 u8 agg_id = TPA_START_AGG_ID(tpa_start); 1076 u16 cons, prod; 1077 struct bnxt_tpa_info *tpa_info; 1078 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf; 1079 struct rx_bd *prod_bd; 1080 dma_addr_t mapping; 1081 1082 cons = tpa_start->rx_tpa_start_cmp_opaque; 1083 prod = rxr->rx_prod; 1084 cons_rx_buf = &rxr->rx_buf_ring[cons]; 1085 prod_rx_buf = &rxr->rx_buf_ring[prod]; 1086 tpa_info = &rxr->rx_tpa[agg_id]; 1087 1088 if (unlikely(cons != rxr->rx_next_cons)) { 1089 bnxt_sched_reset(bp, rxr); 1090 return; 1091 } 1092 /* Store cfa_code in tpa_info to use in tpa_end 1093 * completion processing. 1094 */ 1095 tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1); 1096 prod_rx_buf->data = tpa_info->data; 1097 prod_rx_buf->data_ptr = tpa_info->data_ptr; 1098 1099 mapping = tpa_info->mapping; 1100 prod_rx_buf->mapping = mapping; 1101 1102 prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 1103 1104 prod_bd->rx_bd_haddr = cpu_to_le64(mapping); 1105 1106 tpa_info->data = cons_rx_buf->data; 1107 tpa_info->data_ptr = cons_rx_buf->data_ptr; 1108 cons_rx_buf->data = NULL; 1109 tpa_info->mapping = cons_rx_buf->mapping; 1110 1111 tpa_info->len = 1112 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >> 1113 RX_TPA_START_CMP_LEN_SHIFT; 1114 if (likely(TPA_START_HASH_VALID(tpa_start))) { 1115 u32 hash_type = TPA_START_HASH_TYPE(tpa_start); 1116 1117 tpa_info->hash_type = PKT_HASH_TYPE_L4; 1118 tpa_info->gso_type = SKB_GSO_TCPV4; 1119 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */ 1120 if (hash_type == 3 || TPA_START_IS_IPV6(tpa_start1)) 1121 tpa_info->gso_type = SKB_GSO_TCPV6; 1122 tpa_info->rss_hash = 1123 le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash); 1124 } else { 1125 tpa_info->hash_type = PKT_HASH_TYPE_NONE; 1126 tpa_info->gso_type = 0; 1127 if (netif_msg_rx_err(bp)) 1128 netdev_warn(bp->dev, "TPA packet without valid hash\n"); 1129 } 1130 tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2); 1131 tpa_info->metadata = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata); 1132 tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info); 1133 1134 rxr->rx_prod = NEXT_RX(prod); 1135 cons = NEXT_RX(cons); 1136 rxr->rx_next_cons = NEXT_RX(cons); 1137 cons_rx_buf = &rxr->rx_buf_ring[cons]; 1138 1139 bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data); 1140 rxr->rx_prod = NEXT_RX(rxr->rx_prod); 1141 cons_rx_buf->data = NULL; 1142 } 1143 1144 static void bnxt_abort_tpa(struct bnxt *bp, struct bnxt_napi *bnapi, 1145 u16 cp_cons, u32 agg_bufs) 1146 { 1147 if (agg_bufs) 1148 bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs); 1149 } 1150 1151 static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info, 1152 int payload_off, int tcp_ts, 1153 struct sk_buff *skb) 1154 { 1155 #ifdef CONFIG_INET 1156 struct tcphdr *th; 1157 int len, nw_off; 1158 u16 outer_ip_off, inner_ip_off, inner_mac_off; 1159 u32 hdr_info = tpa_info->hdr_info; 1160 bool loopback = false; 1161 1162 inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info); 1163 inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info); 1164 outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info); 1165 1166 /* If the packet is an internal loopback packet, the offsets will 1167 * have an extra 4 bytes. 1168 */ 1169 if (inner_mac_off == 4) { 1170 loopback = true; 1171 } else if (inner_mac_off > 4) { 1172 __be16 proto = *((__be16 *)(skb->data + inner_ip_off - 1173 ETH_HLEN - 2)); 1174 1175 /* We only support inner iPv4/ipv6. If we don't see the 1176 * correct protocol ID, it must be a loopback packet where 1177 * the offsets are off by 4. 1178 */ 1179 if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6)) 1180 loopback = true; 1181 } 1182 if (loopback) { 1183 /* internal loopback packet, subtract all offsets by 4 */ 1184 inner_ip_off -= 4; 1185 inner_mac_off -= 4; 1186 outer_ip_off -= 4; 1187 } 1188 1189 nw_off = inner_ip_off - ETH_HLEN; 1190 skb_set_network_header(skb, nw_off); 1191 if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) { 1192 struct ipv6hdr *iph = ipv6_hdr(skb); 1193 1194 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr)); 1195 len = skb->len - skb_transport_offset(skb); 1196 th = tcp_hdr(skb); 1197 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0); 1198 } else { 1199 struct iphdr *iph = ip_hdr(skb); 1200 1201 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr)); 1202 len = skb->len - skb_transport_offset(skb); 1203 th = tcp_hdr(skb); 1204 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0); 1205 } 1206 1207 if (inner_mac_off) { /* tunnel */ 1208 struct udphdr *uh = NULL; 1209 __be16 proto = *((__be16 *)(skb->data + outer_ip_off - 1210 ETH_HLEN - 2)); 1211 1212 if (proto == htons(ETH_P_IP)) { 1213 struct iphdr *iph = (struct iphdr *)skb->data; 1214 1215 if (iph->protocol == IPPROTO_UDP) 1216 uh = (struct udphdr *)(iph + 1); 1217 } else { 1218 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data; 1219 1220 if (iph->nexthdr == IPPROTO_UDP) 1221 uh = (struct udphdr *)(iph + 1); 1222 } 1223 if (uh) { 1224 if (uh->check) 1225 skb_shinfo(skb)->gso_type |= 1226 SKB_GSO_UDP_TUNNEL_CSUM; 1227 else 1228 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL; 1229 } 1230 } 1231 #endif 1232 return skb; 1233 } 1234 1235 #define BNXT_IPV4_HDR_SIZE (sizeof(struct iphdr) + sizeof(struct tcphdr)) 1236 #define BNXT_IPV6_HDR_SIZE (sizeof(struct ipv6hdr) + sizeof(struct tcphdr)) 1237 1238 static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info, 1239 int payload_off, int tcp_ts, 1240 struct sk_buff *skb) 1241 { 1242 #ifdef CONFIG_INET 1243 struct tcphdr *th; 1244 int len, nw_off, tcp_opt_len = 0; 1245 1246 if (tcp_ts) 1247 tcp_opt_len = 12; 1248 1249 if (tpa_info->gso_type == SKB_GSO_TCPV4) { 1250 struct iphdr *iph; 1251 1252 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len - 1253 ETH_HLEN; 1254 skb_set_network_header(skb, nw_off); 1255 iph = ip_hdr(skb); 1256 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr)); 1257 len = skb->len - skb_transport_offset(skb); 1258 th = tcp_hdr(skb); 1259 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0); 1260 } else if (tpa_info->gso_type == SKB_GSO_TCPV6) { 1261 struct ipv6hdr *iph; 1262 1263 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len - 1264 ETH_HLEN; 1265 skb_set_network_header(skb, nw_off); 1266 iph = ipv6_hdr(skb); 1267 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr)); 1268 len = skb->len - skb_transport_offset(skb); 1269 th = tcp_hdr(skb); 1270 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0); 1271 } else { 1272 dev_kfree_skb_any(skb); 1273 return NULL; 1274 } 1275 1276 if (nw_off) { /* tunnel */ 1277 struct udphdr *uh = NULL; 1278 1279 if (skb->protocol == htons(ETH_P_IP)) { 1280 struct iphdr *iph = (struct iphdr *)skb->data; 1281 1282 if (iph->protocol == IPPROTO_UDP) 1283 uh = (struct udphdr *)(iph + 1); 1284 } else { 1285 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data; 1286 1287 if (iph->nexthdr == IPPROTO_UDP) 1288 uh = (struct udphdr *)(iph + 1); 1289 } 1290 if (uh) { 1291 if (uh->check) 1292 skb_shinfo(skb)->gso_type |= 1293 SKB_GSO_UDP_TUNNEL_CSUM; 1294 else 1295 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL; 1296 } 1297 } 1298 #endif 1299 return skb; 1300 } 1301 1302 static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp, 1303 struct bnxt_tpa_info *tpa_info, 1304 struct rx_tpa_end_cmp *tpa_end, 1305 struct rx_tpa_end_cmp_ext *tpa_end1, 1306 struct sk_buff *skb) 1307 { 1308 #ifdef CONFIG_INET 1309 int payload_off; 1310 u16 segs; 1311 1312 segs = TPA_END_TPA_SEGS(tpa_end); 1313 if (segs == 1) 1314 return skb; 1315 1316 NAPI_GRO_CB(skb)->count = segs; 1317 skb_shinfo(skb)->gso_size = 1318 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len); 1319 skb_shinfo(skb)->gso_type = tpa_info->gso_type; 1320 payload_off = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) & 1321 RX_TPA_END_CMP_PAYLOAD_OFFSET) >> 1322 RX_TPA_END_CMP_PAYLOAD_OFFSET_SHIFT; 1323 skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb); 1324 if (likely(skb)) 1325 tcp_gro_complete(skb); 1326 #endif 1327 return skb; 1328 } 1329 1330 /* Given the cfa_code of a received packet determine which 1331 * netdev (vf-rep or PF) the packet is destined to. 1332 */ 1333 static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code) 1334 { 1335 struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code); 1336 1337 /* if vf-rep dev is NULL, the must belongs to the PF */ 1338 return dev ? dev : bp->dev; 1339 } 1340 1341 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp, 1342 struct bnxt_napi *bnapi, 1343 u32 *raw_cons, 1344 struct rx_tpa_end_cmp *tpa_end, 1345 struct rx_tpa_end_cmp_ext *tpa_end1, 1346 u8 *event) 1347 { 1348 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 1349 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1350 u8 agg_id = TPA_END_AGG_ID(tpa_end); 1351 u8 *data_ptr, agg_bufs; 1352 u16 cp_cons = RING_CMP(*raw_cons); 1353 unsigned int len; 1354 struct bnxt_tpa_info *tpa_info; 1355 dma_addr_t mapping; 1356 struct sk_buff *skb; 1357 void *data; 1358 1359 if (unlikely(bnapi->in_reset)) { 1360 int rc = bnxt_discard_rx(bp, bnapi, raw_cons, tpa_end); 1361 1362 if (rc < 0) 1363 return ERR_PTR(-EBUSY); 1364 return NULL; 1365 } 1366 1367 tpa_info = &rxr->rx_tpa[agg_id]; 1368 data = tpa_info->data; 1369 data_ptr = tpa_info->data_ptr; 1370 prefetch(data_ptr); 1371 len = tpa_info->len; 1372 mapping = tpa_info->mapping; 1373 1374 agg_bufs = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) & 1375 RX_TPA_END_CMP_AGG_BUFS) >> RX_TPA_END_CMP_AGG_BUFS_SHIFT; 1376 1377 if (agg_bufs) { 1378 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons)) 1379 return ERR_PTR(-EBUSY); 1380 1381 *event |= BNXT_AGG_EVENT; 1382 cp_cons = NEXT_CMP(cp_cons); 1383 } 1384 1385 if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) { 1386 bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs); 1387 if (agg_bufs > MAX_SKB_FRAGS) 1388 netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n", 1389 agg_bufs, (int)MAX_SKB_FRAGS); 1390 return NULL; 1391 } 1392 1393 if (len <= bp->rx_copy_thresh) { 1394 skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping); 1395 if (!skb) { 1396 bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs); 1397 return NULL; 1398 } 1399 } else { 1400 u8 *new_data; 1401 dma_addr_t new_mapping; 1402 1403 new_data = __bnxt_alloc_rx_data(bp, &new_mapping, GFP_ATOMIC); 1404 if (!new_data) { 1405 bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs); 1406 return NULL; 1407 } 1408 1409 tpa_info->data = new_data; 1410 tpa_info->data_ptr = new_data + bp->rx_offset; 1411 tpa_info->mapping = new_mapping; 1412 1413 skb = build_skb(data, 0); 1414 dma_unmap_single_attrs(&bp->pdev->dev, mapping, 1415 bp->rx_buf_use_size, bp->rx_dir, 1416 DMA_ATTR_WEAK_ORDERING); 1417 1418 if (!skb) { 1419 kfree(data); 1420 bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs); 1421 return NULL; 1422 } 1423 skb_reserve(skb, bp->rx_offset); 1424 skb_put(skb, len); 1425 } 1426 1427 if (agg_bufs) { 1428 skb = bnxt_rx_pages(bp, bnapi, skb, cp_cons, agg_bufs); 1429 if (!skb) { 1430 /* Page reuse already handled by bnxt_rx_pages(). */ 1431 return NULL; 1432 } 1433 } 1434 1435 skb->protocol = 1436 eth_type_trans(skb, bnxt_get_pkt_dev(bp, tpa_info->cfa_code)); 1437 1438 if (tpa_info->hash_type != PKT_HASH_TYPE_NONE) 1439 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type); 1440 1441 if ((tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) && 1442 (skb->dev->features & NETIF_F_HW_VLAN_CTAG_RX)) { 1443 u16 vlan_proto = tpa_info->metadata >> 1444 RX_CMP_FLAGS2_METADATA_TPID_SFT; 1445 u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_TCI_MASK; 1446 1447 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag); 1448 } 1449 1450 skb_checksum_none_assert(skb); 1451 if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) { 1452 skb->ip_summed = CHECKSUM_UNNECESSARY; 1453 skb->csum_level = 1454 (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3; 1455 } 1456 1457 if (TPA_END_GRO(tpa_end)) 1458 skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb); 1459 1460 return skb; 1461 } 1462 1463 static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi, 1464 struct sk_buff *skb) 1465 { 1466 if (skb->dev != bp->dev) { 1467 /* this packet belongs to a vf-rep */ 1468 bnxt_vf_rep_rx(bp, skb); 1469 return; 1470 } 1471 skb_record_rx_queue(skb, bnapi->index); 1472 napi_gro_receive(&bnapi->napi, skb); 1473 } 1474 1475 /* returns the following: 1476 * 1 - 1 packet successfully received 1477 * 0 - successful TPA_START, packet not completed yet 1478 * -EBUSY - completion ring does not have all the agg buffers yet 1479 * -ENOMEM - packet aborted due to out of memory 1480 * -EIO - packet aborted due to hw error indicated in BD 1481 */ 1482 static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_napi *bnapi, u32 *raw_cons, 1483 u8 *event) 1484 { 1485 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 1486 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1487 struct net_device *dev = bp->dev; 1488 struct rx_cmp *rxcmp; 1489 struct rx_cmp_ext *rxcmp1; 1490 u32 tmp_raw_cons = *raw_cons; 1491 u16 cfa_code, cons, prod, cp_cons = RING_CMP(tmp_raw_cons); 1492 struct bnxt_sw_rx_bd *rx_buf; 1493 unsigned int len; 1494 u8 *data_ptr, agg_bufs, cmp_type; 1495 dma_addr_t dma_addr; 1496 struct sk_buff *skb; 1497 void *data; 1498 int rc = 0; 1499 u32 misc; 1500 1501 rxcmp = (struct rx_cmp *) 1502 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1503 1504 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons); 1505 cp_cons = RING_CMP(tmp_raw_cons); 1506 rxcmp1 = (struct rx_cmp_ext *) 1507 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1508 1509 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons)) 1510 return -EBUSY; 1511 1512 cmp_type = RX_CMP_TYPE(rxcmp); 1513 1514 prod = rxr->rx_prod; 1515 1516 if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) { 1517 bnxt_tpa_start(bp, rxr, (struct rx_tpa_start_cmp *)rxcmp, 1518 (struct rx_tpa_start_cmp_ext *)rxcmp1); 1519 1520 *event |= BNXT_RX_EVENT; 1521 goto next_rx_no_prod_no_len; 1522 1523 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) { 1524 skb = bnxt_tpa_end(bp, bnapi, &tmp_raw_cons, 1525 (struct rx_tpa_end_cmp *)rxcmp, 1526 (struct rx_tpa_end_cmp_ext *)rxcmp1, event); 1527 1528 if (IS_ERR(skb)) 1529 return -EBUSY; 1530 1531 rc = -ENOMEM; 1532 if (likely(skb)) { 1533 bnxt_deliver_skb(bp, bnapi, skb); 1534 rc = 1; 1535 } 1536 *event |= BNXT_RX_EVENT; 1537 goto next_rx_no_prod_no_len; 1538 } 1539 1540 cons = rxcmp->rx_cmp_opaque; 1541 rx_buf = &rxr->rx_buf_ring[cons]; 1542 data = rx_buf->data; 1543 data_ptr = rx_buf->data_ptr; 1544 if (unlikely(cons != rxr->rx_next_cons)) { 1545 int rc1 = bnxt_discard_rx(bp, bnapi, raw_cons, rxcmp); 1546 1547 bnxt_sched_reset(bp, rxr); 1548 return rc1; 1549 } 1550 prefetch(data_ptr); 1551 1552 misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1); 1553 agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT; 1554 1555 if (agg_bufs) { 1556 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons)) 1557 return -EBUSY; 1558 1559 cp_cons = NEXT_CMP(cp_cons); 1560 *event |= BNXT_AGG_EVENT; 1561 } 1562 *event |= BNXT_RX_EVENT; 1563 1564 rx_buf->data = NULL; 1565 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) { 1566 bnxt_reuse_rx_data(rxr, cons, data); 1567 if (agg_bufs) 1568 bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs); 1569 1570 rc = -EIO; 1571 goto next_rx; 1572 } 1573 1574 len = le32_to_cpu(rxcmp->rx_cmp_len_flags_type) >> RX_CMP_LEN_SHIFT; 1575 dma_addr = rx_buf->mapping; 1576 1577 if (bnxt_rx_xdp(bp, rxr, cons, data, &data_ptr, &len, event)) { 1578 rc = 1; 1579 goto next_rx; 1580 } 1581 1582 if (len <= bp->rx_copy_thresh) { 1583 skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr); 1584 bnxt_reuse_rx_data(rxr, cons, data); 1585 if (!skb) { 1586 rc = -ENOMEM; 1587 goto next_rx; 1588 } 1589 } else { 1590 u32 payload; 1591 1592 if (rx_buf->data_ptr == data_ptr) 1593 payload = misc & RX_CMP_PAYLOAD_OFFSET; 1594 else 1595 payload = 0; 1596 skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr, 1597 payload | len); 1598 if (!skb) { 1599 rc = -ENOMEM; 1600 goto next_rx; 1601 } 1602 } 1603 1604 if (agg_bufs) { 1605 skb = bnxt_rx_pages(bp, bnapi, skb, cp_cons, agg_bufs); 1606 if (!skb) { 1607 rc = -ENOMEM; 1608 goto next_rx; 1609 } 1610 } 1611 1612 if (RX_CMP_HASH_VALID(rxcmp)) { 1613 u32 hash_type = RX_CMP_HASH_TYPE(rxcmp); 1614 enum pkt_hash_types type = PKT_HASH_TYPE_L4; 1615 1616 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */ 1617 if (hash_type != 1 && hash_type != 3) 1618 type = PKT_HASH_TYPE_L3; 1619 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type); 1620 } 1621 1622 cfa_code = RX_CMP_CFA_CODE(rxcmp1); 1623 skb->protocol = eth_type_trans(skb, bnxt_get_pkt_dev(bp, cfa_code)); 1624 1625 if ((rxcmp1->rx_cmp_flags2 & 1626 cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) && 1627 (skb->dev->features & NETIF_F_HW_VLAN_CTAG_RX)) { 1628 u32 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data); 1629 u16 vtag = meta_data & RX_CMP_FLAGS2_METADATA_TCI_MASK; 1630 u16 vlan_proto = meta_data >> RX_CMP_FLAGS2_METADATA_TPID_SFT; 1631 1632 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag); 1633 } 1634 1635 skb_checksum_none_assert(skb); 1636 if (RX_CMP_L4_CS_OK(rxcmp1)) { 1637 if (dev->features & NETIF_F_RXCSUM) { 1638 skb->ip_summed = CHECKSUM_UNNECESSARY; 1639 skb->csum_level = RX_CMP_ENCAP(rxcmp1); 1640 } 1641 } else { 1642 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) { 1643 if (dev->features & NETIF_F_RXCSUM) 1644 cpr->rx_l4_csum_errors++; 1645 } 1646 } 1647 1648 bnxt_deliver_skb(bp, bnapi, skb); 1649 rc = 1; 1650 1651 next_rx: 1652 rxr->rx_prod = NEXT_RX(prod); 1653 rxr->rx_next_cons = NEXT_RX(cons); 1654 1655 cpr->rx_packets += 1; 1656 cpr->rx_bytes += len; 1657 1658 next_rx_no_prod_no_len: 1659 *raw_cons = tmp_raw_cons; 1660 1661 return rc; 1662 } 1663 1664 /* In netpoll mode, if we are using a combined completion ring, we need to 1665 * discard the rx packets and recycle the buffers. 1666 */ 1667 static int bnxt_force_rx_discard(struct bnxt *bp, struct bnxt_napi *bnapi, 1668 u32 *raw_cons, u8 *event) 1669 { 1670 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 1671 u32 tmp_raw_cons = *raw_cons; 1672 struct rx_cmp_ext *rxcmp1; 1673 struct rx_cmp *rxcmp; 1674 u16 cp_cons; 1675 u8 cmp_type; 1676 1677 cp_cons = RING_CMP(tmp_raw_cons); 1678 rxcmp = (struct rx_cmp *) 1679 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1680 1681 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons); 1682 cp_cons = RING_CMP(tmp_raw_cons); 1683 rxcmp1 = (struct rx_cmp_ext *) 1684 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1685 1686 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons)) 1687 return -EBUSY; 1688 1689 cmp_type = RX_CMP_TYPE(rxcmp); 1690 if (cmp_type == CMP_TYPE_RX_L2_CMP) { 1691 rxcmp1->rx_cmp_cfa_code_errors_v2 |= 1692 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR); 1693 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) { 1694 struct rx_tpa_end_cmp_ext *tpa_end1; 1695 1696 tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1; 1697 tpa_end1->rx_tpa_end_cmp_errors_v2 |= 1698 cpu_to_le32(RX_TPA_END_CMP_ERRORS); 1699 } 1700 return bnxt_rx_pkt(bp, bnapi, raw_cons, event); 1701 } 1702 1703 #define BNXT_GET_EVENT_PORT(data) \ 1704 ((data) & \ 1705 ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK) 1706 1707 static int bnxt_async_event_process(struct bnxt *bp, 1708 struct hwrm_async_event_cmpl *cmpl) 1709 { 1710 u16 event_id = le16_to_cpu(cmpl->event_id); 1711 1712 /* TODO CHIMP_FW: Define event id's for link change, error etc */ 1713 switch (event_id) { 1714 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: { 1715 u32 data1 = le32_to_cpu(cmpl->event_data1); 1716 struct bnxt_link_info *link_info = &bp->link_info; 1717 1718 if (BNXT_VF(bp)) 1719 goto async_event_process_exit; 1720 1721 /* print unsupported speed warning in forced speed mode only */ 1722 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED) && 1723 (data1 & 0x20000)) { 1724 u16 fw_speed = link_info->force_link_speed; 1725 u32 speed = bnxt_fw_to_ethtool_speed(fw_speed); 1726 1727 if (speed != SPEED_UNKNOWN) 1728 netdev_warn(bp->dev, "Link speed %d no longer supported\n", 1729 speed); 1730 } 1731 set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event); 1732 } 1733 /* fall through */ 1734 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE: 1735 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event); 1736 break; 1737 case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD: 1738 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event); 1739 break; 1740 case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: { 1741 u32 data1 = le32_to_cpu(cmpl->event_data1); 1742 u16 port_id = BNXT_GET_EVENT_PORT(data1); 1743 1744 if (BNXT_VF(bp)) 1745 break; 1746 1747 if (bp->pf.port_id != port_id) 1748 break; 1749 1750 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event); 1751 break; 1752 } 1753 case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE: 1754 if (BNXT_PF(bp)) 1755 goto async_event_process_exit; 1756 set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event); 1757 break; 1758 default: 1759 goto async_event_process_exit; 1760 } 1761 bnxt_queue_sp_work(bp); 1762 async_event_process_exit: 1763 bnxt_ulp_async_events(bp, cmpl); 1764 return 0; 1765 } 1766 1767 static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp) 1768 { 1769 u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id; 1770 struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp; 1771 struct hwrm_fwd_req_cmpl *fwd_req_cmpl = 1772 (struct hwrm_fwd_req_cmpl *)txcmp; 1773 1774 switch (cmpl_type) { 1775 case CMPL_BASE_TYPE_HWRM_DONE: 1776 seq_id = le16_to_cpu(h_cmpl->sequence_id); 1777 if (seq_id == bp->hwrm_intr_seq_id) 1778 bp->hwrm_intr_seq_id = HWRM_SEQ_ID_INVALID; 1779 else 1780 netdev_err(bp->dev, "Invalid hwrm seq id %d\n", seq_id); 1781 break; 1782 1783 case CMPL_BASE_TYPE_HWRM_FWD_REQ: 1784 vf_id = le16_to_cpu(fwd_req_cmpl->source_id); 1785 1786 if ((vf_id < bp->pf.first_vf_id) || 1787 (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) { 1788 netdev_err(bp->dev, "Msg contains invalid VF id %x\n", 1789 vf_id); 1790 return -EINVAL; 1791 } 1792 1793 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap); 1794 set_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event); 1795 bnxt_queue_sp_work(bp); 1796 break; 1797 1798 case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT: 1799 bnxt_async_event_process(bp, 1800 (struct hwrm_async_event_cmpl *)txcmp); 1801 1802 default: 1803 break; 1804 } 1805 1806 return 0; 1807 } 1808 1809 static irqreturn_t bnxt_msix(int irq, void *dev_instance) 1810 { 1811 struct bnxt_napi *bnapi = dev_instance; 1812 struct bnxt *bp = bnapi->bp; 1813 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 1814 u32 cons = RING_CMP(cpr->cp_raw_cons); 1815 1816 cpr->event_ctr++; 1817 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]); 1818 napi_schedule(&bnapi->napi); 1819 return IRQ_HANDLED; 1820 } 1821 1822 static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr) 1823 { 1824 u32 raw_cons = cpr->cp_raw_cons; 1825 u16 cons = RING_CMP(raw_cons); 1826 struct tx_cmp *txcmp; 1827 1828 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]; 1829 1830 return TX_CMP_VALID(txcmp, raw_cons); 1831 } 1832 1833 static irqreturn_t bnxt_inta(int irq, void *dev_instance) 1834 { 1835 struct bnxt_napi *bnapi = dev_instance; 1836 struct bnxt *bp = bnapi->bp; 1837 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 1838 u32 cons = RING_CMP(cpr->cp_raw_cons); 1839 u32 int_status; 1840 1841 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]); 1842 1843 if (!bnxt_has_work(bp, cpr)) { 1844 int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS); 1845 /* return if erroneous interrupt */ 1846 if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id))) 1847 return IRQ_NONE; 1848 } 1849 1850 /* disable ring IRQ */ 1851 BNXT_CP_DB_IRQ_DIS(cpr->cp_doorbell); 1852 1853 /* Return here if interrupt is shared and is disabled. */ 1854 if (unlikely(atomic_read(&bp->intr_sem) != 0)) 1855 return IRQ_HANDLED; 1856 1857 napi_schedule(&bnapi->napi); 1858 return IRQ_HANDLED; 1859 } 1860 1861 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_napi *bnapi, int budget) 1862 { 1863 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 1864 u32 raw_cons = cpr->cp_raw_cons; 1865 u32 cons; 1866 int tx_pkts = 0; 1867 int rx_pkts = 0; 1868 u8 event = 0; 1869 struct tx_cmp *txcmp; 1870 1871 while (1) { 1872 int rc; 1873 1874 cons = RING_CMP(raw_cons); 1875 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]; 1876 1877 if (!TX_CMP_VALID(txcmp, raw_cons)) 1878 break; 1879 1880 /* The valid test of the entry must be done first before 1881 * reading any further. 1882 */ 1883 dma_rmb(); 1884 if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) { 1885 tx_pkts++; 1886 /* return full budget so NAPI will complete. */ 1887 if (unlikely(tx_pkts > bp->tx_wake_thresh)) 1888 rx_pkts = budget; 1889 } else if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) { 1890 if (likely(budget)) 1891 rc = bnxt_rx_pkt(bp, bnapi, &raw_cons, &event); 1892 else 1893 rc = bnxt_force_rx_discard(bp, bnapi, &raw_cons, 1894 &event); 1895 if (likely(rc >= 0)) 1896 rx_pkts += rc; 1897 /* Increment rx_pkts when rc is -ENOMEM to count towards 1898 * the NAPI budget. Otherwise, we may potentially loop 1899 * here forever if we consistently cannot allocate 1900 * buffers. 1901 */ 1902 else if (rc == -ENOMEM && budget) 1903 rx_pkts++; 1904 else if (rc == -EBUSY) /* partial completion */ 1905 break; 1906 } else if (unlikely((TX_CMP_TYPE(txcmp) == 1907 CMPL_BASE_TYPE_HWRM_DONE) || 1908 (TX_CMP_TYPE(txcmp) == 1909 CMPL_BASE_TYPE_HWRM_FWD_REQ) || 1910 (TX_CMP_TYPE(txcmp) == 1911 CMPL_BASE_TYPE_HWRM_ASYNC_EVENT))) { 1912 bnxt_hwrm_handler(bp, txcmp); 1913 } 1914 raw_cons = NEXT_RAW_CMP(raw_cons); 1915 1916 if (rx_pkts == budget) 1917 break; 1918 } 1919 1920 if (event & BNXT_TX_EVENT) { 1921 struct bnxt_tx_ring_info *txr = bnapi->tx_ring; 1922 void __iomem *db = txr->tx_doorbell; 1923 u16 prod = txr->tx_prod; 1924 1925 /* Sync BD data before updating doorbell */ 1926 wmb(); 1927 1928 bnxt_db_write_relaxed(bp, db, DB_KEY_TX | prod); 1929 } 1930 1931 cpr->cp_raw_cons = raw_cons; 1932 /* ACK completion ring before freeing tx ring and producing new 1933 * buffers in rx/agg rings to prevent overflowing the completion 1934 * ring. 1935 */ 1936 BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons); 1937 1938 if (tx_pkts) 1939 bnapi->tx_int(bp, bnapi, tx_pkts); 1940 1941 if (event & BNXT_RX_EVENT) { 1942 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1943 1944 bnxt_db_write(bp, rxr->rx_doorbell, DB_KEY_RX | rxr->rx_prod); 1945 if (event & BNXT_AGG_EVENT) 1946 bnxt_db_write(bp, rxr->rx_agg_doorbell, 1947 DB_KEY_RX | rxr->rx_agg_prod); 1948 } 1949 return rx_pkts; 1950 } 1951 1952 static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget) 1953 { 1954 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi); 1955 struct bnxt *bp = bnapi->bp; 1956 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 1957 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1958 struct tx_cmp *txcmp; 1959 struct rx_cmp_ext *rxcmp1; 1960 u32 cp_cons, tmp_raw_cons; 1961 u32 raw_cons = cpr->cp_raw_cons; 1962 u32 rx_pkts = 0; 1963 u8 event = 0; 1964 1965 while (1) { 1966 int rc; 1967 1968 cp_cons = RING_CMP(raw_cons); 1969 txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1970 1971 if (!TX_CMP_VALID(txcmp, raw_cons)) 1972 break; 1973 1974 if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) { 1975 tmp_raw_cons = NEXT_RAW_CMP(raw_cons); 1976 cp_cons = RING_CMP(tmp_raw_cons); 1977 rxcmp1 = (struct rx_cmp_ext *) 1978 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1979 1980 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons)) 1981 break; 1982 1983 /* force an error to recycle the buffer */ 1984 rxcmp1->rx_cmp_cfa_code_errors_v2 |= 1985 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR); 1986 1987 rc = bnxt_rx_pkt(bp, bnapi, &raw_cons, &event); 1988 if (likely(rc == -EIO) && budget) 1989 rx_pkts++; 1990 else if (rc == -EBUSY) /* partial completion */ 1991 break; 1992 } else if (unlikely(TX_CMP_TYPE(txcmp) == 1993 CMPL_BASE_TYPE_HWRM_DONE)) { 1994 bnxt_hwrm_handler(bp, txcmp); 1995 } else { 1996 netdev_err(bp->dev, 1997 "Invalid completion received on special ring\n"); 1998 } 1999 raw_cons = NEXT_RAW_CMP(raw_cons); 2000 2001 if (rx_pkts == budget) 2002 break; 2003 } 2004 2005 cpr->cp_raw_cons = raw_cons; 2006 BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons); 2007 bnxt_db_write(bp, rxr->rx_doorbell, DB_KEY_RX | rxr->rx_prod); 2008 2009 if (event & BNXT_AGG_EVENT) 2010 bnxt_db_write(bp, rxr->rx_agg_doorbell, 2011 DB_KEY_RX | rxr->rx_agg_prod); 2012 2013 if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) { 2014 napi_complete_done(napi, rx_pkts); 2015 BNXT_CP_DB_REARM(cpr->cp_doorbell, cpr->cp_raw_cons); 2016 } 2017 return rx_pkts; 2018 } 2019 2020 static int bnxt_poll(struct napi_struct *napi, int budget) 2021 { 2022 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi); 2023 struct bnxt *bp = bnapi->bp; 2024 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2025 int work_done = 0; 2026 2027 while (1) { 2028 work_done += bnxt_poll_work(bp, bnapi, budget - work_done); 2029 2030 if (work_done >= budget) 2031 break; 2032 2033 if (!bnxt_has_work(bp, cpr)) { 2034 if (napi_complete_done(napi, work_done)) 2035 BNXT_CP_DB_REARM(cpr->cp_doorbell, 2036 cpr->cp_raw_cons); 2037 break; 2038 } 2039 } 2040 if (bp->flags & BNXT_FLAG_DIM) { 2041 struct net_dim_sample dim_sample; 2042 2043 net_dim_sample(cpr->event_ctr, 2044 cpr->rx_packets, 2045 cpr->rx_bytes, 2046 &dim_sample); 2047 net_dim(&cpr->dim, dim_sample); 2048 } 2049 mmiowb(); 2050 return work_done; 2051 } 2052 2053 static void bnxt_free_tx_skbs(struct bnxt *bp) 2054 { 2055 int i, max_idx; 2056 struct pci_dev *pdev = bp->pdev; 2057 2058 if (!bp->tx_ring) 2059 return; 2060 2061 max_idx = bp->tx_nr_pages * TX_DESC_CNT; 2062 for (i = 0; i < bp->tx_nr_rings; i++) { 2063 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 2064 int j; 2065 2066 for (j = 0; j < max_idx;) { 2067 struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j]; 2068 struct sk_buff *skb = tx_buf->skb; 2069 int k, last; 2070 2071 if (!skb) { 2072 j++; 2073 continue; 2074 } 2075 2076 tx_buf->skb = NULL; 2077 2078 if (tx_buf->is_push) { 2079 dev_kfree_skb(skb); 2080 j += 2; 2081 continue; 2082 } 2083 2084 dma_unmap_single(&pdev->dev, 2085 dma_unmap_addr(tx_buf, mapping), 2086 skb_headlen(skb), 2087 PCI_DMA_TODEVICE); 2088 2089 last = tx_buf->nr_frags; 2090 j += 2; 2091 for (k = 0; k < last; k++, j++) { 2092 int ring_idx = j & bp->tx_ring_mask; 2093 skb_frag_t *frag = &skb_shinfo(skb)->frags[k]; 2094 2095 tx_buf = &txr->tx_buf_ring[ring_idx]; 2096 dma_unmap_page( 2097 &pdev->dev, 2098 dma_unmap_addr(tx_buf, mapping), 2099 skb_frag_size(frag), PCI_DMA_TODEVICE); 2100 } 2101 dev_kfree_skb(skb); 2102 } 2103 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i)); 2104 } 2105 } 2106 2107 static void bnxt_free_rx_skbs(struct bnxt *bp) 2108 { 2109 int i, max_idx, max_agg_idx; 2110 struct pci_dev *pdev = bp->pdev; 2111 2112 if (!bp->rx_ring) 2113 return; 2114 2115 max_idx = bp->rx_nr_pages * RX_DESC_CNT; 2116 max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT; 2117 for (i = 0; i < bp->rx_nr_rings; i++) { 2118 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 2119 int j; 2120 2121 if (rxr->rx_tpa) { 2122 for (j = 0; j < MAX_TPA; j++) { 2123 struct bnxt_tpa_info *tpa_info = 2124 &rxr->rx_tpa[j]; 2125 u8 *data = tpa_info->data; 2126 2127 if (!data) 2128 continue; 2129 2130 dma_unmap_single_attrs(&pdev->dev, 2131 tpa_info->mapping, 2132 bp->rx_buf_use_size, 2133 bp->rx_dir, 2134 DMA_ATTR_WEAK_ORDERING); 2135 2136 tpa_info->data = NULL; 2137 2138 kfree(data); 2139 } 2140 } 2141 2142 for (j = 0; j < max_idx; j++) { 2143 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[j]; 2144 dma_addr_t mapping = rx_buf->mapping; 2145 void *data = rx_buf->data; 2146 2147 if (!data) 2148 continue; 2149 2150 rx_buf->data = NULL; 2151 2152 if (BNXT_RX_PAGE_MODE(bp)) { 2153 mapping -= bp->rx_dma_offset; 2154 dma_unmap_page_attrs(&pdev->dev, mapping, 2155 PAGE_SIZE, bp->rx_dir, 2156 DMA_ATTR_WEAK_ORDERING); 2157 __free_page(data); 2158 } else { 2159 dma_unmap_single_attrs(&pdev->dev, mapping, 2160 bp->rx_buf_use_size, 2161 bp->rx_dir, 2162 DMA_ATTR_WEAK_ORDERING); 2163 kfree(data); 2164 } 2165 } 2166 2167 for (j = 0; j < max_agg_idx; j++) { 2168 struct bnxt_sw_rx_agg_bd *rx_agg_buf = 2169 &rxr->rx_agg_ring[j]; 2170 struct page *page = rx_agg_buf->page; 2171 2172 if (!page) 2173 continue; 2174 2175 dma_unmap_page_attrs(&pdev->dev, rx_agg_buf->mapping, 2176 BNXT_RX_PAGE_SIZE, 2177 PCI_DMA_FROMDEVICE, 2178 DMA_ATTR_WEAK_ORDERING); 2179 2180 rx_agg_buf->page = NULL; 2181 __clear_bit(j, rxr->rx_agg_bmap); 2182 2183 __free_page(page); 2184 } 2185 if (rxr->rx_page) { 2186 __free_page(rxr->rx_page); 2187 rxr->rx_page = NULL; 2188 } 2189 } 2190 } 2191 2192 static void bnxt_free_skbs(struct bnxt *bp) 2193 { 2194 bnxt_free_tx_skbs(bp); 2195 bnxt_free_rx_skbs(bp); 2196 } 2197 2198 static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_struct *ring) 2199 { 2200 struct pci_dev *pdev = bp->pdev; 2201 int i; 2202 2203 for (i = 0; i < ring->nr_pages; i++) { 2204 if (!ring->pg_arr[i]) 2205 continue; 2206 2207 dma_free_coherent(&pdev->dev, ring->page_size, 2208 ring->pg_arr[i], ring->dma_arr[i]); 2209 2210 ring->pg_arr[i] = NULL; 2211 } 2212 if (ring->pg_tbl) { 2213 dma_free_coherent(&pdev->dev, ring->nr_pages * 8, 2214 ring->pg_tbl, ring->pg_tbl_map); 2215 ring->pg_tbl = NULL; 2216 } 2217 if (ring->vmem_size && *ring->vmem) { 2218 vfree(*ring->vmem); 2219 *ring->vmem = NULL; 2220 } 2221 } 2222 2223 static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_struct *ring) 2224 { 2225 int i; 2226 struct pci_dev *pdev = bp->pdev; 2227 2228 if (ring->nr_pages > 1) { 2229 ring->pg_tbl = dma_alloc_coherent(&pdev->dev, 2230 ring->nr_pages * 8, 2231 &ring->pg_tbl_map, 2232 GFP_KERNEL); 2233 if (!ring->pg_tbl) 2234 return -ENOMEM; 2235 } 2236 2237 for (i = 0; i < ring->nr_pages; i++) { 2238 ring->pg_arr[i] = dma_alloc_coherent(&pdev->dev, 2239 ring->page_size, 2240 &ring->dma_arr[i], 2241 GFP_KERNEL); 2242 if (!ring->pg_arr[i]) 2243 return -ENOMEM; 2244 2245 if (ring->nr_pages > 1) 2246 ring->pg_tbl[i] = cpu_to_le64(ring->dma_arr[i]); 2247 } 2248 2249 if (ring->vmem_size) { 2250 *ring->vmem = vzalloc(ring->vmem_size); 2251 if (!(*ring->vmem)) 2252 return -ENOMEM; 2253 } 2254 return 0; 2255 } 2256 2257 static void bnxt_free_rx_rings(struct bnxt *bp) 2258 { 2259 int i; 2260 2261 if (!bp->rx_ring) 2262 return; 2263 2264 for (i = 0; i < bp->rx_nr_rings; i++) { 2265 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 2266 struct bnxt_ring_struct *ring; 2267 2268 if (rxr->xdp_prog) 2269 bpf_prog_put(rxr->xdp_prog); 2270 2271 if (xdp_rxq_info_is_reg(&rxr->xdp_rxq)) 2272 xdp_rxq_info_unreg(&rxr->xdp_rxq); 2273 2274 kfree(rxr->rx_tpa); 2275 rxr->rx_tpa = NULL; 2276 2277 kfree(rxr->rx_agg_bmap); 2278 rxr->rx_agg_bmap = NULL; 2279 2280 ring = &rxr->rx_ring_struct; 2281 bnxt_free_ring(bp, ring); 2282 2283 ring = &rxr->rx_agg_ring_struct; 2284 bnxt_free_ring(bp, ring); 2285 } 2286 } 2287 2288 static int bnxt_alloc_rx_rings(struct bnxt *bp) 2289 { 2290 int i, rc, agg_rings = 0, tpa_rings = 0; 2291 2292 if (!bp->rx_ring) 2293 return -ENOMEM; 2294 2295 if (bp->flags & BNXT_FLAG_AGG_RINGS) 2296 agg_rings = 1; 2297 2298 if (bp->flags & BNXT_FLAG_TPA) 2299 tpa_rings = 1; 2300 2301 for (i = 0; i < bp->rx_nr_rings; i++) { 2302 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 2303 struct bnxt_ring_struct *ring; 2304 2305 ring = &rxr->rx_ring_struct; 2306 2307 rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i); 2308 if (rc < 0) 2309 return rc; 2310 2311 rc = bnxt_alloc_ring(bp, ring); 2312 if (rc) 2313 return rc; 2314 2315 if (agg_rings) { 2316 u16 mem_size; 2317 2318 ring = &rxr->rx_agg_ring_struct; 2319 rc = bnxt_alloc_ring(bp, ring); 2320 if (rc) 2321 return rc; 2322 2323 ring->grp_idx = i; 2324 rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1; 2325 mem_size = rxr->rx_agg_bmap_size / 8; 2326 rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL); 2327 if (!rxr->rx_agg_bmap) 2328 return -ENOMEM; 2329 2330 if (tpa_rings) { 2331 rxr->rx_tpa = kcalloc(MAX_TPA, 2332 sizeof(struct bnxt_tpa_info), 2333 GFP_KERNEL); 2334 if (!rxr->rx_tpa) 2335 return -ENOMEM; 2336 } 2337 } 2338 } 2339 return 0; 2340 } 2341 2342 static void bnxt_free_tx_rings(struct bnxt *bp) 2343 { 2344 int i; 2345 struct pci_dev *pdev = bp->pdev; 2346 2347 if (!bp->tx_ring) 2348 return; 2349 2350 for (i = 0; i < bp->tx_nr_rings; i++) { 2351 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 2352 struct bnxt_ring_struct *ring; 2353 2354 if (txr->tx_push) { 2355 dma_free_coherent(&pdev->dev, bp->tx_push_size, 2356 txr->tx_push, txr->tx_push_mapping); 2357 txr->tx_push = NULL; 2358 } 2359 2360 ring = &txr->tx_ring_struct; 2361 2362 bnxt_free_ring(bp, ring); 2363 } 2364 } 2365 2366 static int bnxt_alloc_tx_rings(struct bnxt *bp) 2367 { 2368 int i, j, rc; 2369 struct pci_dev *pdev = bp->pdev; 2370 2371 bp->tx_push_size = 0; 2372 if (bp->tx_push_thresh) { 2373 int push_size; 2374 2375 push_size = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) + 2376 bp->tx_push_thresh); 2377 2378 if (push_size > 256) { 2379 push_size = 0; 2380 bp->tx_push_thresh = 0; 2381 } 2382 2383 bp->tx_push_size = push_size; 2384 } 2385 2386 for (i = 0, j = 0; i < bp->tx_nr_rings; i++) { 2387 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 2388 struct bnxt_ring_struct *ring; 2389 u8 qidx; 2390 2391 ring = &txr->tx_ring_struct; 2392 2393 rc = bnxt_alloc_ring(bp, ring); 2394 if (rc) 2395 return rc; 2396 2397 ring->grp_idx = txr->bnapi->index; 2398 if (bp->tx_push_size) { 2399 dma_addr_t mapping; 2400 2401 /* One pre-allocated DMA buffer to backup 2402 * TX push operation 2403 */ 2404 txr->tx_push = dma_alloc_coherent(&pdev->dev, 2405 bp->tx_push_size, 2406 &txr->tx_push_mapping, 2407 GFP_KERNEL); 2408 2409 if (!txr->tx_push) 2410 return -ENOMEM; 2411 2412 mapping = txr->tx_push_mapping + 2413 sizeof(struct tx_push_bd); 2414 txr->data_mapping = cpu_to_le64(mapping); 2415 2416 memset(txr->tx_push, 0, sizeof(struct tx_push_bd)); 2417 } 2418 qidx = bp->tc_to_qidx[j]; 2419 ring->queue_id = bp->q_info[qidx].queue_id; 2420 if (i < bp->tx_nr_rings_xdp) 2421 continue; 2422 if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1)) 2423 j++; 2424 } 2425 return 0; 2426 } 2427 2428 static void bnxt_free_cp_rings(struct bnxt *bp) 2429 { 2430 int i; 2431 2432 if (!bp->bnapi) 2433 return; 2434 2435 for (i = 0; i < bp->cp_nr_rings; i++) { 2436 struct bnxt_napi *bnapi = bp->bnapi[i]; 2437 struct bnxt_cp_ring_info *cpr; 2438 struct bnxt_ring_struct *ring; 2439 2440 if (!bnapi) 2441 continue; 2442 2443 cpr = &bnapi->cp_ring; 2444 ring = &cpr->cp_ring_struct; 2445 2446 bnxt_free_ring(bp, ring); 2447 } 2448 } 2449 2450 static int bnxt_alloc_cp_rings(struct bnxt *bp) 2451 { 2452 int i, rc, ulp_base_vec, ulp_msix; 2453 2454 ulp_msix = bnxt_get_ulp_msix_num(bp); 2455 ulp_base_vec = bnxt_get_ulp_msix_base(bp); 2456 for (i = 0; i < bp->cp_nr_rings; i++) { 2457 struct bnxt_napi *bnapi = bp->bnapi[i]; 2458 struct bnxt_cp_ring_info *cpr; 2459 struct bnxt_ring_struct *ring; 2460 2461 if (!bnapi) 2462 continue; 2463 2464 cpr = &bnapi->cp_ring; 2465 ring = &cpr->cp_ring_struct; 2466 2467 rc = bnxt_alloc_ring(bp, ring); 2468 if (rc) 2469 return rc; 2470 2471 if (ulp_msix && i >= ulp_base_vec) 2472 ring->map_idx = i + ulp_msix; 2473 else 2474 ring->map_idx = i; 2475 } 2476 return 0; 2477 } 2478 2479 static void bnxt_init_ring_struct(struct bnxt *bp) 2480 { 2481 int i; 2482 2483 for (i = 0; i < bp->cp_nr_rings; i++) { 2484 struct bnxt_napi *bnapi = bp->bnapi[i]; 2485 struct bnxt_cp_ring_info *cpr; 2486 struct bnxt_rx_ring_info *rxr; 2487 struct bnxt_tx_ring_info *txr; 2488 struct bnxt_ring_struct *ring; 2489 2490 if (!bnapi) 2491 continue; 2492 2493 cpr = &bnapi->cp_ring; 2494 ring = &cpr->cp_ring_struct; 2495 ring->nr_pages = bp->cp_nr_pages; 2496 ring->page_size = HW_CMPD_RING_SIZE; 2497 ring->pg_arr = (void **)cpr->cp_desc_ring; 2498 ring->dma_arr = cpr->cp_desc_mapping; 2499 ring->vmem_size = 0; 2500 2501 rxr = bnapi->rx_ring; 2502 if (!rxr) 2503 goto skip_rx; 2504 2505 ring = &rxr->rx_ring_struct; 2506 ring->nr_pages = bp->rx_nr_pages; 2507 ring->page_size = HW_RXBD_RING_SIZE; 2508 ring->pg_arr = (void **)rxr->rx_desc_ring; 2509 ring->dma_arr = rxr->rx_desc_mapping; 2510 ring->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages; 2511 ring->vmem = (void **)&rxr->rx_buf_ring; 2512 2513 ring = &rxr->rx_agg_ring_struct; 2514 ring->nr_pages = bp->rx_agg_nr_pages; 2515 ring->page_size = HW_RXBD_RING_SIZE; 2516 ring->pg_arr = (void **)rxr->rx_agg_desc_ring; 2517 ring->dma_arr = rxr->rx_agg_desc_mapping; 2518 ring->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages; 2519 ring->vmem = (void **)&rxr->rx_agg_ring; 2520 2521 skip_rx: 2522 txr = bnapi->tx_ring; 2523 if (!txr) 2524 continue; 2525 2526 ring = &txr->tx_ring_struct; 2527 ring->nr_pages = bp->tx_nr_pages; 2528 ring->page_size = HW_RXBD_RING_SIZE; 2529 ring->pg_arr = (void **)txr->tx_desc_ring; 2530 ring->dma_arr = txr->tx_desc_mapping; 2531 ring->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages; 2532 ring->vmem = (void **)&txr->tx_buf_ring; 2533 } 2534 } 2535 2536 static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type) 2537 { 2538 int i; 2539 u32 prod; 2540 struct rx_bd **rx_buf_ring; 2541 2542 rx_buf_ring = (struct rx_bd **)ring->pg_arr; 2543 for (i = 0, prod = 0; i < ring->nr_pages; i++) { 2544 int j; 2545 struct rx_bd *rxbd; 2546 2547 rxbd = rx_buf_ring[i]; 2548 if (!rxbd) 2549 continue; 2550 2551 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) { 2552 rxbd->rx_bd_len_flags_type = cpu_to_le32(type); 2553 rxbd->rx_bd_opaque = prod; 2554 } 2555 } 2556 } 2557 2558 static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr) 2559 { 2560 struct net_device *dev = bp->dev; 2561 struct bnxt_rx_ring_info *rxr; 2562 struct bnxt_ring_struct *ring; 2563 u32 prod, type; 2564 int i; 2565 2566 type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) | 2567 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP; 2568 2569 if (NET_IP_ALIGN == 2) 2570 type |= RX_BD_FLAGS_SOP; 2571 2572 rxr = &bp->rx_ring[ring_nr]; 2573 ring = &rxr->rx_ring_struct; 2574 bnxt_init_rxbd_pages(ring, type); 2575 2576 if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) { 2577 rxr->xdp_prog = bpf_prog_add(bp->xdp_prog, 1); 2578 if (IS_ERR(rxr->xdp_prog)) { 2579 int rc = PTR_ERR(rxr->xdp_prog); 2580 2581 rxr->xdp_prog = NULL; 2582 return rc; 2583 } 2584 } 2585 prod = rxr->rx_prod; 2586 for (i = 0; i < bp->rx_ring_size; i++) { 2587 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL) != 0) { 2588 netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n", 2589 ring_nr, i, bp->rx_ring_size); 2590 break; 2591 } 2592 prod = NEXT_RX(prod); 2593 } 2594 rxr->rx_prod = prod; 2595 ring->fw_ring_id = INVALID_HW_RING_ID; 2596 2597 ring = &rxr->rx_agg_ring_struct; 2598 ring->fw_ring_id = INVALID_HW_RING_ID; 2599 2600 if (!(bp->flags & BNXT_FLAG_AGG_RINGS)) 2601 return 0; 2602 2603 type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) | 2604 RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP; 2605 2606 bnxt_init_rxbd_pages(ring, type); 2607 2608 prod = rxr->rx_agg_prod; 2609 for (i = 0; i < bp->rx_agg_ring_size; i++) { 2610 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL) != 0) { 2611 netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n", 2612 ring_nr, i, bp->rx_ring_size); 2613 break; 2614 } 2615 prod = NEXT_RX_AGG(prod); 2616 } 2617 rxr->rx_agg_prod = prod; 2618 2619 if (bp->flags & BNXT_FLAG_TPA) { 2620 if (rxr->rx_tpa) { 2621 u8 *data; 2622 dma_addr_t mapping; 2623 2624 for (i = 0; i < MAX_TPA; i++) { 2625 data = __bnxt_alloc_rx_data(bp, &mapping, 2626 GFP_KERNEL); 2627 if (!data) 2628 return -ENOMEM; 2629 2630 rxr->rx_tpa[i].data = data; 2631 rxr->rx_tpa[i].data_ptr = data + bp->rx_offset; 2632 rxr->rx_tpa[i].mapping = mapping; 2633 } 2634 } else { 2635 netdev_err(bp->dev, "No resource allocated for LRO/GRO\n"); 2636 return -ENOMEM; 2637 } 2638 } 2639 2640 return 0; 2641 } 2642 2643 static void bnxt_init_cp_rings(struct bnxt *bp) 2644 { 2645 int i; 2646 2647 for (i = 0; i < bp->cp_nr_rings; i++) { 2648 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring; 2649 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 2650 2651 ring->fw_ring_id = INVALID_HW_RING_ID; 2652 cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks; 2653 cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs; 2654 } 2655 } 2656 2657 static int bnxt_init_rx_rings(struct bnxt *bp) 2658 { 2659 int i, rc = 0; 2660 2661 if (BNXT_RX_PAGE_MODE(bp)) { 2662 bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM; 2663 bp->rx_dma_offset = XDP_PACKET_HEADROOM; 2664 } else { 2665 bp->rx_offset = BNXT_RX_OFFSET; 2666 bp->rx_dma_offset = BNXT_RX_DMA_OFFSET; 2667 } 2668 2669 for (i = 0; i < bp->rx_nr_rings; i++) { 2670 rc = bnxt_init_one_rx_ring(bp, i); 2671 if (rc) 2672 break; 2673 } 2674 2675 return rc; 2676 } 2677 2678 static int bnxt_init_tx_rings(struct bnxt *bp) 2679 { 2680 u16 i; 2681 2682 bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2, 2683 MAX_SKB_FRAGS + 1); 2684 2685 for (i = 0; i < bp->tx_nr_rings; i++) { 2686 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 2687 struct bnxt_ring_struct *ring = &txr->tx_ring_struct; 2688 2689 ring->fw_ring_id = INVALID_HW_RING_ID; 2690 } 2691 2692 return 0; 2693 } 2694 2695 static void bnxt_free_ring_grps(struct bnxt *bp) 2696 { 2697 kfree(bp->grp_info); 2698 bp->grp_info = NULL; 2699 } 2700 2701 static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init) 2702 { 2703 int i; 2704 2705 if (irq_re_init) { 2706 bp->grp_info = kcalloc(bp->cp_nr_rings, 2707 sizeof(struct bnxt_ring_grp_info), 2708 GFP_KERNEL); 2709 if (!bp->grp_info) 2710 return -ENOMEM; 2711 } 2712 for (i = 0; i < bp->cp_nr_rings; i++) { 2713 if (irq_re_init) 2714 bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID; 2715 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID; 2716 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID; 2717 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID; 2718 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID; 2719 } 2720 return 0; 2721 } 2722 2723 static void bnxt_free_vnics(struct bnxt *bp) 2724 { 2725 kfree(bp->vnic_info); 2726 bp->vnic_info = NULL; 2727 bp->nr_vnics = 0; 2728 } 2729 2730 static int bnxt_alloc_vnics(struct bnxt *bp) 2731 { 2732 int num_vnics = 1; 2733 2734 #ifdef CONFIG_RFS_ACCEL 2735 if (bp->flags & BNXT_FLAG_RFS) 2736 num_vnics += bp->rx_nr_rings; 2737 #endif 2738 2739 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 2740 num_vnics++; 2741 2742 bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info), 2743 GFP_KERNEL); 2744 if (!bp->vnic_info) 2745 return -ENOMEM; 2746 2747 bp->nr_vnics = num_vnics; 2748 return 0; 2749 } 2750 2751 static void bnxt_init_vnics(struct bnxt *bp) 2752 { 2753 int i; 2754 2755 for (i = 0; i < bp->nr_vnics; i++) { 2756 struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; 2757 2758 vnic->fw_vnic_id = INVALID_HW_RING_ID; 2759 vnic->fw_rss_cos_lb_ctx[0] = INVALID_HW_RING_ID; 2760 vnic->fw_rss_cos_lb_ctx[1] = INVALID_HW_RING_ID; 2761 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID; 2762 2763 if (bp->vnic_info[i].rss_hash_key) { 2764 if (i == 0) 2765 prandom_bytes(vnic->rss_hash_key, 2766 HW_HASH_KEY_SIZE); 2767 else 2768 memcpy(vnic->rss_hash_key, 2769 bp->vnic_info[0].rss_hash_key, 2770 HW_HASH_KEY_SIZE); 2771 } 2772 } 2773 } 2774 2775 static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg) 2776 { 2777 int pages; 2778 2779 pages = ring_size / desc_per_pg; 2780 2781 if (!pages) 2782 return 1; 2783 2784 pages++; 2785 2786 while (pages & (pages - 1)) 2787 pages++; 2788 2789 return pages; 2790 } 2791 2792 void bnxt_set_tpa_flags(struct bnxt *bp) 2793 { 2794 bp->flags &= ~BNXT_FLAG_TPA; 2795 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS) 2796 return; 2797 if (bp->dev->features & NETIF_F_LRO) 2798 bp->flags |= BNXT_FLAG_LRO; 2799 else if (bp->dev->features & NETIF_F_GRO_HW) 2800 bp->flags |= BNXT_FLAG_GRO; 2801 } 2802 2803 /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must 2804 * be set on entry. 2805 */ 2806 void bnxt_set_ring_params(struct bnxt *bp) 2807 { 2808 u32 ring_size, rx_size, rx_space; 2809 u32 agg_factor = 0, agg_ring_size = 0; 2810 2811 /* 8 for CRC and VLAN */ 2812 rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8); 2813 2814 rx_space = rx_size + NET_SKB_PAD + 2815 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 2816 2817 bp->rx_copy_thresh = BNXT_RX_COPY_THRESH; 2818 ring_size = bp->rx_ring_size; 2819 bp->rx_agg_ring_size = 0; 2820 bp->rx_agg_nr_pages = 0; 2821 2822 if (bp->flags & BNXT_FLAG_TPA) 2823 agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE); 2824 2825 bp->flags &= ~BNXT_FLAG_JUMBO; 2826 if (rx_space > PAGE_SIZE && !(bp->flags & BNXT_FLAG_NO_AGG_RINGS)) { 2827 u32 jumbo_factor; 2828 2829 bp->flags |= BNXT_FLAG_JUMBO; 2830 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT; 2831 if (jumbo_factor > agg_factor) 2832 agg_factor = jumbo_factor; 2833 } 2834 agg_ring_size = ring_size * agg_factor; 2835 2836 if (agg_ring_size) { 2837 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size, 2838 RX_DESC_CNT); 2839 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) { 2840 u32 tmp = agg_ring_size; 2841 2842 bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES; 2843 agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1; 2844 netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n", 2845 tmp, agg_ring_size); 2846 } 2847 bp->rx_agg_ring_size = agg_ring_size; 2848 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1; 2849 rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN); 2850 rx_space = rx_size + NET_SKB_PAD + 2851 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 2852 } 2853 2854 bp->rx_buf_use_size = rx_size; 2855 bp->rx_buf_size = rx_space; 2856 2857 bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT); 2858 bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1; 2859 2860 ring_size = bp->tx_ring_size; 2861 bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT); 2862 bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1; 2863 2864 ring_size = bp->rx_ring_size * (2 + agg_factor) + bp->tx_ring_size; 2865 bp->cp_ring_size = ring_size; 2866 2867 bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT); 2868 if (bp->cp_nr_pages > MAX_CP_PAGES) { 2869 bp->cp_nr_pages = MAX_CP_PAGES; 2870 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1; 2871 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n", 2872 ring_size, bp->cp_ring_size); 2873 } 2874 bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT; 2875 bp->cp_ring_mask = bp->cp_bit - 1; 2876 } 2877 2878 /* Changing allocation mode of RX rings. 2879 * TODO: Update when extending xdp_rxq_info to support allocation modes. 2880 */ 2881 int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode) 2882 { 2883 if (page_mode) { 2884 if (bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU) 2885 return -EOPNOTSUPP; 2886 bp->dev->max_mtu = 2887 min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU); 2888 bp->flags &= ~BNXT_FLAG_AGG_RINGS; 2889 bp->flags |= BNXT_FLAG_NO_AGG_RINGS | BNXT_FLAG_RX_PAGE_MODE; 2890 bp->rx_dir = DMA_BIDIRECTIONAL; 2891 bp->rx_skb_func = bnxt_rx_page_skb; 2892 /* Disable LRO or GRO_HW */ 2893 netdev_update_features(bp->dev); 2894 } else { 2895 bp->dev->max_mtu = bp->max_mtu; 2896 bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE; 2897 bp->rx_dir = DMA_FROM_DEVICE; 2898 bp->rx_skb_func = bnxt_rx_skb; 2899 } 2900 return 0; 2901 } 2902 2903 static void bnxt_free_vnic_attributes(struct bnxt *bp) 2904 { 2905 int i; 2906 struct bnxt_vnic_info *vnic; 2907 struct pci_dev *pdev = bp->pdev; 2908 2909 if (!bp->vnic_info) 2910 return; 2911 2912 for (i = 0; i < bp->nr_vnics; i++) { 2913 vnic = &bp->vnic_info[i]; 2914 2915 kfree(vnic->fw_grp_ids); 2916 vnic->fw_grp_ids = NULL; 2917 2918 kfree(vnic->uc_list); 2919 vnic->uc_list = NULL; 2920 2921 if (vnic->mc_list) { 2922 dma_free_coherent(&pdev->dev, vnic->mc_list_size, 2923 vnic->mc_list, vnic->mc_list_mapping); 2924 vnic->mc_list = NULL; 2925 } 2926 2927 if (vnic->rss_table) { 2928 dma_free_coherent(&pdev->dev, PAGE_SIZE, 2929 vnic->rss_table, 2930 vnic->rss_table_dma_addr); 2931 vnic->rss_table = NULL; 2932 } 2933 2934 vnic->rss_hash_key = NULL; 2935 vnic->flags = 0; 2936 } 2937 } 2938 2939 static int bnxt_alloc_vnic_attributes(struct bnxt *bp) 2940 { 2941 int i, rc = 0, size; 2942 struct bnxt_vnic_info *vnic; 2943 struct pci_dev *pdev = bp->pdev; 2944 int max_rings; 2945 2946 for (i = 0; i < bp->nr_vnics; i++) { 2947 vnic = &bp->vnic_info[i]; 2948 2949 if (vnic->flags & BNXT_VNIC_UCAST_FLAG) { 2950 int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN; 2951 2952 if (mem_size > 0) { 2953 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL); 2954 if (!vnic->uc_list) { 2955 rc = -ENOMEM; 2956 goto out; 2957 } 2958 } 2959 } 2960 2961 if (vnic->flags & BNXT_VNIC_MCAST_FLAG) { 2962 vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN; 2963 vnic->mc_list = 2964 dma_alloc_coherent(&pdev->dev, 2965 vnic->mc_list_size, 2966 &vnic->mc_list_mapping, 2967 GFP_KERNEL); 2968 if (!vnic->mc_list) { 2969 rc = -ENOMEM; 2970 goto out; 2971 } 2972 } 2973 2974 if (vnic->flags & BNXT_VNIC_RSS_FLAG) 2975 max_rings = bp->rx_nr_rings; 2976 else 2977 max_rings = 1; 2978 2979 vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL); 2980 if (!vnic->fw_grp_ids) { 2981 rc = -ENOMEM; 2982 goto out; 2983 } 2984 2985 if ((bp->flags & BNXT_FLAG_NEW_RSS_CAP) && 2986 !(vnic->flags & BNXT_VNIC_RSS_FLAG)) 2987 continue; 2988 2989 /* Allocate rss table and hash key */ 2990 vnic->rss_table = dma_alloc_coherent(&pdev->dev, PAGE_SIZE, 2991 &vnic->rss_table_dma_addr, 2992 GFP_KERNEL); 2993 if (!vnic->rss_table) { 2994 rc = -ENOMEM; 2995 goto out; 2996 } 2997 2998 size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16)); 2999 3000 vnic->rss_hash_key = ((void *)vnic->rss_table) + size; 3001 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size; 3002 } 3003 return 0; 3004 3005 out: 3006 return rc; 3007 } 3008 3009 static void bnxt_free_hwrm_resources(struct bnxt *bp) 3010 { 3011 struct pci_dev *pdev = bp->pdev; 3012 3013 dma_free_coherent(&pdev->dev, PAGE_SIZE, bp->hwrm_cmd_resp_addr, 3014 bp->hwrm_cmd_resp_dma_addr); 3015 3016 bp->hwrm_cmd_resp_addr = NULL; 3017 } 3018 3019 static int bnxt_alloc_hwrm_resources(struct bnxt *bp) 3020 { 3021 struct pci_dev *pdev = bp->pdev; 3022 3023 bp->hwrm_cmd_resp_addr = dma_alloc_coherent(&pdev->dev, PAGE_SIZE, 3024 &bp->hwrm_cmd_resp_dma_addr, 3025 GFP_KERNEL); 3026 if (!bp->hwrm_cmd_resp_addr) 3027 return -ENOMEM; 3028 3029 return 0; 3030 } 3031 3032 static void bnxt_free_hwrm_short_cmd_req(struct bnxt *bp) 3033 { 3034 if (bp->hwrm_short_cmd_req_addr) { 3035 struct pci_dev *pdev = bp->pdev; 3036 3037 dma_free_coherent(&pdev->dev, BNXT_HWRM_MAX_REQ_LEN, 3038 bp->hwrm_short_cmd_req_addr, 3039 bp->hwrm_short_cmd_req_dma_addr); 3040 bp->hwrm_short_cmd_req_addr = NULL; 3041 } 3042 } 3043 3044 static int bnxt_alloc_hwrm_short_cmd_req(struct bnxt *bp) 3045 { 3046 struct pci_dev *pdev = bp->pdev; 3047 3048 bp->hwrm_short_cmd_req_addr = 3049 dma_alloc_coherent(&pdev->dev, BNXT_HWRM_MAX_REQ_LEN, 3050 &bp->hwrm_short_cmd_req_dma_addr, 3051 GFP_KERNEL); 3052 if (!bp->hwrm_short_cmd_req_addr) 3053 return -ENOMEM; 3054 3055 return 0; 3056 } 3057 3058 static void bnxt_free_stats(struct bnxt *bp) 3059 { 3060 u32 size, i; 3061 struct pci_dev *pdev = bp->pdev; 3062 3063 bp->flags &= ~BNXT_FLAG_PORT_STATS; 3064 bp->flags &= ~BNXT_FLAG_PORT_STATS_EXT; 3065 3066 if (bp->hw_rx_port_stats) { 3067 dma_free_coherent(&pdev->dev, bp->hw_port_stats_size, 3068 bp->hw_rx_port_stats, 3069 bp->hw_rx_port_stats_map); 3070 bp->hw_rx_port_stats = NULL; 3071 } 3072 3073 if (bp->hw_rx_port_stats_ext) { 3074 dma_free_coherent(&pdev->dev, sizeof(struct rx_port_stats_ext), 3075 bp->hw_rx_port_stats_ext, 3076 bp->hw_rx_port_stats_ext_map); 3077 bp->hw_rx_port_stats_ext = NULL; 3078 } 3079 3080 if (!bp->bnapi) 3081 return; 3082 3083 size = sizeof(struct ctx_hw_stats); 3084 3085 for (i = 0; i < bp->cp_nr_rings; i++) { 3086 struct bnxt_napi *bnapi = bp->bnapi[i]; 3087 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 3088 3089 if (cpr->hw_stats) { 3090 dma_free_coherent(&pdev->dev, size, cpr->hw_stats, 3091 cpr->hw_stats_map); 3092 cpr->hw_stats = NULL; 3093 } 3094 } 3095 } 3096 3097 static int bnxt_alloc_stats(struct bnxt *bp) 3098 { 3099 u32 size, i; 3100 struct pci_dev *pdev = bp->pdev; 3101 3102 size = sizeof(struct ctx_hw_stats); 3103 3104 for (i = 0; i < bp->cp_nr_rings; i++) { 3105 struct bnxt_napi *bnapi = bp->bnapi[i]; 3106 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 3107 3108 cpr->hw_stats = dma_alloc_coherent(&pdev->dev, size, 3109 &cpr->hw_stats_map, 3110 GFP_KERNEL); 3111 if (!cpr->hw_stats) 3112 return -ENOMEM; 3113 3114 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID; 3115 } 3116 3117 if (BNXT_PF(bp) && bp->chip_num != CHIP_NUM_58700) { 3118 bp->hw_port_stats_size = sizeof(struct rx_port_stats) + 3119 sizeof(struct tx_port_stats) + 1024; 3120 3121 bp->hw_rx_port_stats = 3122 dma_alloc_coherent(&pdev->dev, bp->hw_port_stats_size, 3123 &bp->hw_rx_port_stats_map, 3124 GFP_KERNEL); 3125 if (!bp->hw_rx_port_stats) 3126 return -ENOMEM; 3127 3128 bp->hw_tx_port_stats = (void *)(bp->hw_rx_port_stats + 1) + 3129 512; 3130 bp->hw_tx_port_stats_map = bp->hw_rx_port_stats_map + 3131 sizeof(struct rx_port_stats) + 512; 3132 bp->flags |= BNXT_FLAG_PORT_STATS; 3133 3134 /* Display extended statistics only if FW supports it */ 3135 if (bp->hwrm_spec_code < 0x10804 || 3136 bp->hwrm_spec_code == 0x10900) 3137 return 0; 3138 3139 bp->hw_rx_port_stats_ext = 3140 dma_zalloc_coherent(&pdev->dev, 3141 sizeof(struct rx_port_stats_ext), 3142 &bp->hw_rx_port_stats_ext_map, 3143 GFP_KERNEL); 3144 if (!bp->hw_rx_port_stats_ext) 3145 return 0; 3146 3147 bp->flags |= BNXT_FLAG_PORT_STATS_EXT; 3148 } 3149 return 0; 3150 } 3151 3152 static void bnxt_clear_ring_indices(struct bnxt *bp) 3153 { 3154 int i; 3155 3156 if (!bp->bnapi) 3157 return; 3158 3159 for (i = 0; i < bp->cp_nr_rings; i++) { 3160 struct bnxt_napi *bnapi = bp->bnapi[i]; 3161 struct bnxt_cp_ring_info *cpr; 3162 struct bnxt_rx_ring_info *rxr; 3163 struct bnxt_tx_ring_info *txr; 3164 3165 if (!bnapi) 3166 continue; 3167 3168 cpr = &bnapi->cp_ring; 3169 cpr->cp_raw_cons = 0; 3170 3171 txr = bnapi->tx_ring; 3172 if (txr) { 3173 txr->tx_prod = 0; 3174 txr->tx_cons = 0; 3175 } 3176 3177 rxr = bnapi->rx_ring; 3178 if (rxr) { 3179 rxr->rx_prod = 0; 3180 rxr->rx_agg_prod = 0; 3181 rxr->rx_sw_agg_prod = 0; 3182 rxr->rx_next_cons = 0; 3183 } 3184 } 3185 } 3186 3187 static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool irq_reinit) 3188 { 3189 #ifdef CONFIG_RFS_ACCEL 3190 int i; 3191 3192 /* Under rtnl_lock and all our NAPIs have been disabled. It's 3193 * safe to delete the hash table. 3194 */ 3195 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) { 3196 struct hlist_head *head; 3197 struct hlist_node *tmp; 3198 struct bnxt_ntuple_filter *fltr; 3199 3200 head = &bp->ntp_fltr_hash_tbl[i]; 3201 hlist_for_each_entry_safe(fltr, tmp, head, hash) { 3202 hlist_del(&fltr->hash); 3203 kfree(fltr); 3204 } 3205 } 3206 if (irq_reinit) { 3207 kfree(bp->ntp_fltr_bmap); 3208 bp->ntp_fltr_bmap = NULL; 3209 } 3210 bp->ntp_fltr_count = 0; 3211 #endif 3212 } 3213 3214 static int bnxt_alloc_ntp_fltrs(struct bnxt *bp) 3215 { 3216 #ifdef CONFIG_RFS_ACCEL 3217 int i, rc = 0; 3218 3219 if (!(bp->flags & BNXT_FLAG_RFS)) 3220 return 0; 3221 3222 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) 3223 INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]); 3224 3225 bp->ntp_fltr_count = 0; 3226 bp->ntp_fltr_bmap = kcalloc(BITS_TO_LONGS(BNXT_NTP_FLTR_MAX_FLTR), 3227 sizeof(long), 3228 GFP_KERNEL); 3229 3230 if (!bp->ntp_fltr_bmap) 3231 rc = -ENOMEM; 3232 3233 return rc; 3234 #else 3235 return 0; 3236 #endif 3237 } 3238 3239 static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init) 3240 { 3241 bnxt_free_vnic_attributes(bp); 3242 bnxt_free_tx_rings(bp); 3243 bnxt_free_rx_rings(bp); 3244 bnxt_free_cp_rings(bp); 3245 bnxt_free_ntp_fltrs(bp, irq_re_init); 3246 if (irq_re_init) { 3247 bnxt_free_stats(bp); 3248 bnxt_free_ring_grps(bp); 3249 bnxt_free_vnics(bp); 3250 kfree(bp->tx_ring_map); 3251 bp->tx_ring_map = NULL; 3252 kfree(bp->tx_ring); 3253 bp->tx_ring = NULL; 3254 kfree(bp->rx_ring); 3255 bp->rx_ring = NULL; 3256 kfree(bp->bnapi); 3257 bp->bnapi = NULL; 3258 } else { 3259 bnxt_clear_ring_indices(bp); 3260 } 3261 } 3262 3263 static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init) 3264 { 3265 int i, j, rc, size, arr_size; 3266 void *bnapi; 3267 3268 if (irq_re_init) { 3269 /* Allocate bnapi mem pointer array and mem block for 3270 * all queues 3271 */ 3272 arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) * 3273 bp->cp_nr_rings); 3274 size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi)); 3275 bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL); 3276 if (!bnapi) 3277 return -ENOMEM; 3278 3279 bp->bnapi = bnapi; 3280 bnapi += arr_size; 3281 for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) { 3282 bp->bnapi[i] = bnapi; 3283 bp->bnapi[i]->index = i; 3284 bp->bnapi[i]->bp = bp; 3285 } 3286 3287 bp->rx_ring = kcalloc(bp->rx_nr_rings, 3288 sizeof(struct bnxt_rx_ring_info), 3289 GFP_KERNEL); 3290 if (!bp->rx_ring) 3291 return -ENOMEM; 3292 3293 for (i = 0; i < bp->rx_nr_rings; i++) { 3294 bp->rx_ring[i].bnapi = bp->bnapi[i]; 3295 bp->bnapi[i]->rx_ring = &bp->rx_ring[i]; 3296 } 3297 3298 bp->tx_ring = kcalloc(bp->tx_nr_rings, 3299 sizeof(struct bnxt_tx_ring_info), 3300 GFP_KERNEL); 3301 if (!bp->tx_ring) 3302 return -ENOMEM; 3303 3304 bp->tx_ring_map = kcalloc(bp->tx_nr_rings, sizeof(u16), 3305 GFP_KERNEL); 3306 3307 if (!bp->tx_ring_map) 3308 return -ENOMEM; 3309 3310 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 3311 j = 0; 3312 else 3313 j = bp->rx_nr_rings; 3314 3315 for (i = 0; i < bp->tx_nr_rings; i++, j++) { 3316 bp->tx_ring[i].bnapi = bp->bnapi[j]; 3317 bp->bnapi[j]->tx_ring = &bp->tx_ring[i]; 3318 bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i; 3319 if (i >= bp->tx_nr_rings_xdp) { 3320 bp->tx_ring[i].txq_index = i - 3321 bp->tx_nr_rings_xdp; 3322 bp->bnapi[j]->tx_int = bnxt_tx_int; 3323 } else { 3324 bp->bnapi[j]->flags |= BNXT_NAPI_FLAG_XDP; 3325 bp->bnapi[j]->tx_int = bnxt_tx_int_xdp; 3326 } 3327 } 3328 3329 rc = bnxt_alloc_stats(bp); 3330 if (rc) 3331 goto alloc_mem_err; 3332 3333 rc = bnxt_alloc_ntp_fltrs(bp); 3334 if (rc) 3335 goto alloc_mem_err; 3336 3337 rc = bnxt_alloc_vnics(bp); 3338 if (rc) 3339 goto alloc_mem_err; 3340 } 3341 3342 bnxt_init_ring_struct(bp); 3343 3344 rc = bnxt_alloc_rx_rings(bp); 3345 if (rc) 3346 goto alloc_mem_err; 3347 3348 rc = bnxt_alloc_tx_rings(bp); 3349 if (rc) 3350 goto alloc_mem_err; 3351 3352 rc = bnxt_alloc_cp_rings(bp); 3353 if (rc) 3354 goto alloc_mem_err; 3355 3356 bp->vnic_info[0].flags |= BNXT_VNIC_RSS_FLAG | BNXT_VNIC_MCAST_FLAG | 3357 BNXT_VNIC_UCAST_FLAG; 3358 rc = bnxt_alloc_vnic_attributes(bp); 3359 if (rc) 3360 goto alloc_mem_err; 3361 return 0; 3362 3363 alloc_mem_err: 3364 bnxt_free_mem(bp, true); 3365 return rc; 3366 } 3367 3368 static void bnxt_disable_int(struct bnxt *bp) 3369 { 3370 int i; 3371 3372 if (!bp->bnapi) 3373 return; 3374 3375 for (i = 0; i < bp->cp_nr_rings; i++) { 3376 struct bnxt_napi *bnapi = bp->bnapi[i]; 3377 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 3378 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 3379 3380 if (ring->fw_ring_id != INVALID_HW_RING_ID) 3381 BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons); 3382 } 3383 } 3384 3385 static int bnxt_cp_num_to_irq_num(struct bnxt *bp, int n) 3386 { 3387 struct bnxt_napi *bnapi = bp->bnapi[n]; 3388 struct bnxt_cp_ring_info *cpr; 3389 3390 cpr = &bnapi->cp_ring; 3391 return cpr->cp_ring_struct.map_idx; 3392 } 3393 3394 static void bnxt_disable_int_sync(struct bnxt *bp) 3395 { 3396 int i; 3397 3398 atomic_inc(&bp->intr_sem); 3399 3400 bnxt_disable_int(bp); 3401 for (i = 0; i < bp->cp_nr_rings; i++) { 3402 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 3403 3404 synchronize_irq(bp->irq_tbl[map_idx].vector); 3405 } 3406 } 3407 3408 static void bnxt_enable_int(struct bnxt *bp) 3409 { 3410 int i; 3411 3412 atomic_set(&bp->intr_sem, 0); 3413 for (i = 0; i < bp->cp_nr_rings; i++) { 3414 struct bnxt_napi *bnapi = bp->bnapi[i]; 3415 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 3416 3417 BNXT_CP_DB_REARM(cpr->cp_doorbell, cpr->cp_raw_cons); 3418 } 3419 } 3420 3421 void bnxt_hwrm_cmd_hdr_init(struct bnxt *bp, void *request, u16 req_type, 3422 u16 cmpl_ring, u16 target_id) 3423 { 3424 struct input *req = request; 3425 3426 req->req_type = cpu_to_le16(req_type); 3427 req->cmpl_ring = cpu_to_le16(cmpl_ring); 3428 req->target_id = cpu_to_le16(target_id); 3429 req->resp_addr = cpu_to_le64(bp->hwrm_cmd_resp_dma_addr); 3430 } 3431 3432 static int bnxt_hwrm_do_send_msg(struct bnxt *bp, void *msg, u32 msg_len, 3433 int timeout, bool silent) 3434 { 3435 int i, intr_process, rc, tmo_count; 3436 struct input *req = msg; 3437 u32 *data = msg; 3438 __le32 *resp_len; 3439 u8 *valid; 3440 u16 cp_ring_id, len = 0; 3441 struct hwrm_err_output *resp = bp->hwrm_cmd_resp_addr; 3442 u16 max_req_len = BNXT_HWRM_MAX_REQ_LEN; 3443 struct hwrm_short_input short_input = {0}; 3444 3445 req->seq_id = cpu_to_le16(bp->hwrm_cmd_seq++); 3446 memset(resp, 0, PAGE_SIZE); 3447 cp_ring_id = le16_to_cpu(req->cmpl_ring); 3448 intr_process = (cp_ring_id == INVALID_HW_RING_ID) ? 0 : 1; 3449 3450 if (bp->fw_cap & BNXT_FW_CAP_SHORT_CMD) { 3451 void *short_cmd_req = bp->hwrm_short_cmd_req_addr; 3452 3453 memcpy(short_cmd_req, req, msg_len); 3454 memset(short_cmd_req + msg_len, 0, BNXT_HWRM_MAX_REQ_LEN - 3455 msg_len); 3456 3457 short_input.req_type = req->req_type; 3458 short_input.signature = 3459 cpu_to_le16(SHORT_REQ_SIGNATURE_SHORT_CMD); 3460 short_input.size = cpu_to_le16(msg_len); 3461 short_input.req_addr = 3462 cpu_to_le64(bp->hwrm_short_cmd_req_dma_addr); 3463 3464 data = (u32 *)&short_input; 3465 msg_len = sizeof(short_input); 3466 3467 /* Sync memory write before updating doorbell */ 3468 wmb(); 3469 3470 max_req_len = BNXT_HWRM_SHORT_REQ_LEN; 3471 } 3472 3473 /* Write request msg to hwrm channel */ 3474 __iowrite32_copy(bp->bar0, data, msg_len / 4); 3475 3476 for (i = msg_len; i < max_req_len; i += 4) 3477 writel(0, bp->bar0 + i); 3478 3479 /* currently supports only one outstanding message */ 3480 if (intr_process) 3481 bp->hwrm_intr_seq_id = le16_to_cpu(req->seq_id); 3482 3483 /* Ring channel doorbell */ 3484 writel(1, bp->bar0 + 0x100); 3485 3486 if (!timeout) 3487 timeout = DFLT_HWRM_CMD_TIMEOUT; 3488 /* convert timeout to usec */ 3489 timeout *= 1000; 3490 3491 i = 0; 3492 /* Short timeout for the first few iterations: 3493 * number of loops = number of loops for short timeout + 3494 * number of loops for standard timeout. 3495 */ 3496 tmo_count = HWRM_SHORT_TIMEOUT_COUNTER; 3497 timeout = timeout - HWRM_SHORT_MIN_TIMEOUT * HWRM_SHORT_TIMEOUT_COUNTER; 3498 tmo_count += DIV_ROUND_UP(timeout, HWRM_MIN_TIMEOUT); 3499 resp_len = bp->hwrm_cmd_resp_addr + HWRM_RESP_LEN_OFFSET; 3500 if (intr_process) { 3501 /* Wait until hwrm response cmpl interrupt is processed */ 3502 while (bp->hwrm_intr_seq_id != HWRM_SEQ_ID_INVALID && 3503 i++ < tmo_count) { 3504 /* on first few passes, just barely sleep */ 3505 if (i < HWRM_SHORT_TIMEOUT_COUNTER) 3506 usleep_range(HWRM_SHORT_MIN_TIMEOUT, 3507 HWRM_SHORT_MAX_TIMEOUT); 3508 else 3509 usleep_range(HWRM_MIN_TIMEOUT, 3510 HWRM_MAX_TIMEOUT); 3511 } 3512 3513 if (bp->hwrm_intr_seq_id != HWRM_SEQ_ID_INVALID) { 3514 netdev_err(bp->dev, "Resp cmpl intr err msg: 0x%x\n", 3515 le16_to_cpu(req->req_type)); 3516 return -1; 3517 } 3518 len = (le32_to_cpu(*resp_len) & HWRM_RESP_LEN_MASK) >> 3519 HWRM_RESP_LEN_SFT; 3520 valid = bp->hwrm_cmd_resp_addr + len - 1; 3521 } else { 3522 int j; 3523 3524 /* Check if response len is updated */ 3525 for (i = 0; i < tmo_count; i++) { 3526 len = (le32_to_cpu(*resp_len) & HWRM_RESP_LEN_MASK) >> 3527 HWRM_RESP_LEN_SFT; 3528 if (len) 3529 break; 3530 /* on first few passes, just barely sleep */ 3531 if (i < DFLT_HWRM_CMD_TIMEOUT) 3532 usleep_range(HWRM_SHORT_MIN_TIMEOUT, 3533 HWRM_SHORT_MAX_TIMEOUT); 3534 else 3535 usleep_range(HWRM_MIN_TIMEOUT, 3536 HWRM_MAX_TIMEOUT); 3537 } 3538 3539 if (i >= tmo_count) { 3540 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d\n", 3541 HWRM_TOTAL_TIMEOUT(i), 3542 le16_to_cpu(req->req_type), 3543 le16_to_cpu(req->seq_id), len); 3544 return -1; 3545 } 3546 3547 /* Last byte of resp contains valid bit */ 3548 valid = bp->hwrm_cmd_resp_addr + len - 1; 3549 for (j = 0; j < HWRM_VALID_BIT_DELAY_USEC; j++) { 3550 /* make sure we read from updated DMA memory */ 3551 dma_rmb(); 3552 if (*valid) 3553 break; 3554 udelay(1); 3555 } 3556 3557 if (j >= HWRM_VALID_BIT_DELAY_USEC) { 3558 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d v:%d\n", 3559 HWRM_TOTAL_TIMEOUT(i), 3560 le16_to_cpu(req->req_type), 3561 le16_to_cpu(req->seq_id), len, *valid); 3562 return -1; 3563 } 3564 } 3565 3566 /* Zero valid bit for compatibility. Valid bit in an older spec 3567 * may become a new field in a newer spec. We must make sure that 3568 * a new field not implemented by old spec will read zero. 3569 */ 3570 *valid = 0; 3571 rc = le16_to_cpu(resp->error_code); 3572 if (rc && !silent) 3573 netdev_err(bp->dev, "hwrm req_type 0x%x seq id 0x%x error 0x%x\n", 3574 le16_to_cpu(resp->req_type), 3575 le16_to_cpu(resp->seq_id), rc); 3576 return rc; 3577 } 3578 3579 int _hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout) 3580 { 3581 return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, false); 3582 } 3583 3584 int _hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len, 3585 int timeout) 3586 { 3587 return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true); 3588 } 3589 3590 int hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout) 3591 { 3592 int rc; 3593 3594 mutex_lock(&bp->hwrm_cmd_lock); 3595 rc = _hwrm_send_message(bp, msg, msg_len, timeout); 3596 mutex_unlock(&bp->hwrm_cmd_lock); 3597 return rc; 3598 } 3599 3600 int hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len, 3601 int timeout) 3602 { 3603 int rc; 3604 3605 mutex_lock(&bp->hwrm_cmd_lock); 3606 rc = bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true); 3607 mutex_unlock(&bp->hwrm_cmd_lock); 3608 return rc; 3609 } 3610 3611 int bnxt_hwrm_func_rgtr_async_events(struct bnxt *bp, unsigned long *bmap, 3612 int bmap_size) 3613 { 3614 struct hwrm_func_drv_rgtr_input req = {0}; 3615 DECLARE_BITMAP(async_events_bmap, 256); 3616 u32 *events = (u32 *)async_events_bmap; 3617 int i; 3618 3619 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1); 3620 3621 req.enables = 3622 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD); 3623 3624 memset(async_events_bmap, 0, sizeof(async_events_bmap)); 3625 for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++) 3626 __set_bit(bnxt_async_events_arr[i], async_events_bmap); 3627 3628 if (bmap && bmap_size) { 3629 for (i = 0; i < bmap_size; i++) { 3630 if (test_bit(i, bmap)) 3631 __set_bit(i, async_events_bmap); 3632 } 3633 } 3634 3635 for (i = 0; i < 8; i++) 3636 req.async_event_fwd[i] |= cpu_to_le32(events[i]); 3637 3638 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3639 } 3640 3641 static int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp) 3642 { 3643 struct hwrm_func_drv_rgtr_output *resp = bp->hwrm_cmd_resp_addr; 3644 struct hwrm_func_drv_rgtr_input req = {0}; 3645 int rc; 3646 3647 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1); 3648 3649 req.enables = 3650 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE | 3651 FUNC_DRV_RGTR_REQ_ENABLES_VER); 3652 3653 req.os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX); 3654 req.flags = cpu_to_le32(FUNC_DRV_RGTR_REQ_FLAGS_16BIT_VER_MODE); 3655 req.ver_maj_8b = DRV_VER_MAJ; 3656 req.ver_min_8b = DRV_VER_MIN; 3657 req.ver_upd_8b = DRV_VER_UPD; 3658 req.ver_maj = cpu_to_le16(DRV_VER_MAJ); 3659 req.ver_min = cpu_to_le16(DRV_VER_MIN); 3660 req.ver_upd = cpu_to_le16(DRV_VER_UPD); 3661 3662 if (BNXT_PF(bp)) { 3663 u32 data[8]; 3664 int i; 3665 3666 memset(data, 0, sizeof(data)); 3667 for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) { 3668 u16 cmd = bnxt_vf_req_snif[i]; 3669 unsigned int bit, idx; 3670 3671 idx = cmd / 32; 3672 bit = cmd % 32; 3673 data[idx] |= 1 << bit; 3674 } 3675 3676 for (i = 0; i < 8; i++) 3677 req.vf_req_fwd[i] = cpu_to_le32(data[i]); 3678 3679 req.enables |= 3680 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD); 3681 } 3682 3683 mutex_lock(&bp->hwrm_cmd_lock); 3684 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3685 if (rc) 3686 rc = -EIO; 3687 else if (resp->flags & 3688 cpu_to_le32(FUNC_DRV_RGTR_RESP_FLAGS_IF_CHANGE_SUPPORTED)) 3689 bp->fw_cap |= BNXT_FW_CAP_IF_CHANGE; 3690 mutex_unlock(&bp->hwrm_cmd_lock); 3691 return rc; 3692 } 3693 3694 static int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp) 3695 { 3696 struct hwrm_func_drv_unrgtr_input req = {0}; 3697 3698 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_UNRGTR, -1, -1); 3699 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3700 } 3701 3702 static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type) 3703 { 3704 u32 rc = 0; 3705 struct hwrm_tunnel_dst_port_free_input req = {0}; 3706 3707 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_FREE, -1, -1); 3708 req.tunnel_type = tunnel_type; 3709 3710 switch (tunnel_type) { 3711 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN: 3712 req.tunnel_dst_port_id = bp->vxlan_fw_dst_port_id; 3713 break; 3714 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE: 3715 req.tunnel_dst_port_id = bp->nge_fw_dst_port_id; 3716 break; 3717 default: 3718 break; 3719 } 3720 3721 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3722 if (rc) 3723 netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n", 3724 rc); 3725 return rc; 3726 } 3727 3728 static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port, 3729 u8 tunnel_type) 3730 { 3731 u32 rc = 0; 3732 struct hwrm_tunnel_dst_port_alloc_input req = {0}; 3733 struct hwrm_tunnel_dst_port_alloc_output *resp = bp->hwrm_cmd_resp_addr; 3734 3735 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_ALLOC, -1, -1); 3736 3737 req.tunnel_type = tunnel_type; 3738 req.tunnel_dst_port_val = port; 3739 3740 mutex_lock(&bp->hwrm_cmd_lock); 3741 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3742 if (rc) { 3743 netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n", 3744 rc); 3745 goto err_out; 3746 } 3747 3748 switch (tunnel_type) { 3749 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN: 3750 bp->vxlan_fw_dst_port_id = resp->tunnel_dst_port_id; 3751 break; 3752 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE: 3753 bp->nge_fw_dst_port_id = resp->tunnel_dst_port_id; 3754 break; 3755 default: 3756 break; 3757 } 3758 3759 err_out: 3760 mutex_unlock(&bp->hwrm_cmd_lock); 3761 return rc; 3762 } 3763 3764 static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id) 3765 { 3766 struct hwrm_cfa_l2_set_rx_mask_input req = {0}; 3767 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 3768 3769 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_SET_RX_MASK, -1, -1); 3770 req.vnic_id = cpu_to_le32(vnic->fw_vnic_id); 3771 3772 req.num_mc_entries = cpu_to_le32(vnic->mc_list_count); 3773 req.mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping); 3774 req.mask = cpu_to_le32(vnic->rx_mask); 3775 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3776 } 3777 3778 #ifdef CONFIG_RFS_ACCEL 3779 static int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp, 3780 struct bnxt_ntuple_filter *fltr) 3781 { 3782 struct hwrm_cfa_ntuple_filter_free_input req = {0}; 3783 3784 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_FREE, -1, -1); 3785 req.ntuple_filter_id = fltr->filter_id; 3786 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3787 } 3788 3789 #define BNXT_NTP_FLTR_FLAGS \ 3790 (CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID | \ 3791 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE | \ 3792 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR | \ 3793 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE | \ 3794 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR | \ 3795 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK | \ 3796 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR | \ 3797 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK | \ 3798 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL | \ 3799 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT | \ 3800 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK | \ 3801 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT | \ 3802 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK | \ 3803 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID) 3804 3805 #define BNXT_NTP_TUNNEL_FLTR_FLAG \ 3806 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE 3807 3808 static int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp, 3809 struct bnxt_ntuple_filter *fltr) 3810 { 3811 int rc = 0; 3812 struct hwrm_cfa_ntuple_filter_alloc_input req = {0}; 3813 struct hwrm_cfa_ntuple_filter_alloc_output *resp = 3814 bp->hwrm_cmd_resp_addr; 3815 struct flow_keys *keys = &fltr->fkeys; 3816 struct bnxt_vnic_info *vnic = &bp->vnic_info[fltr->rxq + 1]; 3817 3818 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_ALLOC, -1, -1); 3819 req.l2_filter_id = bp->vnic_info[0].fw_l2_filter_id[fltr->l2_fltr_idx]; 3820 3821 req.enables = cpu_to_le32(BNXT_NTP_FLTR_FLAGS); 3822 3823 req.ethertype = htons(ETH_P_IP); 3824 memcpy(req.src_macaddr, fltr->src_mac_addr, ETH_ALEN); 3825 req.ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4; 3826 req.ip_protocol = keys->basic.ip_proto; 3827 3828 if (keys->basic.n_proto == htons(ETH_P_IPV6)) { 3829 int i; 3830 3831 req.ethertype = htons(ETH_P_IPV6); 3832 req.ip_addr_type = 3833 CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6; 3834 *(struct in6_addr *)&req.src_ipaddr[0] = 3835 keys->addrs.v6addrs.src; 3836 *(struct in6_addr *)&req.dst_ipaddr[0] = 3837 keys->addrs.v6addrs.dst; 3838 for (i = 0; i < 4; i++) { 3839 req.src_ipaddr_mask[i] = cpu_to_be32(0xffffffff); 3840 req.dst_ipaddr_mask[i] = cpu_to_be32(0xffffffff); 3841 } 3842 } else { 3843 req.src_ipaddr[0] = keys->addrs.v4addrs.src; 3844 req.src_ipaddr_mask[0] = cpu_to_be32(0xffffffff); 3845 req.dst_ipaddr[0] = keys->addrs.v4addrs.dst; 3846 req.dst_ipaddr_mask[0] = cpu_to_be32(0xffffffff); 3847 } 3848 if (keys->control.flags & FLOW_DIS_ENCAPSULATION) { 3849 req.enables |= cpu_to_le32(BNXT_NTP_TUNNEL_FLTR_FLAG); 3850 req.tunnel_type = 3851 CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL; 3852 } 3853 3854 req.src_port = keys->ports.src; 3855 req.src_port_mask = cpu_to_be16(0xffff); 3856 req.dst_port = keys->ports.dst; 3857 req.dst_port_mask = cpu_to_be16(0xffff); 3858 3859 req.dst_id = cpu_to_le16(vnic->fw_vnic_id); 3860 mutex_lock(&bp->hwrm_cmd_lock); 3861 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3862 if (!rc) 3863 fltr->filter_id = resp->ntuple_filter_id; 3864 mutex_unlock(&bp->hwrm_cmd_lock); 3865 return rc; 3866 } 3867 #endif 3868 3869 static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx, 3870 u8 *mac_addr) 3871 { 3872 u32 rc = 0; 3873 struct hwrm_cfa_l2_filter_alloc_input req = {0}; 3874 struct hwrm_cfa_l2_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr; 3875 3876 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_ALLOC, -1, -1); 3877 req.flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX); 3878 if (!BNXT_CHIP_TYPE_NITRO_A0(bp)) 3879 req.flags |= 3880 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST); 3881 req.dst_id = cpu_to_le16(bp->vnic_info[vnic_id].fw_vnic_id); 3882 req.enables = 3883 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR | 3884 CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID | 3885 CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK); 3886 memcpy(req.l2_addr, mac_addr, ETH_ALEN); 3887 req.l2_addr_mask[0] = 0xff; 3888 req.l2_addr_mask[1] = 0xff; 3889 req.l2_addr_mask[2] = 0xff; 3890 req.l2_addr_mask[3] = 0xff; 3891 req.l2_addr_mask[4] = 0xff; 3892 req.l2_addr_mask[5] = 0xff; 3893 3894 mutex_lock(&bp->hwrm_cmd_lock); 3895 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3896 if (!rc) 3897 bp->vnic_info[vnic_id].fw_l2_filter_id[idx] = 3898 resp->l2_filter_id; 3899 mutex_unlock(&bp->hwrm_cmd_lock); 3900 return rc; 3901 } 3902 3903 static int bnxt_hwrm_clear_vnic_filter(struct bnxt *bp) 3904 { 3905 u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */ 3906 int rc = 0; 3907 3908 /* Any associated ntuple filters will also be cleared by firmware. */ 3909 mutex_lock(&bp->hwrm_cmd_lock); 3910 for (i = 0; i < num_of_vnics; i++) { 3911 struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; 3912 3913 for (j = 0; j < vnic->uc_filter_count; j++) { 3914 struct hwrm_cfa_l2_filter_free_input req = {0}; 3915 3916 bnxt_hwrm_cmd_hdr_init(bp, &req, 3917 HWRM_CFA_L2_FILTER_FREE, -1, -1); 3918 3919 req.l2_filter_id = vnic->fw_l2_filter_id[j]; 3920 3921 rc = _hwrm_send_message(bp, &req, sizeof(req), 3922 HWRM_CMD_TIMEOUT); 3923 } 3924 vnic->uc_filter_count = 0; 3925 } 3926 mutex_unlock(&bp->hwrm_cmd_lock); 3927 3928 return rc; 3929 } 3930 3931 static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags) 3932 { 3933 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 3934 struct hwrm_vnic_tpa_cfg_input req = {0}; 3935 3936 if (vnic->fw_vnic_id == INVALID_HW_RING_ID) 3937 return 0; 3938 3939 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_TPA_CFG, -1, -1); 3940 3941 if (tpa_flags) { 3942 u16 mss = bp->dev->mtu - 40; 3943 u32 nsegs, n, segs = 0, flags; 3944 3945 flags = VNIC_TPA_CFG_REQ_FLAGS_TPA | 3946 VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA | 3947 VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE | 3948 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN | 3949 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ; 3950 if (tpa_flags & BNXT_FLAG_GRO) 3951 flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO; 3952 3953 req.flags = cpu_to_le32(flags); 3954 3955 req.enables = 3956 cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS | 3957 VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS | 3958 VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN); 3959 3960 /* Number of segs are log2 units, and first packet is not 3961 * included as part of this units. 3962 */ 3963 if (mss <= BNXT_RX_PAGE_SIZE) { 3964 n = BNXT_RX_PAGE_SIZE / mss; 3965 nsegs = (MAX_SKB_FRAGS - 1) * n; 3966 } else { 3967 n = mss / BNXT_RX_PAGE_SIZE; 3968 if (mss & (BNXT_RX_PAGE_SIZE - 1)) 3969 n++; 3970 nsegs = (MAX_SKB_FRAGS - n) / n; 3971 } 3972 3973 segs = ilog2(nsegs); 3974 req.max_agg_segs = cpu_to_le16(segs); 3975 req.max_aggs = cpu_to_le16(VNIC_TPA_CFG_REQ_MAX_AGGS_MAX); 3976 3977 req.min_agg_len = cpu_to_le32(512); 3978 } 3979 req.vnic_id = cpu_to_le16(vnic->fw_vnic_id); 3980 3981 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3982 } 3983 3984 static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss) 3985 { 3986 u32 i, j, max_rings; 3987 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 3988 struct hwrm_vnic_rss_cfg_input req = {0}; 3989 3990 if (vnic->fw_rss_cos_lb_ctx[0] == INVALID_HW_RING_ID) 3991 return 0; 3992 3993 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1); 3994 if (set_rss) { 3995 req.hash_type = cpu_to_le32(bp->rss_hash_cfg); 3996 req.hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT; 3997 if (vnic->flags & BNXT_VNIC_RSS_FLAG) { 3998 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 3999 max_rings = bp->rx_nr_rings - 1; 4000 else 4001 max_rings = bp->rx_nr_rings; 4002 } else { 4003 max_rings = 1; 4004 } 4005 4006 /* Fill the RSS indirection table with ring group ids */ 4007 for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++, j++) { 4008 if (j == max_rings) 4009 j = 0; 4010 vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]); 4011 } 4012 4013 req.ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr); 4014 req.hash_key_tbl_addr = 4015 cpu_to_le64(vnic->rss_hash_key_dma_addr); 4016 } 4017 req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]); 4018 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4019 } 4020 4021 static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id) 4022 { 4023 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 4024 struct hwrm_vnic_plcmodes_cfg_input req = {0}; 4025 4026 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_PLCMODES_CFG, -1, -1); 4027 req.flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT | 4028 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 | 4029 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6); 4030 req.enables = 4031 cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID | 4032 VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID); 4033 /* thresholds not implemented in firmware yet */ 4034 req.jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh); 4035 req.hds_threshold = cpu_to_le16(bp->rx_copy_thresh); 4036 req.vnic_id = cpu_to_le32(vnic->fw_vnic_id); 4037 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4038 } 4039 4040 static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id, 4041 u16 ctx_idx) 4042 { 4043 struct hwrm_vnic_rss_cos_lb_ctx_free_input req = {0}; 4044 4045 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_FREE, -1, -1); 4046 req.rss_cos_lb_ctx_id = 4047 cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx]); 4048 4049 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4050 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID; 4051 } 4052 4053 static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp) 4054 { 4055 int i, j; 4056 4057 for (i = 0; i < bp->nr_vnics; i++) { 4058 struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; 4059 4060 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) { 4061 if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID) 4062 bnxt_hwrm_vnic_ctx_free_one(bp, i, j); 4063 } 4064 } 4065 bp->rsscos_nr_ctxs = 0; 4066 } 4067 4068 static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id, u16 ctx_idx) 4069 { 4070 int rc; 4071 struct hwrm_vnic_rss_cos_lb_ctx_alloc_input req = {0}; 4072 struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp = 4073 bp->hwrm_cmd_resp_addr; 4074 4075 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC, -1, 4076 -1); 4077 4078 mutex_lock(&bp->hwrm_cmd_lock); 4079 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4080 if (!rc) 4081 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = 4082 le16_to_cpu(resp->rss_cos_lb_ctx_id); 4083 mutex_unlock(&bp->hwrm_cmd_lock); 4084 4085 return rc; 4086 } 4087 4088 static u32 bnxt_get_roce_vnic_mode(struct bnxt *bp) 4089 { 4090 if (bp->flags & BNXT_FLAG_ROCE_MIRROR_CAP) 4091 return VNIC_CFG_REQ_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE; 4092 return VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE; 4093 } 4094 4095 int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id) 4096 { 4097 unsigned int ring = 0, grp_idx; 4098 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 4099 struct hwrm_vnic_cfg_input req = {0}; 4100 u16 def_vlan = 0; 4101 4102 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_CFG, -1, -1); 4103 4104 req.enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP); 4105 /* Only RSS support for now TBD: COS & LB */ 4106 if (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID) { 4107 req.rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]); 4108 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE | 4109 VNIC_CFG_REQ_ENABLES_MRU); 4110 } else if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) { 4111 req.rss_rule = 4112 cpu_to_le16(bp->vnic_info[0].fw_rss_cos_lb_ctx[0]); 4113 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE | 4114 VNIC_CFG_REQ_ENABLES_MRU); 4115 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE); 4116 } else { 4117 req.rss_rule = cpu_to_le16(0xffff); 4118 } 4119 4120 if (BNXT_CHIP_TYPE_NITRO_A0(bp) && 4121 (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID)) { 4122 req.cos_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[1]); 4123 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_COS_RULE); 4124 } else { 4125 req.cos_rule = cpu_to_le16(0xffff); 4126 } 4127 4128 if (vnic->flags & BNXT_VNIC_RSS_FLAG) 4129 ring = 0; 4130 else if (vnic->flags & BNXT_VNIC_RFS_FLAG) 4131 ring = vnic_id - 1; 4132 else if ((vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp)) 4133 ring = bp->rx_nr_rings - 1; 4134 4135 grp_idx = bp->rx_ring[ring].bnapi->index; 4136 req.vnic_id = cpu_to_le16(vnic->fw_vnic_id); 4137 req.dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id); 4138 4139 req.lb_rule = cpu_to_le16(0xffff); 4140 req.mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + ETH_FCS_LEN + 4141 VLAN_HLEN); 4142 4143 #ifdef CONFIG_BNXT_SRIOV 4144 if (BNXT_VF(bp)) 4145 def_vlan = bp->vf.vlan; 4146 #endif 4147 if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan) 4148 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE); 4149 if (!vnic_id && bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP)) 4150 req.flags |= cpu_to_le32(bnxt_get_roce_vnic_mode(bp)); 4151 4152 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4153 } 4154 4155 static int bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id) 4156 { 4157 u32 rc = 0; 4158 4159 if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) { 4160 struct hwrm_vnic_free_input req = {0}; 4161 4162 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_FREE, -1, -1); 4163 req.vnic_id = 4164 cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id); 4165 4166 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4167 if (rc) 4168 return rc; 4169 bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID; 4170 } 4171 return rc; 4172 } 4173 4174 static void bnxt_hwrm_vnic_free(struct bnxt *bp) 4175 { 4176 u16 i; 4177 4178 for (i = 0; i < bp->nr_vnics; i++) 4179 bnxt_hwrm_vnic_free_one(bp, i); 4180 } 4181 4182 static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id, 4183 unsigned int start_rx_ring_idx, 4184 unsigned int nr_rings) 4185 { 4186 int rc = 0; 4187 unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings; 4188 struct hwrm_vnic_alloc_input req = {0}; 4189 struct hwrm_vnic_alloc_output *resp = bp->hwrm_cmd_resp_addr; 4190 4191 /* map ring groups to this vnic */ 4192 for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) { 4193 grp_idx = bp->rx_ring[i].bnapi->index; 4194 if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) { 4195 netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n", 4196 j, nr_rings); 4197 break; 4198 } 4199 bp->vnic_info[vnic_id].fw_grp_ids[j] = 4200 bp->grp_info[grp_idx].fw_grp_id; 4201 } 4202 4203 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[0] = INVALID_HW_RING_ID; 4204 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[1] = INVALID_HW_RING_ID; 4205 if (vnic_id == 0) 4206 req.flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT); 4207 4208 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_ALLOC, -1, -1); 4209 4210 mutex_lock(&bp->hwrm_cmd_lock); 4211 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4212 if (!rc) 4213 bp->vnic_info[vnic_id].fw_vnic_id = le32_to_cpu(resp->vnic_id); 4214 mutex_unlock(&bp->hwrm_cmd_lock); 4215 return rc; 4216 } 4217 4218 static int bnxt_hwrm_vnic_qcaps(struct bnxt *bp) 4219 { 4220 struct hwrm_vnic_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 4221 struct hwrm_vnic_qcaps_input req = {0}; 4222 int rc; 4223 4224 if (bp->hwrm_spec_code < 0x10600) 4225 return 0; 4226 4227 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_QCAPS, -1, -1); 4228 mutex_lock(&bp->hwrm_cmd_lock); 4229 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4230 if (!rc) { 4231 u32 flags = le32_to_cpu(resp->flags); 4232 4233 if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP) 4234 bp->flags |= BNXT_FLAG_NEW_RSS_CAP; 4235 if (flags & 4236 VNIC_QCAPS_RESP_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_CAP) 4237 bp->flags |= BNXT_FLAG_ROCE_MIRROR_CAP; 4238 } 4239 mutex_unlock(&bp->hwrm_cmd_lock); 4240 return rc; 4241 } 4242 4243 static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp) 4244 { 4245 u16 i; 4246 u32 rc = 0; 4247 4248 mutex_lock(&bp->hwrm_cmd_lock); 4249 for (i = 0; i < bp->rx_nr_rings; i++) { 4250 struct hwrm_ring_grp_alloc_input req = {0}; 4251 struct hwrm_ring_grp_alloc_output *resp = 4252 bp->hwrm_cmd_resp_addr; 4253 unsigned int grp_idx = bp->rx_ring[i].bnapi->index; 4254 4255 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_ALLOC, -1, -1); 4256 4257 req.cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id); 4258 req.rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id); 4259 req.ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id); 4260 req.sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx); 4261 4262 rc = _hwrm_send_message(bp, &req, sizeof(req), 4263 HWRM_CMD_TIMEOUT); 4264 if (rc) 4265 break; 4266 4267 bp->grp_info[grp_idx].fw_grp_id = 4268 le32_to_cpu(resp->ring_group_id); 4269 } 4270 mutex_unlock(&bp->hwrm_cmd_lock); 4271 return rc; 4272 } 4273 4274 static int bnxt_hwrm_ring_grp_free(struct bnxt *bp) 4275 { 4276 u16 i; 4277 u32 rc = 0; 4278 struct hwrm_ring_grp_free_input req = {0}; 4279 4280 if (!bp->grp_info) 4281 return 0; 4282 4283 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_FREE, -1, -1); 4284 4285 mutex_lock(&bp->hwrm_cmd_lock); 4286 for (i = 0; i < bp->cp_nr_rings; i++) { 4287 if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID) 4288 continue; 4289 req.ring_group_id = 4290 cpu_to_le32(bp->grp_info[i].fw_grp_id); 4291 4292 rc = _hwrm_send_message(bp, &req, sizeof(req), 4293 HWRM_CMD_TIMEOUT); 4294 if (rc) 4295 break; 4296 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID; 4297 } 4298 mutex_unlock(&bp->hwrm_cmd_lock); 4299 return rc; 4300 } 4301 4302 static int hwrm_ring_alloc_send_msg(struct bnxt *bp, 4303 struct bnxt_ring_struct *ring, 4304 u32 ring_type, u32 map_index) 4305 { 4306 int rc = 0, err = 0; 4307 struct hwrm_ring_alloc_input req = {0}; 4308 struct hwrm_ring_alloc_output *resp = bp->hwrm_cmd_resp_addr; 4309 struct bnxt_ring_grp_info *grp_info; 4310 u16 ring_id; 4311 4312 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_ALLOC, -1, -1); 4313 4314 req.enables = 0; 4315 if (ring->nr_pages > 1) { 4316 req.page_tbl_addr = cpu_to_le64(ring->pg_tbl_map); 4317 /* Page size is in log2 units */ 4318 req.page_size = BNXT_PAGE_SHIFT; 4319 req.page_tbl_depth = 1; 4320 } else { 4321 req.page_tbl_addr = cpu_to_le64(ring->dma_arr[0]); 4322 } 4323 req.fbo = 0; 4324 /* Association of ring index with doorbell index and MSIX number */ 4325 req.logical_id = cpu_to_le16(map_index); 4326 4327 switch (ring_type) { 4328 case HWRM_RING_ALLOC_TX: 4329 req.ring_type = RING_ALLOC_REQ_RING_TYPE_TX; 4330 /* Association of transmit ring with completion ring */ 4331 grp_info = &bp->grp_info[ring->grp_idx]; 4332 req.cmpl_ring_id = cpu_to_le16(grp_info->cp_fw_ring_id); 4333 req.length = cpu_to_le32(bp->tx_ring_mask + 1); 4334 req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx); 4335 req.queue_id = cpu_to_le16(ring->queue_id); 4336 break; 4337 case HWRM_RING_ALLOC_RX: 4338 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX; 4339 req.length = cpu_to_le32(bp->rx_ring_mask + 1); 4340 break; 4341 case HWRM_RING_ALLOC_AGG: 4342 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX; 4343 req.length = cpu_to_le32(bp->rx_agg_ring_mask + 1); 4344 break; 4345 case HWRM_RING_ALLOC_CMPL: 4346 req.ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL; 4347 req.length = cpu_to_le32(bp->cp_ring_mask + 1); 4348 if (bp->flags & BNXT_FLAG_USING_MSIX) 4349 req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX; 4350 break; 4351 default: 4352 netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n", 4353 ring_type); 4354 return -1; 4355 } 4356 4357 mutex_lock(&bp->hwrm_cmd_lock); 4358 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4359 err = le16_to_cpu(resp->error_code); 4360 ring_id = le16_to_cpu(resp->ring_id); 4361 mutex_unlock(&bp->hwrm_cmd_lock); 4362 4363 if (rc || err) { 4364 netdev_err(bp->dev, "hwrm_ring_alloc type %d failed. rc:%x err:%x\n", 4365 ring_type, rc, err); 4366 return -EIO; 4367 } 4368 ring->fw_ring_id = ring_id; 4369 return rc; 4370 } 4371 4372 static int bnxt_hwrm_set_async_event_cr(struct bnxt *bp, int idx) 4373 { 4374 int rc; 4375 4376 if (BNXT_PF(bp)) { 4377 struct hwrm_func_cfg_input req = {0}; 4378 4379 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1); 4380 req.fid = cpu_to_le16(0xffff); 4381 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR); 4382 req.async_event_cr = cpu_to_le16(idx); 4383 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4384 } else { 4385 struct hwrm_func_vf_cfg_input req = {0}; 4386 4387 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_VF_CFG, -1, -1); 4388 req.enables = 4389 cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR); 4390 req.async_event_cr = cpu_to_le16(idx); 4391 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4392 } 4393 return rc; 4394 } 4395 4396 static int bnxt_hwrm_ring_alloc(struct bnxt *bp) 4397 { 4398 int i, rc = 0; 4399 4400 for (i = 0; i < bp->cp_nr_rings; i++) { 4401 struct bnxt_napi *bnapi = bp->bnapi[i]; 4402 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 4403 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 4404 u32 map_idx = ring->map_idx; 4405 4406 cpr->cp_doorbell = bp->bar1 + map_idx * 0x80; 4407 rc = hwrm_ring_alloc_send_msg(bp, ring, HWRM_RING_ALLOC_CMPL, 4408 map_idx); 4409 if (rc) 4410 goto err_out; 4411 BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons); 4412 bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id; 4413 4414 if (!i) { 4415 rc = bnxt_hwrm_set_async_event_cr(bp, ring->fw_ring_id); 4416 if (rc) 4417 netdev_warn(bp->dev, "Failed to set async event completion ring.\n"); 4418 } 4419 } 4420 4421 for (i = 0; i < bp->tx_nr_rings; i++) { 4422 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 4423 struct bnxt_ring_struct *ring = &txr->tx_ring_struct; 4424 u32 map_idx = i; 4425 4426 rc = hwrm_ring_alloc_send_msg(bp, ring, HWRM_RING_ALLOC_TX, 4427 map_idx); 4428 if (rc) 4429 goto err_out; 4430 txr->tx_doorbell = bp->bar1 + map_idx * 0x80; 4431 } 4432 4433 for (i = 0; i < bp->rx_nr_rings; i++) { 4434 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 4435 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct; 4436 u32 map_idx = rxr->bnapi->index; 4437 4438 rc = hwrm_ring_alloc_send_msg(bp, ring, HWRM_RING_ALLOC_RX, 4439 map_idx); 4440 if (rc) 4441 goto err_out; 4442 rxr->rx_doorbell = bp->bar1 + map_idx * 0x80; 4443 writel(DB_KEY_RX | rxr->rx_prod, rxr->rx_doorbell); 4444 bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id; 4445 } 4446 4447 if (bp->flags & BNXT_FLAG_AGG_RINGS) { 4448 for (i = 0; i < bp->rx_nr_rings; i++) { 4449 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 4450 struct bnxt_ring_struct *ring = 4451 &rxr->rx_agg_ring_struct; 4452 u32 grp_idx = ring->grp_idx; 4453 u32 map_idx = grp_idx + bp->rx_nr_rings; 4454 4455 rc = hwrm_ring_alloc_send_msg(bp, ring, 4456 HWRM_RING_ALLOC_AGG, 4457 map_idx); 4458 if (rc) 4459 goto err_out; 4460 4461 rxr->rx_agg_doorbell = bp->bar1 + map_idx * 0x80; 4462 writel(DB_KEY_RX | rxr->rx_agg_prod, 4463 rxr->rx_agg_doorbell); 4464 bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id; 4465 } 4466 } 4467 err_out: 4468 return rc; 4469 } 4470 4471 static int hwrm_ring_free_send_msg(struct bnxt *bp, 4472 struct bnxt_ring_struct *ring, 4473 u32 ring_type, int cmpl_ring_id) 4474 { 4475 int rc; 4476 struct hwrm_ring_free_input req = {0}; 4477 struct hwrm_ring_free_output *resp = bp->hwrm_cmd_resp_addr; 4478 u16 error_code; 4479 4480 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_FREE, cmpl_ring_id, -1); 4481 req.ring_type = ring_type; 4482 req.ring_id = cpu_to_le16(ring->fw_ring_id); 4483 4484 mutex_lock(&bp->hwrm_cmd_lock); 4485 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4486 error_code = le16_to_cpu(resp->error_code); 4487 mutex_unlock(&bp->hwrm_cmd_lock); 4488 4489 if (rc || error_code) { 4490 netdev_err(bp->dev, "hwrm_ring_free type %d failed. rc:%x err:%x\n", 4491 ring_type, rc, error_code); 4492 return -EIO; 4493 } 4494 return 0; 4495 } 4496 4497 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path) 4498 { 4499 int i; 4500 4501 if (!bp->bnapi) 4502 return; 4503 4504 for (i = 0; i < bp->tx_nr_rings; i++) { 4505 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 4506 struct bnxt_ring_struct *ring = &txr->tx_ring_struct; 4507 u32 grp_idx = txr->bnapi->index; 4508 u32 cmpl_ring_id = bp->grp_info[grp_idx].cp_fw_ring_id; 4509 4510 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 4511 hwrm_ring_free_send_msg(bp, ring, 4512 RING_FREE_REQ_RING_TYPE_TX, 4513 close_path ? cmpl_ring_id : 4514 INVALID_HW_RING_ID); 4515 ring->fw_ring_id = INVALID_HW_RING_ID; 4516 } 4517 } 4518 4519 for (i = 0; i < bp->rx_nr_rings; i++) { 4520 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 4521 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct; 4522 u32 grp_idx = rxr->bnapi->index; 4523 u32 cmpl_ring_id = bp->grp_info[grp_idx].cp_fw_ring_id; 4524 4525 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 4526 hwrm_ring_free_send_msg(bp, ring, 4527 RING_FREE_REQ_RING_TYPE_RX, 4528 close_path ? cmpl_ring_id : 4529 INVALID_HW_RING_ID); 4530 ring->fw_ring_id = INVALID_HW_RING_ID; 4531 bp->grp_info[grp_idx].rx_fw_ring_id = 4532 INVALID_HW_RING_ID; 4533 } 4534 } 4535 4536 for (i = 0; i < bp->rx_nr_rings; i++) { 4537 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 4538 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct; 4539 u32 grp_idx = rxr->bnapi->index; 4540 u32 cmpl_ring_id = bp->grp_info[grp_idx].cp_fw_ring_id; 4541 4542 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 4543 hwrm_ring_free_send_msg(bp, ring, 4544 RING_FREE_REQ_RING_TYPE_RX, 4545 close_path ? cmpl_ring_id : 4546 INVALID_HW_RING_ID); 4547 ring->fw_ring_id = INVALID_HW_RING_ID; 4548 bp->grp_info[grp_idx].agg_fw_ring_id = 4549 INVALID_HW_RING_ID; 4550 } 4551 } 4552 4553 /* The completion rings are about to be freed. After that the 4554 * IRQ doorbell will not work anymore. So we need to disable 4555 * IRQ here. 4556 */ 4557 bnxt_disable_int_sync(bp); 4558 4559 for (i = 0; i < bp->cp_nr_rings; i++) { 4560 struct bnxt_napi *bnapi = bp->bnapi[i]; 4561 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 4562 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 4563 4564 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 4565 hwrm_ring_free_send_msg(bp, ring, 4566 RING_FREE_REQ_RING_TYPE_L2_CMPL, 4567 INVALID_HW_RING_ID); 4568 ring->fw_ring_id = INVALID_HW_RING_ID; 4569 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID; 4570 } 4571 } 4572 } 4573 4574 static int bnxt_hwrm_get_rings(struct bnxt *bp) 4575 { 4576 struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 4577 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 4578 struct hwrm_func_qcfg_input req = {0}; 4579 int rc; 4580 4581 if (bp->hwrm_spec_code < 0x10601) 4582 return 0; 4583 4584 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1); 4585 req.fid = cpu_to_le16(0xffff); 4586 mutex_lock(&bp->hwrm_cmd_lock); 4587 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4588 if (rc) { 4589 mutex_unlock(&bp->hwrm_cmd_lock); 4590 return -EIO; 4591 } 4592 4593 hw_resc->resv_tx_rings = le16_to_cpu(resp->alloc_tx_rings); 4594 if (BNXT_NEW_RM(bp)) { 4595 u16 cp, stats; 4596 4597 hw_resc->resv_rx_rings = le16_to_cpu(resp->alloc_rx_rings); 4598 hw_resc->resv_hw_ring_grps = 4599 le32_to_cpu(resp->alloc_hw_ring_grps); 4600 hw_resc->resv_vnics = le16_to_cpu(resp->alloc_vnics); 4601 cp = le16_to_cpu(resp->alloc_cmpl_rings); 4602 stats = le16_to_cpu(resp->alloc_stat_ctx); 4603 cp = min_t(u16, cp, stats); 4604 hw_resc->resv_cp_rings = cp; 4605 } 4606 mutex_unlock(&bp->hwrm_cmd_lock); 4607 return 0; 4608 } 4609 4610 /* Caller must hold bp->hwrm_cmd_lock */ 4611 int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings) 4612 { 4613 struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 4614 struct hwrm_func_qcfg_input req = {0}; 4615 int rc; 4616 4617 if (bp->hwrm_spec_code < 0x10601) 4618 return 0; 4619 4620 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1); 4621 req.fid = cpu_to_le16(fid); 4622 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4623 if (!rc) 4624 *tx_rings = le16_to_cpu(resp->alloc_tx_rings); 4625 4626 return rc; 4627 } 4628 4629 static void 4630 __bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, struct hwrm_func_cfg_input *req, 4631 int tx_rings, int rx_rings, int ring_grps, 4632 int cp_rings, int vnics) 4633 { 4634 u32 enables = 0; 4635 4636 bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_CFG, -1, -1); 4637 req->fid = cpu_to_le16(0xffff); 4638 enables |= tx_rings ? FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS : 0; 4639 req->num_tx_rings = cpu_to_le16(tx_rings); 4640 if (BNXT_NEW_RM(bp)) { 4641 enables |= rx_rings ? FUNC_CFG_REQ_ENABLES_NUM_RX_RINGS : 0; 4642 enables |= cp_rings ? FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS | 4643 FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0; 4644 enables |= ring_grps ? 4645 FUNC_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0; 4646 enables |= vnics ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0; 4647 4648 req->num_rx_rings = cpu_to_le16(rx_rings); 4649 req->num_hw_ring_grps = cpu_to_le16(ring_grps); 4650 req->num_cmpl_rings = cpu_to_le16(cp_rings); 4651 req->num_stat_ctxs = req->num_cmpl_rings; 4652 req->num_vnics = cpu_to_le16(vnics); 4653 } 4654 req->enables = cpu_to_le32(enables); 4655 } 4656 4657 static void 4658 __bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, 4659 struct hwrm_func_vf_cfg_input *req, int tx_rings, 4660 int rx_rings, int ring_grps, int cp_rings, 4661 int vnics) 4662 { 4663 u32 enables = 0; 4664 4665 bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_VF_CFG, -1, -1); 4666 enables |= tx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_TX_RINGS : 0; 4667 enables |= rx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_RX_RINGS : 0; 4668 enables |= cp_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS | 4669 FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0; 4670 enables |= ring_grps ? FUNC_VF_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0; 4671 enables |= vnics ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0; 4672 4673 req->num_tx_rings = cpu_to_le16(tx_rings); 4674 req->num_rx_rings = cpu_to_le16(rx_rings); 4675 req->num_hw_ring_grps = cpu_to_le16(ring_grps); 4676 req->num_cmpl_rings = cpu_to_le16(cp_rings); 4677 req->num_stat_ctxs = req->num_cmpl_rings; 4678 req->num_vnics = cpu_to_le16(vnics); 4679 4680 req->enables = cpu_to_le32(enables); 4681 } 4682 4683 static int 4684 bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 4685 int ring_grps, int cp_rings, int vnics) 4686 { 4687 struct hwrm_func_cfg_input req = {0}; 4688 int rc; 4689 4690 __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps, 4691 cp_rings, vnics); 4692 if (!req.enables) 4693 return 0; 4694 4695 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4696 if (rc) 4697 return -ENOMEM; 4698 4699 if (bp->hwrm_spec_code < 0x10601) 4700 bp->hw_resc.resv_tx_rings = tx_rings; 4701 4702 rc = bnxt_hwrm_get_rings(bp); 4703 return rc; 4704 } 4705 4706 static int 4707 bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 4708 int ring_grps, int cp_rings, int vnics) 4709 { 4710 struct hwrm_func_vf_cfg_input req = {0}; 4711 int rc; 4712 4713 if (!BNXT_NEW_RM(bp)) { 4714 bp->hw_resc.resv_tx_rings = tx_rings; 4715 return 0; 4716 } 4717 4718 __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps, 4719 cp_rings, vnics); 4720 req.enables |= cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS | 4721 FUNC_VF_CFG_REQ_ENABLES_NUM_L2_CTXS); 4722 req.num_rsscos_ctxs = cpu_to_le16(BNXT_VF_MAX_RSS_CTX); 4723 req.num_l2_ctxs = cpu_to_le16(BNXT_VF_MAX_L2_CTX); 4724 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4725 if (rc) 4726 return -ENOMEM; 4727 4728 rc = bnxt_hwrm_get_rings(bp); 4729 return rc; 4730 } 4731 4732 static int bnxt_hwrm_reserve_rings(struct bnxt *bp, int tx, int rx, int grp, 4733 int cp, int vnic) 4734 { 4735 if (BNXT_PF(bp)) 4736 return bnxt_hwrm_reserve_pf_rings(bp, tx, rx, grp, cp, vnic); 4737 else 4738 return bnxt_hwrm_reserve_vf_rings(bp, tx, rx, grp, cp, vnic); 4739 } 4740 4741 static int bnxt_cp_rings_in_use(struct bnxt *bp) 4742 { 4743 int cp = bp->cp_nr_rings; 4744 int ulp_msix, ulp_base; 4745 4746 ulp_msix = bnxt_get_ulp_msix_num(bp); 4747 if (ulp_msix) { 4748 ulp_base = bnxt_get_ulp_msix_base(bp); 4749 cp += ulp_msix; 4750 if ((ulp_base + ulp_msix) > cp) 4751 cp = ulp_base + ulp_msix; 4752 } 4753 return cp; 4754 } 4755 4756 static bool bnxt_need_reserve_rings(struct bnxt *bp) 4757 { 4758 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 4759 int cp = bnxt_cp_rings_in_use(bp); 4760 int rx = bp->rx_nr_rings; 4761 int vnic = 1, grp = rx; 4762 4763 if (bp->hwrm_spec_code < 0x10601) 4764 return false; 4765 4766 if (hw_resc->resv_tx_rings != bp->tx_nr_rings) 4767 return true; 4768 4769 if (bp->flags & BNXT_FLAG_RFS) 4770 vnic = rx + 1; 4771 if (bp->flags & BNXT_FLAG_AGG_RINGS) 4772 rx <<= 1; 4773 if (BNXT_NEW_RM(bp) && 4774 (hw_resc->resv_rx_rings != rx || hw_resc->resv_cp_rings != cp || 4775 hw_resc->resv_hw_ring_grps != grp || hw_resc->resv_vnics != vnic)) 4776 return true; 4777 return false; 4778 } 4779 4780 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max, 4781 bool shared); 4782 4783 static int __bnxt_reserve_rings(struct bnxt *bp) 4784 { 4785 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 4786 int cp = bnxt_cp_rings_in_use(bp); 4787 int tx = bp->tx_nr_rings; 4788 int rx = bp->rx_nr_rings; 4789 int grp, rx_rings, rc; 4790 bool sh = false; 4791 int vnic = 1; 4792 4793 if (!bnxt_need_reserve_rings(bp)) 4794 return 0; 4795 4796 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 4797 sh = true; 4798 if (bp->flags & BNXT_FLAG_RFS) 4799 vnic = rx + 1; 4800 if (bp->flags & BNXT_FLAG_AGG_RINGS) 4801 rx <<= 1; 4802 grp = bp->rx_nr_rings; 4803 4804 rc = bnxt_hwrm_reserve_rings(bp, tx, rx, grp, cp, vnic); 4805 if (rc) 4806 return rc; 4807 4808 tx = hw_resc->resv_tx_rings; 4809 if (BNXT_NEW_RM(bp)) { 4810 rx = hw_resc->resv_rx_rings; 4811 cp = hw_resc->resv_cp_rings; 4812 grp = hw_resc->resv_hw_ring_grps; 4813 vnic = hw_resc->resv_vnics; 4814 } 4815 4816 rx_rings = rx; 4817 if (bp->flags & BNXT_FLAG_AGG_RINGS) { 4818 if (rx >= 2) { 4819 rx_rings = rx >> 1; 4820 } else { 4821 if (netif_running(bp->dev)) 4822 return -ENOMEM; 4823 4824 bp->flags &= ~BNXT_FLAG_AGG_RINGS; 4825 bp->flags |= BNXT_FLAG_NO_AGG_RINGS; 4826 bp->dev->hw_features &= ~NETIF_F_LRO; 4827 bp->dev->features &= ~NETIF_F_LRO; 4828 bnxt_set_ring_params(bp); 4829 } 4830 } 4831 rx_rings = min_t(int, rx_rings, grp); 4832 rc = bnxt_trim_rings(bp, &rx_rings, &tx, cp, sh); 4833 if (bp->flags & BNXT_FLAG_AGG_RINGS) 4834 rx = rx_rings << 1; 4835 cp = sh ? max_t(int, tx, rx_rings) : tx + rx_rings; 4836 bp->tx_nr_rings = tx; 4837 bp->rx_nr_rings = rx_rings; 4838 bp->cp_nr_rings = cp; 4839 4840 if (!tx || !rx || !cp || !grp || !vnic) 4841 return -ENOMEM; 4842 4843 return rc; 4844 } 4845 4846 static int bnxt_hwrm_check_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 4847 int ring_grps, int cp_rings, int vnics) 4848 { 4849 struct hwrm_func_vf_cfg_input req = {0}; 4850 u32 flags; 4851 int rc; 4852 4853 if (!BNXT_NEW_RM(bp)) 4854 return 0; 4855 4856 __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps, 4857 cp_rings, vnics); 4858 flags = FUNC_VF_CFG_REQ_FLAGS_TX_ASSETS_TEST | 4859 FUNC_VF_CFG_REQ_FLAGS_RX_ASSETS_TEST | 4860 FUNC_VF_CFG_REQ_FLAGS_CMPL_ASSETS_TEST | 4861 FUNC_VF_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST | 4862 FUNC_VF_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST | 4863 FUNC_VF_CFG_REQ_FLAGS_VNIC_ASSETS_TEST; 4864 4865 req.flags = cpu_to_le32(flags); 4866 rc = hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4867 if (rc) 4868 return -ENOMEM; 4869 return 0; 4870 } 4871 4872 static int bnxt_hwrm_check_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 4873 int ring_grps, int cp_rings, int vnics) 4874 { 4875 struct hwrm_func_cfg_input req = {0}; 4876 u32 flags; 4877 int rc; 4878 4879 __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps, 4880 cp_rings, vnics); 4881 flags = FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST; 4882 if (BNXT_NEW_RM(bp)) 4883 flags |= FUNC_CFG_REQ_FLAGS_RX_ASSETS_TEST | 4884 FUNC_CFG_REQ_FLAGS_CMPL_ASSETS_TEST | 4885 FUNC_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST | 4886 FUNC_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST | 4887 FUNC_CFG_REQ_FLAGS_VNIC_ASSETS_TEST; 4888 4889 req.flags = cpu_to_le32(flags); 4890 rc = hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4891 if (rc) 4892 return -ENOMEM; 4893 return 0; 4894 } 4895 4896 static int bnxt_hwrm_check_rings(struct bnxt *bp, int tx_rings, int rx_rings, 4897 int ring_grps, int cp_rings, int vnics) 4898 { 4899 if (bp->hwrm_spec_code < 0x10801) 4900 return 0; 4901 4902 if (BNXT_PF(bp)) 4903 return bnxt_hwrm_check_pf_rings(bp, tx_rings, rx_rings, 4904 ring_grps, cp_rings, vnics); 4905 4906 return bnxt_hwrm_check_vf_rings(bp, tx_rings, rx_rings, ring_grps, 4907 cp_rings, vnics); 4908 } 4909 4910 static void bnxt_hwrm_set_coal_params(struct bnxt_coal *hw_coal, 4911 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req) 4912 { 4913 u16 val, tmr, max, flags; 4914 4915 max = hw_coal->bufs_per_record * 128; 4916 if (hw_coal->budget) 4917 max = hw_coal->bufs_per_record * hw_coal->budget; 4918 4919 val = clamp_t(u16, hw_coal->coal_bufs, 1, max); 4920 req->num_cmpl_aggr_int = cpu_to_le16(val); 4921 4922 /* This is a 6-bit value and must not be 0, or we'll get non stop IRQ */ 4923 val = min_t(u16, val, 63); 4924 req->num_cmpl_dma_aggr = cpu_to_le16(val); 4925 4926 /* This is a 6-bit value and must not be 0, or we'll get non stop IRQ */ 4927 val = clamp_t(u16, hw_coal->coal_bufs_irq, 1, 63); 4928 req->num_cmpl_dma_aggr_during_int = cpu_to_le16(val); 4929 4930 tmr = BNXT_USEC_TO_COAL_TIMER(hw_coal->coal_ticks); 4931 tmr = max_t(u16, tmr, 1); 4932 req->int_lat_tmr_max = cpu_to_le16(tmr); 4933 4934 /* min timer set to 1/2 of interrupt timer */ 4935 val = tmr / 2; 4936 req->int_lat_tmr_min = cpu_to_le16(val); 4937 4938 /* buf timer set to 1/4 of interrupt timer */ 4939 val = max_t(u16, tmr / 4, 1); 4940 req->cmpl_aggr_dma_tmr = cpu_to_le16(val); 4941 4942 tmr = BNXT_USEC_TO_COAL_TIMER(hw_coal->coal_ticks_irq); 4943 tmr = max_t(u16, tmr, 1); 4944 req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(tmr); 4945 4946 flags = RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET; 4947 if (hw_coal->idle_thresh && hw_coal->coal_ticks < hw_coal->idle_thresh) 4948 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE; 4949 req->flags = cpu_to_le16(flags); 4950 } 4951 4952 int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi) 4953 { 4954 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0}; 4955 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 4956 struct bnxt_coal coal; 4957 unsigned int grp_idx; 4958 4959 /* Tick values in micro seconds. 4960 * 1 coal_buf x bufs_per_record = 1 completion record. 4961 */ 4962 memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal)); 4963 4964 coal.coal_ticks = cpr->rx_ring_coal.coal_ticks; 4965 coal.coal_bufs = cpr->rx_ring_coal.coal_bufs; 4966 4967 if (!bnapi->rx_ring) 4968 return -ENODEV; 4969 4970 bnxt_hwrm_cmd_hdr_init(bp, &req_rx, 4971 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1); 4972 4973 bnxt_hwrm_set_coal_params(&coal, &req_rx); 4974 4975 grp_idx = bnapi->index; 4976 req_rx.ring_id = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id); 4977 4978 return hwrm_send_message(bp, &req_rx, sizeof(req_rx), 4979 HWRM_CMD_TIMEOUT); 4980 } 4981 4982 int bnxt_hwrm_set_coal(struct bnxt *bp) 4983 { 4984 int i, rc = 0; 4985 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0}, 4986 req_tx = {0}, *req; 4987 4988 bnxt_hwrm_cmd_hdr_init(bp, &req_rx, 4989 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1); 4990 bnxt_hwrm_cmd_hdr_init(bp, &req_tx, 4991 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1); 4992 4993 bnxt_hwrm_set_coal_params(&bp->rx_coal, &req_rx); 4994 bnxt_hwrm_set_coal_params(&bp->tx_coal, &req_tx); 4995 4996 mutex_lock(&bp->hwrm_cmd_lock); 4997 for (i = 0; i < bp->cp_nr_rings; i++) { 4998 struct bnxt_napi *bnapi = bp->bnapi[i]; 4999 5000 req = &req_rx; 5001 if (!bnapi->rx_ring) 5002 req = &req_tx; 5003 req->ring_id = cpu_to_le16(bp->grp_info[i].cp_fw_ring_id); 5004 5005 rc = _hwrm_send_message(bp, req, sizeof(*req), 5006 HWRM_CMD_TIMEOUT); 5007 if (rc) 5008 break; 5009 } 5010 mutex_unlock(&bp->hwrm_cmd_lock); 5011 return rc; 5012 } 5013 5014 static int bnxt_hwrm_stat_ctx_free(struct bnxt *bp) 5015 { 5016 int rc = 0, i; 5017 struct hwrm_stat_ctx_free_input req = {0}; 5018 5019 if (!bp->bnapi) 5020 return 0; 5021 5022 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 5023 return 0; 5024 5025 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_FREE, -1, -1); 5026 5027 mutex_lock(&bp->hwrm_cmd_lock); 5028 for (i = 0; i < bp->cp_nr_rings; i++) { 5029 struct bnxt_napi *bnapi = bp->bnapi[i]; 5030 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 5031 5032 if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) { 5033 req.stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id); 5034 5035 rc = _hwrm_send_message(bp, &req, sizeof(req), 5036 HWRM_CMD_TIMEOUT); 5037 if (rc) 5038 break; 5039 5040 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID; 5041 } 5042 } 5043 mutex_unlock(&bp->hwrm_cmd_lock); 5044 return rc; 5045 } 5046 5047 static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp) 5048 { 5049 int rc = 0, i; 5050 struct hwrm_stat_ctx_alloc_input req = {0}; 5051 struct hwrm_stat_ctx_alloc_output *resp = bp->hwrm_cmd_resp_addr; 5052 5053 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 5054 return 0; 5055 5056 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_ALLOC, -1, -1); 5057 5058 req.update_period_ms = cpu_to_le32(bp->stats_coal_ticks / 1000); 5059 5060 mutex_lock(&bp->hwrm_cmd_lock); 5061 for (i = 0; i < bp->cp_nr_rings; i++) { 5062 struct bnxt_napi *bnapi = bp->bnapi[i]; 5063 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 5064 5065 req.stats_dma_addr = cpu_to_le64(cpr->hw_stats_map); 5066 5067 rc = _hwrm_send_message(bp, &req, sizeof(req), 5068 HWRM_CMD_TIMEOUT); 5069 if (rc) 5070 break; 5071 5072 cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id); 5073 5074 bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id; 5075 } 5076 mutex_unlock(&bp->hwrm_cmd_lock); 5077 return rc; 5078 } 5079 5080 static int bnxt_hwrm_func_qcfg(struct bnxt *bp) 5081 { 5082 struct hwrm_func_qcfg_input req = {0}; 5083 struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 5084 u16 flags; 5085 int rc; 5086 5087 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1); 5088 req.fid = cpu_to_le16(0xffff); 5089 mutex_lock(&bp->hwrm_cmd_lock); 5090 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5091 if (rc) 5092 goto func_qcfg_exit; 5093 5094 #ifdef CONFIG_BNXT_SRIOV 5095 if (BNXT_VF(bp)) { 5096 struct bnxt_vf_info *vf = &bp->vf; 5097 5098 vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK; 5099 } 5100 #endif 5101 flags = le16_to_cpu(resp->flags); 5102 if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED | 5103 FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED)) { 5104 bp->fw_cap |= BNXT_FW_CAP_LLDP_AGENT; 5105 if (flags & FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED) 5106 bp->fw_cap |= BNXT_FW_CAP_DCBX_AGENT; 5107 } 5108 if (BNXT_PF(bp) && (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST)) 5109 bp->flags |= BNXT_FLAG_MULTI_HOST; 5110 5111 switch (resp->port_partition_type) { 5112 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0: 5113 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5: 5114 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0: 5115 bp->port_partition_type = resp->port_partition_type; 5116 break; 5117 } 5118 if (bp->hwrm_spec_code < 0x10707 || 5119 resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEB) 5120 bp->br_mode = BRIDGE_MODE_VEB; 5121 else if (resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEPA) 5122 bp->br_mode = BRIDGE_MODE_VEPA; 5123 else 5124 bp->br_mode = BRIDGE_MODE_UNDEF; 5125 5126 bp->max_mtu = le16_to_cpu(resp->max_mtu_configured); 5127 if (!bp->max_mtu) 5128 bp->max_mtu = BNXT_MAX_MTU; 5129 5130 func_qcfg_exit: 5131 mutex_unlock(&bp->hwrm_cmd_lock); 5132 return rc; 5133 } 5134 5135 int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp, bool all) 5136 { 5137 struct hwrm_func_resource_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 5138 struct hwrm_func_resource_qcaps_input req = {0}; 5139 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 5140 int rc; 5141 5142 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESOURCE_QCAPS, -1, -1); 5143 req.fid = cpu_to_le16(0xffff); 5144 5145 mutex_lock(&bp->hwrm_cmd_lock); 5146 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5147 if (rc) { 5148 rc = -EIO; 5149 goto hwrm_func_resc_qcaps_exit; 5150 } 5151 5152 hw_resc->max_tx_sch_inputs = le16_to_cpu(resp->max_tx_scheduler_inputs); 5153 if (!all) 5154 goto hwrm_func_resc_qcaps_exit; 5155 5156 hw_resc->min_rsscos_ctxs = le16_to_cpu(resp->min_rsscos_ctx); 5157 hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx); 5158 hw_resc->min_cp_rings = le16_to_cpu(resp->min_cmpl_rings); 5159 hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings); 5160 hw_resc->min_tx_rings = le16_to_cpu(resp->min_tx_rings); 5161 hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings); 5162 hw_resc->min_rx_rings = le16_to_cpu(resp->min_rx_rings); 5163 hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings); 5164 hw_resc->min_hw_ring_grps = le16_to_cpu(resp->min_hw_ring_grps); 5165 hw_resc->max_hw_ring_grps = le16_to_cpu(resp->max_hw_ring_grps); 5166 hw_resc->min_l2_ctxs = le16_to_cpu(resp->min_l2_ctxs); 5167 hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs); 5168 hw_resc->min_vnics = le16_to_cpu(resp->min_vnics); 5169 hw_resc->max_vnics = le16_to_cpu(resp->max_vnics); 5170 hw_resc->min_stat_ctxs = le16_to_cpu(resp->min_stat_ctx); 5171 hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx); 5172 5173 if (BNXT_PF(bp)) { 5174 struct bnxt_pf_info *pf = &bp->pf; 5175 5176 pf->vf_resv_strategy = 5177 le16_to_cpu(resp->vf_reservation_strategy); 5178 if (pf->vf_resv_strategy > BNXT_VF_RESV_STRATEGY_MINIMAL_STATIC) 5179 pf->vf_resv_strategy = BNXT_VF_RESV_STRATEGY_MAXIMAL; 5180 } 5181 hwrm_func_resc_qcaps_exit: 5182 mutex_unlock(&bp->hwrm_cmd_lock); 5183 return rc; 5184 } 5185 5186 static int __bnxt_hwrm_func_qcaps(struct bnxt *bp) 5187 { 5188 int rc = 0; 5189 struct hwrm_func_qcaps_input req = {0}; 5190 struct hwrm_func_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 5191 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 5192 u32 flags; 5193 5194 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCAPS, -1, -1); 5195 req.fid = cpu_to_le16(0xffff); 5196 5197 mutex_lock(&bp->hwrm_cmd_lock); 5198 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5199 if (rc) 5200 goto hwrm_func_qcaps_exit; 5201 5202 flags = le32_to_cpu(resp->flags); 5203 if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED) 5204 bp->flags |= BNXT_FLAG_ROCEV1_CAP; 5205 if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED) 5206 bp->flags |= BNXT_FLAG_ROCEV2_CAP; 5207 5208 bp->tx_push_thresh = 0; 5209 if (flags & FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED) 5210 bp->tx_push_thresh = BNXT_TX_PUSH_THRESH; 5211 5212 hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx); 5213 hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings); 5214 hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings); 5215 hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings); 5216 hw_resc->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps); 5217 if (!hw_resc->max_hw_ring_grps) 5218 hw_resc->max_hw_ring_grps = hw_resc->max_tx_rings; 5219 hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs); 5220 hw_resc->max_vnics = le16_to_cpu(resp->max_vnics); 5221 hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx); 5222 5223 if (BNXT_PF(bp)) { 5224 struct bnxt_pf_info *pf = &bp->pf; 5225 5226 pf->fw_fid = le16_to_cpu(resp->fid); 5227 pf->port_id = le16_to_cpu(resp->port_id); 5228 bp->dev->dev_port = pf->port_id; 5229 memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN); 5230 pf->first_vf_id = le16_to_cpu(resp->first_vf_id); 5231 pf->max_vfs = le16_to_cpu(resp->max_vfs); 5232 pf->max_encap_records = le32_to_cpu(resp->max_encap_records); 5233 pf->max_decap_records = le32_to_cpu(resp->max_decap_records); 5234 pf->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows); 5235 pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows); 5236 pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows); 5237 pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows); 5238 if (flags & FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED) 5239 bp->flags |= BNXT_FLAG_WOL_CAP; 5240 } else { 5241 #ifdef CONFIG_BNXT_SRIOV 5242 struct bnxt_vf_info *vf = &bp->vf; 5243 5244 vf->fw_fid = le16_to_cpu(resp->fid); 5245 memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN); 5246 #endif 5247 } 5248 5249 hwrm_func_qcaps_exit: 5250 mutex_unlock(&bp->hwrm_cmd_lock); 5251 return rc; 5252 } 5253 5254 static int bnxt_hwrm_func_qcaps(struct bnxt *bp) 5255 { 5256 int rc; 5257 5258 rc = __bnxt_hwrm_func_qcaps(bp); 5259 if (rc) 5260 return rc; 5261 if (bp->hwrm_spec_code >= 0x10803) { 5262 rc = bnxt_hwrm_func_resc_qcaps(bp, true); 5263 if (!rc) 5264 bp->fw_cap |= BNXT_FW_CAP_NEW_RM; 5265 } 5266 return 0; 5267 } 5268 5269 static int bnxt_hwrm_func_reset(struct bnxt *bp) 5270 { 5271 struct hwrm_func_reset_input req = {0}; 5272 5273 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESET, -1, -1); 5274 req.enables = 0; 5275 5276 return hwrm_send_message(bp, &req, sizeof(req), HWRM_RESET_TIMEOUT); 5277 } 5278 5279 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp) 5280 { 5281 int rc = 0; 5282 struct hwrm_queue_qportcfg_input req = {0}; 5283 struct hwrm_queue_qportcfg_output *resp = bp->hwrm_cmd_resp_addr; 5284 u8 i, j, *qptr; 5285 bool no_rdma; 5286 5287 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_QUEUE_QPORTCFG, -1, -1); 5288 5289 mutex_lock(&bp->hwrm_cmd_lock); 5290 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5291 if (rc) 5292 goto qportcfg_exit; 5293 5294 if (!resp->max_configurable_queues) { 5295 rc = -EINVAL; 5296 goto qportcfg_exit; 5297 } 5298 bp->max_tc = resp->max_configurable_queues; 5299 bp->max_lltc = resp->max_configurable_lossless_queues; 5300 if (bp->max_tc > BNXT_MAX_QUEUE) 5301 bp->max_tc = BNXT_MAX_QUEUE; 5302 5303 no_rdma = !(bp->flags & BNXT_FLAG_ROCE_CAP); 5304 qptr = &resp->queue_id0; 5305 for (i = 0, j = 0; i < bp->max_tc; i++) { 5306 bp->q_info[j].queue_id = *qptr++; 5307 bp->q_info[j].queue_profile = *qptr++; 5308 bp->tc_to_qidx[j] = j; 5309 if (!BNXT_CNPQ(bp->q_info[j].queue_profile) || 5310 (no_rdma && BNXT_PF(bp))) 5311 j++; 5312 } 5313 bp->max_tc = max_t(u8, j, 1); 5314 5315 if (resp->queue_cfg_info & QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG) 5316 bp->max_tc = 1; 5317 5318 if (bp->max_lltc > bp->max_tc) 5319 bp->max_lltc = bp->max_tc; 5320 5321 qportcfg_exit: 5322 mutex_unlock(&bp->hwrm_cmd_lock); 5323 return rc; 5324 } 5325 5326 static int bnxt_hwrm_ver_get(struct bnxt *bp) 5327 { 5328 int rc; 5329 struct hwrm_ver_get_input req = {0}; 5330 struct hwrm_ver_get_output *resp = bp->hwrm_cmd_resp_addr; 5331 u32 dev_caps_cfg; 5332 5333 bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN; 5334 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VER_GET, -1, -1); 5335 req.hwrm_intf_maj = HWRM_VERSION_MAJOR; 5336 req.hwrm_intf_min = HWRM_VERSION_MINOR; 5337 req.hwrm_intf_upd = HWRM_VERSION_UPDATE; 5338 mutex_lock(&bp->hwrm_cmd_lock); 5339 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5340 if (rc) 5341 goto hwrm_ver_get_exit; 5342 5343 memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output)); 5344 5345 bp->hwrm_spec_code = resp->hwrm_intf_maj_8b << 16 | 5346 resp->hwrm_intf_min_8b << 8 | 5347 resp->hwrm_intf_upd_8b; 5348 if (resp->hwrm_intf_maj_8b < 1) { 5349 netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n", 5350 resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b, 5351 resp->hwrm_intf_upd_8b); 5352 netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n"); 5353 } 5354 snprintf(bp->fw_ver_str, BC_HWRM_STR_LEN, "%d.%d.%d.%d", 5355 resp->hwrm_fw_maj_8b, resp->hwrm_fw_min_8b, 5356 resp->hwrm_fw_bld_8b, resp->hwrm_fw_rsvd_8b); 5357 5358 bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout); 5359 if (!bp->hwrm_cmd_timeout) 5360 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT; 5361 5362 if (resp->hwrm_intf_maj_8b >= 1) 5363 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len); 5364 5365 bp->chip_num = le16_to_cpu(resp->chip_num); 5366 if (bp->chip_num == CHIP_NUM_58700 && !resp->chip_rev && 5367 !resp->chip_metal) 5368 bp->flags |= BNXT_FLAG_CHIP_NITRO_A0; 5369 5370 dev_caps_cfg = le32_to_cpu(resp->dev_caps_cfg); 5371 if ((dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) && 5372 (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED)) 5373 bp->fw_cap |= BNXT_FW_CAP_SHORT_CMD; 5374 5375 hwrm_ver_get_exit: 5376 mutex_unlock(&bp->hwrm_cmd_lock); 5377 return rc; 5378 } 5379 5380 int bnxt_hwrm_fw_set_time(struct bnxt *bp) 5381 { 5382 struct hwrm_fw_set_time_input req = {0}; 5383 struct tm tm; 5384 time64_t now = ktime_get_real_seconds(); 5385 5386 if ((BNXT_VF(bp) && bp->hwrm_spec_code < 0x10901) || 5387 bp->hwrm_spec_code < 0x10400) 5388 return -EOPNOTSUPP; 5389 5390 time64_to_tm(now, 0, &tm); 5391 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FW_SET_TIME, -1, -1); 5392 req.year = cpu_to_le16(1900 + tm.tm_year); 5393 req.month = 1 + tm.tm_mon; 5394 req.day = tm.tm_mday; 5395 req.hour = tm.tm_hour; 5396 req.minute = tm.tm_min; 5397 req.second = tm.tm_sec; 5398 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5399 } 5400 5401 static int bnxt_hwrm_port_qstats(struct bnxt *bp) 5402 { 5403 int rc; 5404 struct bnxt_pf_info *pf = &bp->pf; 5405 struct hwrm_port_qstats_input req = {0}; 5406 5407 if (!(bp->flags & BNXT_FLAG_PORT_STATS)) 5408 return 0; 5409 5410 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS, -1, -1); 5411 req.port_id = cpu_to_le16(pf->port_id); 5412 req.tx_stat_host_addr = cpu_to_le64(bp->hw_tx_port_stats_map); 5413 req.rx_stat_host_addr = cpu_to_le64(bp->hw_rx_port_stats_map); 5414 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5415 return rc; 5416 } 5417 5418 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp) 5419 { 5420 struct hwrm_port_qstats_ext_input req = {0}; 5421 struct bnxt_pf_info *pf = &bp->pf; 5422 5423 if (!(bp->flags & BNXT_FLAG_PORT_STATS_EXT)) 5424 return 0; 5425 5426 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS_EXT, -1, -1); 5427 req.port_id = cpu_to_le16(pf->port_id); 5428 req.rx_stat_size = cpu_to_le16(sizeof(struct rx_port_stats_ext)); 5429 req.rx_stat_host_addr = cpu_to_le64(bp->hw_rx_port_stats_ext_map); 5430 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5431 } 5432 5433 static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp) 5434 { 5435 if (bp->vxlan_port_cnt) { 5436 bnxt_hwrm_tunnel_dst_port_free( 5437 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN); 5438 } 5439 bp->vxlan_port_cnt = 0; 5440 if (bp->nge_port_cnt) { 5441 bnxt_hwrm_tunnel_dst_port_free( 5442 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE); 5443 } 5444 bp->nge_port_cnt = 0; 5445 } 5446 5447 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa) 5448 { 5449 int rc, i; 5450 u32 tpa_flags = 0; 5451 5452 if (set_tpa) 5453 tpa_flags = bp->flags & BNXT_FLAG_TPA; 5454 for (i = 0; i < bp->nr_vnics; i++) { 5455 rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags); 5456 if (rc) { 5457 netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n", 5458 i, rc); 5459 return rc; 5460 } 5461 } 5462 return 0; 5463 } 5464 5465 static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp) 5466 { 5467 int i; 5468 5469 for (i = 0; i < bp->nr_vnics; i++) 5470 bnxt_hwrm_vnic_set_rss(bp, i, false); 5471 } 5472 5473 static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path, 5474 bool irq_re_init) 5475 { 5476 if (bp->vnic_info) { 5477 bnxt_hwrm_clear_vnic_filter(bp); 5478 /* clear all RSS setting before free vnic ctx */ 5479 bnxt_hwrm_clear_vnic_rss(bp); 5480 bnxt_hwrm_vnic_ctx_free(bp); 5481 /* before free the vnic, undo the vnic tpa settings */ 5482 if (bp->flags & BNXT_FLAG_TPA) 5483 bnxt_set_tpa(bp, false); 5484 bnxt_hwrm_vnic_free(bp); 5485 } 5486 bnxt_hwrm_ring_free(bp, close_path); 5487 bnxt_hwrm_ring_grp_free(bp); 5488 if (irq_re_init) { 5489 bnxt_hwrm_stat_ctx_free(bp); 5490 bnxt_hwrm_free_tunnel_ports(bp); 5491 } 5492 } 5493 5494 static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode) 5495 { 5496 struct hwrm_func_cfg_input req = {0}; 5497 int rc; 5498 5499 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1); 5500 req.fid = cpu_to_le16(0xffff); 5501 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_EVB_MODE); 5502 if (br_mode == BRIDGE_MODE_VEB) 5503 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEB; 5504 else if (br_mode == BRIDGE_MODE_VEPA) 5505 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEPA; 5506 else 5507 return -EINVAL; 5508 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5509 if (rc) 5510 rc = -EIO; 5511 return rc; 5512 } 5513 5514 static int bnxt_hwrm_set_cache_line_size(struct bnxt *bp, int size) 5515 { 5516 struct hwrm_func_cfg_input req = {0}; 5517 int rc; 5518 5519 if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10803) 5520 return 0; 5521 5522 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1); 5523 req.fid = cpu_to_le16(0xffff); 5524 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_CACHE_LINESIZE); 5525 req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_64; 5526 if (size == 128) 5527 req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_128; 5528 5529 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5530 if (rc) 5531 rc = -EIO; 5532 return rc; 5533 } 5534 5535 static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id) 5536 { 5537 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5538 int rc; 5539 5540 if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) 5541 goto skip_rss_ctx; 5542 5543 /* allocate context for vnic */ 5544 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 0); 5545 if (rc) { 5546 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n", 5547 vnic_id, rc); 5548 goto vnic_setup_err; 5549 } 5550 bp->rsscos_nr_ctxs++; 5551 5552 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 5553 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 1); 5554 if (rc) { 5555 netdev_err(bp->dev, "hwrm vnic %d cos ctx alloc failure rc: %x\n", 5556 vnic_id, rc); 5557 goto vnic_setup_err; 5558 } 5559 bp->rsscos_nr_ctxs++; 5560 } 5561 5562 skip_rss_ctx: 5563 /* configure default vnic, ring grp */ 5564 rc = bnxt_hwrm_vnic_cfg(bp, vnic_id); 5565 if (rc) { 5566 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n", 5567 vnic_id, rc); 5568 goto vnic_setup_err; 5569 } 5570 5571 /* Enable RSS hashing on vnic */ 5572 rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true); 5573 if (rc) { 5574 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n", 5575 vnic_id, rc); 5576 goto vnic_setup_err; 5577 } 5578 5579 if (bp->flags & BNXT_FLAG_AGG_RINGS) { 5580 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id); 5581 if (rc) { 5582 netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n", 5583 vnic_id, rc); 5584 } 5585 } 5586 5587 vnic_setup_err: 5588 return rc; 5589 } 5590 5591 static int bnxt_alloc_rfs_vnics(struct bnxt *bp) 5592 { 5593 #ifdef CONFIG_RFS_ACCEL 5594 int i, rc = 0; 5595 5596 for (i = 0; i < bp->rx_nr_rings; i++) { 5597 struct bnxt_vnic_info *vnic; 5598 u16 vnic_id = i + 1; 5599 u16 ring_id = i; 5600 5601 if (vnic_id >= bp->nr_vnics) 5602 break; 5603 5604 vnic = &bp->vnic_info[vnic_id]; 5605 vnic->flags |= BNXT_VNIC_RFS_FLAG; 5606 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP) 5607 vnic->flags |= BNXT_VNIC_RFS_NEW_RSS_FLAG; 5608 rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, ring_id, 1); 5609 if (rc) { 5610 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n", 5611 vnic_id, rc); 5612 break; 5613 } 5614 rc = bnxt_setup_vnic(bp, vnic_id); 5615 if (rc) 5616 break; 5617 } 5618 return rc; 5619 #else 5620 return 0; 5621 #endif 5622 } 5623 5624 /* Allow PF and VF with default VLAN to be in promiscuous mode */ 5625 static bool bnxt_promisc_ok(struct bnxt *bp) 5626 { 5627 #ifdef CONFIG_BNXT_SRIOV 5628 if (BNXT_VF(bp) && !bp->vf.vlan) 5629 return false; 5630 #endif 5631 return true; 5632 } 5633 5634 static int bnxt_setup_nitroa0_vnic(struct bnxt *bp) 5635 { 5636 unsigned int rc = 0; 5637 5638 rc = bnxt_hwrm_vnic_alloc(bp, 1, bp->rx_nr_rings - 1, 1); 5639 if (rc) { 5640 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n", 5641 rc); 5642 return rc; 5643 } 5644 5645 rc = bnxt_hwrm_vnic_cfg(bp, 1); 5646 if (rc) { 5647 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n", 5648 rc); 5649 return rc; 5650 } 5651 return rc; 5652 } 5653 5654 static int bnxt_cfg_rx_mode(struct bnxt *); 5655 static bool bnxt_mc_list_updated(struct bnxt *, u32 *); 5656 5657 static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init) 5658 { 5659 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 5660 int rc = 0; 5661 unsigned int rx_nr_rings = bp->rx_nr_rings; 5662 5663 if (irq_re_init) { 5664 rc = bnxt_hwrm_stat_ctx_alloc(bp); 5665 if (rc) { 5666 netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n", 5667 rc); 5668 goto err_out; 5669 } 5670 } 5671 5672 rc = bnxt_hwrm_ring_alloc(bp); 5673 if (rc) { 5674 netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc); 5675 goto err_out; 5676 } 5677 5678 rc = bnxt_hwrm_ring_grp_alloc(bp); 5679 if (rc) { 5680 netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc); 5681 goto err_out; 5682 } 5683 5684 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 5685 rx_nr_rings--; 5686 5687 /* default vnic 0 */ 5688 rc = bnxt_hwrm_vnic_alloc(bp, 0, 0, rx_nr_rings); 5689 if (rc) { 5690 netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc); 5691 goto err_out; 5692 } 5693 5694 rc = bnxt_setup_vnic(bp, 0); 5695 if (rc) 5696 goto err_out; 5697 5698 if (bp->flags & BNXT_FLAG_RFS) { 5699 rc = bnxt_alloc_rfs_vnics(bp); 5700 if (rc) 5701 goto err_out; 5702 } 5703 5704 if (bp->flags & BNXT_FLAG_TPA) { 5705 rc = bnxt_set_tpa(bp, true); 5706 if (rc) 5707 goto err_out; 5708 } 5709 5710 if (BNXT_VF(bp)) 5711 bnxt_update_vf_mac(bp); 5712 5713 /* Filter for default vnic 0 */ 5714 rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr); 5715 if (rc) { 5716 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc); 5717 goto err_out; 5718 } 5719 vnic->uc_filter_count = 1; 5720 5721 vnic->rx_mask = 0; 5722 if (bp->dev->flags & IFF_BROADCAST) 5723 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST; 5724 5725 if ((bp->dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp)) 5726 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 5727 5728 if (bp->dev->flags & IFF_ALLMULTI) { 5729 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 5730 vnic->mc_list_count = 0; 5731 } else { 5732 u32 mask = 0; 5733 5734 bnxt_mc_list_updated(bp, &mask); 5735 vnic->rx_mask |= mask; 5736 } 5737 5738 rc = bnxt_cfg_rx_mode(bp); 5739 if (rc) 5740 goto err_out; 5741 5742 rc = bnxt_hwrm_set_coal(bp); 5743 if (rc) 5744 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n", 5745 rc); 5746 5747 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 5748 rc = bnxt_setup_nitroa0_vnic(bp); 5749 if (rc) 5750 netdev_err(bp->dev, "Special vnic setup failure for NS2 A0 rc: %x\n", 5751 rc); 5752 } 5753 5754 if (BNXT_VF(bp)) { 5755 bnxt_hwrm_func_qcfg(bp); 5756 netdev_update_features(bp->dev); 5757 } 5758 5759 return 0; 5760 5761 err_out: 5762 bnxt_hwrm_resource_free(bp, 0, true); 5763 5764 return rc; 5765 } 5766 5767 static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init) 5768 { 5769 bnxt_hwrm_resource_free(bp, 1, irq_re_init); 5770 return 0; 5771 } 5772 5773 static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init) 5774 { 5775 bnxt_init_cp_rings(bp); 5776 bnxt_init_rx_rings(bp); 5777 bnxt_init_tx_rings(bp); 5778 bnxt_init_ring_grps(bp, irq_re_init); 5779 bnxt_init_vnics(bp); 5780 5781 return bnxt_init_chip(bp, irq_re_init); 5782 } 5783 5784 static int bnxt_set_real_num_queues(struct bnxt *bp) 5785 { 5786 int rc; 5787 struct net_device *dev = bp->dev; 5788 5789 rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings - 5790 bp->tx_nr_rings_xdp); 5791 if (rc) 5792 return rc; 5793 5794 rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings); 5795 if (rc) 5796 return rc; 5797 5798 #ifdef CONFIG_RFS_ACCEL 5799 if (bp->flags & BNXT_FLAG_RFS) 5800 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings); 5801 #endif 5802 5803 return rc; 5804 } 5805 5806 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max, 5807 bool shared) 5808 { 5809 int _rx = *rx, _tx = *tx; 5810 5811 if (shared) { 5812 *rx = min_t(int, _rx, max); 5813 *tx = min_t(int, _tx, max); 5814 } else { 5815 if (max < 2) 5816 return -ENOMEM; 5817 5818 while (_rx + _tx > max) { 5819 if (_rx > _tx && _rx > 1) 5820 _rx--; 5821 else if (_tx > 1) 5822 _tx--; 5823 } 5824 *rx = _rx; 5825 *tx = _tx; 5826 } 5827 return 0; 5828 } 5829 5830 static void bnxt_setup_msix(struct bnxt *bp) 5831 { 5832 const int len = sizeof(bp->irq_tbl[0].name); 5833 struct net_device *dev = bp->dev; 5834 int tcs, i; 5835 5836 tcs = netdev_get_num_tc(dev); 5837 if (tcs > 1) { 5838 int i, off, count; 5839 5840 for (i = 0; i < tcs; i++) { 5841 count = bp->tx_nr_rings_per_tc; 5842 off = i * count; 5843 netdev_set_tc_queue(dev, i, count, off); 5844 } 5845 } 5846 5847 for (i = 0; i < bp->cp_nr_rings; i++) { 5848 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 5849 char *attr; 5850 5851 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 5852 attr = "TxRx"; 5853 else if (i < bp->rx_nr_rings) 5854 attr = "rx"; 5855 else 5856 attr = "tx"; 5857 5858 snprintf(bp->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name, 5859 attr, i); 5860 bp->irq_tbl[map_idx].handler = bnxt_msix; 5861 } 5862 } 5863 5864 static void bnxt_setup_inta(struct bnxt *bp) 5865 { 5866 const int len = sizeof(bp->irq_tbl[0].name); 5867 5868 if (netdev_get_num_tc(bp->dev)) 5869 netdev_reset_tc(bp->dev); 5870 5871 snprintf(bp->irq_tbl[0].name, len, "%s-%s-%d", bp->dev->name, "TxRx", 5872 0); 5873 bp->irq_tbl[0].handler = bnxt_inta; 5874 } 5875 5876 static int bnxt_setup_int_mode(struct bnxt *bp) 5877 { 5878 int rc; 5879 5880 if (bp->flags & BNXT_FLAG_USING_MSIX) 5881 bnxt_setup_msix(bp); 5882 else 5883 bnxt_setup_inta(bp); 5884 5885 rc = bnxt_set_real_num_queues(bp); 5886 return rc; 5887 } 5888 5889 #ifdef CONFIG_RFS_ACCEL 5890 static unsigned int bnxt_get_max_func_rss_ctxs(struct bnxt *bp) 5891 { 5892 return bp->hw_resc.max_rsscos_ctxs; 5893 } 5894 5895 static unsigned int bnxt_get_max_func_vnics(struct bnxt *bp) 5896 { 5897 return bp->hw_resc.max_vnics; 5898 } 5899 #endif 5900 5901 unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp) 5902 { 5903 return bp->hw_resc.max_stat_ctxs; 5904 } 5905 5906 void bnxt_set_max_func_stat_ctxs(struct bnxt *bp, unsigned int max) 5907 { 5908 bp->hw_resc.max_stat_ctxs = max; 5909 } 5910 5911 unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp) 5912 { 5913 return bp->hw_resc.max_cp_rings; 5914 } 5915 5916 unsigned int bnxt_get_max_func_cp_rings_for_en(struct bnxt *bp) 5917 { 5918 return bp->hw_resc.max_cp_rings - bnxt_get_ulp_msix_num(bp); 5919 } 5920 5921 static unsigned int bnxt_get_max_func_irqs(struct bnxt *bp) 5922 { 5923 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 5924 5925 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_cp_rings); 5926 } 5927 5928 static void bnxt_set_max_func_irqs(struct bnxt *bp, unsigned int max_irqs) 5929 { 5930 bp->hw_resc.max_irqs = max_irqs; 5931 } 5932 5933 int bnxt_get_avail_msix(struct bnxt *bp, int num) 5934 { 5935 int max_cp = bnxt_get_max_func_cp_rings(bp); 5936 int max_irq = bnxt_get_max_func_irqs(bp); 5937 int total_req = bp->cp_nr_rings + num; 5938 int max_idx, avail_msix; 5939 5940 max_idx = min_t(int, bp->total_irqs, max_cp); 5941 avail_msix = max_idx - bp->cp_nr_rings; 5942 if (!BNXT_NEW_RM(bp) || avail_msix >= num) 5943 return avail_msix; 5944 5945 if (max_irq < total_req) { 5946 num = max_irq - bp->cp_nr_rings; 5947 if (num <= 0) 5948 return 0; 5949 } 5950 return num; 5951 } 5952 5953 static int bnxt_get_num_msix(struct bnxt *bp) 5954 { 5955 if (!BNXT_NEW_RM(bp)) 5956 return bnxt_get_max_func_irqs(bp); 5957 5958 return bnxt_cp_rings_in_use(bp); 5959 } 5960 5961 static int bnxt_init_msix(struct bnxt *bp) 5962 { 5963 int i, total_vecs, max, rc = 0, min = 1, ulp_msix; 5964 struct msix_entry *msix_ent; 5965 5966 total_vecs = bnxt_get_num_msix(bp); 5967 max = bnxt_get_max_func_irqs(bp); 5968 if (total_vecs > max) 5969 total_vecs = max; 5970 5971 if (!total_vecs) 5972 return 0; 5973 5974 msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL); 5975 if (!msix_ent) 5976 return -ENOMEM; 5977 5978 for (i = 0; i < total_vecs; i++) { 5979 msix_ent[i].entry = i; 5980 msix_ent[i].vector = 0; 5981 } 5982 5983 if (!(bp->flags & BNXT_FLAG_SHARED_RINGS)) 5984 min = 2; 5985 5986 total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs); 5987 ulp_msix = bnxt_get_ulp_msix_num(bp); 5988 if (total_vecs < 0 || total_vecs < ulp_msix) { 5989 rc = -ENODEV; 5990 goto msix_setup_exit; 5991 } 5992 5993 bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL); 5994 if (bp->irq_tbl) { 5995 for (i = 0; i < total_vecs; i++) 5996 bp->irq_tbl[i].vector = msix_ent[i].vector; 5997 5998 bp->total_irqs = total_vecs; 5999 /* Trim rings based upon num of vectors allocated */ 6000 rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings, 6001 total_vecs - ulp_msix, min == 1); 6002 if (rc) 6003 goto msix_setup_exit; 6004 6005 bp->cp_nr_rings = (min == 1) ? 6006 max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) : 6007 bp->tx_nr_rings + bp->rx_nr_rings; 6008 6009 } else { 6010 rc = -ENOMEM; 6011 goto msix_setup_exit; 6012 } 6013 bp->flags |= BNXT_FLAG_USING_MSIX; 6014 kfree(msix_ent); 6015 return 0; 6016 6017 msix_setup_exit: 6018 netdev_err(bp->dev, "bnxt_init_msix err: %x\n", rc); 6019 kfree(bp->irq_tbl); 6020 bp->irq_tbl = NULL; 6021 pci_disable_msix(bp->pdev); 6022 kfree(msix_ent); 6023 return rc; 6024 } 6025 6026 static int bnxt_init_inta(struct bnxt *bp) 6027 { 6028 bp->irq_tbl = kcalloc(1, sizeof(struct bnxt_irq), GFP_KERNEL); 6029 if (!bp->irq_tbl) 6030 return -ENOMEM; 6031 6032 bp->total_irqs = 1; 6033 bp->rx_nr_rings = 1; 6034 bp->tx_nr_rings = 1; 6035 bp->cp_nr_rings = 1; 6036 bp->flags |= BNXT_FLAG_SHARED_RINGS; 6037 bp->irq_tbl[0].vector = bp->pdev->irq; 6038 return 0; 6039 } 6040 6041 static int bnxt_init_int_mode(struct bnxt *bp) 6042 { 6043 int rc = 0; 6044 6045 if (bp->flags & BNXT_FLAG_MSIX_CAP) 6046 rc = bnxt_init_msix(bp); 6047 6048 if (!(bp->flags & BNXT_FLAG_USING_MSIX) && BNXT_PF(bp)) { 6049 /* fallback to INTA */ 6050 rc = bnxt_init_inta(bp); 6051 } 6052 return rc; 6053 } 6054 6055 static void bnxt_clear_int_mode(struct bnxt *bp) 6056 { 6057 if (bp->flags & BNXT_FLAG_USING_MSIX) 6058 pci_disable_msix(bp->pdev); 6059 6060 kfree(bp->irq_tbl); 6061 bp->irq_tbl = NULL; 6062 bp->flags &= ~BNXT_FLAG_USING_MSIX; 6063 } 6064 6065 int bnxt_reserve_rings(struct bnxt *bp) 6066 { 6067 int tcs = netdev_get_num_tc(bp->dev); 6068 int rc; 6069 6070 if (!bnxt_need_reserve_rings(bp)) 6071 return 0; 6072 6073 rc = __bnxt_reserve_rings(bp); 6074 if (rc) { 6075 netdev_err(bp->dev, "ring reservation failure rc: %d\n", rc); 6076 return rc; 6077 } 6078 if (BNXT_NEW_RM(bp) && (bnxt_get_num_msix(bp) != bp->total_irqs)) { 6079 bnxt_ulp_irq_stop(bp); 6080 bnxt_clear_int_mode(bp); 6081 rc = bnxt_init_int_mode(bp); 6082 bnxt_ulp_irq_restart(bp, rc); 6083 if (rc) 6084 return rc; 6085 } 6086 if (tcs && (bp->tx_nr_rings_per_tc * tcs != bp->tx_nr_rings)) { 6087 netdev_err(bp->dev, "tx ring reservation failure\n"); 6088 netdev_reset_tc(bp->dev); 6089 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 6090 return -ENOMEM; 6091 } 6092 bp->num_stat_ctxs = bp->cp_nr_rings; 6093 return 0; 6094 } 6095 6096 static void bnxt_free_irq(struct bnxt *bp) 6097 { 6098 struct bnxt_irq *irq; 6099 int i; 6100 6101 #ifdef CONFIG_RFS_ACCEL 6102 free_irq_cpu_rmap(bp->dev->rx_cpu_rmap); 6103 bp->dev->rx_cpu_rmap = NULL; 6104 #endif 6105 if (!bp->irq_tbl || !bp->bnapi) 6106 return; 6107 6108 for (i = 0; i < bp->cp_nr_rings; i++) { 6109 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 6110 6111 irq = &bp->irq_tbl[map_idx]; 6112 if (irq->requested) { 6113 if (irq->have_cpumask) { 6114 irq_set_affinity_hint(irq->vector, NULL); 6115 free_cpumask_var(irq->cpu_mask); 6116 irq->have_cpumask = 0; 6117 } 6118 free_irq(irq->vector, bp->bnapi[i]); 6119 } 6120 6121 irq->requested = 0; 6122 } 6123 } 6124 6125 static int bnxt_request_irq(struct bnxt *bp) 6126 { 6127 int i, j, rc = 0; 6128 unsigned long flags = 0; 6129 #ifdef CONFIG_RFS_ACCEL 6130 struct cpu_rmap *rmap; 6131 #endif 6132 6133 rc = bnxt_setup_int_mode(bp); 6134 if (rc) { 6135 netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n", 6136 rc); 6137 return rc; 6138 } 6139 #ifdef CONFIG_RFS_ACCEL 6140 rmap = bp->dev->rx_cpu_rmap; 6141 #endif 6142 if (!(bp->flags & BNXT_FLAG_USING_MSIX)) 6143 flags = IRQF_SHARED; 6144 6145 for (i = 0, j = 0; i < bp->cp_nr_rings; i++) { 6146 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 6147 struct bnxt_irq *irq = &bp->irq_tbl[map_idx]; 6148 6149 #ifdef CONFIG_RFS_ACCEL 6150 if (rmap && bp->bnapi[i]->rx_ring) { 6151 rc = irq_cpu_rmap_add(rmap, irq->vector); 6152 if (rc) 6153 netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n", 6154 j); 6155 j++; 6156 } 6157 #endif 6158 rc = request_irq(irq->vector, irq->handler, flags, irq->name, 6159 bp->bnapi[i]); 6160 if (rc) 6161 break; 6162 6163 irq->requested = 1; 6164 6165 if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) { 6166 int numa_node = dev_to_node(&bp->pdev->dev); 6167 6168 irq->have_cpumask = 1; 6169 cpumask_set_cpu(cpumask_local_spread(i, numa_node), 6170 irq->cpu_mask); 6171 rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask); 6172 if (rc) { 6173 netdev_warn(bp->dev, 6174 "Set affinity failed, IRQ = %d\n", 6175 irq->vector); 6176 break; 6177 } 6178 } 6179 } 6180 return rc; 6181 } 6182 6183 static void bnxt_del_napi(struct bnxt *bp) 6184 { 6185 int i; 6186 6187 if (!bp->bnapi) 6188 return; 6189 6190 for (i = 0; i < bp->cp_nr_rings; i++) { 6191 struct bnxt_napi *bnapi = bp->bnapi[i]; 6192 6193 napi_hash_del(&bnapi->napi); 6194 netif_napi_del(&bnapi->napi); 6195 } 6196 /* We called napi_hash_del() before netif_napi_del(), we need 6197 * to respect an RCU grace period before freeing napi structures. 6198 */ 6199 synchronize_net(); 6200 } 6201 6202 static void bnxt_init_napi(struct bnxt *bp) 6203 { 6204 int i; 6205 unsigned int cp_nr_rings = bp->cp_nr_rings; 6206 struct bnxt_napi *bnapi; 6207 6208 if (bp->flags & BNXT_FLAG_USING_MSIX) { 6209 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 6210 cp_nr_rings--; 6211 for (i = 0; i < cp_nr_rings; i++) { 6212 bnapi = bp->bnapi[i]; 6213 netif_napi_add(bp->dev, &bnapi->napi, 6214 bnxt_poll, 64); 6215 } 6216 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 6217 bnapi = bp->bnapi[cp_nr_rings]; 6218 netif_napi_add(bp->dev, &bnapi->napi, 6219 bnxt_poll_nitroa0, 64); 6220 } 6221 } else { 6222 bnapi = bp->bnapi[0]; 6223 netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll, 64); 6224 } 6225 } 6226 6227 static void bnxt_disable_napi(struct bnxt *bp) 6228 { 6229 int i; 6230 6231 if (!bp->bnapi) 6232 return; 6233 6234 for (i = 0; i < bp->cp_nr_rings; i++) { 6235 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring; 6236 6237 if (bp->bnapi[i]->rx_ring) 6238 cancel_work_sync(&cpr->dim.work); 6239 6240 napi_disable(&bp->bnapi[i]->napi); 6241 } 6242 } 6243 6244 static void bnxt_enable_napi(struct bnxt *bp) 6245 { 6246 int i; 6247 6248 for (i = 0; i < bp->cp_nr_rings; i++) { 6249 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring; 6250 bp->bnapi[i]->in_reset = false; 6251 6252 if (bp->bnapi[i]->rx_ring) { 6253 INIT_WORK(&cpr->dim.work, bnxt_dim_work); 6254 cpr->dim.mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE; 6255 } 6256 napi_enable(&bp->bnapi[i]->napi); 6257 } 6258 } 6259 6260 void bnxt_tx_disable(struct bnxt *bp) 6261 { 6262 int i; 6263 struct bnxt_tx_ring_info *txr; 6264 6265 if (bp->tx_ring) { 6266 for (i = 0; i < bp->tx_nr_rings; i++) { 6267 txr = &bp->tx_ring[i]; 6268 txr->dev_state = BNXT_DEV_STATE_CLOSING; 6269 } 6270 } 6271 /* Stop all TX queues */ 6272 netif_tx_disable(bp->dev); 6273 netif_carrier_off(bp->dev); 6274 } 6275 6276 void bnxt_tx_enable(struct bnxt *bp) 6277 { 6278 int i; 6279 struct bnxt_tx_ring_info *txr; 6280 6281 for (i = 0; i < bp->tx_nr_rings; i++) { 6282 txr = &bp->tx_ring[i]; 6283 txr->dev_state = 0; 6284 } 6285 netif_tx_wake_all_queues(bp->dev); 6286 if (bp->link_info.link_up) 6287 netif_carrier_on(bp->dev); 6288 } 6289 6290 static void bnxt_report_link(struct bnxt *bp) 6291 { 6292 if (bp->link_info.link_up) { 6293 const char *duplex; 6294 const char *flow_ctrl; 6295 u32 speed; 6296 u16 fec; 6297 6298 netif_carrier_on(bp->dev); 6299 if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL) 6300 duplex = "full"; 6301 else 6302 duplex = "half"; 6303 if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH) 6304 flow_ctrl = "ON - receive & transmit"; 6305 else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX) 6306 flow_ctrl = "ON - transmit"; 6307 else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX) 6308 flow_ctrl = "ON - receive"; 6309 else 6310 flow_ctrl = "none"; 6311 speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed); 6312 netdev_info(bp->dev, "NIC Link is Up, %u Mbps %s duplex, Flow control: %s\n", 6313 speed, duplex, flow_ctrl); 6314 if (bp->flags & BNXT_FLAG_EEE_CAP) 6315 netdev_info(bp->dev, "EEE is %s\n", 6316 bp->eee.eee_active ? "active" : 6317 "not active"); 6318 fec = bp->link_info.fec_cfg; 6319 if (!(fec & PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED)) 6320 netdev_info(bp->dev, "FEC autoneg %s encodings: %s\n", 6321 (fec & BNXT_FEC_AUTONEG) ? "on" : "off", 6322 (fec & BNXT_FEC_ENC_BASE_R) ? "BaseR" : 6323 (fec & BNXT_FEC_ENC_RS) ? "RS" : "None"); 6324 } else { 6325 netif_carrier_off(bp->dev); 6326 netdev_err(bp->dev, "NIC Link is Down\n"); 6327 } 6328 } 6329 6330 static int bnxt_hwrm_phy_qcaps(struct bnxt *bp) 6331 { 6332 int rc = 0; 6333 struct hwrm_port_phy_qcaps_input req = {0}; 6334 struct hwrm_port_phy_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 6335 struct bnxt_link_info *link_info = &bp->link_info; 6336 6337 if (bp->hwrm_spec_code < 0x10201) 6338 return 0; 6339 6340 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCAPS, -1, -1); 6341 6342 mutex_lock(&bp->hwrm_cmd_lock); 6343 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6344 if (rc) 6345 goto hwrm_phy_qcaps_exit; 6346 6347 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) { 6348 struct ethtool_eee *eee = &bp->eee; 6349 u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode); 6350 6351 bp->flags |= BNXT_FLAG_EEE_CAP; 6352 eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0); 6353 bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) & 6354 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK; 6355 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) & 6356 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK; 6357 } 6358 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EXTERNAL_LPBK_SUPPORTED) { 6359 if (bp->test_info) 6360 bp->test_info->flags |= BNXT_TEST_FL_EXT_LPBK; 6361 } 6362 if (resp->supported_speeds_auto_mode) 6363 link_info->support_auto_speeds = 6364 le16_to_cpu(resp->supported_speeds_auto_mode); 6365 6366 bp->port_count = resp->port_cnt; 6367 6368 hwrm_phy_qcaps_exit: 6369 mutex_unlock(&bp->hwrm_cmd_lock); 6370 return rc; 6371 } 6372 6373 static int bnxt_update_link(struct bnxt *bp, bool chng_link_state) 6374 { 6375 int rc = 0; 6376 struct bnxt_link_info *link_info = &bp->link_info; 6377 struct hwrm_port_phy_qcfg_input req = {0}; 6378 struct hwrm_port_phy_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 6379 u8 link_up = link_info->link_up; 6380 u16 diff; 6381 6382 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCFG, -1, -1); 6383 6384 mutex_lock(&bp->hwrm_cmd_lock); 6385 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6386 if (rc) { 6387 mutex_unlock(&bp->hwrm_cmd_lock); 6388 return rc; 6389 } 6390 6391 memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp)); 6392 link_info->phy_link_status = resp->link; 6393 link_info->duplex = resp->duplex_cfg; 6394 if (bp->hwrm_spec_code >= 0x10800) 6395 link_info->duplex = resp->duplex_state; 6396 link_info->pause = resp->pause; 6397 link_info->auto_mode = resp->auto_mode; 6398 link_info->auto_pause_setting = resp->auto_pause; 6399 link_info->lp_pause = resp->link_partner_adv_pause; 6400 link_info->force_pause_setting = resp->force_pause; 6401 link_info->duplex_setting = resp->duplex_cfg; 6402 if (link_info->phy_link_status == BNXT_LINK_LINK) 6403 link_info->link_speed = le16_to_cpu(resp->link_speed); 6404 else 6405 link_info->link_speed = 0; 6406 link_info->force_link_speed = le16_to_cpu(resp->force_link_speed); 6407 link_info->support_speeds = le16_to_cpu(resp->support_speeds); 6408 link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask); 6409 link_info->lp_auto_link_speeds = 6410 le16_to_cpu(resp->link_partner_adv_speeds); 6411 link_info->preemphasis = le32_to_cpu(resp->preemphasis); 6412 link_info->phy_ver[0] = resp->phy_maj; 6413 link_info->phy_ver[1] = resp->phy_min; 6414 link_info->phy_ver[2] = resp->phy_bld; 6415 link_info->media_type = resp->media_type; 6416 link_info->phy_type = resp->phy_type; 6417 link_info->transceiver = resp->xcvr_pkg_type; 6418 link_info->phy_addr = resp->eee_config_phy_addr & 6419 PORT_PHY_QCFG_RESP_PHY_ADDR_MASK; 6420 link_info->module_status = resp->module_status; 6421 6422 if (bp->flags & BNXT_FLAG_EEE_CAP) { 6423 struct ethtool_eee *eee = &bp->eee; 6424 u16 fw_speeds; 6425 6426 eee->eee_active = 0; 6427 if (resp->eee_config_phy_addr & 6428 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) { 6429 eee->eee_active = 1; 6430 fw_speeds = le16_to_cpu( 6431 resp->link_partner_adv_eee_link_speed_mask); 6432 eee->lp_advertised = 6433 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0); 6434 } 6435 6436 /* Pull initial EEE config */ 6437 if (!chng_link_state) { 6438 if (resp->eee_config_phy_addr & 6439 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED) 6440 eee->eee_enabled = 1; 6441 6442 fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask); 6443 eee->advertised = 6444 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0); 6445 6446 if (resp->eee_config_phy_addr & 6447 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) { 6448 __le32 tmr; 6449 6450 eee->tx_lpi_enabled = 1; 6451 tmr = resp->xcvr_identifier_type_tx_lpi_timer; 6452 eee->tx_lpi_timer = le32_to_cpu(tmr) & 6453 PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK; 6454 } 6455 } 6456 } 6457 6458 link_info->fec_cfg = PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED; 6459 if (bp->hwrm_spec_code >= 0x10504) 6460 link_info->fec_cfg = le16_to_cpu(resp->fec_cfg); 6461 6462 /* TODO: need to add more logic to report VF link */ 6463 if (chng_link_state) { 6464 if (link_info->phy_link_status == BNXT_LINK_LINK) 6465 link_info->link_up = 1; 6466 else 6467 link_info->link_up = 0; 6468 if (link_up != link_info->link_up) 6469 bnxt_report_link(bp); 6470 } else { 6471 /* alwasy link down if not require to update link state */ 6472 link_info->link_up = 0; 6473 } 6474 mutex_unlock(&bp->hwrm_cmd_lock); 6475 6476 if (!BNXT_SINGLE_PF(bp)) 6477 return 0; 6478 6479 diff = link_info->support_auto_speeds ^ link_info->advertising; 6480 if ((link_info->support_auto_speeds | diff) != 6481 link_info->support_auto_speeds) { 6482 /* An advertised speed is no longer supported, so we need to 6483 * update the advertisement settings. Caller holds RTNL 6484 * so we can modify link settings. 6485 */ 6486 link_info->advertising = link_info->support_auto_speeds; 6487 if (link_info->autoneg & BNXT_AUTONEG_SPEED) 6488 bnxt_hwrm_set_link_setting(bp, true, false); 6489 } 6490 return 0; 6491 } 6492 6493 static void bnxt_get_port_module_status(struct bnxt *bp) 6494 { 6495 struct bnxt_link_info *link_info = &bp->link_info; 6496 struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp; 6497 u8 module_status; 6498 6499 if (bnxt_update_link(bp, true)) 6500 return; 6501 6502 module_status = link_info->module_status; 6503 switch (module_status) { 6504 case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX: 6505 case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN: 6506 case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG: 6507 netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n", 6508 bp->pf.port_id); 6509 if (bp->hwrm_spec_code >= 0x10201) { 6510 netdev_warn(bp->dev, "Module part number %s\n", 6511 resp->phy_vendor_partnumber); 6512 } 6513 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX) 6514 netdev_warn(bp->dev, "TX is disabled\n"); 6515 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN) 6516 netdev_warn(bp->dev, "SFP+ module is shutdown\n"); 6517 } 6518 } 6519 6520 static void 6521 bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req) 6522 { 6523 if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) { 6524 if (bp->hwrm_spec_code >= 0x10201) 6525 req->auto_pause = 6526 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE; 6527 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX) 6528 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX; 6529 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX) 6530 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX; 6531 req->enables |= 6532 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE); 6533 } else { 6534 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX) 6535 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX; 6536 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX) 6537 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX; 6538 req->enables |= 6539 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE); 6540 if (bp->hwrm_spec_code >= 0x10201) { 6541 req->auto_pause = req->force_pause; 6542 req->enables |= cpu_to_le32( 6543 PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE); 6544 } 6545 } 6546 } 6547 6548 static void bnxt_hwrm_set_link_common(struct bnxt *bp, 6549 struct hwrm_port_phy_cfg_input *req) 6550 { 6551 u8 autoneg = bp->link_info.autoneg; 6552 u16 fw_link_speed = bp->link_info.req_link_speed; 6553 u16 advertising = bp->link_info.advertising; 6554 6555 if (autoneg & BNXT_AUTONEG_SPEED) { 6556 req->auto_mode |= 6557 PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK; 6558 6559 req->enables |= cpu_to_le32( 6560 PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK); 6561 req->auto_link_speed_mask = cpu_to_le16(advertising); 6562 6563 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE); 6564 req->flags |= 6565 cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG); 6566 } else { 6567 req->force_link_speed = cpu_to_le16(fw_link_speed); 6568 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE); 6569 } 6570 6571 /* tell chimp that the setting takes effect immediately */ 6572 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY); 6573 } 6574 6575 int bnxt_hwrm_set_pause(struct bnxt *bp) 6576 { 6577 struct hwrm_port_phy_cfg_input req = {0}; 6578 int rc; 6579 6580 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1); 6581 bnxt_hwrm_set_pause_common(bp, &req); 6582 6583 if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) || 6584 bp->link_info.force_link_chng) 6585 bnxt_hwrm_set_link_common(bp, &req); 6586 6587 mutex_lock(&bp->hwrm_cmd_lock); 6588 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6589 if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) { 6590 /* since changing of pause setting doesn't trigger any link 6591 * change event, the driver needs to update the current pause 6592 * result upon successfully return of the phy_cfg command 6593 */ 6594 bp->link_info.pause = 6595 bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl; 6596 bp->link_info.auto_pause_setting = 0; 6597 if (!bp->link_info.force_link_chng) 6598 bnxt_report_link(bp); 6599 } 6600 bp->link_info.force_link_chng = false; 6601 mutex_unlock(&bp->hwrm_cmd_lock); 6602 return rc; 6603 } 6604 6605 static void bnxt_hwrm_set_eee(struct bnxt *bp, 6606 struct hwrm_port_phy_cfg_input *req) 6607 { 6608 struct ethtool_eee *eee = &bp->eee; 6609 6610 if (eee->eee_enabled) { 6611 u16 eee_speeds; 6612 u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE; 6613 6614 if (eee->tx_lpi_enabled) 6615 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE; 6616 else 6617 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE; 6618 6619 req->flags |= cpu_to_le32(flags); 6620 eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised); 6621 req->eee_link_speed_mask = cpu_to_le16(eee_speeds); 6622 req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer); 6623 } else { 6624 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE); 6625 } 6626 } 6627 6628 int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee) 6629 { 6630 struct hwrm_port_phy_cfg_input req = {0}; 6631 6632 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1); 6633 if (set_pause) 6634 bnxt_hwrm_set_pause_common(bp, &req); 6635 6636 bnxt_hwrm_set_link_common(bp, &req); 6637 6638 if (set_eee) 6639 bnxt_hwrm_set_eee(bp, &req); 6640 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6641 } 6642 6643 static int bnxt_hwrm_shutdown_link(struct bnxt *bp) 6644 { 6645 struct hwrm_port_phy_cfg_input req = {0}; 6646 6647 if (!BNXT_SINGLE_PF(bp)) 6648 return 0; 6649 6650 if (pci_num_vf(bp->pdev)) 6651 return 0; 6652 6653 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1); 6654 req.flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN); 6655 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6656 } 6657 6658 static int bnxt_hwrm_if_change(struct bnxt *bp, bool up) 6659 { 6660 struct hwrm_func_drv_if_change_output *resp = bp->hwrm_cmd_resp_addr; 6661 struct hwrm_func_drv_if_change_input req = {0}; 6662 bool resc_reinit = false; 6663 int rc; 6664 6665 if (!(bp->fw_cap & BNXT_FW_CAP_IF_CHANGE)) 6666 return 0; 6667 6668 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_IF_CHANGE, -1, -1); 6669 if (up) 6670 req.flags = cpu_to_le32(FUNC_DRV_IF_CHANGE_REQ_FLAGS_UP); 6671 mutex_lock(&bp->hwrm_cmd_lock); 6672 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6673 if (!rc && (resp->flags & 6674 cpu_to_le32(FUNC_DRV_IF_CHANGE_RESP_FLAGS_RESC_CHANGE))) 6675 resc_reinit = true; 6676 mutex_unlock(&bp->hwrm_cmd_lock); 6677 6678 if (up && resc_reinit && BNXT_NEW_RM(bp)) { 6679 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 6680 6681 rc = bnxt_hwrm_func_resc_qcaps(bp, true); 6682 hw_resc->resv_cp_rings = 0; 6683 hw_resc->resv_tx_rings = 0; 6684 hw_resc->resv_rx_rings = 0; 6685 hw_resc->resv_hw_ring_grps = 0; 6686 hw_resc->resv_vnics = 0; 6687 bp->tx_nr_rings = 0; 6688 bp->rx_nr_rings = 0; 6689 } 6690 return rc; 6691 } 6692 6693 static int bnxt_hwrm_port_led_qcaps(struct bnxt *bp) 6694 { 6695 struct hwrm_port_led_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 6696 struct hwrm_port_led_qcaps_input req = {0}; 6697 struct bnxt_pf_info *pf = &bp->pf; 6698 int rc; 6699 6700 if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10601) 6701 return 0; 6702 6703 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_LED_QCAPS, -1, -1); 6704 req.port_id = cpu_to_le16(pf->port_id); 6705 mutex_lock(&bp->hwrm_cmd_lock); 6706 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6707 if (rc) { 6708 mutex_unlock(&bp->hwrm_cmd_lock); 6709 return rc; 6710 } 6711 if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) { 6712 int i; 6713 6714 bp->num_leds = resp->num_leds; 6715 memcpy(bp->leds, &resp->led0_id, sizeof(bp->leds[0]) * 6716 bp->num_leds); 6717 for (i = 0; i < bp->num_leds; i++) { 6718 struct bnxt_led_info *led = &bp->leds[i]; 6719 __le16 caps = led->led_state_caps; 6720 6721 if (!led->led_group_id || 6722 !BNXT_LED_ALT_BLINK_CAP(caps)) { 6723 bp->num_leds = 0; 6724 break; 6725 } 6726 } 6727 } 6728 mutex_unlock(&bp->hwrm_cmd_lock); 6729 return 0; 6730 } 6731 6732 int bnxt_hwrm_alloc_wol_fltr(struct bnxt *bp) 6733 { 6734 struct hwrm_wol_filter_alloc_input req = {0}; 6735 struct hwrm_wol_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr; 6736 int rc; 6737 6738 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_ALLOC, -1, -1); 6739 req.port_id = cpu_to_le16(bp->pf.port_id); 6740 req.wol_type = WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT; 6741 req.enables = cpu_to_le32(WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS); 6742 memcpy(req.mac_address, bp->dev->dev_addr, ETH_ALEN); 6743 mutex_lock(&bp->hwrm_cmd_lock); 6744 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6745 if (!rc) 6746 bp->wol_filter_id = resp->wol_filter_id; 6747 mutex_unlock(&bp->hwrm_cmd_lock); 6748 return rc; 6749 } 6750 6751 int bnxt_hwrm_free_wol_fltr(struct bnxt *bp) 6752 { 6753 struct hwrm_wol_filter_free_input req = {0}; 6754 int rc; 6755 6756 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_FREE, -1, -1); 6757 req.port_id = cpu_to_le16(bp->pf.port_id); 6758 req.enables = cpu_to_le32(WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID); 6759 req.wol_filter_id = bp->wol_filter_id; 6760 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6761 return rc; 6762 } 6763 6764 static u16 bnxt_hwrm_get_wol_fltrs(struct bnxt *bp, u16 handle) 6765 { 6766 struct hwrm_wol_filter_qcfg_input req = {0}; 6767 struct hwrm_wol_filter_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 6768 u16 next_handle = 0; 6769 int rc; 6770 6771 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_QCFG, -1, -1); 6772 req.port_id = cpu_to_le16(bp->pf.port_id); 6773 req.handle = cpu_to_le16(handle); 6774 mutex_lock(&bp->hwrm_cmd_lock); 6775 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6776 if (!rc) { 6777 next_handle = le16_to_cpu(resp->next_handle); 6778 if (next_handle != 0) { 6779 if (resp->wol_type == 6780 WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT) { 6781 bp->wol = 1; 6782 bp->wol_filter_id = resp->wol_filter_id; 6783 } 6784 } 6785 } 6786 mutex_unlock(&bp->hwrm_cmd_lock); 6787 return next_handle; 6788 } 6789 6790 static void bnxt_get_wol_settings(struct bnxt *bp) 6791 { 6792 u16 handle = 0; 6793 6794 if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_WOL_CAP)) 6795 return; 6796 6797 do { 6798 handle = bnxt_hwrm_get_wol_fltrs(bp, handle); 6799 } while (handle && handle != 0xffff); 6800 } 6801 6802 #ifdef CONFIG_BNXT_HWMON 6803 static ssize_t bnxt_show_temp(struct device *dev, 6804 struct device_attribute *devattr, char *buf) 6805 { 6806 struct hwrm_temp_monitor_query_input req = {0}; 6807 struct hwrm_temp_monitor_query_output *resp; 6808 struct bnxt *bp = dev_get_drvdata(dev); 6809 u32 temp = 0; 6810 6811 resp = bp->hwrm_cmd_resp_addr; 6812 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TEMP_MONITOR_QUERY, -1, -1); 6813 mutex_lock(&bp->hwrm_cmd_lock); 6814 if (!_hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT)) 6815 temp = resp->temp * 1000; /* display millidegree */ 6816 mutex_unlock(&bp->hwrm_cmd_lock); 6817 6818 return sprintf(buf, "%u\n", temp); 6819 } 6820 static SENSOR_DEVICE_ATTR(temp1_input, 0444, bnxt_show_temp, NULL, 0); 6821 6822 static struct attribute *bnxt_attrs[] = { 6823 &sensor_dev_attr_temp1_input.dev_attr.attr, 6824 NULL 6825 }; 6826 ATTRIBUTE_GROUPS(bnxt); 6827 6828 static void bnxt_hwmon_close(struct bnxt *bp) 6829 { 6830 if (bp->hwmon_dev) { 6831 hwmon_device_unregister(bp->hwmon_dev); 6832 bp->hwmon_dev = NULL; 6833 } 6834 } 6835 6836 static void bnxt_hwmon_open(struct bnxt *bp) 6837 { 6838 struct pci_dev *pdev = bp->pdev; 6839 6840 bp->hwmon_dev = hwmon_device_register_with_groups(&pdev->dev, 6841 DRV_MODULE_NAME, bp, 6842 bnxt_groups); 6843 if (IS_ERR(bp->hwmon_dev)) { 6844 bp->hwmon_dev = NULL; 6845 dev_warn(&pdev->dev, "Cannot register hwmon device\n"); 6846 } 6847 } 6848 #else 6849 static void bnxt_hwmon_close(struct bnxt *bp) 6850 { 6851 } 6852 6853 static void bnxt_hwmon_open(struct bnxt *bp) 6854 { 6855 } 6856 #endif 6857 6858 static bool bnxt_eee_config_ok(struct bnxt *bp) 6859 { 6860 struct ethtool_eee *eee = &bp->eee; 6861 struct bnxt_link_info *link_info = &bp->link_info; 6862 6863 if (!(bp->flags & BNXT_FLAG_EEE_CAP)) 6864 return true; 6865 6866 if (eee->eee_enabled) { 6867 u32 advertising = 6868 _bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0); 6869 6870 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) { 6871 eee->eee_enabled = 0; 6872 return false; 6873 } 6874 if (eee->advertised & ~advertising) { 6875 eee->advertised = advertising & eee->supported; 6876 return false; 6877 } 6878 } 6879 return true; 6880 } 6881 6882 static int bnxt_update_phy_setting(struct bnxt *bp) 6883 { 6884 int rc; 6885 bool update_link = false; 6886 bool update_pause = false; 6887 bool update_eee = false; 6888 struct bnxt_link_info *link_info = &bp->link_info; 6889 6890 rc = bnxt_update_link(bp, true); 6891 if (rc) { 6892 netdev_err(bp->dev, "failed to update link (rc: %x)\n", 6893 rc); 6894 return rc; 6895 } 6896 if (!BNXT_SINGLE_PF(bp)) 6897 return 0; 6898 6899 if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) && 6900 (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) != 6901 link_info->req_flow_ctrl) 6902 update_pause = true; 6903 if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) && 6904 link_info->force_pause_setting != link_info->req_flow_ctrl) 6905 update_pause = true; 6906 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) { 6907 if (BNXT_AUTO_MODE(link_info->auto_mode)) 6908 update_link = true; 6909 if (link_info->req_link_speed != link_info->force_link_speed) 6910 update_link = true; 6911 if (link_info->req_duplex != link_info->duplex_setting) 6912 update_link = true; 6913 } else { 6914 if (link_info->auto_mode == BNXT_LINK_AUTO_NONE) 6915 update_link = true; 6916 if (link_info->advertising != link_info->auto_link_speeds) 6917 update_link = true; 6918 } 6919 6920 /* The last close may have shutdown the link, so need to call 6921 * PHY_CFG to bring it back up. 6922 */ 6923 if (!netif_carrier_ok(bp->dev)) 6924 update_link = true; 6925 6926 if (!bnxt_eee_config_ok(bp)) 6927 update_eee = true; 6928 6929 if (update_link) 6930 rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee); 6931 else if (update_pause) 6932 rc = bnxt_hwrm_set_pause(bp); 6933 if (rc) { 6934 netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n", 6935 rc); 6936 return rc; 6937 } 6938 6939 return rc; 6940 } 6941 6942 /* Common routine to pre-map certain register block to different GRC window. 6943 * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows 6944 * in PF and 3 windows in VF that can be customized to map in different 6945 * register blocks. 6946 */ 6947 static void bnxt_preset_reg_win(struct bnxt *bp) 6948 { 6949 if (BNXT_PF(bp)) { 6950 /* CAG registers map to GRC window #4 */ 6951 writel(BNXT_CAG_REG_BASE, 6952 bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12); 6953 } 6954 } 6955 6956 static int bnxt_init_dflt_ring_mode(struct bnxt *bp); 6957 6958 static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) 6959 { 6960 int rc = 0; 6961 6962 bnxt_preset_reg_win(bp); 6963 netif_carrier_off(bp->dev); 6964 if (irq_re_init) { 6965 /* Reserve rings now if none were reserved at driver probe. */ 6966 rc = bnxt_init_dflt_ring_mode(bp); 6967 if (rc) { 6968 netdev_err(bp->dev, "Failed to reserve default rings at open\n"); 6969 return rc; 6970 } 6971 rc = bnxt_reserve_rings(bp); 6972 if (rc) 6973 return rc; 6974 } 6975 if ((bp->flags & BNXT_FLAG_RFS) && 6976 !(bp->flags & BNXT_FLAG_USING_MSIX)) { 6977 /* disable RFS if falling back to INTA */ 6978 bp->dev->hw_features &= ~NETIF_F_NTUPLE; 6979 bp->flags &= ~BNXT_FLAG_RFS; 6980 } 6981 6982 rc = bnxt_alloc_mem(bp, irq_re_init); 6983 if (rc) { 6984 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc); 6985 goto open_err_free_mem; 6986 } 6987 6988 if (irq_re_init) { 6989 bnxt_init_napi(bp); 6990 rc = bnxt_request_irq(bp); 6991 if (rc) { 6992 netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc); 6993 goto open_err_irq; 6994 } 6995 } 6996 6997 bnxt_enable_napi(bp); 6998 bnxt_debug_dev_init(bp); 6999 7000 rc = bnxt_init_nic(bp, irq_re_init); 7001 if (rc) { 7002 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc); 7003 goto open_err; 7004 } 7005 7006 if (link_re_init) { 7007 mutex_lock(&bp->link_lock); 7008 rc = bnxt_update_phy_setting(bp); 7009 mutex_unlock(&bp->link_lock); 7010 if (rc) { 7011 netdev_warn(bp->dev, "failed to update phy settings\n"); 7012 if (BNXT_SINGLE_PF(bp)) { 7013 bp->link_info.phy_retry = true; 7014 bp->link_info.phy_retry_expires = 7015 jiffies + 5 * HZ; 7016 } 7017 } 7018 } 7019 7020 if (irq_re_init) 7021 udp_tunnel_get_rx_info(bp->dev); 7022 7023 set_bit(BNXT_STATE_OPEN, &bp->state); 7024 bnxt_enable_int(bp); 7025 /* Enable TX queues */ 7026 bnxt_tx_enable(bp); 7027 mod_timer(&bp->timer, jiffies + bp->current_interval); 7028 /* Poll link status and check for SFP+ module status */ 7029 bnxt_get_port_module_status(bp); 7030 7031 /* VF-reps may need to be re-opened after the PF is re-opened */ 7032 if (BNXT_PF(bp)) 7033 bnxt_vf_reps_open(bp); 7034 return 0; 7035 7036 open_err: 7037 bnxt_debug_dev_exit(bp); 7038 bnxt_disable_napi(bp); 7039 7040 open_err_irq: 7041 bnxt_del_napi(bp); 7042 7043 open_err_free_mem: 7044 bnxt_free_skbs(bp); 7045 bnxt_free_irq(bp); 7046 bnxt_free_mem(bp, true); 7047 return rc; 7048 } 7049 7050 /* rtnl_lock held */ 7051 int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) 7052 { 7053 int rc = 0; 7054 7055 rc = __bnxt_open_nic(bp, irq_re_init, link_re_init); 7056 if (rc) { 7057 netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc); 7058 dev_close(bp->dev); 7059 } 7060 return rc; 7061 } 7062 7063 /* rtnl_lock held, open the NIC half way by allocating all resources, but 7064 * NAPI, IRQ, and TX are not enabled. This is mainly used for offline 7065 * self tests. 7066 */ 7067 int bnxt_half_open_nic(struct bnxt *bp) 7068 { 7069 int rc = 0; 7070 7071 rc = bnxt_alloc_mem(bp, false); 7072 if (rc) { 7073 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc); 7074 goto half_open_err; 7075 } 7076 rc = bnxt_init_nic(bp, false); 7077 if (rc) { 7078 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc); 7079 goto half_open_err; 7080 } 7081 return 0; 7082 7083 half_open_err: 7084 bnxt_free_skbs(bp); 7085 bnxt_free_mem(bp, false); 7086 dev_close(bp->dev); 7087 return rc; 7088 } 7089 7090 /* rtnl_lock held, this call can only be made after a previous successful 7091 * call to bnxt_half_open_nic(). 7092 */ 7093 void bnxt_half_close_nic(struct bnxt *bp) 7094 { 7095 bnxt_hwrm_resource_free(bp, false, false); 7096 bnxt_free_skbs(bp); 7097 bnxt_free_mem(bp, false); 7098 } 7099 7100 static int bnxt_open(struct net_device *dev) 7101 { 7102 struct bnxt *bp = netdev_priv(dev); 7103 int rc; 7104 7105 bnxt_hwrm_if_change(bp, true); 7106 rc = __bnxt_open_nic(bp, true, true); 7107 if (rc) 7108 bnxt_hwrm_if_change(bp, false); 7109 7110 bnxt_hwmon_open(bp); 7111 7112 return rc; 7113 } 7114 7115 static bool bnxt_drv_busy(struct bnxt *bp) 7116 { 7117 return (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state) || 7118 test_bit(BNXT_STATE_READ_STATS, &bp->state)); 7119 } 7120 7121 static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init, 7122 bool link_re_init) 7123 { 7124 /* Close the VF-reps before closing PF */ 7125 if (BNXT_PF(bp)) 7126 bnxt_vf_reps_close(bp); 7127 7128 /* Change device state to avoid TX queue wake up's */ 7129 bnxt_tx_disable(bp); 7130 7131 clear_bit(BNXT_STATE_OPEN, &bp->state); 7132 smp_mb__after_atomic(); 7133 while (bnxt_drv_busy(bp)) 7134 msleep(20); 7135 7136 /* Flush rings and and disable interrupts */ 7137 bnxt_shutdown_nic(bp, irq_re_init); 7138 7139 /* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */ 7140 7141 bnxt_debug_dev_exit(bp); 7142 bnxt_disable_napi(bp); 7143 del_timer_sync(&bp->timer); 7144 bnxt_free_skbs(bp); 7145 7146 if (irq_re_init) { 7147 bnxt_free_irq(bp); 7148 bnxt_del_napi(bp); 7149 } 7150 bnxt_free_mem(bp, irq_re_init); 7151 } 7152 7153 int bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) 7154 { 7155 int rc = 0; 7156 7157 #ifdef CONFIG_BNXT_SRIOV 7158 if (bp->sriov_cfg) { 7159 rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait, 7160 !bp->sriov_cfg, 7161 BNXT_SRIOV_CFG_WAIT_TMO); 7162 if (rc) 7163 netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete!\n"); 7164 } 7165 #endif 7166 __bnxt_close_nic(bp, irq_re_init, link_re_init); 7167 return rc; 7168 } 7169 7170 static int bnxt_close(struct net_device *dev) 7171 { 7172 struct bnxt *bp = netdev_priv(dev); 7173 7174 bnxt_hwmon_close(bp); 7175 bnxt_close_nic(bp, true, true); 7176 bnxt_hwrm_shutdown_link(bp); 7177 bnxt_hwrm_if_change(bp, false); 7178 return 0; 7179 } 7180 7181 /* rtnl_lock held */ 7182 static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) 7183 { 7184 switch (cmd) { 7185 case SIOCGMIIPHY: 7186 /* fallthru */ 7187 case SIOCGMIIREG: { 7188 if (!netif_running(dev)) 7189 return -EAGAIN; 7190 7191 return 0; 7192 } 7193 7194 case SIOCSMIIREG: 7195 if (!netif_running(dev)) 7196 return -EAGAIN; 7197 7198 return 0; 7199 7200 default: 7201 /* do nothing */ 7202 break; 7203 } 7204 return -EOPNOTSUPP; 7205 } 7206 7207 static void 7208 bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) 7209 { 7210 u32 i; 7211 struct bnxt *bp = netdev_priv(dev); 7212 7213 set_bit(BNXT_STATE_READ_STATS, &bp->state); 7214 /* Make sure bnxt_close_nic() sees that we are reading stats before 7215 * we check the BNXT_STATE_OPEN flag. 7216 */ 7217 smp_mb__after_atomic(); 7218 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 7219 clear_bit(BNXT_STATE_READ_STATS, &bp->state); 7220 return; 7221 } 7222 7223 /* TODO check if we need to synchronize with bnxt_close path */ 7224 for (i = 0; i < bp->cp_nr_rings; i++) { 7225 struct bnxt_napi *bnapi = bp->bnapi[i]; 7226 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 7227 struct ctx_hw_stats *hw_stats = cpr->hw_stats; 7228 7229 stats->rx_packets += le64_to_cpu(hw_stats->rx_ucast_pkts); 7230 stats->rx_packets += le64_to_cpu(hw_stats->rx_mcast_pkts); 7231 stats->rx_packets += le64_to_cpu(hw_stats->rx_bcast_pkts); 7232 7233 stats->tx_packets += le64_to_cpu(hw_stats->tx_ucast_pkts); 7234 stats->tx_packets += le64_to_cpu(hw_stats->tx_mcast_pkts); 7235 stats->tx_packets += le64_to_cpu(hw_stats->tx_bcast_pkts); 7236 7237 stats->rx_bytes += le64_to_cpu(hw_stats->rx_ucast_bytes); 7238 stats->rx_bytes += le64_to_cpu(hw_stats->rx_mcast_bytes); 7239 stats->rx_bytes += le64_to_cpu(hw_stats->rx_bcast_bytes); 7240 7241 stats->tx_bytes += le64_to_cpu(hw_stats->tx_ucast_bytes); 7242 stats->tx_bytes += le64_to_cpu(hw_stats->tx_mcast_bytes); 7243 stats->tx_bytes += le64_to_cpu(hw_stats->tx_bcast_bytes); 7244 7245 stats->rx_missed_errors += 7246 le64_to_cpu(hw_stats->rx_discard_pkts); 7247 7248 stats->multicast += le64_to_cpu(hw_stats->rx_mcast_pkts); 7249 7250 stats->tx_dropped += le64_to_cpu(hw_stats->tx_drop_pkts); 7251 } 7252 7253 if (bp->flags & BNXT_FLAG_PORT_STATS) { 7254 struct rx_port_stats *rx = bp->hw_rx_port_stats; 7255 struct tx_port_stats *tx = bp->hw_tx_port_stats; 7256 7257 stats->rx_crc_errors = le64_to_cpu(rx->rx_fcs_err_frames); 7258 stats->rx_frame_errors = le64_to_cpu(rx->rx_align_err_frames); 7259 stats->rx_length_errors = le64_to_cpu(rx->rx_undrsz_frames) + 7260 le64_to_cpu(rx->rx_ovrsz_frames) + 7261 le64_to_cpu(rx->rx_runt_frames); 7262 stats->rx_errors = le64_to_cpu(rx->rx_false_carrier_frames) + 7263 le64_to_cpu(rx->rx_jbr_frames); 7264 stats->collisions = le64_to_cpu(tx->tx_total_collisions); 7265 stats->tx_fifo_errors = le64_to_cpu(tx->tx_fifo_underruns); 7266 stats->tx_errors = le64_to_cpu(tx->tx_err); 7267 } 7268 clear_bit(BNXT_STATE_READ_STATS, &bp->state); 7269 } 7270 7271 static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask) 7272 { 7273 struct net_device *dev = bp->dev; 7274 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 7275 struct netdev_hw_addr *ha; 7276 u8 *haddr; 7277 int mc_count = 0; 7278 bool update = false; 7279 int off = 0; 7280 7281 netdev_for_each_mc_addr(ha, dev) { 7282 if (mc_count >= BNXT_MAX_MC_ADDRS) { 7283 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 7284 vnic->mc_list_count = 0; 7285 return false; 7286 } 7287 haddr = ha->addr; 7288 if (!ether_addr_equal(haddr, vnic->mc_list + off)) { 7289 memcpy(vnic->mc_list + off, haddr, ETH_ALEN); 7290 update = true; 7291 } 7292 off += ETH_ALEN; 7293 mc_count++; 7294 } 7295 if (mc_count) 7296 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST; 7297 7298 if (mc_count != vnic->mc_list_count) { 7299 vnic->mc_list_count = mc_count; 7300 update = true; 7301 } 7302 return update; 7303 } 7304 7305 static bool bnxt_uc_list_updated(struct bnxt *bp) 7306 { 7307 struct net_device *dev = bp->dev; 7308 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 7309 struct netdev_hw_addr *ha; 7310 int off = 0; 7311 7312 if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1)) 7313 return true; 7314 7315 netdev_for_each_uc_addr(ha, dev) { 7316 if (!ether_addr_equal(ha->addr, vnic->uc_list + off)) 7317 return true; 7318 7319 off += ETH_ALEN; 7320 } 7321 return false; 7322 } 7323 7324 static void bnxt_set_rx_mode(struct net_device *dev) 7325 { 7326 struct bnxt *bp = netdev_priv(dev); 7327 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 7328 u32 mask = vnic->rx_mask; 7329 bool mc_update = false; 7330 bool uc_update; 7331 7332 if (!netif_running(dev)) 7333 return; 7334 7335 mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS | 7336 CFA_L2_SET_RX_MASK_REQ_MASK_MCAST | 7337 CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST | 7338 CFA_L2_SET_RX_MASK_REQ_MASK_BCAST); 7339 7340 if ((dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp)) 7341 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 7342 7343 uc_update = bnxt_uc_list_updated(bp); 7344 7345 if (dev->flags & IFF_BROADCAST) 7346 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST; 7347 if (dev->flags & IFF_ALLMULTI) { 7348 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 7349 vnic->mc_list_count = 0; 7350 } else { 7351 mc_update = bnxt_mc_list_updated(bp, &mask); 7352 } 7353 7354 if (mask != vnic->rx_mask || uc_update || mc_update) { 7355 vnic->rx_mask = mask; 7356 7357 set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event); 7358 bnxt_queue_sp_work(bp); 7359 } 7360 } 7361 7362 static int bnxt_cfg_rx_mode(struct bnxt *bp) 7363 { 7364 struct net_device *dev = bp->dev; 7365 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 7366 struct netdev_hw_addr *ha; 7367 int i, off = 0, rc; 7368 bool uc_update; 7369 7370 netif_addr_lock_bh(dev); 7371 uc_update = bnxt_uc_list_updated(bp); 7372 netif_addr_unlock_bh(dev); 7373 7374 if (!uc_update) 7375 goto skip_uc; 7376 7377 mutex_lock(&bp->hwrm_cmd_lock); 7378 for (i = 1; i < vnic->uc_filter_count; i++) { 7379 struct hwrm_cfa_l2_filter_free_input req = {0}; 7380 7381 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_FREE, -1, 7382 -1); 7383 7384 req.l2_filter_id = vnic->fw_l2_filter_id[i]; 7385 7386 rc = _hwrm_send_message(bp, &req, sizeof(req), 7387 HWRM_CMD_TIMEOUT); 7388 } 7389 mutex_unlock(&bp->hwrm_cmd_lock); 7390 7391 vnic->uc_filter_count = 1; 7392 7393 netif_addr_lock_bh(dev); 7394 if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) { 7395 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 7396 } else { 7397 netdev_for_each_uc_addr(ha, dev) { 7398 memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN); 7399 off += ETH_ALEN; 7400 vnic->uc_filter_count++; 7401 } 7402 } 7403 netif_addr_unlock_bh(dev); 7404 7405 for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) { 7406 rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off); 7407 if (rc) { 7408 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", 7409 rc); 7410 vnic->uc_filter_count = i; 7411 return rc; 7412 } 7413 } 7414 7415 skip_uc: 7416 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0); 7417 if (rc) 7418 netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %x\n", 7419 rc); 7420 7421 return rc; 7422 } 7423 7424 static bool bnxt_can_reserve_rings(struct bnxt *bp) 7425 { 7426 #ifdef CONFIG_BNXT_SRIOV 7427 if (BNXT_NEW_RM(bp) && BNXT_VF(bp)) { 7428 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 7429 7430 /* No minimum rings were provisioned by the PF. Don't 7431 * reserve rings by default when device is down. 7432 */ 7433 if (hw_resc->min_tx_rings || hw_resc->resv_tx_rings) 7434 return true; 7435 7436 if (!netif_running(bp->dev)) 7437 return false; 7438 } 7439 #endif 7440 return true; 7441 } 7442 7443 /* If the chip and firmware supports RFS */ 7444 static bool bnxt_rfs_supported(struct bnxt *bp) 7445 { 7446 if (BNXT_PF(bp) && !BNXT_CHIP_TYPE_NITRO_A0(bp)) 7447 return true; 7448 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP) 7449 return true; 7450 return false; 7451 } 7452 7453 /* If runtime conditions support RFS */ 7454 static bool bnxt_rfs_capable(struct bnxt *bp) 7455 { 7456 #ifdef CONFIG_RFS_ACCEL 7457 int vnics, max_vnics, max_rss_ctxs; 7458 7459 if (!(bp->flags & BNXT_FLAG_MSIX_CAP) || !bnxt_can_reserve_rings(bp)) 7460 return false; 7461 7462 vnics = 1 + bp->rx_nr_rings; 7463 max_vnics = bnxt_get_max_func_vnics(bp); 7464 max_rss_ctxs = bnxt_get_max_func_rss_ctxs(bp); 7465 7466 /* RSS contexts not a limiting factor */ 7467 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP) 7468 max_rss_ctxs = max_vnics; 7469 if (vnics > max_vnics || vnics > max_rss_ctxs) { 7470 if (bp->rx_nr_rings > 1) 7471 netdev_warn(bp->dev, 7472 "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n", 7473 min(max_rss_ctxs - 1, max_vnics - 1)); 7474 return false; 7475 } 7476 7477 if (!BNXT_NEW_RM(bp)) 7478 return true; 7479 7480 if (vnics == bp->hw_resc.resv_vnics) 7481 return true; 7482 7483 bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, vnics); 7484 if (vnics <= bp->hw_resc.resv_vnics) 7485 return true; 7486 7487 netdev_warn(bp->dev, "Unable to reserve resources to support NTUPLE filters.\n"); 7488 bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 1); 7489 return false; 7490 #else 7491 return false; 7492 #endif 7493 } 7494 7495 static netdev_features_t bnxt_fix_features(struct net_device *dev, 7496 netdev_features_t features) 7497 { 7498 struct bnxt *bp = netdev_priv(dev); 7499 7500 if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp)) 7501 features &= ~NETIF_F_NTUPLE; 7502 7503 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS) 7504 features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); 7505 7506 if (!(features & NETIF_F_GRO)) 7507 features &= ~NETIF_F_GRO_HW; 7508 7509 if (features & NETIF_F_GRO_HW) 7510 features &= ~NETIF_F_LRO; 7511 7512 /* Both CTAG and STAG VLAN accelaration on the RX side have to be 7513 * turned on or off together. 7514 */ 7515 if ((features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)) != 7516 (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)) { 7517 if (dev->features & NETIF_F_HW_VLAN_CTAG_RX) 7518 features &= ~(NETIF_F_HW_VLAN_CTAG_RX | 7519 NETIF_F_HW_VLAN_STAG_RX); 7520 else 7521 features |= NETIF_F_HW_VLAN_CTAG_RX | 7522 NETIF_F_HW_VLAN_STAG_RX; 7523 } 7524 #ifdef CONFIG_BNXT_SRIOV 7525 if (BNXT_VF(bp)) { 7526 if (bp->vf.vlan) { 7527 features &= ~(NETIF_F_HW_VLAN_CTAG_RX | 7528 NETIF_F_HW_VLAN_STAG_RX); 7529 } 7530 } 7531 #endif 7532 return features; 7533 } 7534 7535 static int bnxt_set_features(struct net_device *dev, netdev_features_t features) 7536 { 7537 struct bnxt *bp = netdev_priv(dev); 7538 u32 flags = bp->flags; 7539 u32 changes; 7540 int rc = 0; 7541 bool re_init = false; 7542 bool update_tpa = false; 7543 7544 flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS; 7545 if (features & NETIF_F_GRO_HW) 7546 flags |= BNXT_FLAG_GRO; 7547 else if (features & NETIF_F_LRO) 7548 flags |= BNXT_FLAG_LRO; 7549 7550 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS) 7551 flags &= ~BNXT_FLAG_TPA; 7552 7553 if (features & NETIF_F_HW_VLAN_CTAG_RX) 7554 flags |= BNXT_FLAG_STRIP_VLAN; 7555 7556 if (features & NETIF_F_NTUPLE) 7557 flags |= BNXT_FLAG_RFS; 7558 7559 changes = flags ^ bp->flags; 7560 if (changes & BNXT_FLAG_TPA) { 7561 update_tpa = true; 7562 if ((bp->flags & BNXT_FLAG_TPA) == 0 || 7563 (flags & BNXT_FLAG_TPA) == 0) 7564 re_init = true; 7565 } 7566 7567 if (changes & ~BNXT_FLAG_TPA) 7568 re_init = true; 7569 7570 if (flags != bp->flags) { 7571 u32 old_flags = bp->flags; 7572 7573 bp->flags = flags; 7574 7575 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 7576 if (update_tpa) 7577 bnxt_set_ring_params(bp); 7578 return rc; 7579 } 7580 7581 if (re_init) { 7582 bnxt_close_nic(bp, false, false); 7583 if (update_tpa) 7584 bnxt_set_ring_params(bp); 7585 7586 return bnxt_open_nic(bp, false, false); 7587 } 7588 if (update_tpa) { 7589 rc = bnxt_set_tpa(bp, 7590 (flags & BNXT_FLAG_TPA) ? 7591 true : false); 7592 if (rc) 7593 bp->flags = old_flags; 7594 } 7595 } 7596 return rc; 7597 } 7598 7599 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi) 7600 { 7601 struct bnxt_tx_ring_info *txr = bnapi->tx_ring; 7602 int i = bnapi->index; 7603 7604 if (!txr) 7605 return; 7606 7607 netdev_info(bnapi->bp->dev, "[%d]: tx{fw_ring: %d prod: %x cons: %x}\n", 7608 i, txr->tx_ring_struct.fw_ring_id, txr->tx_prod, 7609 txr->tx_cons); 7610 } 7611 7612 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi) 7613 { 7614 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 7615 int i = bnapi->index; 7616 7617 if (!rxr) 7618 return; 7619 7620 netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n", 7621 i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod, 7622 rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod, 7623 rxr->rx_sw_agg_prod); 7624 } 7625 7626 static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi) 7627 { 7628 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 7629 int i = bnapi->index; 7630 7631 netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n", 7632 i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons); 7633 } 7634 7635 static void bnxt_dbg_dump_states(struct bnxt *bp) 7636 { 7637 int i; 7638 struct bnxt_napi *bnapi; 7639 7640 for (i = 0; i < bp->cp_nr_rings; i++) { 7641 bnapi = bp->bnapi[i]; 7642 if (netif_msg_drv(bp)) { 7643 bnxt_dump_tx_sw_state(bnapi); 7644 bnxt_dump_rx_sw_state(bnapi); 7645 bnxt_dump_cp_sw_state(bnapi); 7646 } 7647 } 7648 } 7649 7650 static void bnxt_reset_task(struct bnxt *bp, bool silent) 7651 { 7652 if (!silent) 7653 bnxt_dbg_dump_states(bp); 7654 if (netif_running(bp->dev)) { 7655 int rc; 7656 7657 if (!silent) 7658 bnxt_ulp_stop(bp); 7659 bnxt_close_nic(bp, false, false); 7660 rc = bnxt_open_nic(bp, false, false); 7661 if (!silent && !rc) 7662 bnxt_ulp_start(bp); 7663 } 7664 } 7665 7666 static void bnxt_tx_timeout(struct net_device *dev) 7667 { 7668 struct bnxt *bp = netdev_priv(dev); 7669 7670 netdev_err(bp->dev, "TX timeout detected, starting reset task!\n"); 7671 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event); 7672 bnxt_queue_sp_work(bp); 7673 } 7674 7675 #ifdef CONFIG_NET_POLL_CONTROLLER 7676 static void bnxt_poll_controller(struct net_device *dev) 7677 { 7678 struct bnxt *bp = netdev_priv(dev); 7679 int i; 7680 7681 /* Only process tx rings/combined rings in netpoll mode. */ 7682 for (i = 0; i < bp->tx_nr_rings; i++) { 7683 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 7684 7685 napi_schedule(&txr->bnapi->napi); 7686 } 7687 } 7688 #endif 7689 7690 static void bnxt_timer(struct timer_list *t) 7691 { 7692 struct bnxt *bp = from_timer(bp, t, timer); 7693 struct net_device *dev = bp->dev; 7694 7695 if (!netif_running(dev)) 7696 return; 7697 7698 if (atomic_read(&bp->intr_sem) != 0) 7699 goto bnxt_restart_timer; 7700 7701 if (bp->link_info.link_up && (bp->flags & BNXT_FLAG_PORT_STATS) && 7702 bp->stats_coal_ticks) { 7703 set_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event); 7704 bnxt_queue_sp_work(bp); 7705 } 7706 7707 if (bnxt_tc_flower_enabled(bp)) { 7708 set_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event); 7709 bnxt_queue_sp_work(bp); 7710 } 7711 7712 if (bp->link_info.phy_retry) { 7713 if (time_after(jiffies, bp->link_info.phy_retry_expires)) { 7714 bp->link_info.phy_retry = 0; 7715 netdev_warn(bp->dev, "failed to update phy settings after maximum retries.\n"); 7716 } else { 7717 set_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event); 7718 bnxt_queue_sp_work(bp); 7719 } 7720 } 7721 bnxt_restart_timer: 7722 mod_timer(&bp->timer, jiffies + bp->current_interval); 7723 } 7724 7725 static void bnxt_rtnl_lock_sp(struct bnxt *bp) 7726 { 7727 /* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK 7728 * set. If the device is being closed, bnxt_close() may be holding 7729 * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear. So we 7730 * must clear BNXT_STATE_IN_SP_TASK before holding rtnl(). 7731 */ 7732 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 7733 rtnl_lock(); 7734 } 7735 7736 static void bnxt_rtnl_unlock_sp(struct bnxt *bp) 7737 { 7738 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 7739 rtnl_unlock(); 7740 } 7741 7742 /* Only called from bnxt_sp_task() */ 7743 static void bnxt_reset(struct bnxt *bp, bool silent) 7744 { 7745 bnxt_rtnl_lock_sp(bp); 7746 if (test_bit(BNXT_STATE_OPEN, &bp->state)) 7747 bnxt_reset_task(bp, silent); 7748 bnxt_rtnl_unlock_sp(bp); 7749 } 7750 7751 static void bnxt_cfg_ntp_filters(struct bnxt *); 7752 7753 static void bnxt_sp_task(struct work_struct *work) 7754 { 7755 struct bnxt *bp = container_of(work, struct bnxt, sp_task); 7756 7757 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 7758 smp_mb__after_atomic(); 7759 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 7760 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 7761 return; 7762 } 7763 7764 if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event)) 7765 bnxt_cfg_rx_mode(bp); 7766 7767 if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event)) 7768 bnxt_cfg_ntp_filters(bp); 7769 if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event)) 7770 bnxt_hwrm_exec_fwd_req(bp); 7771 if (test_and_clear_bit(BNXT_VXLAN_ADD_PORT_SP_EVENT, &bp->sp_event)) { 7772 bnxt_hwrm_tunnel_dst_port_alloc( 7773 bp, bp->vxlan_port, 7774 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN); 7775 } 7776 if (test_and_clear_bit(BNXT_VXLAN_DEL_PORT_SP_EVENT, &bp->sp_event)) { 7777 bnxt_hwrm_tunnel_dst_port_free( 7778 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN); 7779 } 7780 if (test_and_clear_bit(BNXT_GENEVE_ADD_PORT_SP_EVENT, &bp->sp_event)) { 7781 bnxt_hwrm_tunnel_dst_port_alloc( 7782 bp, bp->nge_port, 7783 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE); 7784 } 7785 if (test_and_clear_bit(BNXT_GENEVE_DEL_PORT_SP_EVENT, &bp->sp_event)) { 7786 bnxt_hwrm_tunnel_dst_port_free( 7787 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE); 7788 } 7789 if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event)) { 7790 bnxt_hwrm_port_qstats(bp); 7791 bnxt_hwrm_port_qstats_ext(bp); 7792 } 7793 7794 if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) { 7795 int rc; 7796 7797 mutex_lock(&bp->link_lock); 7798 if (test_and_clear_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, 7799 &bp->sp_event)) 7800 bnxt_hwrm_phy_qcaps(bp); 7801 7802 rc = bnxt_update_link(bp, true); 7803 mutex_unlock(&bp->link_lock); 7804 if (rc) 7805 netdev_err(bp->dev, "SP task can't update link (rc: %x)\n", 7806 rc); 7807 } 7808 if (test_and_clear_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event)) { 7809 int rc; 7810 7811 mutex_lock(&bp->link_lock); 7812 rc = bnxt_update_phy_setting(bp); 7813 mutex_unlock(&bp->link_lock); 7814 if (rc) { 7815 netdev_warn(bp->dev, "update phy settings retry failed\n"); 7816 } else { 7817 bp->link_info.phy_retry = false; 7818 netdev_info(bp->dev, "update phy settings retry succeeded\n"); 7819 } 7820 } 7821 if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event)) { 7822 mutex_lock(&bp->link_lock); 7823 bnxt_get_port_module_status(bp); 7824 mutex_unlock(&bp->link_lock); 7825 } 7826 7827 if (test_and_clear_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event)) 7828 bnxt_tc_flow_stats_work(bp); 7829 7830 /* These functions below will clear BNXT_STATE_IN_SP_TASK. They 7831 * must be the last functions to be called before exiting. 7832 */ 7833 if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event)) 7834 bnxt_reset(bp, false); 7835 7836 if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event)) 7837 bnxt_reset(bp, true); 7838 7839 smp_mb__before_atomic(); 7840 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 7841 } 7842 7843 /* Under rtnl_lock */ 7844 int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs, 7845 int tx_xdp) 7846 { 7847 int max_rx, max_tx, tx_sets = 1; 7848 int tx_rings_needed; 7849 int rx_rings = rx; 7850 int cp, vnics, rc; 7851 7852 if (tcs) 7853 tx_sets = tcs; 7854 7855 rc = bnxt_get_max_rings(bp, &max_rx, &max_tx, sh); 7856 if (rc) 7857 return rc; 7858 7859 if (max_rx < rx) 7860 return -ENOMEM; 7861 7862 tx_rings_needed = tx * tx_sets + tx_xdp; 7863 if (max_tx < tx_rings_needed) 7864 return -ENOMEM; 7865 7866 vnics = 1; 7867 if (bp->flags & BNXT_FLAG_RFS) 7868 vnics += rx_rings; 7869 7870 if (bp->flags & BNXT_FLAG_AGG_RINGS) 7871 rx_rings <<= 1; 7872 cp = sh ? max_t(int, tx_rings_needed, rx) : tx_rings_needed + rx; 7873 if (BNXT_NEW_RM(bp)) 7874 cp += bnxt_get_ulp_msix_num(bp); 7875 return bnxt_hwrm_check_rings(bp, tx_rings_needed, rx_rings, rx, cp, 7876 vnics); 7877 } 7878 7879 static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev) 7880 { 7881 if (bp->bar2) { 7882 pci_iounmap(pdev, bp->bar2); 7883 bp->bar2 = NULL; 7884 } 7885 7886 if (bp->bar1) { 7887 pci_iounmap(pdev, bp->bar1); 7888 bp->bar1 = NULL; 7889 } 7890 7891 if (bp->bar0) { 7892 pci_iounmap(pdev, bp->bar0); 7893 bp->bar0 = NULL; 7894 } 7895 } 7896 7897 static void bnxt_cleanup_pci(struct bnxt *bp) 7898 { 7899 bnxt_unmap_bars(bp, bp->pdev); 7900 pci_release_regions(bp->pdev); 7901 pci_disable_device(bp->pdev); 7902 } 7903 7904 static void bnxt_init_dflt_coal(struct bnxt *bp) 7905 { 7906 struct bnxt_coal *coal; 7907 7908 /* Tick values in micro seconds. 7909 * 1 coal_buf x bufs_per_record = 1 completion record. 7910 */ 7911 coal = &bp->rx_coal; 7912 coal->coal_ticks = 14; 7913 coal->coal_bufs = 30; 7914 coal->coal_ticks_irq = 1; 7915 coal->coal_bufs_irq = 2; 7916 coal->idle_thresh = 50; 7917 coal->bufs_per_record = 2; 7918 coal->budget = 64; /* NAPI budget */ 7919 7920 coal = &bp->tx_coal; 7921 coal->coal_ticks = 28; 7922 coal->coal_bufs = 30; 7923 coal->coal_ticks_irq = 2; 7924 coal->coal_bufs_irq = 2; 7925 coal->bufs_per_record = 1; 7926 7927 bp->stats_coal_ticks = BNXT_DEF_STATS_COAL_TICKS; 7928 } 7929 7930 static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev) 7931 { 7932 int rc; 7933 struct bnxt *bp = netdev_priv(dev); 7934 7935 SET_NETDEV_DEV(dev, &pdev->dev); 7936 7937 /* enable device (incl. PCI PM wakeup), and bus-mastering */ 7938 rc = pci_enable_device(pdev); 7939 if (rc) { 7940 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n"); 7941 goto init_err; 7942 } 7943 7944 if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) { 7945 dev_err(&pdev->dev, 7946 "Cannot find PCI device base address, aborting\n"); 7947 rc = -ENODEV; 7948 goto init_err_disable; 7949 } 7950 7951 rc = pci_request_regions(pdev, DRV_MODULE_NAME); 7952 if (rc) { 7953 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n"); 7954 goto init_err_disable; 7955 } 7956 7957 if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 && 7958 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) { 7959 dev_err(&pdev->dev, "System does not support DMA, aborting\n"); 7960 goto init_err_disable; 7961 } 7962 7963 pci_set_master(pdev); 7964 7965 bp->dev = dev; 7966 bp->pdev = pdev; 7967 7968 bp->bar0 = pci_ioremap_bar(pdev, 0); 7969 if (!bp->bar0) { 7970 dev_err(&pdev->dev, "Cannot map device registers, aborting\n"); 7971 rc = -ENOMEM; 7972 goto init_err_release; 7973 } 7974 7975 bp->bar1 = pci_ioremap_bar(pdev, 2); 7976 if (!bp->bar1) { 7977 dev_err(&pdev->dev, "Cannot map doorbell registers, aborting\n"); 7978 rc = -ENOMEM; 7979 goto init_err_release; 7980 } 7981 7982 bp->bar2 = pci_ioremap_bar(pdev, 4); 7983 if (!bp->bar2) { 7984 dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n"); 7985 rc = -ENOMEM; 7986 goto init_err_release; 7987 } 7988 7989 pci_enable_pcie_error_reporting(pdev); 7990 7991 INIT_WORK(&bp->sp_task, bnxt_sp_task); 7992 7993 spin_lock_init(&bp->ntp_fltr_lock); 7994 7995 bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE; 7996 bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE; 7997 7998 bnxt_init_dflt_coal(bp); 7999 8000 timer_setup(&bp->timer, bnxt_timer, 0); 8001 bp->current_interval = BNXT_TIMER_INTERVAL; 8002 8003 clear_bit(BNXT_STATE_OPEN, &bp->state); 8004 return 0; 8005 8006 init_err_release: 8007 bnxt_unmap_bars(bp, pdev); 8008 pci_release_regions(pdev); 8009 8010 init_err_disable: 8011 pci_disable_device(pdev); 8012 8013 init_err: 8014 return rc; 8015 } 8016 8017 /* rtnl_lock held */ 8018 static int bnxt_change_mac_addr(struct net_device *dev, void *p) 8019 { 8020 struct sockaddr *addr = p; 8021 struct bnxt *bp = netdev_priv(dev); 8022 int rc = 0; 8023 8024 if (!is_valid_ether_addr(addr->sa_data)) 8025 return -EADDRNOTAVAIL; 8026 8027 if (ether_addr_equal(addr->sa_data, dev->dev_addr)) 8028 return 0; 8029 8030 rc = bnxt_approve_mac(bp, addr->sa_data); 8031 if (rc) 8032 return rc; 8033 8034 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len); 8035 if (netif_running(dev)) { 8036 bnxt_close_nic(bp, false, false); 8037 rc = bnxt_open_nic(bp, false, false); 8038 } 8039 8040 return rc; 8041 } 8042 8043 /* rtnl_lock held */ 8044 static int bnxt_change_mtu(struct net_device *dev, int new_mtu) 8045 { 8046 struct bnxt *bp = netdev_priv(dev); 8047 8048 if (netif_running(dev)) 8049 bnxt_close_nic(bp, false, false); 8050 8051 dev->mtu = new_mtu; 8052 bnxt_set_ring_params(bp); 8053 8054 if (netif_running(dev)) 8055 return bnxt_open_nic(bp, false, false); 8056 8057 return 0; 8058 } 8059 8060 int bnxt_setup_mq_tc(struct net_device *dev, u8 tc) 8061 { 8062 struct bnxt *bp = netdev_priv(dev); 8063 bool sh = false; 8064 int rc; 8065 8066 if (tc > bp->max_tc) { 8067 netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n", 8068 tc, bp->max_tc); 8069 return -EINVAL; 8070 } 8071 8072 if (netdev_get_num_tc(dev) == tc) 8073 return 0; 8074 8075 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 8076 sh = true; 8077 8078 rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings, 8079 sh, tc, bp->tx_nr_rings_xdp); 8080 if (rc) 8081 return rc; 8082 8083 /* Needs to close the device and do hw resource re-allocations */ 8084 if (netif_running(bp->dev)) 8085 bnxt_close_nic(bp, true, false); 8086 8087 if (tc) { 8088 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc; 8089 netdev_set_num_tc(dev, tc); 8090 } else { 8091 bp->tx_nr_rings = bp->tx_nr_rings_per_tc; 8092 netdev_reset_tc(dev); 8093 } 8094 bp->tx_nr_rings += bp->tx_nr_rings_xdp; 8095 bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) : 8096 bp->tx_nr_rings + bp->rx_nr_rings; 8097 bp->num_stat_ctxs = bp->cp_nr_rings; 8098 8099 if (netif_running(bp->dev)) 8100 return bnxt_open_nic(bp, true, false); 8101 8102 return 0; 8103 } 8104 8105 static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 8106 void *cb_priv) 8107 { 8108 struct bnxt *bp = cb_priv; 8109 8110 if (!bnxt_tc_flower_enabled(bp) || 8111 !tc_cls_can_offload_and_chain0(bp->dev, type_data)) 8112 return -EOPNOTSUPP; 8113 8114 switch (type) { 8115 case TC_SETUP_CLSFLOWER: 8116 return bnxt_tc_setup_flower(bp, bp->pf.fw_fid, type_data); 8117 default: 8118 return -EOPNOTSUPP; 8119 } 8120 } 8121 8122 static int bnxt_setup_tc_block(struct net_device *dev, 8123 struct tc_block_offload *f) 8124 { 8125 struct bnxt *bp = netdev_priv(dev); 8126 8127 if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS) 8128 return -EOPNOTSUPP; 8129 8130 switch (f->command) { 8131 case TC_BLOCK_BIND: 8132 return tcf_block_cb_register(f->block, bnxt_setup_tc_block_cb, 8133 bp, bp, f->extack); 8134 case TC_BLOCK_UNBIND: 8135 tcf_block_cb_unregister(f->block, bnxt_setup_tc_block_cb, bp); 8136 return 0; 8137 default: 8138 return -EOPNOTSUPP; 8139 } 8140 } 8141 8142 static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type, 8143 void *type_data) 8144 { 8145 switch (type) { 8146 case TC_SETUP_BLOCK: 8147 return bnxt_setup_tc_block(dev, type_data); 8148 case TC_SETUP_QDISC_MQPRIO: { 8149 struct tc_mqprio_qopt *mqprio = type_data; 8150 8151 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 8152 8153 return bnxt_setup_mq_tc(dev, mqprio->num_tc); 8154 } 8155 default: 8156 return -EOPNOTSUPP; 8157 } 8158 } 8159 8160 #ifdef CONFIG_RFS_ACCEL 8161 static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1, 8162 struct bnxt_ntuple_filter *f2) 8163 { 8164 struct flow_keys *keys1 = &f1->fkeys; 8165 struct flow_keys *keys2 = &f2->fkeys; 8166 8167 if (keys1->addrs.v4addrs.src == keys2->addrs.v4addrs.src && 8168 keys1->addrs.v4addrs.dst == keys2->addrs.v4addrs.dst && 8169 keys1->ports.ports == keys2->ports.ports && 8170 keys1->basic.ip_proto == keys2->basic.ip_proto && 8171 keys1->basic.n_proto == keys2->basic.n_proto && 8172 keys1->control.flags == keys2->control.flags && 8173 ether_addr_equal(f1->src_mac_addr, f2->src_mac_addr) && 8174 ether_addr_equal(f1->dst_mac_addr, f2->dst_mac_addr)) 8175 return true; 8176 8177 return false; 8178 } 8179 8180 static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb, 8181 u16 rxq_index, u32 flow_id) 8182 { 8183 struct bnxt *bp = netdev_priv(dev); 8184 struct bnxt_ntuple_filter *fltr, *new_fltr; 8185 struct flow_keys *fkeys; 8186 struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb); 8187 int rc = 0, idx, bit_id, l2_idx = 0; 8188 struct hlist_head *head; 8189 8190 if (!ether_addr_equal(dev->dev_addr, eth->h_dest)) { 8191 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 8192 int off = 0, j; 8193 8194 netif_addr_lock_bh(dev); 8195 for (j = 0; j < vnic->uc_filter_count; j++, off += ETH_ALEN) { 8196 if (ether_addr_equal(eth->h_dest, 8197 vnic->uc_list + off)) { 8198 l2_idx = j + 1; 8199 break; 8200 } 8201 } 8202 netif_addr_unlock_bh(dev); 8203 if (!l2_idx) 8204 return -EINVAL; 8205 } 8206 new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC); 8207 if (!new_fltr) 8208 return -ENOMEM; 8209 8210 fkeys = &new_fltr->fkeys; 8211 if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) { 8212 rc = -EPROTONOSUPPORT; 8213 goto err_free; 8214 } 8215 8216 if ((fkeys->basic.n_proto != htons(ETH_P_IP) && 8217 fkeys->basic.n_proto != htons(ETH_P_IPV6)) || 8218 ((fkeys->basic.ip_proto != IPPROTO_TCP) && 8219 (fkeys->basic.ip_proto != IPPROTO_UDP))) { 8220 rc = -EPROTONOSUPPORT; 8221 goto err_free; 8222 } 8223 if (fkeys->basic.n_proto == htons(ETH_P_IPV6) && 8224 bp->hwrm_spec_code < 0x10601) { 8225 rc = -EPROTONOSUPPORT; 8226 goto err_free; 8227 } 8228 if ((fkeys->control.flags & FLOW_DIS_ENCAPSULATION) && 8229 bp->hwrm_spec_code < 0x10601) { 8230 rc = -EPROTONOSUPPORT; 8231 goto err_free; 8232 } 8233 8234 memcpy(new_fltr->dst_mac_addr, eth->h_dest, ETH_ALEN); 8235 memcpy(new_fltr->src_mac_addr, eth->h_source, ETH_ALEN); 8236 8237 idx = skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK; 8238 head = &bp->ntp_fltr_hash_tbl[idx]; 8239 rcu_read_lock(); 8240 hlist_for_each_entry_rcu(fltr, head, hash) { 8241 if (bnxt_fltr_match(fltr, new_fltr)) { 8242 rcu_read_unlock(); 8243 rc = 0; 8244 goto err_free; 8245 } 8246 } 8247 rcu_read_unlock(); 8248 8249 spin_lock_bh(&bp->ntp_fltr_lock); 8250 bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap, 8251 BNXT_NTP_FLTR_MAX_FLTR, 0); 8252 if (bit_id < 0) { 8253 spin_unlock_bh(&bp->ntp_fltr_lock); 8254 rc = -ENOMEM; 8255 goto err_free; 8256 } 8257 8258 new_fltr->sw_id = (u16)bit_id; 8259 new_fltr->flow_id = flow_id; 8260 new_fltr->l2_fltr_idx = l2_idx; 8261 new_fltr->rxq = rxq_index; 8262 hlist_add_head_rcu(&new_fltr->hash, head); 8263 bp->ntp_fltr_count++; 8264 spin_unlock_bh(&bp->ntp_fltr_lock); 8265 8266 set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event); 8267 bnxt_queue_sp_work(bp); 8268 8269 return new_fltr->sw_id; 8270 8271 err_free: 8272 kfree(new_fltr); 8273 return rc; 8274 } 8275 8276 static void bnxt_cfg_ntp_filters(struct bnxt *bp) 8277 { 8278 int i; 8279 8280 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) { 8281 struct hlist_head *head; 8282 struct hlist_node *tmp; 8283 struct bnxt_ntuple_filter *fltr; 8284 int rc; 8285 8286 head = &bp->ntp_fltr_hash_tbl[i]; 8287 hlist_for_each_entry_safe(fltr, tmp, head, hash) { 8288 bool del = false; 8289 8290 if (test_bit(BNXT_FLTR_VALID, &fltr->state)) { 8291 if (rps_may_expire_flow(bp->dev, fltr->rxq, 8292 fltr->flow_id, 8293 fltr->sw_id)) { 8294 bnxt_hwrm_cfa_ntuple_filter_free(bp, 8295 fltr); 8296 del = true; 8297 } 8298 } else { 8299 rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp, 8300 fltr); 8301 if (rc) 8302 del = true; 8303 else 8304 set_bit(BNXT_FLTR_VALID, &fltr->state); 8305 } 8306 8307 if (del) { 8308 spin_lock_bh(&bp->ntp_fltr_lock); 8309 hlist_del_rcu(&fltr->hash); 8310 bp->ntp_fltr_count--; 8311 spin_unlock_bh(&bp->ntp_fltr_lock); 8312 synchronize_rcu(); 8313 clear_bit(fltr->sw_id, bp->ntp_fltr_bmap); 8314 kfree(fltr); 8315 } 8316 } 8317 } 8318 if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event)) 8319 netdev_info(bp->dev, "Receive PF driver unload event!"); 8320 } 8321 8322 #else 8323 8324 static void bnxt_cfg_ntp_filters(struct bnxt *bp) 8325 { 8326 } 8327 8328 #endif /* CONFIG_RFS_ACCEL */ 8329 8330 static void bnxt_udp_tunnel_add(struct net_device *dev, 8331 struct udp_tunnel_info *ti) 8332 { 8333 struct bnxt *bp = netdev_priv(dev); 8334 8335 if (ti->sa_family != AF_INET6 && ti->sa_family != AF_INET) 8336 return; 8337 8338 if (!netif_running(dev)) 8339 return; 8340 8341 switch (ti->type) { 8342 case UDP_TUNNEL_TYPE_VXLAN: 8343 if (bp->vxlan_port_cnt && bp->vxlan_port != ti->port) 8344 return; 8345 8346 bp->vxlan_port_cnt++; 8347 if (bp->vxlan_port_cnt == 1) { 8348 bp->vxlan_port = ti->port; 8349 set_bit(BNXT_VXLAN_ADD_PORT_SP_EVENT, &bp->sp_event); 8350 bnxt_queue_sp_work(bp); 8351 } 8352 break; 8353 case UDP_TUNNEL_TYPE_GENEVE: 8354 if (bp->nge_port_cnt && bp->nge_port != ti->port) 8355 return; 8356 8357 bp->nge_port_cnt++; 8358 if (bp->nge_port_cnt == 1) { 8359 bp->nge_port = ti->port; 8360 set_bit(BNXT_GENEVE_ADD_PORT_SP_EVENT, &bp->sp_event); 8361 } 8362 break; 8363 default: 8364 return; 8365 } 8366 8367 bnxt_queue_sp_work(bp); 8368 } 8369 8370 static void bnxt_udp_tunnel_del(struct net_device *dev, 8371 struct udp_tunnel_info *ti) 8372 { 8373 struct bnxt *bp = netdev_priv(dev); 8374 8375 if (ti->sa_family != AF_INET6 && ti->sa_family != AF_INET) 8376 return; 8377 8378 if (!netif_running(dev)) 8379 return; 8380 8381 switch (ti->type) { 8382 case UDP_TUNNEL_TYPE_VXLAN: 8383 if (!bp->vxlan_port_cnt || bp->vxlan_port != ti->port) 8384 return; 8385 bp->vxlan_port_cnt--; 8386 8387 if (bp->vxlan_port_cnt != 0) 8388 return; 8389 8390 set_bit(BNXT_VXLAN_DEL_PORT_SP_EVENT, &bp->sp_event); 8391 break; 8392 case UDP_TUNNEL_TYPE_GENEVE: 8393 if (!bp->nge_port_cnt || bp->nge_port != ti->port) 8394 return; 8395 bp->nge_port_cnt--; 8396 8397 if (bp->nge_port_cnt != 0) 8398 return; 8399 8400 set_bit(BNXT_GENEVE_DEL_PORT_SP_EVENT, &bp->sp_event); 8401 break; 8402 default: 8403 return; 8404 } 8405 8406 bnxt_queue_sp_work(bp); 8407 } 8408 8409 static int bnxt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 8410 struct net_device *dev, u32 filter_mask, 8411 int nlflags) 8412 { 8413 struct bnxt *bp = netdev_priv(dev); 8414 8415 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bp->br_mode, 0, 0, 8416 nlflags, filter_mask, NULL); 8417 } 8418 8419 static int bnxt_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh, 8420 u16 flags) 8421 { 8422 struct bnxt *bp = netdev_priv(dev); 8423 struct nlattr *attr, *br_spec; 8424 int rem, rc = 0; 8425 8426 if (bp->hwrm_spec_code < 0x10708 || !BNXT_SINGLE_PF(bp)) 8427 return -EOPNOTSUPP; 8428 8429 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 8430 if (!br_spec) 8431 return -EINVAL; 8432 8433 nla_for_each_nested(attr, br_spec, rem) { 8434 u16 mode; 8435 8436 if (nla_type(attr) != IFLA_BRIDGE_MODE) 8437 continue; 8438 8439 if (nla_len(attr) < sizeof(mode)) 8440 return -EINVAL; 8441 8442 mode = nla_get_u16(attr); 8443 if (mode == bp->br_mode) 8444 break; 8445 8446 rc = bnxt_hwrm_set_br_mode(bp, mode); 8447 if (!rc) 8448 bp->br_mode = mode; 8449 break; 8450 } 8451 return rc; 8452 } 8453 8454 static int bnxt_get_phys_port_name(struct net_device *dev, char *buf, 8455 size_t len) 8456 { 8457 struct bnxt *bp = netdev_priv(dev); 8458 int rc; 8459 8460 /* The PF and it's VF-reps only support the switchdev framework */ 8461 if (!BNXT_PF(bp)) 8462 return -EOPNOTSUPP; 8463 8464 rc = snprintf(buf, len, "p%d", bp->pf.port_id); 8465 8466 if (rc >= len) 8467 return -EOPNOTSUPP; 8468 return 0; 8469 } 8470 8471 int bnxt_port_attr_get(struct bnxt *bp, struct switchdev_attr *attr) 8472 { 8473 if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV) 8474 return -EOPNOTSUPP; 8475 8476 /* The PF and it's VF-reps only support the switchdev framework */ 8477 if (!BNXT_PF(bp)) 8478 return -EOPNOTSUPP; 8479 8480 switch (attr->id) { 8481 case SWITCHDEV_ATTR_ID_PORT_PARENT_ID: 8482 attr->u.ppid.id_len = sizeof(bp->switch_id); 8483 memcpy(attr->u.ppid.id, bp->switch_id, attr->u.ppid.id_len); 8484 break; 8485 default: 8486 return -EOPNOTSUPP; 8487 } 8488 return 0; 8489 } 8490 8491 static int bnxt_swdev_port_attr_get(struct net_device *dev, 8492 struct switchdev_attr *attr) 8493 { 8494 return bnxt_port_attr_get(netdev_priv(dev), attr); 8495 } 8496 8497 static const struct switchdev_ops bnxt_switchdev_ops = { 8498 .switchdev_port_attr_get = bnxt_swdev_port_attr_get 8499 }; 8500 8501 static const struct net_device_ops bnxt_netdev_ops = { 8502 .ndo_open = bnxt_open, 8503 .ndo_start_xmit = bnxt_start_xmit, 8504 .ndo_stop = bnxt_close, 8505 .ndo_get_stats64 = bnxt_get_stats64, 8506 .ndo_set_rx_mode = bnxt_set_rx_mode, 8507 .ndo_do_ioctl = bnxt_ioctl, 8508 .ndo_validate_addr = eth_validate_addr, 8509 .ndo_set_mac_address = bnxt_change_mac_addr, 8510 .ndo_change_mtu = bnxt_change_mtu, 8511 .ndo_fix_features = bnxt_fix_features, 8512 .ndo_set_features = bnxt_set_features, 8513 .ndo_tx_timeout = bnxt_tx_timeout, 8514 #ifdef CONFIG_BNXT_SRIOV 8515 .ndo_get_vf_config = bnxt_get_vf_config, 8516 .ndo_set_vf_mac = bnxt_set_vf_mac, 8517 .ndo_set_vf_vlan = bnxt_set_vf_vlan, 8518 .ndo_set_vf_rate = bnxt_set_vf_bw, 8519 .ndo_set_vf_link_state = bnxt_set_vf_link_state, 8520 .ndo_set_vf_spoofchk = bnxt_set_vf_spoofchk, 8521 .ndo_set_vf_trust = bnxt_set_vf_trust, 8522 #endif 8523 #ifdef CONFIG_NET_POLL_CONTROLLER 8524 .ndo_poll_controller = bnxt_poll_controller, 8525 #endif 8526 .ndo_setup_tc = bnxt_setup_tc, 8527 #ifdef CONFIG_RFS_ACCEL 8528 .ndo_rx_flow_steer = bnxt_rx_flow_steer, 8529 #endif 8530 .ndo_udp_tunnel_add = bnxt_udp_tunnel_add, 8531 .ndo_udp_tunnel_del = bnxt_udp_tunnel_del, 8532 .ndo_bpf = bnxt_xdp, 8533 .ndo_bridge_getlink = bnxt_bridge_getlink, 8534 .ndo_bridge_setlink = bnxt_bridge_setlink, 8535 .ndo_get_phys_port_name = bnxt_get_phys_port_name 8536 }; 8537 8538 static void bnxt_remove_one(struct pci_dev *pdev) 8539 { 8540 struct net_device *dev = pci_get_drvdata(pdev); 8541 struct bnxt *bp = netdev_priv(dev); 8542 8543 if (BNXT_PF(bp)) { 8544 bnxt_sriov_disable(bp); 8545 bnxt_dl_unregister(bp); 8546 } 8547 8548 pci_disable_pcie_error_reporting(pdev); 8549 unregister_netdev(dev); 8550 bnxt_shutdown_tc(bp); 8551 bnxt_cancel_sp_work(bp); 8552 bp->sp_event = 0; 8553 8554 bnxt_clear_int_mode(bp); 8555 bnxt_hwrm_func_drv_unrgtr(bp); 8556 bnxt_free_hwrm_resources(bp); 8557 bnxt_free_hwrm_short_cmd_req(bp); 8558 bnxt_ethtool_free(bp); 8559 bnxt_dcb_free(bp); 8560 kfree(bp->edev); 8561 bp->edev = NULL; 8562 bnxt_cleanup_pci(bp); 8563 free_netdev(dev); 8564 } 8565 8566 static int bnxt_probe_phy(struct bnxt *bp) 8567 { 8568 int rc = 0; 8569 struct bnxt_link_info *link_info = &bp->link_info; 8570 8571 rc = bnxt_hwrm_phy_qcaps(bp); 8572 if (rc) { 8573 netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n", 8574 rc); 8575 return rc; 8576 } 8577 mutex_init(&bp->link_lock); 8578 8579 rc = bnxt_update_link(bp, false); 8580 if (rc) { 8581 netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n", 8582 rc); 8583 return rc; 8584 } 8585 8586 /* Older firmware does not have supported_auto_speeds, so assume 8587 * that all supported speeds can be autonegotiated. 8588 */ 8589 if (link_info->auto_link_speeds && !link_info->support_auto_speeds) 8590 link_info->support_auto_speeds = link_info->support_speeds; 8591 8592 /*initialize the ethool setting copy with NVM settings */ 8593 if (BNXT_AUTO_MODE(link_info->auto_mode)) { 8594 link_info->autoneg = BNXT_AUTONEG_SPEED; 8595 if (bp->hwrm_spec_code >= 0x10201) { 8596 if (link_info->auto_pause_setting & 8597 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE) 8598 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL; 8599 } else { 8600 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL; 8601 } 8602 link_info->advertising = link_info->auto_link_speeds; 8603 } else { 8604 link_info->req_link_speed = link_info->force_link_speed; 8605 link_info->req_duplex = link_info->duplex_setting; 8606 } 8607 if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) 8608 link_info->req_flow_ctrl = 8609 link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH; 8610 else 8611 link_info->req_flow_ctrl = link_info->force_pause_setting; 8612 return rc; 8613 } 8614 8615 static int bnxt_get_max_irq(struct pci_dev *pdev) 8616 { 8617 u16 ctrl; 8618 8619 if (!pdev->msix_cap) 8620 return 1; 8621 8622 pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl); 8623 return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1; 8624 } 8625 8626 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, 8627 int *max_cp) 8628 { 8629 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 8630 int max_ring_grps = 0; 8631 8632 *max_tx = hw_resc->max_tx_rings; 8633 *max_rx = hw_resc->max_rx_rings; 8634 *max_cp = min_t(int, bnxt_get_max_func_cp_rings_for_en(bp), 8635 hw_resc->max_irqs); 8636 *max_cp = min_t(int, *max_cp, hw_resc->max_stat_ctxs); 8637 max_ring_grps = hw_resc->max_hw_ring_grps; 8638 if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) { 8639 *max_cp -= 1; 8640 *max_rx -= 2; 8641 } 8642 if (bp->flags & BNXT_FLAG_AGG_RINGS) 8643 *max_rx >>= 1; 8644 *max_rx = min_t(int, *max_rx, max_ring_grps); 8645 } 8646 8647 int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared) 8648 { 8649 int rx, tx, cp; 8650 8651 _bnxt_get_max_rings(bp, &rx, &tx, &cp); 8652 *max_rx = rx; 8653 *max_tx = tx; 8654 if (!rx || !tx || !cp) 8655 return -ENOMEM; 8656 8657 return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared); 8658 } 8659 8660 static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx, 8661 bool shared) 8662 { 8663 int rc; 8664 8665 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared); 8666 if (rc && (bp->flags & BNXT_FLAG_AGG_RINGS)) { 8667 /* Not enough rings, try disabling agg rings. */ 8668 bp->flags &= ~BNXT_FLAG_AGG_RINGS; 8669 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared); 8670 if (rc) { 8671 /* set BNXT_FLAG_AGG_RINGS back for consistency */ 8672 bp->flags |= BNXT_FLAG_AGG_RINGS; 8673 return rc; 8674 } 8675 bp->flags |= BNXT_FLAG_NO_AGG_RINGS; 8676 bp->dev->hw_features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); 8677 bp->dev->features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); 8678 bnxt_set_ring_params(bp); 8679 } 8680 8681 if (bp->flags & BNXT_FLAG_ROCE_CAP) { 8682 int max_cp, max_stat, max_irq; 8683 8684 /* Reserve minimum resources for RoCE */ 8685 max_cp = bnxt_get_max_func_cp_rings(bp); 8686 max_stat = bnxt_get_max_func_stat_ctxs(bp); 8687 max_irq = bnxt_get_max_func_irqs(bp); 8688 if (max_cp <= BNXT_MIN_ROCE_CP_RINGS || 8689 max_irq <= BNXT_MIN_ROCE_CP_RINGS || 8690 max_stat <= BNXT_MIN_ROCE_STAT_CTXS) 8691 return 0; 8692 8693 max_cp -= BNXT_MIN_ROCE_CP_RINGS; 8694 max_irq -= BNXT_MIN_ROCE_CP_RINGS; 8695 max_stat -= BNXT_MIN_ROCE_STAT_CTXS; 8696 max_cp = min_t(int, max_cp, max_irq); 8697 max_cp = min_t(int, max_cp, max_stat); 8698 rc = bnxt_trim_rings(bp, max_rx, max_tx, max_cp, shared); 8699 if (rc) 8700 rc = 0; 8701 } 8702 return rc; 8703 } 8704 8705 /* In initial default shared ring setting, each shared ring must have a 8706 * RX/TX ring pair. 8707 */ 8708 static void bnxt_trim_dflt_sh_rings(struct bnxt *bp) 8709 { 8710 bp->cp_nr_rings = min_t(int, bp->tx_nr_rings_per_tc, bp->rx_nr_rings); 8711 bp->rx_nr_rings = bp->cp_nr_rings; 8712 bp->tx_nr_rings_per_tc = bp->cp_nr_rings; 8713 bp->tx_nr_rings = bp->tx_nr_rings_per_tc; 8714 } 8715 8716 static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh) 8717 { 8718 int dflt_rings, max_rx_rings, max_tx_rings, rc; 8719 8720 if (!bnxt_can_reserve_rings(bp)) 8721 return 0; 8722 8723 if (sh) 8724 bp->flags |= BNXT_FLAG_SHARED_RINGS; 8725 dflt_rings = netif_get_num_default_rss_queues(); 8726 /* Reduce default rings on multi-port cards so that total default 8727 * rings do not exceed CPU count. 8728 */ 8729 if (bp->port_count > 1) { 8730 int max_rings = 8731 max_t(int, num_online_cpus() / bp->port_count, 1); 8732 8733 dflt_rings = min_t(int, dflt_rings, max_rings); 8734 } 8735 rc = bnxt_get_dflt_rings(bp, &max_rx_rings, &max_tx_rings, sh); 8736 if (rc) 8737 return rc; 8738 bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings); 8739 bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings); 8740 if (sh) 8741 bnxt_trim_dflt_sh_rings(bp); 8742 else 8743 bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings; 8744 bp->tx_nr_rings = bp->tx_nr_rings_per_tc; 8745 8746 rc = __bnxt_reserve_rings(bp); 8747 if (rc) 8748 netdev_warn(bp->dev, "Unable to reserve tx rings\n"); 8749 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 8750 if (sh) 8751 bnxt_trim_dflt_sh_rings(bp); 8752 8753 /* Rings may have been trimmed, re-reserve the trimmed rings. */ 8754 if (bnxt_need_reserve_rings(bp)) { 8755 rc = __bnxt_reserve_rings(bp); 8756 if (rc) 8757 netdev_warn(bp->dev, "2nd rings reservation failed.\n"); 8758 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 8759 } 8760 bp->num_stat_ctxs = bp->cp_nr_rings; 8761 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 8762 bp->rx_nr_rings++; 8763 bp->cp_nr_rings++; 8764 } 8765 return rc; 8766 } 8767 8768 static int bnxt_init_dflt_ring_mode(struct bnxt *bp) 8769 { 8770 int rc; 8771 8772 if (bp->tx_nr_rings) 8773 return 0; 8774 8775 bnxt_ulp_irq_stop(bp); 8776 bnxt_clear_int_mode(bp); 8777 rc = bnxt_set_dflt_rings(bp, true); 8778 if (rc) { 8779 netdev_err(bp->dev, "Not enough rings available.\n"); 8780 goto init_dflt_ring_err; 8781 } 8782 rc = bnxt_init_int_mode(bp); 8783 if (rc) 8784 goto init_dflt_ring_err; 8785 8786 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 8787 if (bnxt_rfs_supported(bp) && bnxt_rfs_capable(bp)) { 8788 bp->flags |= BNXT_FLAG_RFS; 8789 bp->dev->features |= NETIF_F_NTUPLE; 8790 } 8791 init_dflt_ring_err: 8792 bnxt_ulp_irq_restart(bp, rc); 8793 return rc; 8794 } 8795 8796 int bnxt_restore_pf_fw_resources(struct bnxt *bp) 8797 { 8798 int rc; 8799 8800 ASSERT_RTNL(); 8801 bnxt_hwrm_func_qcaps(bp); 8802 8803 if (netif_running(bp->dev)) 8804 __bnxt_close_nic(bp, true, false); 8805 8806 bnxt_ulp_irq_stop(bp); 8807 bnxt_clear_int_mode(bp); 8808 rc = bnxt_init_int_mode(bp); 8809 bnxt_ulp_irq_restart(bp, rc); 8810 8811 if (netif_running(bp->dev)) { 8812 if (rc) 8813 dev_close(bp->dev); 8814 else 8815 rc = bnxt_open_nic(bp, true, false); 8816 } 8817 8818 return rc; 8819 } 8820 8821 static int bnxt_init_mac_addr(struct bnxt *bp) 8822 { 8823 int rc = 0; 8824 8825 if (BNXT_PF(bp)) { 8826 memcpy(bp->dev->dev_addr, bp->pf.mac_addr, ETH_ALEN); 8827 } else { 8828 #ifdef CONFIG_BNXT_SRIOV 8829 struct bnxt_vf_info *vf = &bp->vf; 8830 8831 if (is_valid_ether_addr(vf->mac_addr)) { 8832 /* overwrite netdev dev_addr with admin VF MAC */ 8833 memcpy(bp->dev->dev_addr, vf->mac_addr, ETH_ALEN); 8834 } else { 8835 eth_hw_addr_random(bp->dev); 8836 } 8837 rc = bnxt_approve_mac(bp, bp->dev->dev_addr); 8838 #endif 8839 } 8840 return rc; 8841 } 8842 8843 static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) 8844 { 8845 static int version_printed; 8846 struct net_device *dev; 8847 struct bnxt *bp; 8848 int rc, max_irqs; 8849 8850 if (pci_is_bridge(pdev)) 8851 return -ENODEV; 8852 8853 if (version_printed++ == 0) 8854 pr_info("%s", version); 8855 8856 max_irqs = bnxt_get_max_irq(pdev); 8857 dev = alloc_etherdev_mq(sizeof(*bp), max_irqs); 8858 if (!dev) 8859 return -ENOMEM; 8860 8861 bp = netdev_priv(dev); 8862 8863 if (bnxt_vf_pciid(ent->driver_data)) 8864 bp->flags |= BNXT_FLAG_VF; 8865 8866 if (pdev->msix_cap) 8867 bp->flags |= BNXT_FLAG_MSIX_CAP; 8868 8869 rc = bnxt_init_board(pdev, dev); 8870 if (rc < 0) 8871 goto init_err_free; 8872 8873 dev->netdev_ops = &bnxt_netdev_ops; 8874 dev->watchdog_timeo = BNXT_TX_TIMEOUT; 8875 dev->ethtool_ops = &bnxt_ethtool_ops; 8876 SWITCHDEV_SET_OPS(dev, &bnxt_switchdev_ops); 8877 pci_set_drvdata(pdev, dev); 8878 8879 rc = bnxt_alloc_hwrm_resources(bp); 8880 if (rc) 8881 goto init_err_pci_clean; 8882 8883 mutex_init(&bp->hwrm_cmd_lock); 8884 rc = bnxt_hwrm_ver_get(bp); 8885 if (rc) 8886 goto init_err_pci_clean; 8887 8888 if (bp->fw_cap & BNXT_FW_CAP_SHORT_CMD) { 8889 rc = bnxt_alloc_hwrm_short_cmd_req(bp); 8890 if (rc) 8891 goto init_err_pci_clean; 8892 } 8893 8894 rc = bnxt_hwrm_func_reset(bp); 8895 if (rc) 8896 goto init_err_pci_clean; 8897 8898 bnxt_hwrm_fw_set_time(bp); 8899 8900 dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG | 8901 NETIF_F_TSO | NETIF_F_TSO6 | 8902 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE | 8903 NETIF_F_GSO_IPXIP4 | 8904 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM | 8905 NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH | 8906 NETIF_F_RXCSUM | NETIF_F_GRO; 8907 8908 if (!BNXT_CHIP_TYPE_NITRO_A0(bp)) 8909 dev->hw_features |= NETIF_F_LRO; 8910 8911 dev->hw_enc_features = 8912 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG | 8913 NETIF_F_TSO | NETIF_F_TSO6 | 8914 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE | 8915 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM | 8916 NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL; 8917 dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM | 8918 NETIF_F_GSO_GRE_CSUM; 8919 dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA; 8920 dev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX | 8921 NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX; 8922 if (!BNXT_CHIP_TYPE_NITRO_A0(bp)) 8923 dev->hw_features |= NETIF_F_GRO_HW; 8924 dev->features |= dev->hw_features | NETIF_F_HIGHDMA; 8925 if (dev->features & NETIF_F_GRO_HW) 8926 dev->features &= ~NETIF_F_LRO; 8927 dev->priv_flags |= IFF_UNICAST_FLT; 8928 8929 #ifdef CONFIG_BNXT_SRIOV 8930 init_waitqueue_head(&bp->sriov_cfg_wait); 8931 mutex_init(&bp->sriov_lock); 8932 #endif 8933 bp->gro_func = bnxt_gro_func_5730x; 8934 if (BNXT_CHIP_P4_PLUS(bp)) 8935 bp->gro_func = bnxt_gro_func_5731x; 8936 else 8937 bp->flags |= BNXT_FLAG_DOUBLE_DB; 8938 8939 rc = bnxt_hwrm_func_drv_rgtr(bp); 8940 if (rc) 8941 goto init_err_pci_clean; 8942 8943 rc = bnxt_hwrm_func_rgtr_async_events(bp, NULL, 0); 8944 if (rc) 8945 goto init_err_pci_clean; 8946 8947 bp->ulp_probe = bnxt_ulp_probe; 8948 8949 /* Get the MAX capabilities for this function */ 8950 rc = bnxt_hwrm_func_qcaps(bp); 8951 if (rc) { 8952 netdev_err(bp->dev, "hwrm query capability failure rc: %x\n", 8953 rc); 8954 rc = -1; 8955 goto init_err_pci_clean; 8956 } 8957 rc = bnxt_init_mac_addr(bp); 8958 if (rc) { 8959 dev_err(&pdev->dev, "Unable to initialize mac address.\n"); 8960 rc = -EADDRNOTAVAIL; 8961 goto init_err_pci_clean; 8962 } 8963 rc = bnxt_hwrm_queue_qportcfg(bp); 8964 if (rc) { 8965 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %x\n", 8966 rc); 8967 rc = -1; 8968 goto init_err_pci_clean; 8969 } 8970 8971 bnxt_hwrm_func_qcfg(bp); 8972 bnxt_hwrm_port_led_qcaps(bp); 8973 bnxt_ethtool_init(bp); 8974 bnxt_dcb_init(bp); 8975 8976 /* MTU range: 60 - FW defined max */ 8977 dev->min_mtu = ETH_ZLEN; 8978 dev->max_mtu = bp->max_mtu; 8979 8980 rc = bnxt_probe_phy(bp); 8981 if (rc) 8982 goto init_err_pci_clean; 8983 8984 bnxt_set_rx_skb_mode(bp, false); 8985 bnxt_set_tpa_flags(bp); 8986 bnxt_set_ring_params(bp); 8987 bnxt_set_max_func_irqs(bp, max_irqs); 8988 rc = bnxt_set_dflt_rings(bp, true); 8989 if (rc) { 8990 netdev_err(bp->dev, "Not enough rings available.\n"); 8991 rc = -ENOMEM; 8992 goto init_err_pci_clean; 8993 } 8994 8995 /* Default RSS hash cfg. */ 8996 bp->rss_hash_cfg = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 | 8997 VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 | 8998 VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 | 8999 VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6; 9000 if (BNXT_CHIP_P4_PLUS(bp) && bp->hwrm_spec_code >= 0x10501) { 9001 bp->flags |= BNXT_FLAG_UDP_RSS_CAP; 9002 bp->rss_hash_cfg |= VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 | 9003 VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6; 9004 } 9005 9006 bnxt_hwrm_vnic_qcaps(bp); 9007 if (bnxt_rfs_supported(bp)) { 9008 dev->hw_features |= NETIF_F_NTUPLE; 9009 if (bnxt_rfs_capable(bp)) { 9010 bp->flags |= BNXT_FLAG_RFS; 9011 dev->features |= NETIF_F_NTUPLE; 9012 } 9013 } 9014 9015 if (dev->hw_features & NETIF_F_HW_VLAN_CTAG_RX) 9016 bp->flags |= BNXT_FLAG_STRIP_VLAN; 9017 9018 rc = bnxt_init_int_mode(bp); 9019 if (rc) 9020 goto init_err_pci_clean; 9021 9022 /* No TC has been set yet and rings may have been trimmed due to 9023 * limited MSIX, so we re-initialize the TX rings per TC. 9024 */ 9025 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 9026 9027 bnxt_get_wol_settings(bp); 9028 if (bp->flags & BNXT_FLAG_WOL_CAP) 9029 device_set_wakeup_enable(&pdev->dev, bp->wol); 9030 else 9031 device_set_wakeup_capable(&pdev->dev, false); 9032 9033 bnxt_hwrm_set_cache_line_size(bp, cache_line_size()); 9034 9035 if (BNXT_PF(bp)) { 9036 if (!bnxt_pf_wq) { 9037 bnxt_pf_wq = 9038 create_singlethread_workqueue("bnxt_pf_wq"); 9039 if (!bnxt_pf_wq) { 9040 dev_err(&pdev->dev, "Unable to create workqueue.\n"); 9041 goto init_err_pci_clean; 9042 } 9043 } 9044 bnxt_init_tc(bp); 9045 } 9046 9047 rc = register_netdev(dev); 9048 if (rc) 9049 goto init_err_cleanup_tc; 9050 9051 if (BNXT_PF(bp)) 9052 bnxt_dl_register(bp); 9053 9054 netdev_info(dev, "%s found at mem %lx, node addr %pM\n", 9055 board_info[ent->driver_data].name, 9056 (long)pci_resource_start(pdev, 0), dev->dev_addr); 9057 pcie_print_link_status(pdev); 9058 9059 return 0; 9060 9061 init_err_cleanup_tc: 9062 bnxt_shutdown_tc(bp); 9063 bnxt_clear_int_mode(bp); 9064 9065 init_err_pci_clean: 9066 bnxt_cleanup_pci(bp); 9067 9068 init_err_free: 9069 free_netdev(dev); 9070 return rc; 9071 } 9072 9073 static void bnxt_shutdown(struct pci_dev *pdev) 9074 { 9075 struct net_device *dev = pci_get_drvdata(pdev); 9076 struct bnxt *bp; 9077 9078 if (!dev) 9079 return; 9080 9081 rtnl_lock(); 9082 bp = netdev_priv(dev); 9083 if (!bp) 9084 goto shutdown_exit; 9085 9086 if (netif_running(dev)) 9087 dev_close(dev); 9088 9089 bnxt_ulp_shutdown(bp); 9090 9091 if (system_state == SYSTEM_POWER_OFF) { 9092 bnxt_clear_int_mode(bp); 9093 pci_wake_from_d3(pdev, bp->wol); 9094 pci_set_power_state(pdev, PCI_D3hot); 9095 } 9096 9097 shutdown_exit: 9098 rtnl_unlock(); 9099 } 9100 9101 #ifdef CONFIG_PM_SLEEP 9102 static int bnxt_suspend(struct device *device) 9103 { 9104 struct pci_dev *pdev = to_pci_dev(device); 9105 struct net_device *dev = pci_get_drvdata(pdev); 9106 struct bnxt *bp = netdev_priv(dev); 9107 int rc = 0; 9108 9109 rtnl_lock(); 9110 if (netif_running(dev)) { 9111 netif_device_detach(dev); 9112 rc = bnxt_close(dev); 9113 } 9114 bnxt_hwrm_func_drv_unrgtr(bp); 9115 rtnl_unlock(); 9116 return rc; 9117 } 9118 9119 static int bnxt_resume(struct device *device) 9120 { 9121 struct pci_dev *pdev = to_pci_dev(device); 9122 struct net_device *dev = pci_get_drvdata(pdev); 9123 struct bnxt *bp = netdev_priv(dev); 9124 int rc = 0; 9125 9126 rtnl_lock(); 9127 if (bnxt_hwrm_ver_get(bp) || bnxt_hwrm_func_drv_rgtr(bp)) { 9128 rc = -ENODEV; 9129 goto resume_exit; 9130 } 9131 rc = bnxt_hwrm_func_reset(bp); 9132 if (rc) { 9133 rc = -EBUSY; 9134 goto resume_exit; 9135 } 9136 bnxt_get_wol_settings(bp); 9137 if (netif_running(dev)) { 9138 rc = bnxt_open(dev); 9139 if (!rc) 9140 netif_device_attach(dev); 9141 } 9142 9143 resume_exit: 9144 rtnl_unlock(); 9145 return rc; 9146 } 9147 9148 static SIMPLE_DEV_PM_OPS(bnxt_pm_ops, bnxt_suspend, bnxt_resume); 9149 #define BNXT_PM_OPS (&bnxt_pm_ops) 9150 9151 #else 9152 9153 #define BNXT_PM_OPS NULL 9154 9155 #endif /* CONFIG_PM_SLEEP */ 9156 9157 /** 9158 * bnxt_io_error_detected - called when PCI error is detected 9159 * @pdev: Pointer to PCI device 9160 * @state: The current pci connection state 9161 * 9162 * This function is called after a PCI bus error affecting 9163 * this device has been detected. 9164 */ 9165 static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev, 9166 pci_channel_state_t state) 9167 { 9168 struct net_device *netdev = pci_get_drvdata(pdev); 9169 struct bnxt *bp = netdev_priv(netdev); 9170 9171 netdev_info(netdev, "PCI I/O error detected\n"); 9172 9173 rtnl_lock(); 9174 netif_device_detach(netdev); 9175 9176 bnxt_ulp_stop(bp); 9177 9178 if (state == pci_channel_io_perm_failure) { 9179 rtnl_unlock(); 9180 return PCI_ERS_RESULT_DISCONNECT; 9181 } 9182 9183 if (netif_running(netdev)) 9184 bnxt_close(netdev); 9185 9186 pci_disable_device(pdev); 9187 rtnl_unlock(); 9188 9189 /* Request a slot slot reset. */ 9190 return PCI_ERS_RESULT_NEED_RESET; 9191 } 9192 9193 /** 9194 * bnxt_io_slot_reset - called after the pci bus has been reset. 9195 * @pdev: Pointer to PCI device 9196 * 9197 * Restart the card from scratch, as if from a cold-boot. 9198 * At this point, the card has exprienced a hard reset, 9199 * followed by fixups by BIOS, and has its config space 9200 * set up identically to what it was at cold boot. 9201 */ 9202 static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev) 9203 { 9204 struct net_device *netdev = pci_get_drvdata(pdev); 9205 struct bnxt *bp = netdev_priv(netdev); 9206 int err = 0; 9207 pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT; 9208 9209 netdev_info(bp->dev, "PCI Slot Reset\n"); 9210 9211 rtnl_lock(); 9212 9213 if (pci_enable_device(pdev)) { 9214 dev_err(&pdev->dev, 9215 "Cannot re-enable PCI device after reset.\n"); 9216 } else { 9217 pci_set_master(pdev); 9218 9219 err = bnxt_hwrm_func_reset(bp); 9220 if (!err && netif_running(netdev)) 9221 err = bnxt_open(netdev); 9222 9223 if (!err) { 9224 result = PCI_ERS_RESULT_RECOVERED; 9225 bnxt_ulp_start(bp); 9226 } 9227 } 9228 9229 if (result != PCI_ERS_RESULT_RECOVERED && netif_running(netdev)) 9230 dev_close(netdev); 9231 9232 rtnl_unlock(); 9233 9234 err = pci_cleanup_aer_uncorrect_error_status(pdev); 9235 if (err) { 9236 dev_err(&pdev->dev, 9237 "pci_cleanup_aer_uncorrect_error_status failed 0x%0x\n", 9238 err); /* non-fatal, continue */ 9239 } 9240 9241 return PCI_ERS_RESULT_RECOVERED; 9242 } 9243 9244 /** 9245 * bnxt_io_resume - called when traffic can start flowing again. 9246 * @pdev: Pointer to PCI device 9247 * 9248 * This callback is called when the error recovery driver tells 9249 * us that its OK to resume normal operation. 9250 */ 9251 static void bnxt_io_resume(struct pci_dev *pdev) 9252 { 9253 struct net_device *netdev = pci_get_drvdata(pdev); 9254 9255 rtnl_lock(); 9256 9257 netif_device_attach(netdev); 9258 9259 rtnl_unlock(); 9260 } 9261 9262 static const struct pci_error_handlers bnxt_err_handler = { 9263 .error_detected = bnxt_io_error_detected, 9264 .slot_reset = bnxt_io_slot_reset, 9265 .resume = bnxt_io_resume 9266 }; 9267 9268 static struct pci_driver bnxt_pci_driver = { 9269 .name = DRV_MODULE_NAME, 9270 .id_table = bnxt_pci_tbl, 9271 .probe = bnxt_init_one, 9272 .remove = bnxt_remove_one, 9273 .shutdown = bnxt_shutdown, 9274 .driver.pm = BNXT_PM_OPS, 9275 .err_handler = &bnxt_err_handler, 9276 #if defined(CONFIG_BNXT_SRIOV) 9277 .sriov_configure = bnxt_sriov_configure, 9278 #endif 9279 }; 9280 9281 static int __init bnxt_init(void) 9282 { 9283 bnxt_debug_init(); 9284 return pci_register_driver(&bnxt_pci_driver); 9285 } 9286 9287 static void __exit bnxt_exit(void) 9288 { 9289 pci_unregister_driver(&bnxt_pci_driver); 9290 if (bnxt_pf_wq) 9291 destroy_workqueue(bnxt_pf_wq); 9292 bnxt_debug_exit(); 9293 } 9294 9295 module_init(bnxt_init); 9296 module_exit(bnxt_exit); 9297