1 /* Broadcom NetXtreme-C/E network driver. 2 * 3 * Copyright (c) 2014-2016 Broadcom Corporation 4 * Copyright (c) 2016-2019 Broadcom Limited 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License as published by 8 * the Free Software Foundation. 9 */ 10 11 #include <linux/module.h> 12 13 #include <linux/stringify.h> 14 #include <linux/kernel.h> 15 #include <linux/timer.h> 16 #include <linux/errno.h> 17 #include <linux/ioport.h> 18 #include <linux/slab.h> 19 #include <linux/vmalloc.h> 20 #include <linux/interrupt.h> 21 #include <linux/pci.h> 22 #include <linux/netdevice.h> 23 #include <linux/etherdevice.h> 24 #include <linux/skbuff.h> 25 #include <linux/dma-mapping.h> 26 #include <linux/bitops.h> 27 #include <linux/io.h> 28 #include <linux/irq.h> 29 #include <linux/delay.h> 30 #include <asm/byteorder.h> 31 #include <asm/page.h> 32 #include <linux/time.h> 33 #include <linux/mii.h> 34 #include <linux/mdio.h> 35 #include <linux/if.h> 36 #include <linux/if_vlan.h> 37 #include <linux/if_bridge.h> 38 #include <linux/rtc.h> 39 #include <linux/bpf.h> 40 #include <net/ip.h> 41 #include <net/tcp.h> 42 #include <net/udp.h> 43 #include <net/checksum.h> 44 #include <net/ip6_checksum.h> 45 #include <net/udp_tunnel.h> 46 #include <linux/workqueue.h> 47 #include <linux/prefetch.h> 48 #include <linux/cache.h> 49 #include <linux/log2.h> 50 #include <linux/aer.h> 51 #include <linux/bitmap.h> 52 #include <linux/cpu_rmap.h> 53 #include <linux/cpumask.h> 54 #include <net/pkt_cls.h> 55 #include <linux/hwmon.h> 56 #include <linux/hwmon-sysfs.h> 57 #include <net/page_pool.h> 58 59 #include "bnxt_hsi.h" 60 #include "bnxt.h" 61 #include "bnxt_ulp.h" 62 #include "bnxt_sriov.h" 63 #include "bnxt_ethtool.h" 64 #include "bnxt_dcb.h" 65 #include "bnxt_xdp.h" 66 #include "bnxt_vfr.h" 67 #include "bnxt_tc.h" 68 #include "bnxt_devlink.h" 69 #include "bnxt_debugfs.h" 70 71 #define BNXT_TX_TIMEOUT (5 * HZ) 72 #define BNXT_DEF_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_HW) 73 74 MODULE_LICENSE("GPL"); 75 MODULE_DESCRIPTION("Broadcom BCM573xx network driver"); 76 77 #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN) 78 #define BNXT_RX_DMA_OFFSET NET_SKB_PAD 79 #define BNXT_RX_COPY_THRESH 256 80 81 #define BNXT_TX_PUSH_THRESH 164 82 83 enum board_idx { 84 BCM57301, 85 BCM57302, 86 BCM57304, 87 BCM57417_NPAR, 88 BCM58700, 89 BCM57311, 90 BCM57312, 91 BCM57402, 92 BCM57404, 93 BCM57406, 94 BCM57402_NPAR, 95 BCM57407, 96 BCM57412, 97 BCM57414, 98 BCM57416, 99 BCM57417, 100 BCM57412_NPAR, 101 BCM57314, 102 BCM57417_SFP, 103 BCM57416_SFP, 104 BCM57404_NPAR, 105 BCM57406_NPAR, 106 BCM57407_SFP, 107 BCM57407_NPAR, 108 BCM57414_NPAR, 109 BCM57416_NPAR, 110 BCM57452, 111 BCM57454, 112 BCM5745x_NPAR, 113 BCM57508, 114 BCM57504, 115 BCM57502, 116 BCM57508_NPAR, 117 BCM57504_NPAR, 118 BCM57502_NPAR, 119 BCM58802, 120 BCM58804, 121 BCM58808, 122 NETXTREME_E_VF, 123 NETXTREME_C_VF, 124 NETXTREME_S_VF, 125 NETXTREME_E_P5_VF, 126 }; 127 128 /* indexed by enum above */ 129 static const struct { 130 char *name; 131 } board_info[] = { 132 [BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" }, 133 [BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" }, 134 [BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" }, 135 [BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" }, 136 [BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" }, 137 [BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" }, 138 [BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" }, 139 [BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" }, 140 [BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" }, 141 [BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" }, 142 [BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" }, 143 [BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" }, 144 [BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" }, 145 [BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" }, 146 [BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" }, 147 [BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" }, 148 [BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" }, 149 [BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" }, 150 [BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" }, 151 [BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" }, 152 [BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" }, 153 [BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" }, 154 [BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" }, 155 [BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" }, 156 [BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" }, 157 [BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" }, 158 [BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" }, 159 [BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" }, 160 [BCM5745x_NPAR] = { "Broadcom BCM5745x NetXtreme-E Ethernet Partition" }, 161 [BCM57508] = { "Broadcom BCM57508 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" }, 162 [BCM57504] = { "Broadcom BCM57504 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" }, 163 [BCM57502] = { "Broadcom BCM57502 NetXtreme-E 10Gb/25Gb/50Gb Ethernet" }, 164 [BCM57508_NPAR] = { "Broadcom BCM57508 NetXtreme-E Ethernet Partition" }, 165 [BCM57504_NPAR] = { "Broadcom BCM57504 NetXtreme-E Ethernet Partition" }, 166 [BCM57502_NPAR] = { "Broadcom BCM57502 NetXtreme-E Ethernet Partition" }, 167 [BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" }, 168 [BCM58804] = { "Broadcom BCM58804 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" }, 169 [BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" }, 170 [NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" }, 171 [NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" }, 172 [NETXTREME_S_VF] = { "Broadcom NetXtreme-S Ethernet Virtual Function" }, 173 [NETXTREME_E_P5_VF] = { "Broadcom BCM5750X NetXtreme-E Ethernet Virtual Function" }, 174 }; 175 176 static const struct pci_device_id bnxt_pci_tbl[] = { 177 { PCI_VDEVICE(BROADCOM, 0x1604), .driver_data = BCM5745x_NPAR }, 178 { PCI_VDEVICE(BROADCOM, 0x1605), .driver_data = BCM5745x_NPAR }, 179 { PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 }, 180 { PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR }, 181 { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 }, 182 { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 }, 183 { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 }, 184 { PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR }, 185 { PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 }, 186 { PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 }, 187 { PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 }, 188 { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 }, 189 { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 }, 190 { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 }, 191 { PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR }, 192 { PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 }, 193 { PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 }, 194 { PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 }, 195 { PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 }, 196 { PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 }, 197 { PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR }, 198 { PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 }, 199 { PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP }, 200 { PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP }, 201 { PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR }, 202 { PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR }, 203 { PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP }, 204 { PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR }, 205 { PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR }, 206 { PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR }, 207 { PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR }, 208 { PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR }, 209 { PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR }, 210 { PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 }, 211 { PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 }, 212 { PCI_VDEVICE(BROADCOM, 0x1750), .driver_data = BCM57508 }, 213 { PCI_VDEVICE(BROADCOM, 0x1751), .driver_data = BCM57504 }, 214 { PCI_VDEVICE(BROADCOM, 0x1752), .driver_data = BCM57502 }, 215 { PCI_VDEVICE(BROADCOM, 0x1800), .driver_data = BCM57508_NPAR }, 216 { PCI_VDEVICE(BROADCOM, 0x1801), .driver_data = BCM57504_NPAR }, 217 { PCI_VDEVICE(BROADCOM, 0x1802), .driver_data = BCM57502_NPAR }, 218 { PCI_VDEVICE(BROADCOM, 0x1803), .driver_data = BCM57508_NPAR }, 219 { PCI_VDEVICE(BROADCOM, 0x1804), .driver_data = BCM57504_NPAR }, 220 { PCI_VDEVICE(BROADCOM, 0x1805), .driver_data = BCM57502_NPAR }, 221 { PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 }, 222 { PCI_VDEVICE(BROADCOM, 0xd804), .driver_data = BCM58804 }, 223 #ifdef CONFIG_BNXT_SRIOV 224 { PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF }, 225 { PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF }, 226 { PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF }, 227 { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF }, 228 { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF }, 229 { PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF }, 230 { PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF }, 231 { PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF }, 232 { PCI_VDEVICE(BROADCOM, 0x1806), .driver_data = NETXTREME_E_P5_VF }, 233 { PCI_VDEVICE(BROADCOM, 0x1807), .driver_data = NETXTREME_E_P5_VF }, 234 { PCI_VDEVICE(BROADCOM, 0xd800), .driver_data = NETXTREME_S_VF }, 235 #endif 236 { 0 } 237 }; 238 239 MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl); 240 241 static const u16 bnxt_vf_req_snif[] = { 242 HWRM_FUNC_CFG, 243 HWRM_FUNC_VF_CFG, 244 HWRM_PORT_PHY_QCFG, 245 HWRM_CFA_L2_FILTER_ALLOC, 246 }; 247 248 static const u16 bnxt_async_events_arr[] = { 249 ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE, 250 ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE, 251 ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD, 252 ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED, 253 ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE, 254 ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE, 255 ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE, 256 ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY, 257 ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY, 258 ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION, 259 ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG, 260 ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST, 261 }; 262 263 static struct workqueue_struct *bnxt_pf_wq; 264 265 static bool bnxt_vf_pciid(enum board_idx idx) 266 { 267 return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF || 268 idx == NETXTREME_S_VF || idx == NETXTREME_E_P5_VF); 269 } 270 271 #define DB_CP_REARM_FLAGS (DB_KEY_CP | DB_IDX_VALID) 272 #define DB_CP_FLAGS (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS) 273 #define DB_CP_IRQ_DIS_FLAGS (DB_KEY_CP | DB_IRQ_DIS) 274 275 #define BNXT_CP_DB_IRQ_DIS(db) \ 276 writel(DB_CP_IRQ_DIS_FLAGS, db) 277 278 #define BNXT_DB_CQ(db, idx) \ 279 writel(DB_CP_FLAGS | RING_CMP(idx), (db)->doorbell) 280 281 #define BNXT_DB_NQ_P5(db, idx) \ 282 writeq((db)->db_key64 | DBR_TYPE_NQ | RING_CMP(idx), (db)->doorbell) 283 284 #define BNXT_DB_CQ_ARM(db, idx) \ 285 writel(DB_CP_REARM_FLAGS | RING_CMP(idx), (db)->doorbell) 286 287 #define BNXT_DB_NQ_ARM_P5(db, idx) \ 288 writeq((db)->db_key64 | DBR_TYPE_NQ_ARM | RING_CMP(idx), (db)->doorbell) 289 290 static void bnxt_db_nq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx) 291 { 292 if (bp->flags & BNXT_FLAG_CHIP_P5) 293 BNXT_DB_NQ_P5(db, idx); 294 else 295 BNXT_DB_CQ(db, idx); 296 } 297 298 static void bnxt_db_nq_arm(struct bnxt *bp, struct bnxt_db_info *db, u32 idx) 299 { 300 if (bp->flags & BNXT_FLAG_CHIP_P5) 301 BNXT_DB_NQ_ARM_P5(db, idx); 302 else 303 BNXT_DB_CQ_ARM(db, idx); 304 } 305 306 static void bnxt_db_cq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx) 307 { 308 if (bp->flags & BNXT_FLAG_CHIP_P5) 309 writeq(db->db_key64 | DBR_TYPE_CQ_ARMALL | RING_CMP(idx), 310 db->doorbell); 311 else 312 BNXT_DB_CQ(db, idx); 313 } 314 315 const u16 bnxt_lhint_arr[] = { 316 TX_BD_FLAGS_LHINT_512_AND_SMALLER, 317 TX_BD_FLAGS_LHINT_512_TO_1023, 318 TX_BD_FLAGS_LHINT_1024_TO_2047, 319 TX_BD_FLAGS_LHINT_1024_TO_2047, 320 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 321 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 322 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 323 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 324 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 325 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 326 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 327 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 328 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 329 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 330 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 331 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 332 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 333 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 334 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 335 }; 336 337 static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb) 338 { 339 struct metadata_dst *md_dst = skb_metadata_dst(skb); 340 341 if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX) 342 return 0; 343 344 return md_dst->u.port_info.port_id; 345 } 346 347 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev) 348 { 349 struct bnxt *bp = netdev_priv(dev); 350 struct tx_bd *txbd; 351 struct tx_bd_ext *txbd1; 352 struct netdev_queue *txq; 353 int i; 354 dma_addr_t mapping; 355 unsigned int length, pad = 0; 356 u32 len, free_size, vlan_tag_flags, cfa_action, flags; 357 u16 prod, last_frag; 358 struct pci_dev *pdev = bp->pdev; 359 struct bnxt_tx_ring_info *txr; 360 struct bnxt_sw_tx_bd *tx_buf; 361 362 i = skb_get_queue_mapping(skb); 363 if (unlikely(i >= bp->tx_nr_rings)) { 364 dev_kfree_skb_any(skb); 365 return NETDEV_TX_OK; 366 } 367 368 txq = netdev_get_tx_queue(dev, i); 369 txr = &bp->tx_ring[bp->tx_ring_map[i]]; 370 prod = txr->tx_prod; 371 372 free_size = bnxt_tx_avail(bp, txr); 373 if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) { 374 netif_tx_stop_queue(txq); 375 return NETDEV_TX_BUSY; 376 } 377 378 length = skb->len; 379 len = skb_headlen(skb); 380 last_frag = skb_shinfo(skb)->nr_frags; 381 382 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 383 384 txbd->tx_bd_opaque = prod; 385 386 tx_buf = &txr->tx_buf_ring[prod]; 387 tx_buf->skb = skb; 388 tx_buf->nr_frags = last_frag; 389 390 vlan_tag_flags = 0; 391 cfa_action = bnxt_xmit_get_cfa_action(skb); 392 if (skb_vlan_tag_present(skb)) { 393 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN | 394 skb_vlan_tag_get(skb); 395 /* Currently supports 8021Q, 8021AD vlan offloads 396 * QINQ1, QINQ2, QINQ3 vlan headers are deprecated 397 */ 398 if (skb->vlan_proto == htons(ETH_P_8021Q)) 399 vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT; 400 } 401 402 if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh) { 403 struct tx_push_buffer *tx_push_buf = txr->tx_push; 404 struct tx_push_bd *tx_push = &tx_push_buf->push_bd; 405 struct tx_bd_ext *tx_push1 = &tx_push->txbd2; 406 void __iomem *db = txr->tx_db.doorbell; 407 void *pdata = tx_push_buf->data; 408 u64 *end; 409 int j, push_len; 410 411 /* Set COAL_NOW to be ready quickly for the next push */ 412 tx_push->tx_bd_len_flags_type = 413 cpu_to_le32((length << TX_BD_LEN_SHIFT) | 414 TX_BD_TYPE_LONG_TX_BD | 415 TX_BD_FLAGS_LHINT_512_AND_SMALLER | 416 TX_BD_FLAGS_COAL_NOW | 417 TX_BD_FLAGS_PACKET_END | 418 (2 << TX_BD_FLAGS_BD_CNT_SHIFT)); 419 420 if (skb->ip_summed == CHECKSUM_PARTIAL) 421 tx_push1->tx_bd_hsize_lflags = 422 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM); 423 else 424 tx_push1->tx_bd_hsize_lflags = 0; 425 426 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags); 427 tx_push1->tx_bd_cfa_action = 428 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT); 429 430 end = pdata + length; 431 end = PTR_ALIGN(end, 8) - 1; 432 *end = 0; 433 434 skb_copy_from_linear_data(skb, pdata, len); 435 pdata += len; 436 for (j = 0; j < last_frag; j++) { 437 skb_frag_t *frag = &skb_shinfo(skb)->frags[j]; 438 void *fptr; 439 440 fptr = skb_frag_address_safe(frag); 441 if (!fptr) 442 goto normal_tx; 443 444 memcpy(pdata, fptr, skb_frag_size(frag)); 445 pdata += skb_frag_size(frag); 446 } 447 448 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type; 449 txbd->tx_bd_haddr = txr->data_mapping; 450 prod = NEXT_TX(prod); 451 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 452 memcpy(txbd, tx_push1, sizeof(*txbd)); 453 prod = NEXT_TX(prod); 454 tx_push->doorbell = 455 cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | prod); 456 txr->tx_prod = prod; 457 458 tx_buf->is_push = 1; 459 netdev_tx_sent_queue(txq, skb->len); 460 wmb(); /* Sync is_push and byte queue before pushing data */ 461 462 push_len = (length + sizeof(*tx_push) + 7) / 8; 463 if (push_len > 16) { 464 __iowrite64_copy(db, tx_push_buf, 16); 465 __iowrite32_copy(db + 4, tx_push_buf + 1, 466 (push_len - 16) << 1); 467 } else { 468 __iowrite64_copy(db, tx_push_buf, push_len); 469 } 470 471 goto tx_done; 472 } 473 474 normal_tx: 475 if (length < BNXT_MIN_PKT_SIZE) { 476 pad = BNXT_MIN_PKT_SIZE - length; 477 if (skb_pad(skb, pad)) { 478 /* SKB already freed. */ 479 tx_buf->skb = NULL; 480 return NETDEV_TX_OK; 481 } 482 length = BNXT_MIN_PKT_SIZE; 483 } 484 485 mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE); 486 487 if (unlikely(dma_mapping_error(&pdev->dev, mapping))) { 488 dev_kfree_skb_any(skb); 489 tx_buf->skb = NULL; 490 return NETDEV_TX_OK; 491 } 492 493 dma_unmap_addr_set(tx_buf, mapping, mapping); 494 flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD | 495 ((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT); 496 497 txbd->tx_bd_haddr = cpu_to_le64(mapping); 498 499 prod = NEXT_TX(prod); 500 txbd1 = (struct tx_bd_ext *) 501 &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 502 503 txbd1->tx_bd_hsize_lflags = 0; 504 if (skb_is_gso(skb)) { 505 u32 hdr_len; 506 507 if (skb->encapsulation) 508 hdr_len = skb_inner_network_offset(skb) + 509 skb_inner_network_header_len(skb) + 510 inner_tcp_hdrlen(skb); 511 else 512 hdr_len = skb_transport_offset(skb) + 513 tcp_hdrlen(skb); 514 515 txbd1->tx_bd_hsize_lflags = cpu_to_le32(TX_BD_FLAGS_LSO | 516 TX_BD_FLAGS_T_IPID | 517 (hdr_len << (TX_BD_HSIZE_SHIFT - 1))); 518 length = skb_shinfo(skb)->gso_size; 519 txbd1->tx_bd_mss = cpu_to_le32(length); 520 length += hdr_len; 521 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { 522 txbd1->tx_bd_hsize_lflags = 523 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM); 524 txbd1->tx_bd_mss = 0; 525 } 526 527 length >>= 9; 528 if (unlikely(length >= ARRAY_SIZE(bnxt_lhint_arr))) { 529 dev_warn_ratelimited(&pdev->dev, "Dropped oversize %d bytes TX packet.\n", 530 skb->len); 531 i = 0; 532 goto tx_dma_error; 533 } 534 flags |= bnxt_lhint_arr[length]; 535 txbd->tx_bd_len_flags_type = cpu_to_le32(flags); 536 537 txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags); 538 txbd1->tx_bd_cfa_action = 539 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT); 540 for (i = 0; i < last_frag; i++) { 541 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 542 543 prod = NEXT_TX(prod); 544 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 545 546 len = skb_frag_size(frag); 547 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len, 548 DMA_TO_DEVICE); 549 550 if (unlikely(dma_mapping_error(&pdev->dev, mapping))) 551 goto tx_dma_error; 552 553 tx_buf = &txr->tx_buf_ring[prod]; 554 dma_unmap_addr_set(tx_buf, mapping, mapping); 555 556 txbd->tx_bd_haddr = cpu_to_le64(mapping); 557 558 flags = len << TX_BD_LEN_SHIFT; 559 txbd->tx_bd_len_flags_type = cpu_to_le32(flags); 560 } 561 562 flags &= ~TX_BD_LEN; 563 txbd->tx_bd_len_flags_type = 564 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags | 565 TX_BD_FLAGS_PACKET_END); 566 567 netdev_tx_sent_queue(txq, skb->len); 568 569 /* Sync BD data before updating doorbell */ 570 wmb(); 571 572 prod = NEXT_TX(prod); 573 txr->tx_prod = prod; 574 575 if (!netdev_xmit_more() || netif_xmit_stopped(txq)) 576 bnxt_db_write(bp, &txr->tx_db, prod); 577 578 tx_done: 579 580 if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) { 581 if (netdev_xmit_more() && !tx_buf->is_push) 582 bnxt_db_write(bp, &txr->tx_db, prod); 583 584 netif_tx_stop_queue(txq); 585 586 /* netif_tx_stop_queue() must be done before checking 587 * tx index in bnxt_tx_avail() below, because in 588 * bnxt_tx_int(), we update tx index before checking for 589 * netif_tx_queue_stopped(). 590 */ 591 smp_mb(); 592 if (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh) 593 netif_tx_wake_queue(txq); 594 } 595 return NETDEV_TX_OK; 596 597 tx_dma_error: 598 last_frag = i; 599 600 /* start back at beginning and unmap skb */ 601 prod = txr->tx_prod; 602 tx_buf = &txr->tx_buf_ring[prod]; 603 tx_buf->skb = NULL; 604 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping), 605 skb_headlen(skb), PCI_DMA_TODEVICE); 606 prod = NEXT_TX(prod); 607 608 /* unmap remaining mapped pages */ 609 for (i = 0; i < last_frag; i++) { 610 prod = NEXT_TX(prod); 611 tx_buf = &txr->tx_buf_ring[prod]; 612 dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping), 613 skb_frag_size(&skb_shinfo(skb)->frags[i]), 614 PCI_DMA_TODEVICE); 615 } 616 617 dev_kfree_skb_any(skb); 618 return NETDEV_TX_OK; 619 } 620 621 static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts) 622 { 623 struct bnxt_tx_ring_info *txr = bnapi->tx_ring; 624 struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index); 625 u16 cons = txr->tx_cons; 626 struct pci_dev *pdev = bp->pdev; 627 int i; 628 unsigned int tx_bytes = 0; 629 630 for (i = 0; i < nr_pkts; i++) { 631 struct bnxt_sw_tx_bd *tx_buf; 632 struct sk_buff *skb; 633 int j, last; 634 635 tx_buf = &txr->tx_buf_ring[cons]; 636 cons = NEXT_TX(cons); 637 skb = tx_buf->skb; 638 tx_buf->skb = NULL; 639 640 if (tx_buf->is_push) { 641 tx_buf->is_push = 0; 642 goto next_tx_int; 643 } 644 645 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping), 646 skb_headlen(skb), PCI_DMA_TODEVICE); 647 last = tx_buf->nr_frags; 648 649 for (j = 0; j < last; j++) { 650 cons = NEXT_TX(cons); 651 tx_buf = &txr->tx_buf_ring[cons]; 652 dma_unmap_page( 653 &pdev->dev, 654 dma_unmap_addr(tx_buf, mapping), 655 skb_frag_size(&skb_shinfo(skb)->frags[j]), 656 PCI_DMA_TODEVICE); 657 } 658 659 next_tx_int: 660 cons = NEXT_TX(cons); 661 662 tx_bytes += skb->len; 663 dev_kfree_skb_any(skb); 664 } 665 666 netdev_tx_completed_queue(txq, nr_pkts, tx_bytes); 667 txr->tx_cons = cons; 668 669 /* Need to make the tx_cons update visible to bnxt_start_xmit() 670 * before checking for netif_tx_queue_stopped(). Without the 671 * memory barrier, there is a small possibility that bnxt_start_xmit() 672 * will miss it and cause the queue to be stopped forever. 673 */ 674 smp_mb(); 675 676 if (unlikely(netif_tx_queue_stopped(txq)) && 677 (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)) { 678 __netif_tx_lock(txq, smp_processor_id()); 679 if (netif_tx_queue_stopped(txq) && 680 bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh && 681 txr->dev_state != BNXT_DEV_STATE_CLOSING) 682 netif_tx_wake_queue(txq); 683 __netif_tx_unlock(txq); 684 } 685 } 686 687 static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping, 688 struct bnxt_rx_ring_info *rxr, 689 gfp_t gfp) 690 { 691 struct device *dev = &bp->pdev->dev; 692 struct page *page; 693 694 page = page_pool_dev_alloc_pages(rxr->page_pool); 695 if (!page) 696 return NULL; 697 698 *mapping = dma_map_page_attrs(dev, page, 0, PAGE_SIZE, bp->rx_dir, 699 DMA_ATTR_WEAK_ORDERING); 700 if (dma_mapping_error(dev, *mapping)) { 701 page_pool_recycle_direct(rxr->page_pool, page); 702 return NULL; 703 } 704 *mapping += bp->rx_dma_offset; 705 return page; 706 } 707 708 static inline u8 *__bnxt_alloc_rx_data(struct bnxt *bp, dma_addr_t *mapping, 709 gfp_t gfp) 710 { 711 u8 *data; 712 struct pci_dev *pdev = bp->pdev; 713 714 data = kmalloc(bp->rx_buf_size, gfp); 715 if (!data) 716 return NULL; 717 718 *mapping = dma_map_single_attrs(&pdev->dev, data + bp->rx_dma_offset, 719 bp->rx_buf_use_size, bp->rx_dir, 720 DMA_ATTR_WEAK_ORDERING); 721 722 if (dma_mapping_error(&pdev->dev, *mapping)) { 723 kfree(data); 724 data = NULL; 725 } 726 return data; 727 } 728 729 int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, 730 u16 prod, gfp_t gfp) 731 { 732 struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 733 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod]; 734 dma_addr_t mapping; 735 736 if (BNXT_RX_PAGE_MODE(bp)) { 737 struct page *page = 738 __bnxt_alloc_rx_page(bp, &mapping, rxr, gfp); 739 740 if (!page) 741 return -ENOMEM; 742 743 rx_buf->data = page; 744 rx_buf->data_ptr = page_address(page) + bp->rx_offset; 745 } else { 746 u8 *data = __bnxt_alloc_rx_data(bp, &mapping, gfp); 747 748 if (!data) 749 return -ENOMEM; 750 751 rx_buf->data = data; 752 rx_buf->data_ptr = data + bp->rx_offset; 753 } 754 rx_buf->mapping = mapping; 755 756 rxbd->rx_bd_haddr = cpu_to_le64(mapping); 757 return 0; 758 } 759 760 void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data) 761 { 762 u16 prod = rxr->rx_prod; 763 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf; 764 struct rx_bd *cons_bd, *prod_bd; 765 766 prod_rx_buf = &rxr->rx_buf_ring[prod]; 767 cons_rx_buf = &rxr->rx_buf_ring[cons]; 768 769 prod_rx_buf->data = data; 770 prod_rx_buf->data_ptr = cons_rx_buf->data_ptr; 771 772 prod_rx_buf->mapping = cons_rx_buf->mapping; 773 774 prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 775 cons_bd = &rxr->rx_desc_ring[RX_RING(cons)][RX_IDX(cons)]; 776 777 prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr; 778 } 779 780 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx) 781 { 782 u16 next, max = rxr->rx_agg_bmap_size; 783 784 next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx); 785 if (next >= max) 786 next = find_first_zero_bit(rxr->rx_agg_bmap, max); 787 return next; 788 } 789 790 static inline int bnxt_alloc_rx_page(struct bnxt *bp, 791 struct bnxt_rx_ring_info *rxr, 792 u16 prod, gfp_t gfp) 793 { 794 struct rx_bd *rxbd = 795 &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 796 struct bnxt_sw_rx_agg_bd *rx_agg_buf; 797 struct pci_dev *pdev = bp->pdev; 798 struct page *page; 799 dma_addr_t mapping; 800 u16 sw_prod = rxr->rx_sw_agg_prod; 801 unsigned int offset = 0; 802 803 if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) { 804 page = rxr->rx_page; 805 if (!page) { 806 page = alloc_page(gfp); 807 if (!page) 808 return -ENOMEM; 809 rxr->rx_page = page; 810 rxr->rx_page_offset = 0; 811 } 812 offset = rxr->rx_page_offset; 813 rxr->rx_page_offset += BNXT_RX_PAGE_SIZE; 814 if (rxr->rx_page_offset == PAGE_SIZE) 815 rxr->rx_page = NULL; 816 else 817 get_page(page); 818 } else { 819 page = alloc_page(gfp); 820 if (!page) 821 return -ENOMEM; 822 } 823 824 mapping = dma_map_page_attrs(&pdev->dev, page, offset, 825 BNXT_RX_PAGE_SIZE, PCI_DMA_FROMDEVICE, 826 DMA_ATTR_WEAK_ORDERING); 827 if (dma_mapping_error(&pdev->dev, mapping)) { 828 __free_page(page); 829 return -EIO; 830 } 831 832 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap))) 833 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod); 834 835 __set_bit(sw_prod, rxr->rx_agg_bmap); 836 rx_agg_buf = &rxr->rx_agg_ring[sw_prod]; 837 rxr->rx_sw_agg_prod = NEXT_RX_AGG(sw_prod); 838 839 rx_agg_buf->page = page; 840 rx_agg_buf->offset = offset; 841 rx_agg_buf->mapping = mapping; 842 rxbd->rx_bd_haddr = cpu_to_le64(mapping); 843 rxbd->rx_bd_opaque = sw_prod; 844 return 0; 845 } 846 847 static struct rx_agg_cmp *bnxt_get_agg(struct bnxt *bp, 848 struct bnxt_cp_ring_info *cpr, 849 u16 cp_cons, u16 curr) 850 { 851 struct rx_agg_cmp *agg; 852 853 cp_cons = RING_CMP(ADV_RAW_CMP(cp_cons, curr)); 854 agg = (struct rx_agg_cmp *) 855 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 856 return agg; 857 } 858 859 static struct rx_agg_cmp *bnxt_get_tpa_agg_p5(struct bnxt *bp, 860 struct bnxt_rx_ring_info *rxr, 861 u16 agg_id, u16 curr) 862 { 863 struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[agg_id]; 864 865 return &tpa_info->agg_arr[curr]; 866 } 867 868 static void bnxt_reuse_rx_agg_bufs(struct bnxt_cp_ring_info *cpr, u16 idx, 869 u16 start, u32 agg_bufs, bool tpa) 870 { 871 struct bnxt_napi *bnapi = cpr->bnapi; 872 struct bnxt *bp = bnapi->bp; 873 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 874 u16 prod = rxr->rx_agg_prod; 875 u16 sw_prod = rxr->rx_sw_agg_prod; 876 bool p5_tpa = false; 877 u32 i; 878 879 if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa) 880 p5_tpa = true; 881 882 for (i = 0; i < agg_bufs; i++) { 883 u16 cons; 884 struct rx_agg_cmp *agg; 885 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf; 886 struct rx_bd *prod_bd; 887 struct page *page; 888 889 if (p5_tpa) 890 agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, start + i); 891 else 892 agg = bnxt_get_agg(bp, cpr, idx, start + i); 893 cons = agg->rx_agg_cmp_opaque; 894 __clear_bit(cons, rxr->rx_agg_bmap); 895 896 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap))) 897 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod); 898 899 __set_bit(sw_prod, rxr->rx_agg_bmap); 900 prod_rx_buf = &rxr->rx_agg_ring[sw_prod]; 901 cons_rx_buf = &rxr->rx_agg_ring[cons]; 902 903 /* It is possible for sw_prod to be equal to cons, so 904 * set cons_rx_buf->page to NULL first. 905 */ 906 page = cons_rx_buf->page; 907 cons_rx_buf->page = NULL; 908 prod_rx_buf->page = page; 909 prod_rx_buf->offset = cons_rx_buf->offset; 910 911 prod_rx_buf->mapping = cons_rx_buf->mapping; 912 913 prod_bd = &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 914 915 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping); 916 prod_bd->rx_bd_opaque = sw_prod; 917 918 prod = NEXT_RX_AGG(prod); 919 sw_prod = NEXT_RX_AGG(sw_prod); 920 } 921 rxr->rx_agg_prod = prod; 922 rxr->rx_sw_agg_prod = sw_prod; 923 } 924 925 static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp, 926 struct bnxt_rx_ring_info *rxr, 927 u16 cons, void *data, u8 *data_ptr, 928 dma_addr_t dma_addr, 929 unsigned int offset_and_len) 930 { 931 unsigned int payload = offset_and_len >> 16; 932 unsigned int len = offset_and_len & 0xffff; 933 skb_frag_t *frag; 934 struct page *page = data; 935 u16 prod = rxr->rx_prod; 936 struct sk_buff *skb; 937 int off, err; 938 939 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC); 940 if (unlikely(err)) { 941 bnxt_reuse_rx_data(rxr, cons, data); 942 return NULL; 943 } 944 dma_addr -= bp->rx_dma_offset; 945 dma_unmap_page_attrs(&bp->pdev->dev, dma_addr, PAGE_SIZE, bp->rx_dir, 946 DMA_ATTR_WEAK_ORDERING); 947 page_pool_release_page(rxr->page_pool, page); 948 949 if (unlikely(!payload)) 950 payload = eth_get_headlen(bp->dev, data_ptr, len); 951 952 skb = napi_alloc_skb(&rxr->bnapi->napi, payload); 953 if (!skb) { 954 __free_page(page); 955 return NULL; 956 } 957 958 off = (void *)data_ptr - page_address(page); 959 skb_add_rx_frag(skb, 0, page, off, len, PAGE_SIZE); 960 memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN, 961 payload + NET_IP_ALIGN); 962 963 frag = &skb_shinfo(skb)->frags[0]; 964 skb_frag_size_sub(frag, payload); 965 skb_frag_off_add(frag, payload); 966 skb->data_len -= payload; 967 skb->tail += payload; 968 969 return skb; 970 } 971 972 static struct sk_buff *bnxt_rx_skb(struct bnxt *bp, 973 struct bnxt_rx_ring_info *rxr, u16 cons, 974 void *data, u8 *data_ptr, 975 dma_addr_t dma_addr, 976 unsigned int offset_and_len) 977 { 978 u16 prod = rxr->rx_prod; 979 struct sk_buff *skb; 980 int err; 981 982 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC); 983 if (unlikely(err)) { 984 bnxt_reuse_rx_data(rxr, cons, data); 985 return NULL; 986 } 987 988 skb = build_skb(data, 0); 989 dma_unmap_single_attrs(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size, 990 bp->rx_dir, DMA_ATTR_WEAK_ORDERING); 991 if (!skb) { 992 kfree(data); 993 return NULL; 994 } 995 996 skb_reserve(skb, bp->rx_offset); 997 skb_put(skb, offset_and_len & 0xffff); 998 return skb; 999 } 1000 1001 static struct sk_buff *bnxt_rx_pages(struct bnxt *bp, 1002 struct bnxt_cp_ring_info *cpr, 1003 struct sk_buff *skb, u16 idx, 1004 u32 agg_bufs, bool tpa) 1005 { 1006 struct bnxt_napi *bnapi = cpr->bnapi; 1007 struct pci_dev *pdev = bp->pdev; 1008 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1009 u16 prod = rxr->rx_agg_prod; 1010 bool p5_tpa = false; 1011 u32 i; 1012 1013 if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa) 1014 p5_tpa = true; 1015 1016 for (i = 0; i < agg_bufs; i++) { 1017 u16 cons, frag_len; 1018 struct rx_agg_cmp *agg; 1019 struct bnxt_sw_rx_agg_bd *cons_rx_buf; 1020 struct page *page; 1021 dma_addr_t mapping; 1022 1023 if (p5_tpa) 1024 agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, i); 1025 else 1026 agg = bnxt_get_agg(bp, cpr, idx, i); 1027 cons = agg->rx_agg_cmp_opaque; 1028 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) & 1029 RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT; 1030 1031 cons_rx_buf = &rxr->rx_agg_ring[cons]; 1032 skb_fill_page_desc(skb, i, cons_rx_buf->page, 1033 cons_rx_buf->offset, frag_len); 1034 __clear_bit(cons, rxr->rx_agg_bmap); 1035 1036 /* It is possible for bnxt_alloc_rx_page() to allocate 1037 * a sw_prod index that equals the cons index, so we 1038 * need to clear the cons entry now. 1039 */ 1040 mapping = cons_rx_buf->mapping; 1041 page = cons_rx_buf->page; 1042 cons_rx_buf->page = NULL; 1043 1044 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) { 1045 struct skb_shared_info *shinfo; 1046 unsigned int nr_frags; 1047 1048 shinfo = skb_shinfo(skb); 1049 nr_frags = --shinfo->nr_frags; 1050 __skb_frag_set_page(&shinfo->frags[nr_frags], NULL); 1051 1052 dev_kfree_skb(skb); 1053 1054 cons_rx_buf->page = page; 1055 1056 /* Update prod since possibly some pages have been 1057 * allocated already. 1058 */ 1059 rxr->rx_agg_prod = prod; 1060 bnxt_reuse_rx_agg_bufs(cpr, idx, i, agg_bufs - i, tpa); 1061 return NULL; 1062 } 1063 1064 dma_unmap_page_attrs(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE, 1065 PCI_DMA_FROMDEVICE, 1066 DMA_ATTR_WEAK_ORDERING); 1067 1068 skb->data_len += frag_len; 1069 skb->len += frag_len; 1070 skb->truesize += PAGE_SIZE; 1071 1072 prod = NEXT_RX_AGG(prod); 1073 } 1074 rxr->rx_agg_prod = prod; 1075 return skb; 1076 } 1077 1078 static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 1079 u8 agg_bufs, u32 *raw_cons) 1080 { 1081 u16 last; 1082 struct rx_agg_cmp *agg; 1083 1084 *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs); 1085 last = RING_CMP(*raw_cons); 1086 agg = (struct rx_agg_cmp *) 1087 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)]; 1088 return RX_AGG_CMP_VALID(agg, *raw_cons); 1089 } 1090 1091 static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data, 1092 unsigned int len, 1093 dma_addr_t mapping) 1094 { 1095 struct bnxt *bp = bnapi->bp; 1096 struct pci_dev *pdev = bp->pdev; 1097 struct sk_buff *skb; 1098 1099 skb = napi_alloc_skb(&bnapi->napi, len); 1100 if (!skb) 1101 return NULL; 1102 1103 dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copy_thresh, 1104 bp->rx_dir); 1105 1106 memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN, 1107 len + NET_IP_ALIGN); 1108 1109 dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copy_thresh, 1110 bp->rx_dir); 1111 1112 skb_put(skb, len); 1113 return skb; 1114 } 1115 1116 static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 1117 u32 *raw_cons, void *cmp) 1118 { 1119 struct rx_cmp *rxcmp = cmp; 1120 u32 tmp_raw_cons = *raw_cons; 1121 u8 cmp_type, agg_bufs = 0; 1122 1123 cmp_type = RX_CMP_TYPE(rxcmp); 1124 1125 if (cmp_type == CMP_TYPE_RX_L2_CMP) { 1126 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) & 1127 RX_CMP_AGG_BUFS) >> 1128 RX_CMP_AGG_BUFS_SHIFT; 1129 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) { 1130 struct rx_tpa_end_cmp *tpa_end = cmp; 1131 1132 if (bp->flags & BNXT_FLAG_CHIP_P5) 1133 return 0; 1134 1135 agg_bufs = TPA_END_AGG_BUFS(tpa_end); 1136 } 1137 1138 if (agg_bufs) { 1139 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons)) 1140 return -EBUSY; 1141 } 1142 *raw_cons = tmp_raw_cons; 1143 return 0; 1144 } 1145 1146 static void bnxt_queue_fw_reset_work(struct bnxt *bp, unsigned long delay) 1147 { 1148 if (!(test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))) 1149 return; 1150 1151 if (BNXT_PF(bp)) 1152 queue_delayed_work(bnxt_pf_wq, &bp->fw_reset_task, delay); 1153 else 1154 schedule_delayed_work(&bp->fw_reset_task, delay); 1155 } 1156 1157 static void bnxt_queue_sp_work(struct bnxt *bp) 1158 { 1159 if (BNXT_PF(bp)) 1160 queue_work(bnxt_pf_wq, &bp->sp_task); 1161 else 1162 schedule_work(&bp->sp_task); 1163 } 1164 1165 static void bnxt_sched_reset(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) 1166 { 1167 if (!rxr->bnapi->in_reset) { 1168 rxr->bnapi->in_reset = true; 1169 if (bp->flags & BNXT_FLAG_CHIP_P5) 1170 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event); 1171 else 1172 set_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event); 1173 bnxt_queue_sp_work(bp); 1174 } 1175 rxr->rx_next_cons = 0xffff; 1176 } 1177 1178 static u16 bnxt_alloc_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id) 1179 { 1180 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map; 1181 u16 idx = agg_id & MAX_TPA_P5_MASK; 1182 1183 if (test_bit(idx, map->agg_idx_bmap)) 1184 idx = find_first_zero_bit(map->agg_idx_bmap, 1185 BNXT_AGG_IDX_BMAP_SIZE); 1186 __set_bit(idx, map->agg_idx_bmap); 1187 map->agg_id_tbl[agg_id] = idx; 1188 return idx; 1189 } 1190 1191 static void bnxt_free_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx) 1192 { 1193 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map; 1194 1195 __clear_bit(idx, map->agg_idx_bmap); 1196 } 1197 1198 static u16 bnxt_lookup_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id) 1199 { 1200 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map; 1201 1202 return map->agg_id_tbl[agg_id]; 1203 } 1204 1205 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, 1206 struct rx_tpa_start_cmp *tpa_start, 1207 struct rx_tpa_start_cmp_ext *tpa_start1) 1208 { 1209 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf; 1210 struct bnxt_tpa_info *tpa_info; 1211 u16 cons, prod, agg_id; 1212 struct rx_bd *prod_bd; 1213 dma_addr_t mapping; 1214 1215 if (bp->flags & BNXT_FLAG_CHIP_P5) { 1216 agg_id = TPA_START_AGG_ID_P5(tpa_start); 1217 agg_id = bnxt_alloc_agg_idx(rxr, agg_id); 1218 } else { 1219 agg_id = TPA_START_AGG_ID(tpa_start); 1220 } 1221 cons = tpa_start->rx_tpa_start_cmp_opaque; 1222 prod = rxr->rx_prod; 1223 cons_rx_buf = &rxr->rx_buf_ring[cons]; 1224 prod_rx_buf = &rxr->rx_buf_ring[prod]; 1225 tpa_info = &rxr->rx_tpa[agg_id]; 1226 1227 if (unlikely(cons != rxr->rx_next_cons || 1228 TPA_START_ERROR(tpa_start))) { 1229 netdev_warn(bp->dev, "TPA cons %x, expected cons %x, error code %x\n", 1230 cons, rxr->rx_next_cons, 1231 TPA_START_ERROR_CODE(tpa_start1)); 1232 bnxt_sched_reset(bp, rxr); 1233 return; 1234 } 1235 /* Store cfa_code in tpa_info to use in tpa_end 1236 * completion processing. 1237 */ 1238 tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1); 1239 prod_rx_buf->data = tpa_info->data; 1240 prod_rx_buf->data_ptr = tpa_info->data_ptr; 1241 1242 mapping = tpa_info->mapping; 1243 prod_rx_buf->mapping = mapping; 1244 1245 prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 1246 1247 prod_bd->rx_bd_haddr = cpu_to_le64(mapping); 1248 1249 tpa_info->data = cons_rx_buf->data; 1250 tpa_info->data_ptr = cons_rx_buf->data_ptr; 1251 cons_rx_buf->data = NULL; 1252 tpa_info->mapping = cons_rx_buf->mapping; 1253 1254 tpa_info->len = 1255 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >> 1256 RX_TPA_START_CMP_LEN_SHIFT; 1257 if (likely(TPA_START_HASH_VALID(tpa_start))) { 1258 u32 hash_type = TPA_START_HASH_TYPE(tpa_start); 1259 1260 tpa_info->hash_type = PKT_HASH_TYPE_L4; 1261 tpa_info->gso_type = SKB_GSO_TCPV4; 1262 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */ 1263 if (hash_type == 3 || TPA_START_IS_IPV6(tpa_start1)) 1264 tpa_info->gso_type = SKB_GSO_TCPV6; 1265 tpa_info->rss_hash = 1266 le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash); 1267 } else { 1268 tpa_info->hash_type = PKT_HASH_TYPE_NONE; 1269 tpa_info->gso_type = 0; 1270 netif_warn(bp, rx_err, bp->dev, "TPA packet without valid hash\n"); 1271 } 1272 tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2); 1273 tpa_info->metadata = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata); 1274 tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info); 1275 tpa_info->agg_count = 0; 1276 1277 rxr->rx_prod = NEXT_RX(prod); 1278 cons = NEXT_RX(cons); 1279 rxr->rx_next_cons = NEXT_RX(cons); 1280 cons_rx_buf = &rxr->rx_buf_ring[cons]; 1281 1282 bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data); 1283 rxr->rx_prod = NEXT_RX(rxr->rx_prod); 1284 cons_rx_buf->data = NULL; 1285 } 1286 1287 static void bnxt_abort_tpa(struct bnxt_cp_ring_info *cpr, u16 idx, u32 agg_bufs) 1288 { 1289 if (agg_bufs) 1290 bnxt_reuse_rx_agg_bufs(cpr, idx, 0, agg_bufs, true); 1291 } 1292 1293 #ifdef CONFIG_INET 1294 static void bnxt_gro_tunnel(struct sk_buff *skb, __be16 ip_proto) 1295 { 1296 struct udphdr *uh = NULL; 1297 1298 if (ip_proto == htons(ETH_P_IP)) { 1299 struct iphdr *iph = (struct iphdr *)skb->data; 1300 1301 if (iph->protocol == IPPROTO_UDP) 1302 uh = (struct udphdr *)(iph + 1); 1303 } else { 1304 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data; 1305 1306 if (iph->nexthdr == IPPROTO_UDP) 1307 uh = (struct udphdr *)(iph + 1); 1308 } 1309 if (uh) { 1310 if (uh->check) 1311 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM; 1312 else 1313 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL; 1314 } 1315 } 1316 #endif 1317 1318 static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info, 1319 int payload_off, int tcp_ts, 1320 struct sk_buff *skb) 1321 { 1322 #ifdef CONFIG_INET 1323 struct tcphdr *th; 1324 int len, nw_off; 1325 u16 outer_ip_off, inner_ip_off, inner_mac_off; 1326 u32 hdr_info = tpa_info->hdr_info; 1327 bool loopback = false; 1328 1329 inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info); 1330 inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info); 1331 outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info); 1332 1333 /* If the packet is an internal loopback packet, the offsets will 1334 * have an extra 4 bytes. 1335 */ 1336 if (inner_mac_off == 4) { 1337 loopback = true; 1338 } else if (inner_mac_off > 4) { 1339 __be16 proto = *((__be16 *)(skb->data + inner_ip_off - 1340 ETH_HLEN - 2)); 1341 1342 /* We only support inner iPv4/ipv6. If we don't see the 1343 * correct protocol ID, it must be a loopback packet where 1344 * the offsets are off by 4. 1345 */ 1346 if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6)) 1347 loopback = true; 1348 } 1349 if (loopback) { 1350 /* internal loopback packet, subtract all offsets by 4 */ 1351 inner_ip_off -= 4; 1352 inner_mac_off -= 4; 1353 outer_ip_off -= 4; 1354 } 1355 1356 nw_off = inner_ip_off - ETH_HLEN; 1357 skb_set_network_header(skb, nw_off); 1358 if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) { 1359 struct ipv6hdr *iph = ipv6_hdr(skb); 1360 1361 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr)); 1362 len = skb->len - skb_transport_offset(skb); 1363 th = tcp_hdr(skb); 1364 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0); 1365 } else { 1366 struct iphdr *iph = ip_hdr(skb); 1367 1368 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr)); 1369 len = skb->len - skb_transport_offset(skb); 1370 th = tcp_hdr(skb); 1371 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0); 1372 } 1373 1374 if (inner_mac_off) { /* tunnel */ 1375 __be16 proto = *((__be16 *)(skb->data + outer_ip_off - 1376 ETH_HLEN - 2)); 1377 1378 bnxt_gro_tunnel(skb, proto); 1379 } 1380 #endif 1381 return skb; 1382 } 1383 1384 static struct sk_buff *bnxt_gro_func_5750x(struct bnxt_tpa_info *tpa_info, 1385 int payload_off, int tcp_ts, 1386 struct sk_buff *skb) 1387 { 1388 #ifdef CONFIG_INET 1389 u16 outer_ip_off, inner_ip_off, inner_mac_off; 1390 u32 hdr_info = tpa_info->hdr_info; 1391 int iphdr_len, nw_off; 1392 1393 inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info); 1394 inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info); 1395 outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info); 1396 1397 nw_off = inner_ip_off - ETH_HLEN; 1398 skb_set_network_header(skb, nw_off); 1399 iphdr_len = (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) ? 1400 sizeof(struct ipv6hdr) : sizeof(struct iphdr); 1401 skb_set_transport_header(skb, nw_off + iphdr_len); 1402 1403 if (inner_mac_off) { /* tunnel */ 1404 __be16 proto = *((__be16 *)(skb->data + outer_ip_off - 1405 ETH_HLEN - 2)); 1406 1407 bnxt_gro_tunnel(skb, proto); 1408 } 1409 #endif 1410 return skb; 1411 } 1412 1413 #define BNXT_IPV4_HDR_SIZE (sizeof(struct iphdr) + sizeof(struct tcphdr)) 1414 #define BNXT_IPV6_HDR_SIZE (sizeof(struct ipv6hdr) + sizeof(struct tcphdr)) 1415 1416 static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info, 1417 int payload_off, int tcp_ts, 1418 struct sk_buff *skb) 1419 { 1420 #ifdef CONFIG_INET 1421 struct tcphdr *th; 1422 int len, nw_off, tcp_opt_len = 0; 1423 1424 if (tcp_ts) 1425 tcp_opt_len = 12; 1426 1427 if (tpa_info->gso_type == SKB_GSO_TCPV4) { 1428 struct iphdr *iph; 1429 1430 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len - 1431 ETH_HLEN; 1432 skb_set_network_header(skb, nw_off); 1433 iph = ip_hdr(skb); 1434 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr)); 1435 len = skb->len - skb_transport_offset(skb); 1436 th = tcp_hdr(skb); 1437 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0); 1438 } else if (tpa_info->gso_type == SKB_GSO_TCPV6) { 1439 struct ipv6hdr *iph; 1440 1441 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len - 1442 ETH_HLEN; 1443 skb_set_network_header(skb, nw_off); 1444 iph = ipv6_hdr(skb); 1445 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr)); 1446 len = skb->len - skb_transport_offset(skb); 1447 th = tcp_hdr(skb); 1448 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0); 1449 } else { 1450 dev_kfree_skb_any(skb); 1451 return NULL; 1452 } 1453 1454 if (nw_off) /* tunnel */ 1455 bnxt_gro_tunnel(skb, skb->protocol); 1456 #endif 1457 return skb; 1458 } 1459 1460 static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp, 1461 struct bnxt_tpa_info *tpa_info, 1462 struct rx_tpa_end_cmp *tpa_end, 1463 struct rx_tpa_end_cmp_ext *tpa_end1, 1464 struct sk_buff *skb) 1465 { 1466 #ifdef CONFIG_INET 1467 int payload_off; 1468 u16 segs; 1469 1470 segs = TPA_END_TPA_SEGS(tpa_end); 1471 if (segs == 1) 1472 return skb; 1473 1474 NAPI_GRO_CB(skb)->count = segs; 1475 skb_shinfo(skb)->gso_size = 1476 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len); 1477 skb_shinfo(skb)->gso_type = tpa_info->gso_type; 1478 if (bp->flags & BNXT_FLAG_CHIP_P5) 1479 payload_off = TPA_END_PAYLOAD_OFF_P5(tpa_end1); 1480 else 1481 payload_off = TPA_END_PAYLOAD_OFF(tpa_end); 1482 skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb); 1483 if (likely(skb)) 1484 tcp_gro_complete(skb); 1485 #endif 1486 return skb; 1487 } 1488 1489 /* Given the cfa_code of a received packet determine which 1490 * netdev (vf-rep or PF) the packet is destined to. 1491 */ 1492 static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code) 1493 { 1494 struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code); 1495 1496 /* if vf-rep dev is NULL, the must belongs to the PF */ 1497 return dev ? dev : bp->dev; 1498 } 1499 1500 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp, 1501 struct bnxt_cp_ring_info *cpr, 1502 u32 *raw_cons, 1503 struct rx_tpa_end_cmp *tpa_end, 1504 struct rx_tpa_end_cmp_ext *tpa_end1, 1505 u8 *event) 1506 { 1507 struct bnxt_napi *bnapi = cpr->bnapi; 1508 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1509 u8 *data_ptr, agg_bufs; 1510 unsigned int len; 1511 struct bnxt_tpa_info *tpa_info; 1512 dma_addr_t mapping; 1513 struct sk_buff *skb; 1514 u16 idx = 0, agg_id; 1515 void *data; 1516 bool gro; 1517 1518 if (unlikely(bnapi->in_reset)) { 1519 int rc = bnxt_discard_rx(bp, cpr, raw_cons, tpa_end); 1520 1521 if (rc < 0) 1522 return ERR_PTR(-EBUSY); 1523 return NULL; 1524 } 1525 1526 if (bp->flags & BNXT_FLAG_CHIP_P5) { 1527 agg_id = TPA_END_AGG_ID_P5(tpa_end); 1528 agg_id = bnxt_lookup_agg_idx(rxr, agg_id); 1529 agg_bufs = TPA_END_AGG_BUFS_P5(tpa_end1); 1530 tpa_info = &rxr->rx_tpa[agg_id]; 1531 if (unlikely(agg_bufs != tpa_info->agg_count)) { 1532 netdev_warn(bp->dev, "TPA end agg_buf %d != expected agg_bufs %d\n", 1533 agg_bufs, tpa_info->agg_count); 1534 agg_bufs = tpa_info->agg_count; 1535 } 1536 tpa_info->agg_count = 0; 1537 *event |= BNXT_AGG_EVENT; 1538 bnxt_free_agg_idx(rxr, agg_id); 1539 idx = agg_id; 1540 gro = !!(bp->flags & BNXT_FLAG_GRO); 1541 } else { 1542 agg_id = TPA_END_AGG_ID(tpa_end); 1543 agg_bufs = TPA_END_AGG_BUFS(tpa_end); 1544 tpa_info = &rxr->rx_tpa[agg_id]; 1545 idx = RING_CMP(*raw_cons); 1546 if (agg_bufs) { 1547 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons)) 1548 return ERR_PTR(-EBUSY); 1549 1550 *event |= BNXT_AGG_EVENT; 1551 idx = NEXT_CMP(idx); 1552 } 1553 gro = !!TPA_END_GRO(tpa_end); 1554 } 1555 data = tpa_info->data; 1556 data_ptr = tpa_info->data_ptr; 1557 prefetch(data_ptr); 1558 len = tpa_info->len; 1559 mapping = tpa_info->mapping; 1560 1561 if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) { 1562 bnxt_abort_tpa(cpr, idx, agg_bufs); 1563 if (agg_bufs > MAX_SKB_FRAGS) 1564 netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n", 1565 agg_bufs, (int)MAX_SKB_FRAGS); 1566 return NULL; 1567 } 1568 1569 if (len <= bp->rx_copy_thresh) { 1570 skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping); 1571 if (!skb) { 1572 bnxt_abort_tpa(cpr, idx, agg_bufs); 1573 return NULL; 1574 } 1575 } else { 1576 u8 *new_data; 1577 dma_addr_t new_mapping; 1578 1579 new_data = __bnxt_alloc_rx_data(bp, &new_mapping, GFP_ATOMIC); 1580 if (!new_data) { 1581 bnxt_abort_tpa(cpr, idx, agg_bufs); 1582 return NULL; 1583 } 1584 1585 tpa_info->data = new_data; 1586 tpa_info->data_ptr = new_data + bp->rx_offset; 1587 tpa_info->mapping = new_mapping; 1588 1589 skb = build_skb(data, 0); 1590 dma_unmap_single_attrs(&bp->pdev->dev, mapping, 1591 bp->rx_buf_use_size, bp->rx_dir, 1592 DMA_ATTR_WEAK_ORDERING); 1593 1594 if (!skb) { 1595 kfree(data); 1596 bnxt_abort_tpa(cpr, idx, agg_bufs); 1597 return NULL; 1598 } 1599 skb_reserve(skb, bp->rx_offset); 1600 skb_put(skb, len); 1601 } 1602 1603 if (agg_bufs) { 1604 skb = bnxt_rx_pages(bp, cpr, skb, idx, agg_bufs, true); 1605 if (!skb) { 1606 /* Page reuse already handled by bnxt_rx_pages(). */ 1607 return NULL; 1608 } 1609 } 1610 1611 skb->protocol = 1612 eth_type_trans(skb, bnxt_get_pkt_dev(bp, tpa_info->cfa_code)); 1613 1614 if (tpa_info->hash_type != PKT_HASH_TYPE_NONE) 1615 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type); 1616 1617 if ((tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) && 1618 (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) { 1619 u16 vlan_proto = tpa_info->metadata >> 1620 RX_CMP_FLAGS2_METADATA_TPID_SFT; 1621 u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_TCI_MASK; 1622 1623 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag); 1624 } 1625 1626 skb_checksum_none_assert(skb); 1627 if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) { 1628 skb->ip_summed = CHECKSUM_UNNECESSARY; 1629 skb->csum_level = 1630 (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3; 1631 } 1632 1633 if (gro) 1634 skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb); 1635 1636 return skb; 1637 } 1638 1639 static void bnxt_tpa_agg(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, 1640 struct rx_agg_cmp *rx_agg) 1641 { 1642 u16 agg_id = TPA_AGG_AGG_ID(rx_agg); 1643 struct bnxt_tpa_info *tpa_info; 1644 1645 agg_id = bnxt_lookup_agg_idx(rxr, agg_id); 1646 tpa_info = &rxr->rx_tpa[agg_id]; 1647 BUG_ON(tpa_info->agg_count >= MAX_SKB_FRAGS); 1648 tpa_info->agg_arr[tpa_info->agg_count++] = *rx_agg; 1649 } 1650 1651 static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi, 1652 struct sk_buff *skb) 1653 { 1654 if (skb->dev != bp->dev) { 1655 /* this packet belongs to a vf-rep */ 1656 bnxt_vf_rep_rx(bp, skb); 1657 return; 1658 } 1659 skb_record_rx_queue(skb, bnapi->index); 1660 napi_gro_receive(&bnapi->napi, skb); 1661 } 1662 1663 /* returns the following: 1664 * 1 - 1 packet successfully received 1665 * 0 - successful TPA_START, packet not completed yet 1666 * -EBUSY - completion ring does not have all the agg buffers yet 1667 * -ENOMEM - packet aborted due to out of memory 1668 * -EIO - packet aborted due to hw error indicated in BD 1669 */ 1670 static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 1671 u32 *raw_cons, u8 *event) 1672 { 1673 struct bnxt_napi *bnapi = cpr->bnapi; 1674 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1675 struct net_device *dev = bp->dev; 1676 struct rx_cmp *rxcmp; 1677 struct rx_cmp_ext *rxcmp1; 1678 u32 tmp_raw_cons = *raw_cons; 1679 u16 cfa_code, cons, prod, cp_cons = RING_CMP(tmp_raw_cons); 1680 struct bnxt_sw_rx_bd *rx_buf; 1681 unsigned int len; 1682 u8 *data_ptr, agg_bufs, cmp_type; 1683 dma_addr_t dma_addr; 1684 struct sk_buff *skb; 1685 void *data; 1686 int rc = 0; 1687 u32 misc; 1688 1689 rxcmp = (struct rx_cmp *) 1690 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1691 1692 cmp_type = RX_CMP_TYPE(rxcmp); 1693 1694 if (cmp_type == CMP_TYPE_RX_TPA_AGG_CMP) { 1695 bnxt_tpa_agg(bp, rxr, (struct rx_agg_cmp *)rxcmp); 1696 goto next_rx_no_prod_no_len; 1697 } 1698 1699 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons); 1700 cp_cons = RING_CMP(tmp_raw_cons); 1701 rxcmp1 = (struct rx_cmp_ext *) 1702 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1703 1704 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons)) 1705 return -EBUSY; 1706 1707 prod = rxr->rx_prod; 1708 1709 if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) { 1710 bnxt_tpa_start(bp, rxr, (struct rx_tpa_start_cmp *)rxcmp, 1711 (struct rx_tpa_start_cmp_ext *)rxcmp1); 1712 1713 *event |= BNXT_RX_EVENT; 1714 goto next_rx_no_prod_no_len; 1715 1716 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) { 1717 skb = bnxt_tpa_end(bp, cpr, &tmp_raw_cons, 1718 (struct rx_tpa_end_cmp *)rxcmp, 1719 (struct rx_tpa_end_cmp_ext *)rxcmp1, event); 1720 1721 if (IS_ERR(skb)) 1722 return -EBUSY; 1723 1724 rc = -ENOMEM; 1725 if (likely(skb)) { 1726 bnxt_deliver_skb(bp, bnapi, skb); 1727 rc = 1; 1728 } 1729 *event |= BNXT_RX_EVENT; 1730 goto next_rx_no_prod_no_len; 1731 } 1732 1733 cons = rxcmp->rx_cmp_opaque; 1734 if (unlikely(cons != rxr->rx_next_cons)) { 1735 int rc1 = bnxt_discard_rx(bp, cpr, raw_cons, rxcmp); 1736 1737 /* 0xffff is forced error, don't print it */ 1738 if (rxr->rx_next_cons != 0xffff) 1739 netdev_warn(bp->dev, "RX cons %x != expected cons %x\n", 1740 cons, rxr->rx_next_cons); 1741 bnxt_sched_reset(bp, rxr); 1742 return rc1; 1743 } 1744 rx_buf = &rxr->rx_buf_ring[cons]; 1745 data = rx_buf->data; 1746 data_ptr = rx_buf->data_ptr; 1747 prefetch(data_ptr); 1748 1749 misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1); 1750 agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT; 1751 1752 if (agg_bufs) { 1753 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons)) 1754 return -EBUSY; 1755 1756 cp_cons = NEXT_CMP(cp_cons); 1757 *event |= BNXT_AGG_EVENT; 1758 } 1759 *event |= BNXT_RX_EVENT; 1760 1761 rx_buf->data = NULL; 1762 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) { 1763 u32 rx_err = le32_to_cpu(rxcmp1->rx_cmp_cfa_code_errors_v2); 1764 1765 bnxt_reuse_rx_data(rxr, cons, data); 1766 if (agg_bufs) 1767 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, agg_bufs, 1768 false); 1769 1770 rc = -EIO; 1771 if (rx_err & RX_CMPL_ERRORS_BUFFER_ERROR_MASK) { 1772 bnapi->cp_ring.sw_stats.rx.rx_buf_errors++; 1773 if (!(bp->flags & BNXT_FLAG_CHIP_P5) && 1774 !(bp->fw_cap & BNXT_FW_CAP_RING_MONITOR)) { 1775 netdev_warn_once(bp->dev, "RX buffer error %x\n", 1776 rx_err); 1777 bnxt_sched_reset(bp, rxr); 1778 } 1779 } 1780 goto next_rx_no_len; 1781 } 1782 1783 len = le32_to_cpu(rxcmp->rx_cmp_len_flags_type) >> RX_CMP_LEN_SHIFT; 1784 dma_addr = rx_buf->mapping; 1785 1786 if (bnxt_rx_xdp(bp, rxr, cons, data, &data_ptr, &len, event)) { 1787 rc = 1; 1788 goto next_rx; 1789 } 1790 1791 if (len <= bp->rx_copy_thresh) { 1792 skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr); 1793 bnxt_reuse_rx_data(rxr, cons, data); 1794 if (!skb) { 1795 if (agg_bufs) 1796 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, 1797 agg_bufs, false); 1798 rc = -ENOMEM; 1799 goto next_rx; 1800 } 1801 } else { 1802 u32 payload; 1803 1804 if (rx_buf->data_ptr == data_ptr) 1805 payload = misc & RX_CMP_PAYLOAD_OFFSET; 1806 else 1807 payload = 0; 1808 skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr, 1809 payload | len); 1810 if (!skb) { 1811 rc = -ENOMEM; 1812 goto next_rx; 1813 } 1814 } 1815 1816 if (agg_bufs) { 1817 skb = bnxt_rx_pages(bp, cpr, skb, cp_cons, agg_bufs, false); 1818 if (!skb) { 1819 rc = -ENOMEM; 1820 goto next_rx; 1821 } 1822 } 1823 1824 if (RX_CMP_HASH_VALID(rxcmp)) { 1825 u32 hash_type = RX_CMP_HASH_TYPE(rxcmp); 1826 enum pkt_hash_types type = PKT_HASH_TYPE_L4; 1827 1828 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */ 1829 if (hash_type != 1 && hash_type != 3) 1830 type = PKT_HASH_TYPE_L3; 1831 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type); 1832 } 1833 1834 cfa_code = RX_CMP_CFA_CODE(rxcmp1); 1835 skb->protocol = eth_type_trans(skb, bnxt_get_pkt_dev(bp, cfa_code)); 1836 1837 if ((rxcmp1->rx_cmp_flags2 & 1838 cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) && 1839 (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) { 1840 u32 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data); 1841 u16 vtag = meta_data & RX_CMP_FLAGS2_METADATA_TCI_MASK; 1842 u16 vlan_proto = meta_data >> RX_CMP_FLAGS2_METADATA_TPID_SFT; 1843 1844 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag); 1845 } 1846 1847 skb_checksum_none_assert(skb); 1848 if (RX_CMP_L4_CS_OK(rxcmp1)) { 1849 if (dev->features & NETIF_F_RXCSUM) { 1850 skb->ip_summed = CHECKSUM_UNNECESSARY; 1851 skb->csum_level = RX_CMP_ENCAP(rxcmp1); 1852 } 1853 } else { 1854 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) { 1855 if (dev->features & NETIF_F_RXCSUM) 1856 bnapi->cp_ring.sw_stats.rx.rx_l4_csum_errors++; 1857 } 1858 } 1859 1860 bnxt_deliver_skb(bp, bnapi, skb); 1861 rc = 1; 1862 1863 next_rx: 1864 cpr->rx_packets += 1; 1865 cpr->rx_bytes += len; 1866 1867 next_rx_no_len: 1868 rxr->rx_prod = NEXT_RX(prod); 1869 rxr->rx_next_cons = NEXT_RX(cons); 1870 1871 next_rx_no_prod_no_len: 1872 *raw_cons = tmp_raw_cons; 1873 1874 return rc; 1875 } 1876 1877 /* In netpoll mode, if we are using a combined completion ring, we need to 1878 * discard the rx packets and recycle the buffers. 1879 */ 1880 static int bnxt_force_rx_discard(struct bnxt *bp, 1881 struct bnxt_cp_ring_info *cpr, 1882 u32 *raw_cons, u8 *event) 1883 { 1884 u32 tmp_raw_cons = *raw_cons; 1885 struct rx_cmp_ext *rxcmp1; 1886 struct rx_cmp *rxcmp; 1887 u16 cp_cons; 1888 u8 cmp_type; 1889 1890 cp_cons = RING_CMP(tmp_raw_cons); 1891 rxcmp = (struct rx_cmp *) 1892 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1893 1894 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons); 1895 cp_cons = RING_CMP(tmp_raw_cons); 1896 rxcmp1 = (struct rx_cmp_ext *) 1897 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1898 1899 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons)) 1900 return -EBUSY; 1901 1902 cmp_type = RX_CMP_TYPE(rxcmp); 1903 if (cmp_type == CMP_TYPE_RX_L2_CMP) { 1904 rxcmp1->rx_cmp_cfa_code_errors_v2 |= 1905 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR); 1906 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) { 1907 struct rx_tpa_end_cmp_ext *tpa_end1; 1908 1909 tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1; 1910 tpa_end1->rx_tpa_end_cmp_errors_v2 |= 1911 cpu_to_le32(RX_TPA_END_CMP_ERRORS); 1912 } 1913 return bnxt_rx_pkt(bp, cpr, raw_cons, event); 1914 } 1915 1916 u32 bnxt_fw_health_readl(struct bnxt *bp, int reg_idx) 1917 { 1918 struct bnxt_fw_health *fw_health = bp->fw_health; 1919 u32 reg = fw_health->regs[reg_idx]; 1920 u32 reg_type, reg_off, val = 0; 1921 1922 reg_type = BNXT_FW_HEALTH_REG_TYPE(reg); 1923 reg_off = BNXT_FW_HEALTH_REG_OFF(reg); 1924 switch (reg_type) { 1925 case BNXT_FW_HEALTH_REG_TYPE_CFG: 1926 pci_read_config_dword(bp->pdev, reg_off, &val); 1927 break; 1928 case BNXT_FW_HEALTH_REG_TYPE_GRC: 1929 reg_off = fw_health->mapped_regs[reg_idx]; 1930 fallthrough; 1931 case BNXT_FW_HEALTH_REG_TYPE_BAR0: 1932 val = readl(bp->bar0 + reg_off); 1933 break; 1934 case BNXT_FW_HEALTH_REG_TYPE_BAR1: 1935 val = readl(bp->bar1 + reg_off); 1936 break; 1937 } 1938 if (reg_idx == BNXT_FW_RESET_INPROG_REG) 1939 val &= fw_health->fw_reset_inprog_reg_mask; 1940 return val; 1941 } 1942 1943 static u16 bnxt_agg_ring_id_to_grp_idx(struct bnxt *bp, u16 ring_id) 1944 { 1945 int i; 1946 1947 for (i = 0; i < bp->rx_nr_rings; i++) { 1948 u16 grp_idx = bp->rx_ring[i].bnapi->index; 1949 struct bnxt_ring_grp_info *grp_info; 1950 1951 grp_info = &bp->grp_info[grp_idx]; 1952 if (grp_info->agg_fw_ring_id == ring_id) 1953 return grp_idx; 1954 } 1955 return INVALID_HW_RING_ID; 1956 } 1957 1958 #define BNXT_GET_EVENT_PORT(data) \ 1959 ((data) & \ 1960 ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK) 1961 1962 #define BNXT_EVENT_RING_TYPE(data2) \ 1963 ((data2) & \ 1964 ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_MASK) 1965 1966 #define BNXT_EVENT_RING_TYPE_RX(data2) \ 1967 (BNXT_EVENT_RING_TYPE(data2) == \ 1968 ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_RX) 1969 1970 static int bnxt_async_event_process(struct bnxt *bp, 1971 struct hwrm_async_event_cmpl *cmpl) 1972 { 1973 u16 event_id = le16_to_cpu(cmpl->event_id); 1974 u32 data1 = le32_to_cpu(cmpl->event_data1); 1975 u32 data2 = le32_to_cpu(cmpl->event_data2); 1976 1977 /* TODO CHIMP_FW: Define event id's for link change, error etc */ 1978 switch (event_id) { 1979 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: { 1980 struct bnxt_link_info *link_info = &bp->link_info; 1981 1982 if (BNXT_VF(bp)) 1983 goto async_event_process_exit; 1984 1985 /* print unsupported speed warning in forced speed mode only */ 1986 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED) && 1987 (data1 & 0x20000)) { 1988 u16 fw_speed = link_info->force_link_speed; 1989 u32 speed = bnxt_fw_to_ethtool_speed(fw_speed); 1990 1991 if (speed != SPEED_UNKNOWN) 1992 netdev_warn(bp->dev, "Link speed %d no longer supported\n", 1993 speed); 1994 } 1995 set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event); 1996 } 1997 fallthrough; 1998 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE: 1999 case ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE: 2000 set_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT, &bp->sp_event); 2001 fallthrough; 2002 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE: 2003 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event); 2004 break; 2005 case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD: 2006 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event); 2007 break; 2008 case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: { 2009 u16 port_id = BNXT_GET_EVENT_PORT(data1); 2010 2011 if (BNXT_VF(bp)) 2012 break; 2013 2014 if (bp->pf.port_id != port_id) 2015 break; 2016 2017 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event); 2018 break; 2019 } 2020 case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE: 2021 if (BNXT_PF(bp)) 2022 goto async_event_process_exit; 2023 set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event); 2024 break; 2025 case ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY: { 2026 char *fatal_str = "non-fatal"; 2027 2028 if (!bp->fw_health) 2029 goto async_event_process_exit; 2030 2031 bp->fw_reset_timestamp = jiffies; 2032 bp->fw_reset_min_dsecs = cmpl->timestamp_lo; 2033 if (!bp->fw_reset_min_dsecs) 2034 bp->fw_reset_min_dsecs = BNXT_DFLT_FW_RST_MIN_DSECS; 2035 bp->fw_reset_max_dsecs = le16_to_cpu(cmpl->timestamp_hi); 2036 if (!bp->fw_reset_max_dsecs) 2037 bp->fw_reset_max_dsecs = BNXT_DFLT_FW_RST_MAX_DSECS; 2038 if (EVENT_DATA1_RESET_NOTIFY_FATAL(data1)) { 2039 fatal_str = "fatal"; 2040 set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state); 2041 } 2042 netif_warn(bp, hw, bp->dev, 2043 "Firmware %s reset event, data1: 0x%x, data2: 0x%x, min wait %u ms, max wait %u ms\n", 2044 fatal_str, data1, data2, 2045 bp->fw_reset_min_dsecs * 100, 2046 bp->fw_reset_max_dsecs * 100); 2047 set_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event); 2048 break; 2049 } 2050 case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY: { 2051 struct bnxt_fw_health *fw_health = bp->fw_health; 2052 2053 if (!fw_health) 2054 goto async_event_process_exit; 2055 2056 fw_health->enabled = EVENT_DATA1_RECOVERY_ENABLED(data1); 2057 fw_health->master = EVENT_DATA1_RECOVERY_MASTER_FUNC(data1); 2058 if (!fw_health->enabled) { 2059 netif_info(bp, drv, bp->dev, 2060 "Error recovery info: error recovery[0]\n"); 2061 break; 2062 } 2063 fw_health->tmr_multiplier = 2064 DIV_ROUND_UP(fw_health->polling_dsecs * HZ, 2065 bp->current_interval * 10); 2066 fw_health->tmr_counter = fw_health->tmr_multiplier; 2067 fw_health->last_fw_heartbeat = 2068 bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG); 2069 fw_health->last_fw_reset_cnt = 2070 bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG); 2071 netif_info(bp, drv, bp->dev, 2072 "Error recovery info: error recovery[1], master[%d], reset count[%u], health status: 0x%x\n", 2073 fw_health->master, fw_health->last_fw_reset_cnt, 2074 bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG)); 2075 goto async_event_process_exit; 2076 } 2077 case ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION: 2078 netif_notice(bp, hw, bp->dev, 2079 "Received firmware debug notification, data1: 0x%x, data2: 0x%x\n", 2080 data1, data2); 2081 goto async_event_process_exit; 2082 case ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG: { 2083 struct bnxt_rx_ring_info *rxr; 2084 u16 grp_idx; 2085 2086 if (bp->flags & BNXT_FLAG_CHIP_P5) 2087 goto async_event_process_exit; 2088 2089 netdev_warn(bp->dev, "Ring monitor event, ring type %lu id 0x%x\n", 2090 BNXT_EVENT_RING_TYPE(data2), data1); 2091 if (!BNXT_EVENT_RING_TYPE_RX(data2)) 2092 goto async_event_process_exit; 2093 2094 grp_idx = bnxt_agg_ring_id_to_grp_idx(bp, data1); 2095 if (grp_idx == INVALID_HW_RING_ID) { 2096 netdev_warn(bp->dev, "Unknown RX agg ring id 0x%x\n", 2097 data1); 2098 goto async_event_process_exit; 2099 } 2100 rxr = bp->bnapi[grp_idx]->rx_ring; 2101 bnxt_sched_reset(bp, rxr); 2102 goto async_event_process_exit; 2103 } 2104 case ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST: { 2105 struct bnxt_fw_health *fw_health = bp->fw_health; 2106 2107 netif_notice(bp, hw, bp->dev, 2108 "Received firmware echo request, data1: 0x%x, data2: 0x%x\n", 2109 data1, data2); 2110 if (fw_health) { 2111 fw_health->echo_req_data1 = data1; 2112 fw_health->echo_req_data2 = data2; 2113 set_bit(BNXT_FW_ECHO_REQUEST_SP_EVENT, &bp->sp_event); 2114 break; 2115 } 2116 goto async_event_process_exit; 2117 } 2118 default: 2119 goto async_event_process_exit; 2120 } 2121 bnxt_queue_sp_work(bp); 2122 async_event_process_exit: 2123 bnxt_ulp_async_events(bp, cmpl); 2124 return 0; 2125 } 2126 2127 static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp) 2128 { 2129 u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id; 2130 struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp; 2131 struct hwrm_fwd_req_cmpl *fwd_req_cmpl = 2132 (struct hwrm_fwd_req_cmpl *)txcmp; 2133 2134 switch (cmpl_type) { 2135 case CMPL_BASE_TYPE_HWRM_DONE: 2136 seq_id = le16_to_cpu(h_cmpl->sequence_id); 2137 if (seq_id == bp->hwrm_intr_seq_id) 2138 bp->hwrm_intr_seq_id = (u16)~bp->hwrm_intr_seq_id; 2139 else 2140 netdev_err(bp->dev, "Invalid hwrm seq id %d\n", seq_id); 2141 break; 2142 2143 case CMPL_BASE_TYPE_HWRM_FWD_REQ: 2144 vf_id = le16_to_cpu(fwd_req_cmpl->source_id); 2145 2146 if ((vf_id < bp->pf.first_vf_id) || 2147 (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) { 2148 netdev_err(bp->dev, "Msg contains invalid VF id %x\n", 2149 vf_id); 2150 return -EINVAL; 2151 } 2152 2153 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap); 2154 set_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event); 2155 bnxt_queue_sp_work(bp); 2156 break; 2157 2158 case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT: 2159 bnxt_async_event_process(bp, 2160 (struct hwrm_async_event_cmpl *)txcmp); 2161 2162 default: 2163 break; 2164 } 2165 2166 return 0; 2167 } 2168 2169 static irqreturn_t bnxt_msix(int irq, void *dev_instance) 2170 { 2171 struct bnxt_napi *bnapi = dev_instance; 2172 struct bnxt *bp = bnapi->bp; 2173 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2174 u32 cons = RING_CMP(cpr->cp_raw_cons); 2175 2176 cpr->event_ctr++; 2177 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]); 2178 napi_schedule(&bnapi->napi); 2179 return IRQ_HANDLED; 2180 } 2181 2182 static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr) 2183 { 2184 u32 raw_cons = cpr->cp_raw_cons; 2185 u16 cons = RING_CMP(raw_cons); 2186 struct tx_cmp *txcmp; 2187 2188 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]; 2189 2190 return TX_CMP_VALID(txcmp, raw_cons); 2191 } 2192 2193 static irqreturn_t bnxt_inta(int irq, void *dev_instance) 2194 { 2195 struct bnxt_napi *bnapi = dev_instance; 2196 struct bnxt *bp = bnapi->bp; 2197 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2198 u32 cons = RING_CMP(cpr->cp_raw_cons); 2199 u32 int_status; 2200 2201 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]); 2202 2203 if (!bnxt_has_work(bp, cpr)) { 2204 int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS); 2205 /* return if erroneous interrupt */ 2206 if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id))) 2207 return IRQ_NONE; 2208 } 2209 2210 /* disable ring IRQ */ 2211 BNXT_CP_DB_IRQ_DIS(cpr->cp_db.doorbell); 2212 2213 /* Return here if interrupt is shared and is disabled. */ 2214 if (unlikely(atomic_read(&bp->intr_sem) != 0)) 2215 return IRQ_HANDLED; 2216 2217 napi_schedule(&bnapi->napi); 2218 return IRQ_HANDLED; 2219 } 2220 2221 static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 2222 int budget) 2223 { 2224 struct bnxt_napi *bnapi = cpr->bnapi; 2225 u32 raw_cons = cpr->cp_raw_cons; 2226 u32 cons; 2227 int tx_pkts = 0; 2228 int rx_pkts = 0; 2229 u8 event = 0; 2230 struct tx_cmp *txcmp; 2231 2232 cpr->has_more_work = 0; 2233 cpr->had_work_done = 1; 2234 while (1) { 2235 int rc; 2236 2237 cons = RING_CMP(raw_cons); 2238 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]; 2239 2240 if (!TX_CMP_VALID(txcmp, raw_cons)) 2241 break; 2242 2243 /* The valid test of the entry must be done first before 2244 * reading any further. 2245 */ 2246 dma_rmb(); 2247 if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) { 2248 tx_pkts++; 2249 /* return full budget so NAPI will complete. */ 2250 if (unlikely(tx_pkts > bp->tx_wake_thresh)) { 2251 rx_pkts = budget; 2252 raw_cons = NEXT_RAW_CMP(raw_cons); 2253 if (budget) 2254 cpr->has_more_work = 1; 2255 break; 2256 } 2257 } else if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) { 2258 if (likely(budget)) 2259 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event); 2260 else 2261 rc = bnxt_force_rx_discard(bp, cpr, &raw_cons, 2262 &event); 2263 if (likely(rc >= 0)) 2264 rx_pkts += rc; 2265 /* Increment rx_pkts when rc is -ENOMEM to count towards 2266 * the NAPI budget. Otherwise, we may potentially loop 2267 * here forever if we consistently cannot allocate 2268 * buffers. 2269 */ 2270 else if (rc == -ENOMEM && budget) 2271 rx_pkts++; 2272 else if (rc == -EBUSY) /* partial completion */ 2273 break; 2274 } else if (unlikely((TX_CMP_TYPE(txcmp) == 2275 CMPL_BASE_TYPE_HWRM_DONE) || 2276 (TX_CMP_TYPE(txcmp) == 2277 CMPL_BASE_TYPE_HWRM_FWD_REQ) || 2278 (TX_CMP_TYPE(txcmp) == 2279 CMPL_BASE_TYPE_HWRM_ASYNC_EVENT))) { 2280 bnxt_hwrm_handler(bp, txcmp); 2281 } 2282 raw_cons = NEXT_RAW_CMP(raw_cons); 2283 2284 if (rx_pkts && rx_pkts == budget) { 2285 cpr->has_more_work = 1; 2286 break; 2287 } 2288 } 2289 2290 if (event & BNXT_REDIRECT_EVENT) 2291 xdp_do_flush_map(); 2292 2293 if (event & BNXT_TX_EVENT) { 2294 struct bnxt_tx_ring_info *txr = bnapi->tx_ring; 2295 u16 prod = txr->tx_prod; 2296 2297 /* Sync BD data before updating doorbell */ 2298 wmb(); 2299 2300 bnxt_db_write_relaxed(bp, &txr->tx_db, prod); 2301 } 2302 2303 cpr->cp_raw_cons = raw_cons; 2304 bnapi->tx_pkts += tx_pkts; 2305 bnapi->events |= event; 2306 return rx_pkts; 2307 } 2308 2309 static void __bnxt_poll_work_done(struct bnxt *bp, struct bnxt_napi *bnapi) 2310 { 2311 if (bnapi->tx_pkts) { 2312 bnapi->tx_int(bp, bnapi, bnapi->tx_pkts); 2313 bnapi->tx_pkts = 0; 2314 } 2315 2316 if ((bnapi->events & BNXT_RX_EVENT) && !(bnapi->in_reset)) { 2317 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 2318 2319 if (bnapi->events & BNXT_AGG_EVENT) 2320 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); 2321 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 2322 } 2323 bnapi->events = 0; 2324 } 2325 2326 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 2327 int budget) 2328 { 2329 struct bnxt_napi *bnapi = cpr->bnapi; 2330 int rx_pkts; 2331 2332 rx_pkts = __bnxt_poll_work(bp, cpr, budget); 2333 2334 /* ACK completion ring before freeing tx ring and producing new 2335 * buffers in rx/agg rings to prevent overflowing the completion 2336 * ring. 2337 */ 2338 bnxt_db_cq(bp, &cpr->cp_db, cpr->cp_raw_cons); 2339 2340 __bnxt_poll_work_done(bp, bnapi); 2341 return rx_pkts; 2342 } 2343 2344 static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget) 2345 { 2346 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi); 2347 struct bnxt *bp = bnapi->bp; 2348 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2349 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 2350 struct tx_cmp *txcmp; 2351 struct rx_cmp_ext *rxcmp1; 2352 u32 cp_cons, tmp_raw_cons; 2353 u32 raw_cons = cpr->cp_raw_cons; 2354 u32 rx_pkts = 0; 2355 u8 event = 0; 2356 2357 while (1) { 2358 int rc; 2359 2360 cp_cons = RING_CMP(raw_cons); 2361 txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 2362 2363 if (!TX_CMP_VALID(txcmp, raw_cons)) 2364 break; 2365 2366 if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) { 2367 tmp_raw_cons = NEXT_RAW_CMP(raw_cons); 2368 cp_cons = RING_CMP(tmp_raw_cons); 2369 rxcmp1 = (struct rx_cmp_ext *) 2370 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 2371 2372 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons)) 2373 break; 2374 2375 /* force an error to recycle the buffer */ 2376 rxcmp1->rx_cmp_cfa_code_errors_v2 |= 2377 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR); 2378 2379 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event); 2380 if (likely(rc == -EIO) && budget) 2381 rx_pkts++; 2382 else if (rc == -EBUSY) /* partial completion */ 2383 break; 2384 } else if (unlikely(TX_CMP_TYPE(txcmp) == 2385 CMPL_BASE_TYPE_HWRM_DONE)) { 2386 bnxt_hwrm_handler(bp, txcmp); 2387 } else { 2388 netdev_err(bp->dev, 2389 "Invalid completion received on special ring\n"); 2390 } 2391 raw_cons = NEXT_RAW_CMP(raw_cons); 2392 2393 if (rx_pkts == budget) 2394 break; 2395 } 2396 2397 cpr->cp_raw_cons = raw_cons; 2398 BNXT_DB_CQ(&cpr->cp_db, cpr->cp_raw_cons); 2399 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 2400 2401 if (event & BNXT_AGG_EVENT) 2402 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); 2403 2404 if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) { 2405 napi_complete_done(napi, rx_pkts); 2406 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons); 2407 } 2408 return rx_pkts; 2409 } 2410 2411 static int bnxt_poll(struct napi_struct *napi, int budget) 2412 { 2413 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi); 2414 struct bnxt *bp = bnapi->bp; 2415 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2416 int work_done = 0; 2417 2418 if (unlikely(test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))) { 2419 napi_complete(napi); 2420 return 0; 2421 } 2422 while (1) { 2423 work_done += bnxt_poll_work(bp, cpr, budget - work_done); 2424 2425 if (work_done >= budget) { 2426 if (!budget) 2427 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons); 2428 break; 2429 } 2430 2431 if (!bnxt_has_work(bp, cpr)) { 2432 if (napi_complete_done(napi, work_done)) 2433 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons); 2434 break; 2435 } 2436 } 2437 if (bp->flags & BNXT_FLAG_DIM) { 2438 struct dim_sample dim_sample = {}; 2439 2440 dim_update_sample(cpr->event_ctr, 2441 cpr->rx_packets, 2442 cpr->rx_bytes, 2443 &dim_sample); 2444 net_dim(&cpr->dim, dim_sample); 2445 } 2446 return work_done; 2447 } 2448 2449 static int __bnxt_poll_cqs(struct bnxt *bp, struct bnxt_napi *bnapi, int budget) 2450 { 2451 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2452 int i, work_done = 0; 2453 2454 for (i = 0; i < 2; i++) { 2455 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i]; 2456 2457 if (cpr2) { 2458 work_done += __bnxt_poll_work(bp, cpr2, 2459 budget - work_done); 2460 cpr->has_more_work |= cpr2->has_more_work; 2461 } 2462 } 2463 return work_done; 2464 } 2465 2466 static void __bnxt_poll_cqs_done(struct bnxt *bp, struct bnxt_napi *bnapi, 2467 u64 dbr_type) 2468 { 2469 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2470 int i; 2471 2472 for (i = 0; i < 2; i++) { 2473 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i]; 2474 struct bnxt_db_info *db; 2475 2476 if (cpr2 && cpr2->had_work_done) { 2477 db = &cpr2->cp_db; 2478 writeq(db->db_key64 | dbr_type | 2479 RING_CMP(cpr2->cp_raw_cons), db->doorbell); 2480 cpr2->had_work_done = 0; 2481 } 2482 } 2483 __bnxt_poll_work_done(bp, bnapi); 2484 } 2485 2486 static int bnxt_poll_p5(struct napi_struct *napi, int budget) 2487 { 2488 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi); 2489 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2490 u32 raw_cons = cpr->cp_raw_cons; 2491 struct bnxt *bp = bnapi->bp; 2492 struct nqe_cn *nqcmp; 2493 int work_done = 0; 2494 u32 cons; 2495 2496 if (unlikely(test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))) { 2497 napi_complete(napi); 2498 return 0; 2499 } 2500 if (cpr->has_more_work) { 2501 cpr->has_more_work = 0; 2502 work_done = __bnxt_poll_cqs(bp, bnapi, budget); 2503 } 2504 while (1) { 2505 cons = RING_CMP(raw_cons); 2506 nqcmp = &cpr->nq_desc_ring[CP_RING(cons)][CP_IDX(cons)]; 2507 2508 if (!NQ_CMP_VALID(nqcmp, raw_cons)) { 2509 if (cpr->has_more_work) 2510 break; 2511 2512 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ_ARMALL); 2513 cpr->cp_raw_cons = raw_cons; 2514 if (napi_complete_done(napi, work_done)) 2515 BNXT_DB_NQ_ARM_P5(&cpr->cp_db, 2516 cpr->cp_raw_cons); 2517 return work_done; 2518 } 2519 2520 /* The valid test of the entry must be done first before 2521 * reading any further. 2522 */ 2523 dma_rmb(); 2524 2525 if (nqcmp->type == cpu_to_le16(NQ_CN_TYPE_CQ_NOTIFICATION)) { 2526 u32 idx = le32_to_cpu(nqcmp->cq_handle_low); 2527 struct bnxt_cp_ring_info *cpr2; 2528 2529 cpr2 = cpr->cp_ring_arr[idx]; 2530 work_done += __bnxt_poll_work(bp, cpr2, 2531 budget - work_done); 2532 cpr->has_more_work |= cpr2->has_more_work; 2533 } else { 2534 bnxt_hwrm_handler(bp, (struct tx_cmp *)nqcmp); 2535 } 2536 raw_cons = NEXT_RAW_CMP(raw_cons); 2537 } 2538 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ); 2539 if (raw_cons != cpr->cp_raw_cons) { 2540 cpr->cp_raw_cons = raw_cons; 2541 BNXT_DB_NQ_P5(&cpr->cp_db, raw_cons); 2542 } 2543 return work_done; 2544 } 2545 2546 static void bnxt_free_tx_skbs(struct bnxt *bp) 2547 { 2548 int i, max_idx; 2549 struct pci_dev *pdev = bp->pdev; 2550 2551 if (!bp->tx_ring) 2552 return; 2553 2554 max_idx = bp->tx_nr_pages * TX_DESC_CNT; 2555 for (i = 0; i < bp->tx_nr_rings; i++) { 2556 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 2557 int j; 2558 2559 for (j = 0; j < max_idx;) { 2560 struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j]; 2561 struct sk_buff *skb; 2562 int k, last; 2563 2564 if (i < bp->tx_nr_rings_xdp && 2565 tx_buf->action == XDP_REDIRECT) { 2566 dma_unmap_single(&pdev->dev, 2567 dma_unmap_addr(tx_buf, mapping), 2568 dma_unmap_len(tx_buf, len), 2569 PCI_DMA_TODEVICE); 2570 xdp_return_frame(tx_buf->xdpf); 2571 tx_buf->action = 0; 2572 tx_buf->xdpf = NULL; 2573 j++; 2574 continue; 2575 } 2576 2577 skb = tx_buf->skb; 2578 if (!skb) { 2579 j++; 2580 continue; 2581 } 2582 2583 tx_buf->skb = NULL; 2584 2585 if (tx_buf->is_push) { 2586 dev_kfree_skb(skb); 2587 j += 2; 2588 continue; 2589 } 2590 2591 dma_unmap_single(&pdev->dev, 2592 dma_unmap_addr(tx_buf, mapping), 2593 skb_headlen(skb), 2594 PCI_DMA_TODEVICE); 2595 2596 last = tx_buf->nr_frags; 2597 j += 2; 2598 for (k = 0; k < last; k++, j++) { 2599 int ring_idx = j & bp->tx_ring_mask; 2600 skb_frag_t *frag = &skb_shinfo(skb)->frags[k]; 2601 2602 tx_buf = &txr->tx_buf_ring[ring_idx]; 2603 dma_unmap_page( 2604 &pdev->dev, 2605 dma_unmap_addr(tx_buf, mapping), 2606 skb_frag_size(frag), PCI_DMA_TODEVICE); 2607 } 2608 dev_kfree_skb(skb); 2609 } 2610 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i)); 2611 } 2612 } 2613 2614 static void bnxt_free_one_rx_ring_skbs(struct bnxt *bp, int ring_nr) 2615 { 2616 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr]; 2617 struct pci_dev *pdev = bp->pdev; 2618 struct bnxt_tpa_idx_map *map; 2619 int i, max_idx, max_agg_idx; 2620 2621 max_idx = bp->rx_nr_pages * RX_DESC_CNT; 2622 max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT; 2623 if (!rxr->rx_tpa) 2624 goto skip_rx_tpa_free; 2625 2626 for (i = 0; i < bp->max_tpa; i++) { 2627 struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[i]; 2628 u8 *data = tpa_info->data; 2629 2630 if (!data) 2631 continue; 2632 2633 dma_unmap_single_attrs(&pdev->dev, tpa_info->mapping, 2634 bp->rx_buf_use_size, bp->rx_dir, 2635 DMA_ATTR_WEAK_ORDERING); 2636 2637 tpa_info->data = NULL; 2638 2639 kfree(data); 2640 } 2641 2642 skip_rx_tpa_free: 2643 for (i = 0; i < max_idx; i++) { 2644 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[i]; 2645 dma_addr_t mapping = rx_buf->mapping; 2646 void *data = rx_buf->data; 2647 2648 if (!data) 2649 continue; 2650 2651 rx_buf->data = NULL; 2652 if (BNXT_RX_PAGE_MODE(bp)) { 2653 mapping -= bp->rx_dma_offset; 2654 dma_unmap_page_attrs(&pdev->dev, mapping, PAGE_SIZE, 2655 bp->rx_dir, 2656 DMA_ATTR_WEAK_ORDERING); 2657 page_pool_recycle_direct(rxr->page_pool, data); 2658 } else { 2659 dma_unmap_single_attrs(&pdev->dev, mapping, 2660 bp->rx_buf_use_size, bp->rx_dir, 2661 DMA_ATTR_WEAK_ORDERING); 2662 kfree(data); 2663 } 2664 } 2665 for (i = 0; i < max_agg_idx; i++) { 2666 struct bnxt_sw_rx_agg_bd *rx_agg_buf = &rxr->rx_agg_ring[i]; 2667 struct page *page = rx_agg_buf->page; 2668 2669 if (!page) 2670 continue; 2671 2672 dma_unmap_page_attrs(&pdev->dev, rx_agg_buf->mapping, 2673 BNXT_RX_PAGE_SIZE, PCI_DMA_FROMDEVICE, 2674 DMA_ATTR_WEAK_ORDERING); 2675 2676 rx_agg_buf->page = NULL; 2677 __clear_bit(i, rxr->rx_agg_bmap); 2678 2679 __free_page(page); 2680 } 2681 if (rxr->rx_page) { 2682 __free_page(rxr->rx_page); 2683 rxr->rx_page = NULL; 2684 } 2685 map = rxr->rx_tpa_idx_map; 2686 if (map) 2687 memset(map->agg_idx_bmap, 0, sizeof(map->agg_idx_bmap)); 2688 } 2689 2690 static void bnxt_free_rx_skbs(struct bnxt *bp) 2691 { 2692 int i; 2693 2694 if (!bp->rx_ring) 2695 return; 2696 2697 for (i = 0; i < bp->rx_nr_rings; i++) 2698 bnxt_free_one_rx_ring_skbs(bp, i); 2699 } 2700 2701 static void bnxt_free_skbs(struct bnxt *bp) 2702 { 2703 bnxt_free_tx_skbs(bp); 2704 bnxt_free_rx_skbs(bp); 2705 } 2706 2707 static void bnxt_init_ctx_mem(struct bnxt_mem_init *mem_init, void *p, int len) 2708 { 2709 u8 init_val = mem_init->init_val; 2710 u16 offset = mem_init->offset; 2711 u8 *p2 = p; 2712 int i; 2713 2714 if (!init_val) 2715 return; 2716 if (offset == BNXT_MEM_INVALID_OFFSET) { 2717 memset(p, init_val, len); 2718 return; 2719 } 2720 for (i = 0; i < len; i += mem_init->size) 2721 *(p2 + i + offset) = init_val; 2722 } 2723 2724 static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem) 2725 { 2726 struct pci_dev *pdev = bp->pdev; 2727 int i; 2728 2729 for (i = 0; i < rmem->nr_pages; i++) { 2730 if (!rmem->pg_arr[i]) 2731 continue; 2732 2733 dma_free_coherent(&pdev->dev, rmem->page_size, 2734 rmem->pg_arr[i], rmem->dma_arr[i]); 2735 2736 rmem->pg_arr[i] = NULL; 2737 } 2738 if (rmem->pg_tbl) { 2739 size_t pg_tbl_size = rmem->nr_pages * 8; 2740 2741 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG) 2742 pg_tbl_size = rmem->page_size; 2743 dma_free_coherent(&pdev->dev, pg_tbl_size, 2744 rmem->pg_tbl, rmem->pg_tbl_map); 2745 rmem->pg_tbl = NULL; 2746 } 2747 if (rmem->vmem_size && *rmem->vmem) { 2748 vfree(*rmem->vmem); 2749 *rmem->vmem = NULL; 2750 } 2751 } 2752 2753 static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem) 2754 { 2755 struct pci_dev *pdev = bp->pdev; 2756 u64 valid_bit = 0; 2757 int i; 2758 2759 if (rmem->flags & (BNXT_RMEM_VALID_PTE_FLAG | BNXT_RMEM_RING_PTE_FLAG)) 2760 valid_bit = PTU_PTE_VALID; 2761 if ((rmem->nr_pages > 1 || rmem->depth > 0) && !rmem->pg_tbl) { 2762 size_t pg_tbl_size = rmem->nr_pages * 8; 2763 2764 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG) 2765 pg_tbl_size = rmem->page_size; 2766 rmem->pg_tbl = dma_alloc_coherent(&pdev->dev, pg_tbl_size, 2767 &rmem->pg_tbl_map, 2768 GFP_KERNEL); 2769 if (!rmem->pg_tbl) 2770 return -ENOMEM; 2771 } 2772 2773 for (i = 0; i < rmem->nr_pages; i++) { 2774 u64 extra_bits = valid_bit; 2775 2776 rmem->pg_arr[i] = dma_alloc_coherent(&pdev->dev, 2777 rmem->page_size, 2778 &rmem->dma_arr[i], 2779 GFP_KERNEL); 2780 if (!rmem->pg_arr[i]) 2781 return -ENOMEM; 2782 2783 if (rmem->mem_init) 2784 bnxt_init_ctx_mem(rmem->mem_init, rmem->pg_arr[i], 2785 rmem->page_size); 2786 if (rmem->nr_pages > 1 || rmem->depth > 0) { 2787 if (i == rmem->nr_pages - 2 && 2788 (rmem->flags & BNXT_RMEM_RING_PTE_FLAG)) 2789 extra_bits |= PTU_PTE_NEXT_TO_LAST; 2790 else if (i == rmem->nr_pages - 1 && 2791 (rmem->flags & BNXT_RMEM_RING_PTE_FLAG)) 2792 extra_bits |= PTU_PTE_LAST; 2793 rmem->pg_tbl[i] = 2794 cpu_to_le64(rmem->dma_arr[i] | extra_bits); 2795 } 2796 } 2797 2798 if (rmem->vmem_size) { 2799 *rmem->vmem = vzalloc(rmem->vmem_size); 2800 if (!(*rmem->vmem)) 2801 return -ENOMEM; 2802 } 2803 return 0; 2804 } 2805 2806 static void bnxt_free_tpa_info(struct bnxt *bp) 2807 { 2808 int i; 2809 2810 for (i = 0; i < bp->rx_nr_rings; i++) { 2811 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 2812 2813 kfree(rxr->rx_tpa_idx_map); 2814 rxr->rx_tpa_idx_map = NULL; 2815 if (rxr->rx_tpa) { 2816 kfree(rxr->rx_tpa[0].agg_arr); 2817 rxr->rx_tpa[0].agg_arr = NULL; 2818 } 2819 kfree(rxr->rx_tpa); 2820 rxr->rx_tpa = NULL; 2821 } 2822 } 2823 2824 static int bnxt_alloc_tpa_info(struct bnxt *bp) 2825 { 2826 int i, j, total_aggs = 0; 2827 2828 bp->max_tpa = MAX_TPA; 2829 if (bp->flags & BNXT_FLAG_CHIP_P5) { 2830 if (!bp->max_tpa_v2) 2831 return 0; 2832 bp->max_tpa = max_t(u16, bp->max_tpa_v2, MAX_TPA_P5); 2833 total_aggs = bp->max_tpa * MAX_SKB_FRAGS; 2834 } 2835 2836 for (i = 0; i < bp->rx_nr_rings; i++) { 2837 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 2838 struct rx_agg_cmp *agg; 2839 2840 rxr->rx_tpa = kcalloc(bp->max_tpa, sizeof(struct bnxt_tpa_info), 2841 GFP_KERNEL); 2842 if (!rxr->rx_tpa) 2843 return -ENOMEM; 2844 2845 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 2846 continue; 2847 agg = kcalloc(total_aggs, sizeof(*agg), GFP_KERNEL); 2848 rxr->rx_tpa[0].agg_arr = agg; 2849 if (!agg) 2850 return -ENOMEM; 2851 for (j = 1; j < bp->max_tpa; j++) 2852 rxr->rx_tpa[j].agg_arr = agg + j * MAX_SKB_FRAGS; 2853 rxr->rx_tpa_idx_map = kzalloc(sizeof(*rxr->rx_tpa_idx_map), 2854 GFP_KERNEL); 2855 if (!rxr->rx_tpa_idx_map) 2856 return -ENOMEM; 2857 } 2858 return 0; 2859 } 2860 2861 static void bnxt_free_rx_rings(struct bnxt *bp) 2862 { 2863 int i; 2864 2865 if (!bp->rx_ring) 2866 return; 2867 2868 bnxt_free_tpa_info(bp); 2869 for (i = 0; i < bp->rx_nr_rings; i++) { 2870 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 2871 struct bnxt_ring_struct *ring; 2872 2873 if (rxr->xdp_prog) 2874 bpf_prog_put(rxr->xdp_prog); 2875 2876 if (xdp_rxq_info_is_reg(&rxr->xdp_rxq)) 2877 xdp_rxq_info_unreg(&rxr->xdp_rxq); 2878 2879 page_pool_destroy(rxr->page_pool); 2880 rxr->page_pool = NULL; 2881 2882 kfree(rxr->rx_agg_bmap); 2883 rxr->rx_agg_bmap = NULL; 2884 2885 ring = &rxr->rx_ring_struct; 2886 bnxt_free_ring(bp, &ring->ring_mem); 2887 2888 ring = &rxr->rx_agg_ring_struct; 2889 bnxt_free_ring(bp, &ring->ring_mem); 2890 } 2891 } 2892 2893 static int bnxt_alloc_rx_page_pool(struct bnxt *bp, 2894 struct bnxt_rx_ring_info *rxr) 2895 { 2896 struct page_pool_params pp = { 0 }; 2897 2898 pp.pool_size = bp->rx_ring_size; 2899 pp.nid = dev_to_node(&bp->pdev->dev); 2900 pp.dev = &bp->pdev->dev; 2901 pp.dma_dir = DMA_BIDIRECTIONAL; 2902 2903 rxr->page_pool = page_pool_create(&pp); 2904 if (IS_ERR(rxr->page_pool)) { 2905 int err = PTR_ERR(rxr->page_pool); 2906 2907 rxr->page_pool = NULL; 2908 return err; 2909 } 2910 return 0; 2911 } 2912 2913 static int bnxt_alloc_rx_rings(struct bnxt *bp) 2914 { 2915 int i, rc = 0, agg_rings = 0; 2916 2917 if (!bp->rx_ring) 2918 return -ENOMEM; 2919 2920 if (bp->flags & BNXT_FLAG_AGG_RINGS) 2921 agg_rings = 1; 2922 2923 for (i = 0; i < bp->rx_nr_rings; i++) { 2924 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 2925 struct bnxt_ring_struct *ring; 2926 2927 ring = &rxr->rx_ring_struct; 2928 2929 rc = bnxt_alloc_rx_page_pool(bp, rxr); 2930 if (rc) 2931 return rc; 2932 2933 rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i, 0); 2934 if (rc < 0) 2935 return rc; 2936 2937 rc = xdp_rxq_info_reg_mem_model(&rxr->xdp_rxq, 2938 MEM_TYPE_PAGE_POOL, 2939 rxr->page_pool); 2940 if (rc) { 2941 xdp_rxq_info_unreg(&rxr->xdp_rxq); 2942 return rc; 2943 } 2944 2945 rc = bnxt_alloc_ring(bp, &ring->ring_mem); 2946 if (rc) 2947 return rc; 2948 2949 ring->grp_idx = i; 2950 if (agg_rings) { 2951 u16 mem_size; 2952 2953 ring = &rxr->rx_agg_ring_struct; 2954 rc = bnxt_alloc_ring(bp, &ring->ring_mem); 2955 if (rc) 2956 return rc; 2957 2958 ring->grp_idx = i; 2959 rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1; 2960 mem_size = rxr->rx_agg_bmap_size / 8; 2961 rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL); 2962 if (!rxr->rx_agg_bmap) 2963 return -ENOMEM; 2964 } 2965 } 2966 if (bp->flags & BNXT_FLAG_TPA) 2967 rc = bnxt_alloc_tpa_info(bp); 2968 return rc; 2969 } 2970 2971 static void bnxt_free_tx_rings(struct bnxt *bp) 2972 { 2973 int i; 2974 struct pci_dev *pdev = bp->pdev; 2975 2976 if (!bp->tx_ring) 2977 return; 2978 2979 for (i = 0; i < bp->tx_nr_rings; i++) { 2980 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 2981 struct bnxt_ring_struct *ring; 2982 2983 if (txr->tx_push) { 2984 dma_free_coherent(&pdev->dev, bp->tx_push_size, 2985 txr->tx_push, txr->tx_push_mapping); 2986 txr->tx_push = NULL; 2987 } 2988 2989 ring = &txr->tx_ring_struct; 2990 2991 bnxt_free_ring(bp, &ring->ring_mem); 2992 } 2993 } 2994 2995 static int bnxt_alloc_tx_rings(struct bnxt *bp) 2996 { 2997 int i, j, rc; 2998 struct pci_dev *pdev = bp->pdev; 2999 3000 bp->tx_push_size = 0; 3001 if (bp->tx_push_thresh) { 3002 int push_size; 3003 3004 push_size = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) + 3005 bp->tx_push_thresh); 3006 3007 if (push_size > 256) { 3008 push_size = 0; 3009 bp->tx_push_thresh = 0; 3010 } 3011 3012 bp->tx_push_size = push_size; 3013 } 3014 3015 for (i = 0, j = 0; i < bp->tx_nr_rings; i++) { 3016 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 3017 struct bnxt_ring_struct *ring; 3018 u8 qidx; 3019 3020 ring = &txr->tx_ring_struct; 3021 3022 rc = bnxt_alloc_ring(bp, &ring->ring_mem); 3023 if (rc) 3024 return rc; 3025 3026 ring->grp_idx = txr->bnapi->index; 3027 if (bp->tx_push_size) { 3028 dma_addr_t mapping; 3029 3030 /* One pre-allocated DMA buffer to backup 3031 * TX push operation 3032 */ 3033 txr->tx_push = dma_alloc_coherent(&pdev->dev, 3034 bp->tx_push_size, 3035 &txr->tx_push_mapping, 3036 GFP_KERNEL); 3037 3038 if (!txr->tx_push) 3039 return -ENOMEM; 3040 3041 mapping = txr->tx_push_mapping + 3042 sizeof(struct tx_push_bd); 3043 txr->data_mapping = cpu_to_le64(mapping); 3044 } 3045 qidx = bp->tc_to_qidx[j]; 3046 ring->queue_id = bp->q_info[qidx].queue_id; 3047 if (i < bp->tx_nr_rings_xdp) 3048 continue; 3049 if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1)) 3050 j++; 3051 } 3052 return 0; 3053 } 3054 3055 static void bnxt_free_cp_rings(struct bnxt *bp) 3056 { 3057 int i; 3058 3059 if (!bp->bnapi) 3060 return; 3061 3062 for (i = 0; i < bp->cp_nr_rings; i++) { 3063 struct bnxt_napi *bnapi = bp->bnapi[i]; 3064 struct bnxt_cp_ring_info *cpr; 3065 struct bnxt_ring_struct *ring; 3066 int j; 3067 3068 if (!bnapi) 3069 continue; 3070 3071 cpr = &bnapi->cp_ring; 3072 ring = &cpr->cp_ring_struct; 3073 3074 bnxt_free_ring(bp, &ring->ring_mem); 3075 3076 for (j = 0; j < 2; j++) { 3077 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j]; 3078 3079 if (cpr2) { 3080 ring = &cpr2->cp_ring_struct; 3081 bnxt_free_ring(bp, &ring->ring_mem); 3082 kfree(cpr2); 3083 cpr->cp_ring_arr[j] = NULL; 3084 } 3085 } 3086 } 3087 } 3088 3089 static struct bnxt_cp_ring_info *bnxt_alloc_cp_sub_ring(struct bnxt *bp) 3090 { 3091 struct bnxt_ring_mem_info *rmem; 3092 struct bnxt_ring_struct *ring; 3093 struct bnxt_cp_ring_info *cpr; 3094 int rc; 3095 3096 cpr = kzalloc(sizeof(*cpr), GFP_KERNEL); 3097 if (!cpr) 3098 return NULL; 3099 3100 ring = &cpr->cp_ring_struct; 3101 rmem = &ring->ring_mem; 3102 rmem->nr_pages = bp->cp_nr_pages; 3103 rmem->page_size = HW_CMPD_RING_SIZE; 3104 rmem->pg_arr = (void **)cpr->cp_desc_ring; 3105 rmem->dma_arr = cpr->cp_desc_mapping; 3106 rmem->flags = BNXT_RMEM_RING_PTE_FLAG; 3107 rc = bnxt_alloc_ring(bp, rmem); 3108 if (rc) { 3109 bnxt_free_ring(bp, rmem); 3110 kfree(cpr); 3111 cpr = NULL; 3112 } 3113 return cpr; 3114 } 3115 3116 static int bnxt_alloc_cp_rings(struct bnxt *bp) 3117 { 3118 bool sh = !!(bp->flags & BNXT_FLAG_SHARED_RINGS); 3119 int i, rc, ulp_base_vec, ulp_msix; 3120 3121 ulp_msix = bnxt_get_ulp_msix_num(bp); 3122 ulp_base_vec = bnxt_get_ulp_msix_base(bp); 3123 for (i = 0; i < bp->cp_nr_rings; i++) { 3124 struct bnxt_napi *bnapi = bp->bnapi[i]; 3125 struct bnxt_cp_ring_info *cpr; 3126 struct bnxt_ring_struct *ring; 3127 3128 if (!bnapi) 3129 continue; 3130 3131 cpr = &bnapi->cp_ring; 3132 cpr->bnapi = bnapi; 3133 ring = &cpr->cp_ring_struct; 3134 3135 rc = bnxt_alloc_ring(bp, &ring->ring_mem); 3136 if (rc) 3137 return rc; 3138 3139 if (ulp_msix && i >= ulp_base_vec) 3140 ring->map_idx = i + ulp_msix; 3141 else 3142 ring->map_idx = i; 3143 3144 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 3145 continue; 3146 3147 if (i < bp->rx_nr_rings) { 3148 struct bnxt_cp_ring_info *cpr2 = 3149 bnxt_alloc_cp_sub_ring(bp); 3150 3151 cpr->cp_ring_arr[BNXT_RX_HDL] = cpr2; 3152 if (!cpr2) 3153 return -ENOMEM; 3154 cpr2->bnapi = bnapi; 3155 } 3156 if ((sh && i < bp->tx_nr_rings) || 3157 (!sh && i >= bp->rx_nr_rings)) { 3158 struct bnxt_cp_ring_info *cpr2 = 3159 bnxt_alloc_cp_sub_ring(bp); 3160 3161 cpr->cp_ring_arr[BNXT_TX_HDL] = cpr2; 3162 if (!cpr2) 3163 return -ENOMEM; 3164 cpr2->bnapi = bnapi; 3165 } 3166 } 3167 return 0; 3168 } 3169 3170 static void bnxt_init_ring_struct(struct bnxt *bp) 3171 { 3172 int i; 3173 3174 for (i = 0; i < bp->cp_nr_rings; i++) { 3175 struct bnxt_napi *bnapi = bp->bnapi[i]; 3176 struct bnxt_ring_mem_info *rmem; 3177 struct bnxt_cp_ring_info *cpr; 3178 struct bnxt_rx_ring_info *rxr; 3179 struct bnxt_tx_ring_info *txr; 3180 struct bnxt_ring_struct *ring; 3181 3182 if (!bnapi) 3183 continue; 3184 3185 cpr = &bnapi->cp_ring; 3186 ring = &cpr->cp_ring_struct; 3187 rmem = &ring->ring_mem; 3188 rmem->nr_pages = bp->cp_nr_pages; 3189 rmem->page_size = HW_CMPD_RING_SIZE; 3190 rmem->pg_arr = (void **)cpr->cp_desc_ring; 3191 rmem->dma_arr = cpr->cp_desc_mapping; 3192 rmem->vmem_size = 0; 3193 3194 rxr = bnapi->rx_ring; 3195 if (!rxr) 3196 goto skip_rx; 3197 3198 ring = &rxr->rx_ring_struct; 3199 rmem = &ring->ring_mem; 3200 rmem->nr_pages = bp->rx_nr_pages; 3201 rmem->page_size = HW_RXBD_RING_SIZE; 3202 rmem->pg_arr = (void **)rxr->rx_desc_ring; 3203 rmem->dma_arr = rxr->rx_desc_mapping; 3204 rmem->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages; 3205 rmem->vmem = (void **)&rxr->rx_buf_ring; 3206 3207 ring = &rxr->rx_agg_ring_struct; 3208 rmem = &ring->ring_mem; 3209 rmem->nr_pages = bp->rx_agg_nr_pages; 3210 rmem->page_size = HW_RXBD_RING_SIZE; 3211 rmem->pg_arr = (void **)rxr->rx_agg_desc_ring; 3212 rmem->dma_arr = rxr->rx_agg_desc_mapping; 3213 rmem->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages; 3214 rmem->vmem = (void **)&rxr->rx_agg_ring; 3215 3216 skip_rx: 3217 txr = bnapi->tx_ring; 3218 if (!txr) 3219 continue; 3220 3221 ring = &txr->tx_ring_struct; 3222 rmem = &ring->ring_mem; 3223 rmem->nr_pages = bp->tx_nr_pages; 3224 rmem->page_size = HW_RXBD_RING_SIZE; 3225 rmem->pg_arr = (void **)txr->tx_desc_ring; 3226 rmem->dma_arr = txr->tx_desc_mapping; 3227 rmem->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages; 3228 rmem->vmem = (void **)&txr->tx_buf_ring; 3229 } 3230 } 3231 3232 static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type) 3233 { 3234 int i; 3235 u32 prod; 3236 struct rx_bd **rx_buf_ring; 3237 3238 rx_buf_ring = (struct rx_bd **)ring->ring_mem.pg_arr; 3239 for (i = 0, prod = 0; i < ring->ring_mem.nr_pages; i++) { 3240 int j; 3241 struct rx_bd *rxbd; 3242 3243 rxbd = rx_buf_ring[i]; 3244 if (!rxbd) 3245 continue; 3246 3247 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) { 3248 rxbd->rx_bd_len_flags_type = cpu_to_le32(type); 3249 rxbd->rx_bd_opaque = prod; 3250 } 3251 } 3252 } 3253 3254 static int bnxt_alloc_one_rx_ring(struct bnxt *bp, int ring_nr) 3255 { 3256 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr]; 3257 struct net_device *dev = bp->dev; 3258 u32 prod; 3259 int i; 3260 3261 prod = rxr->rx_prod; 3262 for (i = 0; i < bp->rx_ring_size; i++) { 3263 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL)) { 3264 netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n", 3265 ring_nr, i, bp->rx_ring_size); 3266 break; 3267 } 3268 prod = NEXT_RX(prod); 3269 } 3270 rxr->rx_prod = prod; 3271 3272 if (!(bp->flags & BNXT_FLAG_AGG_RINGS)) 3273 return 0; 3274 3275 prod = rxr->rx_agg_prod; 3276 for (i = 0; i < bp->rx_agg_ring_size; i++) { 3277 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL)) { 3278 netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n", 3279 ring_nr, i, bp->rx_ring_size); 3280 break; 3281 } 3282 prod = NEXT_RX_AGG(prod); 3283 } 3284 rxr->rx_agg_prod = prod; 3285 3286 if (rxr->rx_tpa) { 3287 dma_addr_t mapping; 3288 u8 *data; 3289 3290 for (i = 0; i < bp->max_tpa; i++) { 3291 data = __bnxt_alloc_rx_data(bp, &mapping, GFP_KERNEL); 3292 if (!data) 3293 return -ENOMEM; 3294 3295 rxr->rx_tpa[i].data = data; 3296 rxr->rx_tpa[i].data_ptr = data + bp->rx_offset; 3297 rxr->rx_tpa[i].mapping = mapping; 3298 } 3299 } 3300 return 0; 3301 } 3302 3303 static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr) 3304 { 3305 struct bnxt_rx_ring_info *rxr; 3306 struct bnxt_ring_struct *ring; 3307 u32 type; 3308 3309 type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) | 3310 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP; 3311 3312 if (NET_IP_ALIGN == 2) 3313 type |= RX_BD_FLAGS_SOP; 3314 3315 rxr = &bp->rx_ring[ring_nr]; 3316 ring = &rxr->rx_ring_struct; 3317 bnxt_init_rxbd_pages(ring, type); 3318 3319 if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) { 3320 bpf_prog_add(bp->xdp_prog, 1); 3321 rxr->xdp_prog = bp->xdp_prog; 3322 } 3323 ring->fw_ring_id = INVALID_HW_RING_ID; 3324 3325 ring = &rxr->rx_agg_ring_struct; 3326 ring->fw_ring_id = INVALID_HW_RING_ID; 3327 3328 if ((bp->flags & BNXT_FLAG_AGG_RINGS)) { 3329 type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) | 3330 RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP; 3331 3332 bnxt_init_rxbd_pages(ring, type); 3333 } 3334 3335 return bnxt_alloc_one_rx_ring(bp, ring_nr); 3336 } 3337 3338 static void bnxt_init_cp_rings(struct bnxt *bp) 3339 { 3340 int i, j; 3341 3342 for (i = 0; i < bp->cp_nr_rings; i++) { 3343 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring; 3344 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 3345 3346 ring->fw_ring_id = INVALID_HW_RING_ID; 3347 cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks; 3348 cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs; 3349 for (j = 0; j < 2; j++) { 3350 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j]; 3351 3352 if (!cpr2) 3353 continue; 3354 3355 ring = &cpr2->cp_ring_struct; 3356 ring->fw_ring_id = INVALID_HW_RING_ID; 3357 cpr2->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks; 3358 cpr2->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs; 3359 } 3360 } 3361 } 3362 3363 static int bnxt_init_rx_rings(struct bnxt *bp) 3364 { 3365 int i, rc = 0; 3366 3367 if (BNXT_RX_PAGE_MODE(bp)) { 3368 bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM; 3369 bp->rx_dma_offset = XDP_PACKET_HEADROOM; 3370 } else { 3371 bp->rx_offset = BNXT_RX_OFFSET; 3372 bp->rx_dma_offset = BNXT_RX_DMA_OFFSET; 3373 } 3374 3375 for (i = 0; i < bp->rx_nr_rings; i++) { 3376 rc = bnxt_init_one_rx_ring(bp, i); 3377 if (rc) 3378 break; 3379 } 3380 3381 return rc; 3382 } 3383 3384 static int bnxt_init_tx_rings(struct bnxt *bp) 3385 { 3386 u16 i; 3387 3388 bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2, 3389 MAX_SKB_FRAGS + 1); 3390 3391 for (i = 0; i < bp->tx_nr_rings; i++) { 3392 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 3393 struct bnxt_ring_struct *ring = &txr->tx_ring_struct; 3394 3395 ring->fw_ring_id = INVALID_HW_RING_ID; 3396 } 3397 3398 return 0; 3399 } 3400 3401 static void bnxt_free_ring_grps(struct bnxt *bp) 3402 { 3403 kfree(bp->grp_info); 3404 bp->grp_info = NULL; 3405 } 3406 3407 static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init) 3408 { 3409 int i; 3410 3411 if (irq_re_init) { 3412 bp->grp_info = kcalloc(bp->cp_nr_rings, 3413 sizeof(struct bnxt_ring_grp_info), 3414 GFP_KERNEL); 3415 if (!bp->grp_info) 3416 return -ENOMEM; 3417 } 3418 for (i = 0; i < bp->cp_nr_rings; i++) { 3419 if (irq_re_init) 3420 bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID; 3421 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID; 3422 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID; 3423 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID; 3424 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID; 3425 } 3426 return 0; 3427 } 3428 3429 static void bnxt_free_vnics(struct bnxt *bp) 3430 { 3431 kfree(bp->vnic_info); 3432 bp->vnic_info = NULL; 3433 bp->nr_vnics = 0; 3434 } 3435 3436 static int bnxt_alloc_vnics(struct bnxt *bp) 3437 { 3438 int num_vnics = 1; 3439 3440 #ifdef CONFIG_RFS_ACCEL 3441 if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS) 3442 num_vnics += bp->rx_nr_rings; 3443 #endif 3444 3445 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 3446 num_vnics++; 3447 3448 bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info), 3449 GFP_KERNEL); 3450 if (!bp->vnic_info) 3451 return -ENOMEM; 3452 3453 bp->nr_vnics = num_vnics; 3454 return 0; 3455 } 3456 3457 static void bnxt_init_vnics(struct bnxt *bp) 3458 { 3459 int i; 3460 3461 for (i = 0; i < bp->nr_vnics; i++) { 3462 struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; 3463 int j; 3464 3465 vnic->fw_vnic_id = INVALID_HW_RING_ID; 3466 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) 3467 vnic->fw_rss_cos_lb_ctx[j] = INVALID_HW_RING_ID; 3468 3469 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID; 3470 3471 if (bp->vnic_info[i].rss_hash_key) { 3472 if (i == 0) 3473 prandom_bytes(vnic->rss_hash_key, 3474 HW_HASH_KEY_SIZE); 3475 else 3476 memcpy(vnic->rss_hash_key, 3477 bp->vnic_info[0].rss_hash_key, 3478 HW_HASH_KEY_SIZE); 3479 } 3480 } 3481 } 3482 3483 static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg) 3484 { 3485 int pages; 3486 3487 pages = ring_size / desc_per_pg; 3488 3489 if (!pages) 3490 return 1; 3491 3492 pages++; 3493 3494 while (pages & (pages - 1)) 3495 pages++; 3496 3497 return pages; 3498 } 3499 3500 void bnxt_set_tpa_flags(struct bnxt *bp) 3501 { 3502 bp->flags &= ~BNXT_FLAG_TPA; 3503 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS) 3504 return; 3505 if (bp->dev->features & NETIF_F_LRO) 3506 bp->flags |= BNXT_FLAG_LRO; 3507 else if (bp->dev->features & NETIF_F_GRO_HW) 3508 bp->flags |= BNXT_FLAG_GRO; 3509 } 3510 3511 /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must 3512 * be set on entry. 3513 */ 3514 void bnxt_set_ring_params(struct bnxt *bp) 3515 { 3516 u32 ring_size, rx_size, rx_space, max_rx_cmpl; 3517 u32 agg_factor = 0, agg_ring_size = 0; 3518 3519 /* 8 for CRC and VLAN */ 3520 rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8); 3521 3522 rx_space = rx_size + NET_SKB_PAD + 3523 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 3524 3525 bp->rx_copy_thresh = BNXT_RX_COPY_THRESH; 3526 ring_size = bp->rx_ring_size; 3527 bp->rx_agg_ring_size = 0; 3528 bp->rx_agg_nr_pages = 0; 3529 3530 if (bp->flags & BNXT_FLAG_TPA) 3531 agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE); 3532 3533 bp->flags &= ~BNXT_FLAG_JUMBO; 3534 if (rx_space > PAGE_SIZE && !(bp->flags & BNXT_FLAG_NO_AGG_RINGS)) { 3535 u32 jumbo_factor; 3536 3537 bp->flags |= BNXT_FLAG_JUMBO; 3538 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT; 3539 if (jumbo_factor > agg_factor) 3540 agg_factor = jumbo_factor; 3541 } 3542 agg_ring_size = ring_size * agg_factor; 3543 3544 if (agg_ring_size) { 3545 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size, 3546 RX_DESC_CNT); 3547 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) { 3548 u32 tmp = agg_ring_size; 3549 3550 bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES; 3551 agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1; 3552 netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n", 3553 tmp, agg_ring_size); 3554 } 3555 bp->rx_agg_ring_size = agg_ring_size; 3556 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1; 3557 rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN); 3558 rx_space = rx_size + NET_SKB_PAD + 3559 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 3560 } 3561 3562 bp->rx_buf_use_size = rx_size; 3563 bp->rx_buf_size = rx_space; 3564 3565 bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT); 3566 bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1; 3567 3568 ring_size = bp->tx_ring_size; 3569 bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT); 3570 bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1; 3571 3572 max_rx_cmpl = bp->rx_ring_size; 3573 /* MAX TPA needs to be added because TPA_START completions are 3574 * immediately recycled, so the TPA completions are not bound by 3575 * the RX ring size. 3576 */ 3577 if (bp->flags & BNXT_FLAG_TPA) 3578 max_rx_cmpl += bp->max_tpa; 3579 /* RX and TPA completions are 32-byte, all others are 16-byte */ 3580 ring_size = max_rx_cmpl * 2 + agg_ring_size + bp->tx_ring_size; 3581 bp->cp_ring_size = ring_size; 3582 3583 bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT); 3584 if (bp->cp_nr_pages > MAX_CP_PAGES) { 3585 bp->cp_nr_pages = MAX_CP_PAGES; 3586 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1; 3587 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n", 3588 ring_size, bp->cp_ring_size); 3589 } 3590 bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT; 3591 bp->cp_ring_mask = bp->cp_bit - 1; 3592 } 3593 3594 /* Changing allocation mode of RX rings. 3595 * TODO: Update when extending xdp_rxq_info to support allocation modes. 3596 */ 3597 int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode) 3598 { 3599 if (page_mode) { 3600 if (bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU) 3601 return -EOPNOTSUPP; 3602 bp->dev->max_mtu = 3603 min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU); 3604 bp->flags &= ~BNXT_FLAG_AGG_RINGS; 3605 bp->flags |= BNXT_FLAG_NO_AGG_RINGS | BNXT_FLAG_RX_PAGE_MODE; 3606 bp->rx_dir = DMA_BIDIRECTIONAL; 3607 bp->rx_skb_func = bnxt_rx_page_skb; 3608 /* Disable LRO or GRO_HW */ 3609 netdev_update_features(bp->dev); 3610 } else { 3611 bp->dev->max_mtu = bp->max_mtu; 3612 bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE; 3613 bp->rx_dir = DMA_FROM_DEVICE; 3614 bp->rx_skb_func = bnxt_rx_skb; 3615 } 3616 return 0; 3617 } 3618 3619 static void bnxt_free_vnic_attributes(struct bnxt *bp) 3620 { 3621 int i; 3622 struct bnxt_vnic_info *vnic; 3623 struct pci_dev *pdev = bp->pdev; 3624 3625 if (!bp->vnic_info) 3626 return; 3627 3628 for (i = 0; i < bp->nr_vnics; i++) { 3629 vnic = &bp->vnic_info[i]; 3630 3631 kfree(vnic->fw_grp_ids); 3632 vnic->fw_grp_ids = NULL; 3633 3634 kfree(vnic->uc_list); 3635 vnic->uc_list = NULL; 3636 3637 if (vnic->mc_list) { 3638 dma_free_coherent(&pdev->dev, vnic->mc_list_size, 3639 vnic->mc_list, vnic->mc_list_mapping); 3640 vnic->mc_list = NULL; 3641 } 3642 3643 if (vnic->rss_table) { 3644 dma_free_coherent(&pdev->dev, vnic->rss_table_size, 3645 vnic->rss_table, 3646 vnic->rss_table_dma_addr); 3647 vnic->rss_table = NULL; 3648 } 3649 3650 vnic->rss_hash_key = NULL; 3651 vnic->flags = 0; 3652 } 3653 } 3654 3655 static int bnxt_alloc_vnic_attributes(struct bnxt *bp) 3656 { 3657 int i, rc = 0, size; 3658 struct bnxt_vnic_info *vnic; 3659 struct pci_dev *pdev = bp->pdev; 3660 int max_rings; 3661 3662 for (i = 0; i < bp->nr_vnics; i++) { 3663 vnic = &bp->vnic_info[i]; 3664 3665 if (vnic->flags & BNXT_VNIC_UCAST_FLAG) { 3666 int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN; 3667 3668 if (mem_size > 0) { 3669 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL); 3670 if (!vnic->uc_list) { 3671 rc = -ENOMEM; 3672 goto out; 3673 } 3674 } 3675 } 3676 3677 if (vnic->flags & BNXT_VNIC_MCAST_FLAG) { 3678 vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN; 3679 vnic->mc_list = 3680 dma_alloc_coherent(&pdev->dev, 3681 vnic->mc_list_size, 3682 &vnic->mc_list_mapping, 3683 GFP_KERNEL); 3684 if (!vnic->mc_list) { 3685 rc = -ENOMEM; 3686 goto out; 3687 } 3688 } 3689 3690 if (bp->flags & BNXT_FLAG_CHIP_P5) 3691 goto vnic_skip_grps; 3692 3693 if (vnic->flags & BNXT_VNIC_RSS_FLAG) 3694 max_rings = bp->rx_nr_rings; 3695 else 3696 max_rings = 1; 3697 3698 vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL); 3699 if (!vnic->fw_grp_ids) { 3700 rc = -ENOMEM; 3701 goto out; 3702 } 3703 vnic_skip_grps: 3704 if ((bp->flags & BNXT_FLAG_NEW_RSS_CAP) && 3705 !(vnic->flags & BNXT_VNIC_RSS_FLAG)) 3706 continue; 3707 3708 /* Allocate rss table and hash key */ 3709 size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16)); 3710 if (bp->flags & BNXT_FLAG_CHIP_P5) 3711 size = L1_CACHE_ALIGN(BNXT_MAX_RSS_TABLE_SIZE_P5); 3712 3713 vnic->rss_table_size = size + HW_HASH_KEY_SIZE; 3714 vnic->rss_table = dma_alloc_coherent(&pdev->dev, 3715 vnic->rss_table_size, 3716 &vnic->rss_table_dma_addr, 3717 GFP_KERNEL); 3718 if (!vnic->rss_table) { 3719 rc = -ENOMEM; 3720 goto out; 3721 } 3722 3723 vnic->rss_hash_key = ((void *)vnic->rss_table) + size; 3724 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size; 3725 } 3726 return 0; 3727 3728 out: 3729 return rc; 3730 } 3731 3732 static void bnxt_free_hwrm_resources(struct bnxt *bp) 3733 { 3734 struct pci_dev *pdev = bp->pdev; 3735 3736 if (bp->hwrm_cmd_resp_addr) { 3737 dma_free_coherent(&pdev->dev, PAGE_SIZE, bp->hwrm_cmd_resp_addr, 3738 bp->hwrm_cmd_resp_dma_addr); 3739 bp->hwrm_cmd_resp_addr = NULL; 3740 } 3741 3742 if (bp->hwrm_cmd_kong_resp_addr) { 3743 dma_free_coherent(&pdev->dev, PAGE_SIZE, 3744 bp->hwrm_cmd_kong_resp_addr, 3745 bp->hwrm_cmd_kong_resp_dma_addr); 3746 bp->hwrm_cmd_kong_resp_addr = NULL; 3747 } 3748 } 3749 3750 static int bnxt_alloc_kong_hwrm_resources(struct bnxt *bp) 3751 { 3752 struct pci_dev *pdev = bp->pdev; 3753 3754 if (bp->hwrm_cmd_kong_resp_addr) 3755 return 0; 3756 3757 bp->hwrm_cmd_kong_resp_addr = 3758 dma_alloc_coherent(&pdev->dev, PAGE_SIZE, 3759 &bp->hwrm_cmd_kong_resp_dma_addr, 3760 GFP_KERNEL); 3761 if (!bp->hwrm_cmd_kong_resp_addr) 3762 return -ENOMEM; 3763 3764 return 0; 3765 } 3766 3767 static int bnxt_alloc_hwrm_resources(struct bnxt *bp) 3768 { 3769 struct pci_dev *pdev = bp->pdev; 3770 3771 bp->hwrm_cmd_resp_addr = dma_alloc_coherent(&pdev->dev, PAGE_SIZE, 3772 &bp->hwrm_cmd_resp_dma_addr, 3773 GFP_KERNEL); 3774 if (!bp->hwrm_cmd_resp_addr) 3775 return -ENOMEM; 3776 3777 return 0; 3778 } 3779 3780 static void bnxt_free_hwrm_short_cmd_req(struct bnxt *bp) 3781 { 3782 if (bp->hwrm_short_cmd_req_addr) { 3783 struct pci_dev *pdev = bp->pdev; 3784 3785 dma_free_coherent(&pdev->dev, bp->hwrm_max_ext_req_len, 3786 bp->hwrm_short_cmd_req_addr, 3787 bp->hwrm_short_cmd_req_dma_addr); 3788 bp->hwrm_short_cmd_req_addr = NULL; 3789 } 3790 } 3791 3792 static int bnxt_alloc_hwrm_short_cmd_req(struct bnxt *bp) 3793 { 3794 struct pci_dev *pdev = bp->pdev; 3795 3796 if (bp->hwrm_short_cmd_req_addr) 3797 return 0; 3798 3799 bp->hwrm_short_cmd_req_addr = 3800 dma_alloc_coherent(&pdev->dev, bp->hwrm_max_ext_req_len, 3801 &bp->hwrm_short_cmd_req_dma_addr, 3802 GFP_KERNEL); 3803 if (!bp->hwrm_short_cmd_req_addr) 3804 return -ENOMEM; 3805 3806 return 0; 3807 } 3808 3809 static void bnxt_free_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats) 3810 { 3811 kfree(stats->hw_masks); 3812 stats->hw_masks = NULL; 3813 kfree(stats->sw_stats); 3814 stats->sw_stats = NULL; 3815 if (stats->hw_stats) { 3816 dma_free_coherent(&bp->pdev->dev, stats->len, stats->hw_stats, 3817 stats->hw_stats_map); 3818 stats->hw_stats = NULL; 3819 } 3820 } 3821 3822 static int bnxt_alloc_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats, 3823 bool alloc_masks) 3824 { 3825 stats->hw_stats = dma_alloc_coherent(&bp->pdev->dev, stats->len, 3826 &stats->hw_stats_map, GFP_KERNEL); 3827 if (!stats->hw_stats) 3828 return -ENOMEM; 3829 3830 stats->sw_stats = kzalloc(stats->len, GFP_KERNEL); 3831 if (!stats->sw_stats) 3832 goto stats_mem_err; 3833 3834 if (alloc_masks) { 3835 stats->hw_masks = kzalloc(stats->len, GFP_KERNEL); 3836 if (!stats->hw_masks) 3837 goto stats_mem_err; 3838 } 3839 return 0; 3840 3841 stats_mem_err: 3842 bnxt_free_stats_mem(bp, stats); 3843 return -ENOMEM; 3844 } 3845 3846 static void bnxt_fill_masks(u64 *mask_arr, u64 mask, int count) 3847 { 3848 int i; 3849 3850 for (i = 0; i < count; i++) 3851 mask_arr[i] = mask; 3852 } 3853 3854 static void bnxt_copy_hw_masks(u64 *mask_arr, __le64 *hw_mask_arr, int count) 3855 { 3856 int i; 3857 3858 for (i = 0; i < count; i++) 3859 mask_arr[i] = le64_to_cpu(hw_mask_arr[i]); 3860 } 3861 3862 static int bnxt_hwrm_func_qstat_ext(struct bnxt *bp, 3863 struct bnxt_stats_mem *stats) 3864 { 3865 struct hwrm_func_qstats_ext_output *resp = bp->hwrm_cmd_resp_addr; 3866 struct hwrm_func_qstats_ext_input req = {0}; 3867 __le64 *hw_masks; 3868 int rc; 3869 3870 if (!(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED) || 3871 !(bp->flags & BNXT_FLAG_CHIP_P5)) 3872 return -EOPNOTSUPP; 3873 3874 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QSTATS_EXT, -1, -1); 3875 req.fid = cpu_to_le16(0xffff); 3876 req.flags = FUNC_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK; 3877 mutex_lock(&bp->hwrm_cmd_lock); 3878 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3879 if (rc) 3880 goto qstat_exit; 3881 3882 hw_masks = &resp->rx_ucast_pkts; 3883 bnxt_copy_hw_masks(stats->hw_masks, hw_masks, stats->len / 8); 3884 3885 qstat_exit: 3886 mutex_unlock(&bp->hwrm_cmd_lock); 3887 return rc; 3888 } 3889 3890 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags); 3891 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags); 3892 3893 static void bnxt_init_stats(struct bnxt *bp) 3894 { 3895 struct bnxt_napi *bnapi = bp->bnapi[0]; 3896 struct bnxt_cp_ring_info *cpr; 3897 struct bnxt_stats_mem *stats; 3898 __le64 *rx_stats, *tx_stats; 3899 int rc, rx_count, tx_count; 3900 u64 *rx_masks, *tx_masks; 3901 u64 mask; 3902 u8 flags; 3903 3904 cpr = &bnapi->cp_ring; 3905 stats = &cpr->stats; 3906 rc = bnxt_hwrm_func_qstat_ext(bp, stats); 3907 if (rc) { 3908 if (bp->flags & BNXT_FLAG_CHIP_P5) 3909 mask = (1ULL << 48) - 1; 3910 else 3911 mask = -1ULL; 3912 bnxt_fill_masks(stats->hw_masks, mask, stats->len / 8); 3913 } 3914 if (bp->flags & BNXT_FLAG_PORT_STATS) { 3915 stats = &bp->port_stats; 3916 rx_stats = stats->hw_stats; 3917 rx_masks = stats->hw_masks; 3918 rx_count = sizeof(struct rx_port_stats) / 8; 3919 tx_stats = rx_stats + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 3920 tx_masks = rx_masks + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 3921 tx_count = sizeof(struct tx_port_stats) / 8; 3922 3923 flags = PORT_QSTATS_REQ_FLAGS_COUNTER_MASK; 3924 rc = bnxt_hwrm_port_qstats(bp, flags); 3925 if (rc) { 3926 mask = (1ULL << 40) - 1; 3927 3928 bnxt_fill_masks(rx_masks, mask, rx_count); 3929 bnxt_fill_masks(tx_masks, mask, tx_count); 3930 } else { 3931 bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count); 3932 bnxt_copy_hw_masks(tx_masks, tx_stats, tx_count); 3933 bnxt_hwrm_port_qstats(bp, 0); 3934 } 3935 } 3936 if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) { 3937 stats = &bp->rx_port_stats_ext; 3938 rx_stats = stats->hw_stats; 3939 rx_masks = stats->hw_masks; 3940 rx_count = sizeof(struct rx_port_stats_ext) / 8; 3941 stats = &bp->tx_port_stats_ext; 3942 tx_stats = stats->hw_stats; 3943 tx_masks = stats->hw_masks; 3944 tx_count = sizeof(struct tx_port_stats_ext) / 8; 3945 3946 flags = PORT_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK; 3947 rc = bnxt_hwrm_port_qstats_ext(bp, flags); 3948 if (rc) { 3949 mask = (1ULL << 40) - 1; 3950 3951 bnxt_fill_masks(rx_masks, mask, rx_count); 3952 if (tx_stats) 3953 bnxt_fill_masks(tx_masks, mask, tx_count); 3954 } else { 3955 bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count); 3956 if (tx_stats) 3957 bnxt_copy_hw_masks(tx_masks, tx_stats, 3958 tx_count); 3959 bnxt_hwrm_port_qstats_ext(bp, 0); 3960 } 3961 } 3962 } 3963 3964 static void bnxt_free_port_stats(struct bnxt *bp) 3965 { 3966 bp->flags &= ~BNXT_FLAG_PORT_STATS; 3967 bp->flags &= ~BNXT_FLAG_PORT_STATS_EXT; 3968 3969 bnxt_free_stats_mem(bp, &bp->port_stats); 3970 bnxt_free_stats_mem(bp, &bp->rx_port_stats_ext); 3971 bnxt_free_stats_mem(bp, &bp->tx_port_stats_ext); 3972 } 3973 3974 static void bnxt_free_ring_stats(struct bnxt *bp) 3975 { 3976 int i; 3977 3978 if (!bp->bnapi) 3979 return; 3980 3981 for (i = 0; i < bp->cp_nr_rings; i++) { 3982 struct bnxt_napi *bnapi = bp->bnapi[i]; 3983 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 3984 3985 bnxt_free_stats_mem(bp, &cpr->stats); 3986 } 3987 } 3988 3989 static int bnxt_alloc_stats(struct bnxt *bp) 3990 { 3991 u32 size, i; 3992 int rc; 3993 3994 size = bp->hw_ring_stats_size; 3995 3996 for (i = 0; i < bp->cp_nr_rings; i++) { 3997 struct bnxt_napi *bnapi = bp->bnapi[i]; 3998 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 3999 4000 cpr->stats.len = size; 4001 rc = bnxt_alloc_stats_mem(bp, &cpr->stats, !i); 4002 if (rc) 4003 return rc; 4004 4005 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID; 4006 } 4007 4008 if (BNXT_VF(bp) || bp->chip_num == CHIP_NUM_58700) 4009 return 0; 4010 4011 if (bp->port_stats.hw_stats) 4012 goto alloc_ext_stats; 4013 4014 bp->port_stats.len = BNXT_PORT_STATS_SIZE; 4015 rc = bnxt_alloc_stats_mem(bp, &bp->port_stats, true); 4016 if (rc) 4017 return rc; 4018 4019 bp->flags |= BNXT_FLAG_PORT_STATS; 4020 4021 alloc_ext_stats: 4022 /* Display extended statistics only if FW supports it */ 4023 if (bp->hwrm_spec_code < 0x10804 || bp->hwrm_spec_code == 0x10900) 4024 if (!(bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED)) 4025 return 0; 4026 4027 if (bp->rx_port_stats_ext.hw_stats) 4028 goto alloc_tx_ext_stats; 4029 4030 bp->rx_port_stats_ext.len = sizeof(struct rx_port_stats_ext); 4031 rc = bnxt_alloc_stats_mem(bp, &bp->rx_port_stats_ext, true); 4032 /* Extended stats are optional */ 4033 if (rc) 4034 return 0; 4035 4036 alloc_tx_ext_stats: 4037 if (bp->tx_port_stats_ext.hw_stats) 4038 return 0; 4039 4040 if (bp->hwrm_spec_code >= 0x10902 || 4041 (bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED)) { 4042 bp->tx_port_stats_ext.len = sizeof(struct tx_port_stats_ext); 4043 rc = bnxt_alloc_stats_mem(bp, &bp->tx_port_stats_ext, true); 4044 /* Extended stats are optional */ 4045 if (rc) 4046 return 0; 4047 } 4048 bp->flags |= BNXT_FLAG_PORT_STATS_EXT; 4049 return 0; 4050 } 4051 4052 static void bnxt_clear_ring_indices(struct bnxt *bp) 4053 { 4054 int i; 4055 4056 if (!bp->bnapi) 4057 return; 4058 4059 for (i = 0; i < bp->cp_nr_rings; i++) { 4060 struct bnxt_napi *bnapi = bp->bnapi[i]; 4061 struct bnxt_cp_ring_info *cpr; 4062 struct bnxt_rx_ring_info *rxr; 4063 struct bnxt_tx_ring_info *txr; 4064 4065 if (!bnapi) 4066 continue; 4067 4068 cpr = &bnapi->cp_ring; 4069 cpr->cp_raw_cons = 0; 4070 4071 txr = bnapi->tx_ring; 4072 if (txr) { 4073 txr->tx_prod = 0; 4074 txr->tx_cons = 0; 4075 } 4076 4077 rxr = bnapi->rx_ring; 4078 if (rxr) { 4079 rxr->rx_prod = 0; 4080 rxr->rx_agg_prod = 0; 4081 rxr->rx_sw_agg_prod = 0; 4082 rxr->rx_next_cons = 0; 4083 } 4084 } 4085 } 4086 4087 static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool irq_reinit) 4088 { 4089 #ifdef CONFIG_RFS_ACCEL 4090 int i; 4091 4092 /* Under rtnl_lock and all our NAPIs have been disabled. It's 4093 * safe to delete the hash table. 4094 */ 4095 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) { 4096 struct hlist_head *head; 4097 struct hlist_node *tmp; 4098 struct bnxt_ntuple_filter *fltr; 4099 4100 head = &bp->ntp_fltr_hash_tbl[i]; 4101 hlist_for_each_entry_safe(fltr, tmp, head, hash) { 4102 hlist_del(&fltr->hash); 4103 kfree(fltr); 4104 } 4105 } 4106 if (irq_reinit) { 4107 kfree(bp->ntp_fltr_bmap); 4108 bp->ntp_fltr_bmap = NULL; 4109 } 4110 bp->ntp_fltr_count = 0; 4111 #endif 4112 } 4113 4114 static int bnxt_alloc_ntp_fltrs(struct bnxt *bp) 4115 { 4116 #ifdef CONFIG_RFS_ACCEL 4117 int i, rc = 0; 4118 4119 if (!(bp->flags & BNXT_FLAG_RFS)) 4120 return 0; 4121 4122 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) 4123 INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]); 4124 4125 bp->ntp_fltr_count = 0; 4126 bp->ntp_fltr_bmap = kcalloc(BITS_TO_LONGS(BNXT_NTP_FLTR_MAX_FLTR), 4127 sizeof(long), 4128 GFP_KERNEL); 4129 4130 if (!bp->ntp_fltr_bmap) 4131 rc = -ENOMEM; 4132 4133 return rc; 4134 #else 4135 return 0; 4136 #endif 4137 } 4138 4139 static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init) 4140 { 4141 bnxt_free_vnic_attributes(bp); 4142 bnxt_free_tx_rings(bp); 4143 bnxt_free_rx_rings(bp); 4144 bnxt_free_cp_rings(bp); 4145 bnxt_free_ntp_fltrs(bp, irq_re_init); 4146 if (irq_re_init) { 4147 bnxt_free_ring_stats(bp); 4148 if (!(bp->fw_cap & BNXT_FW_CAP_PORT_STATS_NO_RESET) || 4149 test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 4150 bnxt_free_port_stats(bp); 4151 bnxt_free_ring_grps(bp); 4152 bnxt_free_vnics(bp); 4153 kfree(bp->tx_ring_map); 4154 bp->tx_ring_map = NULL; 4155 kfree(bp->tx_ring); 4156 bp->tx_ring = NULL; 4157 kfree(bp->rx_ring); 4158 bp->rx_ring = NULL; 4159 kfree(bp->bnapi); 4160 bp->bnapi = NULL; 4161 } else { 4162 bnxt_clear_ring_indices(bp); 4163 } 4164 } 4165 4166 static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init) 4167 { 4168 int i, j, rc, size, arr_size; 4169 void *bnapi; 4170 4171 if (irq_re_init) { 4172 /* Allocate bnapi mem pointer array and mem block for 4173 * all queues 4174 */ 4175 arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) * 4176 bp->cp_nr_rings); 4177 size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi)); 4178 bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL); 4179 if (!bnapi) 4180 return -ENOMEM; 4181 4182 bp->bnapi = bnapi; 4183 bnapi += arr_size; 4184 for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) { 4185 bp->bnapi[i] = bnapi; 4186 bp->bnapi[i]->index = i; 4187 bp->bnapi[i]->bp = bp; 4188 if (bp->flags & BNXT_FLAG_CHIP_P5) { 4189 struct bnxt_cp_ring_info *cpr = 4190 &bp->bnapi[i]->cp_ring; 4191 4192 cpr->cp_ring_struct.ring_mem.flags = 4193 BNXT_RMEM_RING_PTE_FLAG; 4194 } 4195 } 4196 4197 bp->rx_ring = kcalloc(bp->rx_nr_rings, 4198 sizeof(struct bnxt_rx_ring_info), 4199 GFP_KERNEL); 4200 if (!bp->rx_ring) 4201 return -ENOMEM; 4202 4203 for (i = 0; i < bp->rx_nr_rings; i++) { 4204 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 4205 4206 if (bp->flags & BNXT_FLAG_CHIP_P5) { 4207 rxr->rx_ring_struct.ring_mem.flags = 4208 BNXT_RMEM_RING_PTE_FLAG; 4209 rxr->rx_agg_ring_struct.ring_mem.flags = 4210 BNXT_RMEM_RING_PTE_FLAG; 4211 } 4212 rxr->bnapi = bp->bnapi[i]; 4213 bp->bnapi[i]->rx_ring = &bp->rx_ring[i]; 4214 } 4215 4216 bp->tx_ring = kcalloc(bp->tx_nr_rings, 4217 sizeof(struct bnxt_tx_ring_info), 4218 GFP_KERNEL); 4219 if (!bp->tx_ring) 4220 return -ENOMEM; 4221 4222 bp->tx_ring_map = kcalloc(bp->tx_nr_rings, sizeof(u16), 4223 GFP_KERNEL); 4224 4225 if (!bp->tx_ring_map) 4226 return -ENOMEM; 4227 4228 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 4229 j = 0; 4230 else 4231 j = bp->rx_nr_rings; 4232 4233 for (i = 0; i < bp->tx_nr_rings; i++, j++) { 4234 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 4235 4236 if (bp->flags & BNXT_FLAG_CHIP_P5) 4237 txr->tx_ring_struct.ring_mem.flags = 4238 BNXT_RMEM_RING_PTE_FLAG; 4239 txr->bnapi = bp->bnapi[j]; 4240 bp->bnapi[j]->tx_ring = txr; 4241 bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i; 4242 if (i >= bp->tx_nr_rings_xdp) { 4243 txr->txq_index = i - bp->tx_nr_rings_xdp; 4244 bp->bnapi[j]->tx_int = bnxt_tx_int; 4245 } else { 4246 bp->bnapi[j]->flags |= BNXT_NAPI_FLAG_XDP; 4247 bp->bnapi[j]->tx_int = bnxt_tx_int_xdp; 4248 } 4249 } 4250 4251 rc = bnxt_alloc_stats(bp); 4252 if (rc) 4253 goto alloc_mem_err; 4254 bnxt_init_stats(bp); 4255 4256 rc = bnxt_alloc_ntp_fltrs(bp); 4257 if (rc) 4258 goto alloc_mem_err; 4259 4260 rc = bnxt_alloc_vnics(bp); 4261 if (rc) 4262 goto alloc_mem_err; 4263 } 4264 4265 bnxt_init_ring_struct(bp); 4266 4267 rc = bnxt_alloc_rx_rings(bp); 4268 if (rc) 4269 goto alloc_mem_err; 4270 4271 rc = bnxt_alloc_tx_rings(bp); 4272 if (rc) 4273 goto alloc_mem_err; 4274 4275 rc = bnxt_alloc_cp_rings(bp); 4276 if (rc) 4277 goto alloc_mem_err; 4278 4279 bp->vnic_info[0].flags |= BNXT_VNIC_RSS_FLAG | BNXT_VNIC_MCAST_FLAG | 4280 BNXT_VNIC_UCAST_FLAG; 4281 rc = bnxt_alloc_vnic_attributes(bp); 4282 if (rc) 4283 goto alloc_mem_err; 4284 return 0; 4285 4286 alloc_mem_err: 4287 bnxt_free_mem(bp, true); 4288 return rc; 4289 } 4290 4291 static void bnxt_disable_int(struct bnxt *bp) 4292 { 4293 int i; 4294 4295 if (!bp->bnapi) 4296 return; 4297 4298 for (i = 0; i < bp->cp_nr_rings; i++) { 4299 struct bnxt_napi *bnapi = bp->bnapi[i]; 4300 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 4301 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 4302 4303 if (ring->fw_ring_id != INVALID_HW_RING_ID) 4304 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons); 4305 } 4306 } 4307 4308 static int bnxt_cp_num_to_irq_num(struct bnxt *bp, int n) 4309 { 4310 struct bnxt_napi *bnapi = bp->bnapi[n]; 4311 struct bnxt_cp_ring_info *cpr; 4312 4313 cpr = &bnapi->cp_ring; 4314 return cpr->cp_ring_struct.map_idx; 4315 } 4316 4317 static void bnxt_disable_int_sync(struct bnxt *bp) 4318 { 4319 int i; 4320 4321 if (!bp->irq_tbl) 4322 return; 4323 4324 atomic_inc(&bp->intr_sem); 4325 4326 bnxt_disable_int(bp); 4327 for (i = 0; i < bp->cp_nr_rings; i++) { 4328 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 4329 4330 synchronize_irq(bp->irq_tbl[map_idx].vector); 4331 } 4332 } 4333 4334 static void bnxt_enable_int(struct bnxt *bp) 4335 { 4336 int i; 4337 4338 atomic_set(&bp->intr_sem, 0); 4339 for (i = 0; i < bp->cp_nr_rings; i++) { 4340 struct bnxt_napi *bnapi = bp->bnapi[i]; 4341 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 4342 4343 bnxt_db_nq_arm(bp, &cpr->cp_db, cpr->cp_raw_cons); 4344 } 4345 } 4346 4347 void bnxt_hwrm_cmd_hdr_init(struct bnxt *bp, void *request, u16 req_type, 4348 u16 cmpl_ring, u16 target_id) 4349 { 4350 struct input *req = request; 4351 4352 req->req_type = cpu_to_le16(req_type); 4353 req->cmpl_ring = cpu_to_le16(cmpl_ring); 4354 req->target_id = cpu_to_le16(target_id); 4355 if (bnxt_kong_hwrm_message(bp, req)) 4356 req->resp_addr = cpu_to_le64(bp->hwrm_cmd_kong_resp_dma_addr); 4357 else 4358 req->resp_addr = cpu_to_le64(bp->hwrm_cmd_resp_dma_addr); 4359 } 4360 4361 static int bnxt_hwrm_to_stderr(u32 hwrm_err) 4362 { 4363 switch (hwrm_err) { 4364 case HWRM_ERR_CODE_SUCCESS: 4365 return 0; 4366 case HWRM_ERR_CODE_RESOURCE_LOCKED: 4367 return -EROFS; 4368 case HWRM_ERR_CODE_RESOURCE_ACCESS_DENIED: 4369 return -EACCES; 4370 case HWRM_ERR_CODE_RESOURCE_ALLOC_ERROR: 4371 return -ENOSPC; 4372 case HWRM_ERR_CODE_INVALID_PARAMS: 4373 case HWRM_ERR_CODE_INVALID_FLAGS: 4374 case HWRM_ERR_CODE_INVALID_ENABLES: 4375 case HWRM_ERR_CODE_UNSUPPORTED_TLV: 4376 case HWRM_ERR_CODE_UNSUPPORTED_OPTION_ERR: 4377 return -EINVAL; 4378 case HWRM_ERR_CODE_NO_BUFFER: 4379 return -ENOMEM; 4380 case HWRM_ERR_CODE_HOT_RESET_PROGRESS: 4381 case HWRM_ERR_CODE_BUSY: 4382 return -EAGAIN; 4383 case HWRM_ERR_CODE_CMD_NOT_SUPPORTED: 4384 return -EOPNOTSUPP; 4385 default: 4386 return -EIO; 4387 } 4388 } 4389 4390 static int bnxt_hwrm_do_send_msg(struct bnxt *bp, void *msg, u32 msg_len, 4391 int timeout, bool silent) 4392 { 4393 int i, intr_process, rc, tmo_count; 4394 struct input *req = msg; 4395 u32 *data = msg; 4396 u8 *valid; 4397 u16 cp_ring_id, len = 0; 4398 struct hwrm_err_output *resp = bp->hwrm_cmd_resp_addr; 4399 u16 max_req_len = BNXT_HWRM_MAX_REQ_LEN; 4400 struct hwrm_short_input short_input = {0}; 4401 u32 doorbell_offset = BNXT_GRCPF_REG_CHIMP_COMM_TRIGGER; 4402 u32 bar_offset = BNXT_GRCPF_REG_CHIMP_COMM; 4403 u16 dst = BNXT_HWRM_CHNL_CHIMP; 4404 4405 if (BNXT_NO_FW_ACCESS(bp) && 4406 le16_to_cpu(req->req_type) != HWRM_FUNC_RESET) 4407 return -EBUSY; 4408 4409 if (msg_len > BNXT_HWRM_MAX_REQ_LEN) { 4410 if (msg_len > bp->hwrm_max_ext_req_len || 4411 !bp->hwrm_short_cmd_req_addr) 4412 return -EINVAL; 4413 } 4414 4415 if (bnxt_hwrm_kong_chnl(bp, req)) { 4416 dst = BNXT_HWRM_CHNL_KONG; 4417 bar_offset = BNXT_GRCPF_REG_KONG_COMM; 4418 doorbell_offset = BNXT_GRCPF_REG_KONG_COMM_TRIGGER; 4419 resp = bp->hwrm_cmd_kong_resp_addr; 4420 } 4421 4422 memset(resp, 0, PAGE_SIZE); 4423 cp_ring_id = le16_to_cpu(req->cmpl_ring); 4424 intr_process = (cp_ring_id == INVALID_HW_RING_ID) ? 0 : 1; 4425 4426 req->seq_id = cpu_to_le16(bnxt_get_hwrm_seq_id(bp, dst)); 4427 /* currently supports only one outstanding message */ 4428 if (intr_process) 4429 bp->hwrm_intr_seq_id = le16_to_cpu(req->seq_id); 4430 4431 if ((bp->fw_cap & BNXT_FW_CAP_SHORT_CMD) || 4432 msg_len > BNXT_HWRM_MAX_REQ_LEN) { 4433 void *short_cmd_req = bp->hwrm_short_cmd_req_addr; 4434 u16 max_msg_len; 4435 4436 /* Set boundary for maximum extended request length for short 4437 * cmd format. If passed up from device use the max supported 4438 * internal req length. 4439 */ 4440 max_msg_len = bp->hwrm_max_ext_req_len; 4441 4442 memcpy(short_cmd_req, req, msg_len); 4443 if (msg_len < max_msg_len) 4444 memset(short_cmd_req + msg_len, 0, 4445 max_msg_len - msg_len); 4446 4447 short_input.req_type = req->req_type; 4448 short_input.signature = 4449 cpu_to_le16(SHORT_REQ_SIGNATURE_SHORT_CMD); 4450 short_input.size = cpu_to_le16(msg_len); 4451 short_input.req_addr = 4452 cpu_to_le64(bp->hwrm_short_cmd_req_dma_addr); 4453 4454 data = (u32 *)&short_input; 4455 msg_len = sizeof(short_input); 4456 4457 /* Sync memory write before updating doorbell */ 4458 wmb(); 4459 4460 max_req_len = BNXT_HWRM_SHORT_REQ_LEN; 4461 } 4462 4463 /* Write request msg to hwrm channel */ 4464 __iowrite32_copy(bp->bar0 + bar_offset, data, msg_len / 4); 4465 4466 for (i = msg_len; i < max_req_len; i += 4) 4467 writel(0, bp->bar0 + bar_offset + i); 4468 4469 /* Ring channel doorbell */ 4470 writel(1, bp->bar0 + doorbell_offset); 4471 4472 if (!pci_is_enabled(bp->pdev)) 4473 return 0; 4474 4475 if (!timeout) 4476 timeout = DFLT_HWRM_CMD_TIMEOUT; 4477 /* Limit timeout to an upper limit */ 4478 timeout = min(timeout, HWRM_CMD_MAX_TIMEOUT); 4479 /* convert timeout to usec */ 4480 timeout *= 1000; 4481 4482 i = 0; 4483 /* Short timeout for the first few iterations: 4484 * number of loops = number of loops for short timeout + 4485 * number of loops for standard timeout. 4486 */ 4487 tmo_count = HWRM_SHORT_TIMEOUT_COUNTER; 4488 timeout = timeout - HWRM_SHORT_MIN_TIMEOUT * HWRM_SHORT_TIMEOUT_COUNTER; 4489 tmo_count += DIV_ROUND_UP(timeout, HWRM_MIN_TIMEOUT); 4490 4491 if (intr_process) { 4492 u16 seq_id = bp->hwrm_intr_seq_id; 4493 4494 /* Wait until hwrm response cmpl interrupt is processed */ 4495 while (bp->hwrm_intr_seq_id != (u16)~seq_id && 4496 i++ < tmo_count) { 4497 /* Abort the wait for completion if the FW health 4498 * check has failed. 4499 */ 4500 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) 4501 return -EBUSY; 4502 /* on first few passes, just barely sleep */ 4503 if (i < HWRM_SHORT_TIMEOUT_COUNTER) 4504 usleep_range(HWRM_SHORT_MIN_TIMEOUT, 4505 HWRM_SHORT_MAX_TIMEOUT); 4506 else 4507 usleep_range(HWRM_MIN_TIMEOUT, 4508 HWRM_MAX_TIMEOUT); 4509 } 4510 4511 if (bp->hwrm_intr_seq_id != (u16)~seq_id) { 4512 if (!silent) 4513 netdev_err(bp->dev, "Resp cmpl intr err msg: 0x%x\n", 4514 le16_to_cpu(req->req_type)); 4515 return -EBUSY; 4516 } 4517 len = le16_to_cpu(resp->resp_len); 4518 valid = ((u8 *)resp) + len - 1; 4519 } else { 4520 int j; 4521 4522 /* Check if response len is updated */ 4523 for (i = 0; i < tmo_count; i++) { 4524 /* Abort the wait for completion if the FW health 4525 * check has failed. 4526 */ 4527 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) 4528 return -EBUSY; 4529 len = le16_to_cpu(resp->resp_len); 4530 if (len) 4531 break; 4532 /* on first few passes, just barely sleep */ 4533 if (i < HWRM_SHORT_TIMEOUT_COUNTER) 4534 usleep_range(HWRM_SHORT_MIN_TIMEOUT, 4535 HWRM_SHORT_MAX_TIMEOUT); 4536 else 4537 usleep_range(HWRM_MIN_TIMEOUT, 4538 HWRM_MAX_TIMEOUT); 4539 } 4540 4541 if (i >= tmo_count) { 4542 if (!silent) 4543 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d\n", 4544 HWRM_TOTAL_TIMEOUT(i), 4545 le16_to_cpu(req->req_type), 4546 le16_to_cpu(req->seq_id), len); 4547 return -EBUSY; 4548 } 4549 4550 /* Last byte of resp contains valid bit */ 4551 valid = ((u8 *)resp) + len - 1; 4552 for (j = 0; j < HWRM_VALID_BIT_DELAY_USEC; j++) { 4553 /* make sure we read from updated DMA memory */ 4554 dma_rmb(); 4555 if (*valid) 4556 break; 4557 usleep_range(1, 5); 4558 } 4559 4560 if (j >= HWRM_VALID_BIT_DELAY_USEC) { 4561 if (!silent) 4562 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d v:%d\n", 4563 HWRM_TOTAL_TIMEOUT(i), 4564 le16_to_cpu(req->req_type), 4565 le16_to_cpu(req->seq_id), len, 4566 *valid); 4567 return -EBUSY; 4568 } 4569 } 4570 4571 /* Zero valid bit for compatibility. Valid bit in an older spec 4572 * may become a new field in a newer spec. We must make sure that 4573 * a new field not implemented by old spec will read zero. 4574 */ 4575 *valid = 0; 4576 rc = le16_to_cpu(resp->error_code); 4577 if (rc && !silent) 4578 netdev_err(bp->dev, "hwrm req_type 0x%x seq id 0x%x error 0x%x\n", 4579 le16_to_cpu(resp->req_type), 4580 le16_to_cpu(resp->seq_id), rc); 4581 return bnxt_hwrm_to_stderr(rc); 4582 } 4583 4584 int _hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout) 4585 { 4586 return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, false); 4587 } 4588 4589 int _hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len, 4590 int timeout) 4591 { 4592 return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true); 4593 } 4594 4595 int hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout) 4596 { 4597 int rc; 4598 4599 mutex_lock(&bp->hwrm_cmd_lock); 4600 rc = _hwrm_send_message(bp, msg, msg_len, timeout); 4601 mutex_unlock(&bp->hwrm_cmd_lock); 4602 return rc; 4603 } 4604 4605 int hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len, 4606 int timeout) 4607 { 4608 int rc; 4609 4610 mutex_lock(&bp->hwrm_cmd_lock); 4611 rc = bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true); 4612 mutex_unlock(&bp->hwrm_cmd_lock); 4613 return rc; 4614 } 4615 4616 int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp, unsigned long *bmap, int bmap_size, 4617 bool async_only) 4618 { 4619 struct hwrm_func_drv_rgtr_output *resp = bp->hwrm_cmd_resp_addr; 4620 struct hwrm_func_drv_rgtr_input req = {0}; 4621 DECLARE_BITMAP(async_events_bmap, 256); 4622 u32 *events = (u32 *)async_events_bmap; 4623 u32 flags; 4624 int rc, i; 4625 4626 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1); 4627 4628 req.enables = 4629 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE | 4630 FUNC_DRV_RGTR_REQ_ENABLES_VER | 4631 FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD); 4632 4633 req.os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX); 4634 flags = FUNC_DRV_RGTR_REQ_FLAGS_16BIT_VER_MODE; 4635 if (bp->fw_cap & BNXT_FW_CAP_HOT_RESET) 4636 flags |= FUNC_DRV_RGTR_REQ_FLAGS_HOT_RESET_SUPPORT; 4637 if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) 4638 flags |= FUNC_DRV_RGTR_REQ_FLAGS_ERROR_RECOVERY_SUPPORT | 4639 FUNC_DRV_RGTR_REQ_FLAGS_MASTER_SUPPORT; 4640 req.flags = cpu_to_le32(flags); 4641 req.ver_maj_8b = DRV_VER_MAJ; 4642 req.ver_min_8b = DRV_VER_MIN; 4643 req.ver_upd_8b = DRV_VER_UPD; 4644 req.ver_maj = cpu_to_le16(DRV_VER_MAJ); 4645 req.ver_min = cpu_to_le16(DRV_VER_MIN); 4646 req.ver_upd = cpu_to_le16(DRV_VER_UPD); 4647 4648 if (BNXT_PF(bp)) { 4649 u32 data[8]; 4650 int i; 4651 4652 memset(data, 0, sizeof(data)); 4653 for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) { 4654 u16 cmd = bnxt_vf_req_snif[i]; 4655 unsigned int bit, idx; 4656 4657 idx = cmd / 32; 4658 bit = cmd % 32; 4659 data[idx] |= 1 << bit; 4660 } 4661 4662 for (i = 0; i < 8; i++) 4663 req.vf_req_fwd[i] = cpu_to_le32(data[i]); 4664 4665 req.enables |= 4666 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD); 4667 } 4668 4669 if (bp->fw_cap & BNXT_FW_CAP_OVS_64BIT_HANDLE) 4670 req.flags |= cpu_to_le32( 4671 FUNC_DRV_RGTR_REQ_FLAGS_FLOW_HANDLE_64BIT_MODE); 4672 4673 memset(async_events_bmap, 0, sizeof(async_events_bmap)); 4674 for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++) { 4675 u16 event_id = bnxt_async_events_arr[i]; 4676 4677 if (event_id == ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY && 4678 !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) 4679 continue; 4680 __set_bit(bnxt_async_events_arr[i], async_events_bmap); 4681 } 4682 if (bmap && bmap_size) { 4683 for (i = 0; i < bmap_size; i++) { 4684 if (test_bit(i, bmap)) 4685 __set_bit(i, async_events_bmap); 4686 } 4687 } 4688 for (i = 0; i < 8; i++) 4689 req.async_event_fwd[i] |= cpu_to_le32(events[i]); 4690 4691 if (async_only) 4692 req.enables = 4693 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD); 4694 4695 mutex_lock(&bp->hwrm_cmd_lock); 4696 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4697 if (!rc) { 4698 set_bit(BNXT_STATE_DRV_REGISTERED, &bp->state); 4699 if (resp->flags & 4700 cpu_to_le32(FUNC_DRV_RGTR_RESP_FLAGS_IF_CHANGE_SUPPORTED)) 4701 bp->fw_cap |= BNXT_FW_CAP_IF_CHANGE; 4702 } 4703 mutex_unlock(&bp->hwrm_cmd_lock); 4704 return rc; 4705 } 4706 4707 static int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp) 4708 { 4709 struct hwrm_func_drv_unrgtr_input req = {0}; 4710 4711 if (!test_and_clear_bit(BNXT_STATE_DRV_REGISTERED, &bp->state)) 4712 return 0; 4713 4714 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_UNRGTR, -1, -1); 4715 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4716 } 4717 4718 static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type) 4719 { 4720 u32 rc = 0; 4721 struct hwrm_tunnel_dst_port_free_input req = {0}; 4722 4723 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_FREE, -1, -1); 4724 req.tunnel_type = tunnel_type; 4725 4726 switch (tunnel_type) { 4727 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN: 4728 req.tunnel_dst_port_id = cpu_to_le16(bp->vxlan_fw_dst_port_id); 4729 bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID; 4730 break; 4731 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE: 4732 req.tunnel_dst_port_id = cpu_to_le16(bp->nge_fw_dst_port_id); 4733 bp->nge_fw_dst_port_id = INVALID_HW_RING_ID; 4734 break; 4735 default: 4736 break; 4737 } 4738 4739 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4740 if (rc) 4741 netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n", 4742 rc); 4743 return rc; 4744 } 4745 4746 static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port, 4747 u8 tunnel_type) 4748 { 4749 u32 rc = 0; 4750 struct hwrm_tunnel_dst_port_alloc_input req = {0}; 4751 struct hwrm_tunnel_dst_port_alloc_output *resp = bp->hwrm_cmd_resp_addr; 4752 4753 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_ALLOC, -1, -1); 4754 4755 req.tunnel_type = tunnel_type; 4756 req.tunnel_dst_port_val = port; 4757 4758 mutex_lock(&bp->hwrm_cmd_lock); 4759 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4760 if (rc) { 4761 netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n", 4762 rc); 4763 goto err_out; 4764 } 4765 4766 switch (tunnel_type) { 4767 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN: 4768 bp->vxlan_fw_dst_port_id = 4769 le16_to_cpu(resp->tunnel_dst_port_id); 4770 break; 4771 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE: 4772 bp->nge_fw_dst_port_id = le16_to_cpu(resp->tunnel_dst_port_id); 4773 break; 4774 default: 4775 break; 4776 } 4777 4778 err_out: 4779 mutex_unlock(&bp->hwrm_cmd_lock); 4780 return rc; 4781 } 4782 4783 static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id) 4784 { 4785 struct hwrm_cfa_l2_set_rx_mask_input req = {0}; 4786 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 4787 4788 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_SET_RX_MASK, -1, -1); 4789 req.vnic_id = cpu_to_le32(vnic->fw_vnic_id); 4790 4791 req.num_mc_entries = cpu_to_le32(vnic->mc_list_count); 4792 req.mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping); 4793 req.mask = cpu_to_le32(vnic->rx_mask); 4794 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4795 } 4796 4797 #ifdef CONFIG_RFS_ACCEL 4798 static int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp, 4799 struct bnxt_ntuple_filter *fltr) 4800 { 4801 struct hwrm_cfa_ntuple_filter_free_input req = {0}; 4802 4803 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_FREE, -1, -1); 4804 req.ntuple_filter_id = fltr->filter_id; 4805 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4806 } 4807 4808 #define BNXT_NTP_FLTR_FLAGS \ 4809 (CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID | \ 4810 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE | \ 4811 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR | \ 4812 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE | \ 4813 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR | \ 4814 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK | \ 4815 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR | \ 4816 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK | \ 4817 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL | \ 4818 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT | \ 4819 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK | \ 4820 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT | \ 4821 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK | \ 4822 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID) 4823 4824 #define BNXT_NTP_TUNNEL_FLTR_FLAG \ 4825 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE 4826 4827 static int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp, 4828 struct bnxt_ntuple_filter *fltr) 4829 { 4830 struct hwrm_cfa_ntuple_filter_alloc_input req = {0}; 4831 struct hwrm_cfa_ntuple_filter_alloc_output *resp; 4832 struct flow_keys *keys = &fltr->fkeys; 4833 struct bnxt_vnic_info *vnic; 4834 u32 flags = 0; 4835 int rc = 0; 4836 4837 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_ALLOC, -1, -1); 4838 req.l2_filter_id = bp->vnic_info[0].fw_l2_filter_id[fltr->l2_fltr_idx]; 4839 4840 if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2) { 4841 flags = CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DEST_RFS_RING_IDX; 4842 req.dst_id = cpu_to_le16(fltr->rxq); 4843 } else { 4844 vnic = &bp->vnic_info[fltr->rxq + 1]; 4845 req.dst_id = cpu_to_le16(vnic->fw_vnic_id); 4846 } 4847 req.flags = cpu_to_le32(flags); 4848 req.enables = cpu_to_le32(BNXT_NTP_FLTR_FLAGS); 4849 4850 req.ethertype = htons(ETH_P_IP); 4851 memcpy(req.src_macaddr, fltr->src_mac_addr, ETH_ALEN); 4852 req.ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4; 4853 req.ip_protocol = keys->basic.ip_proto; 4854 4855 if (keys->basic.n_proto == htons(ETH_P_IPV6)) { 4856 int i; 4857 4858 req.ethertype = htons(ETH_P_IPV6); 4859 req.ip_addr_type = 4860 CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6; 4861 *(struct in6_addr *)&req.src_ipaddr[0] = 4862 keys->addrs.v6addrs.src; 4863 *(struct in6_addr *)&req.dst_ipaddr[0] = 4864 keys->addrs.v6addrs.dst; 4865 for (i = 0; i < 4; i++) { 4866 req.src_ipaddr_mask[i] = cpu_to_be32(0xffffffff); 4867 req.dst_ipaddr_mask[i] = cpu_to_be32(0xffffffff); 4868 } 4869 } else { 4870 req.src_ipaddr[0] = keys->addrs.v4addrs.src; 4871 req.src_ipaddr_mask[0] = cpu_to_be32(0xffffffff); 4872 req.dst_ipaddr[0] = keys->addrs.v4addrs.dst; 4873 req.dst_ipaddr_mask[0] = cpu_to_be32(0xffffffff); 4874 } 4875 if (keys->control.flags & FLOW_DIS_ENCAPSULATION) { 4876 req.enables |= cpu_to_le32(BNXT_NTP_TUNNEL_FLTR_FLAG); 4877 req.tunnel_type = 4878 CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL; 4879 } 4880 4881 req.src_port = keys->ports.src; 4882 req.src_port_mask = cpu_to_be16(0xffff); 4883 req.dst_port = keys->ports.dst; 4884 req.dst_port_mask = cpu_to_be16(0xffff); 4885 4886 mutex_lock(&bp->hwrm_cmd_lock); 4887 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4888 if (!rc) { 4889 resp = bnxt_get_hwrm_resp_addr(bp, &req); 4890 fltr->filter_id = resp->ntuple_filter_id; 4891 } 4892 mutex_unlock(&bp->hwrm_cmd_lock); 4893 return rc; 4894 } 4895 #endif 4896 4897 static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx, 4898 u8 *mac_addr) 4899 { 4900 u32 rc = 0; 4901 struct hwrm_cfa_l2_filter_alloc_input req = {0}; 4902 struct hwrm_cfa_l2_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr; 4903 4904 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_ALLOC, -1, -1); 4905 req.flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX); 4906 if (!BNXT_CHIP_TYPE_NITRO_A0(bp)) 4907 req.flags |= 4908 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST); 4909 req.dst_id = cpu_to_le16(bp->vnic_info[vnic_id].fw_vnic_id); 4910 req.enables = 4911 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR | 4912 CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID | 4913 CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK); 4914 memcpy(req.l2_addr, mac_addr, ETH_ALEN); 4915 req.l2_addr_mask[0] = 0xff; 4916 req.l2_addr_mask[1] = 0xff; 4917 req.l2_addr_mask[2] = 0xff; 4918 req.l2_addr_mask[3] = 0xff; 4919 req.l2_addr_mask[4] = 0xff; 4920 req.l2_addr_mask[5] = 0xff; 4921 4922 mutex_lock(&bp->hwrm_cmd_lock); 4923 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4924 if (!rc) 4925 bp->vnic_info[vnic_id].fw_l2_filter_id[idx] = 4926 resp->l2_filter_id; 4927 mutex_unlock(&bp->hwrm_cmd_lock); 4928 return rc; 4929 } 4930 4931 static int bnxt_hwrm_clear_vnic_filter(struct bnxt *bp) 4932 { 4933 u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */ 4934 int rc = 0; 4935 4936 /* Any associated ntuple filters will also be cleared by firmware. */ 4937 mutex_lock(&bp->hwrm_cmd_lock); 4938 for (i = 0; i < num_of_vnics; i++) { 4939 struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; 4940 4941 for (j = 0; j < vnic->uc_filter_count; j++) { 4942 struct hwrm_cfa_l2_filter_free_input req = {0}; 4943 4944 bnxt_hwrm_cmd_hdr_init(bp, &req, 4945 HWRM_CFA_L2_FILTER_FREE, -1, -1); 4946 4947 req.l2_filter_id = vnic->fw_l2_filter_id[j]; 4948 4949 rc = _hwrm_send_message(bp, &req, sizeof(req), 4950 HWRM_CMD_TIMEOUT); 4951 } 4952 vnic->uc_filter_count = 0; 4953 } 4954 mutex_unlock(&bp->hwrm_cmd_lock); 4955 4956 return rc; 4957 } 4958 4959 static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags) 4960 { 4961 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 4962 u16 max_aggs = VNIC_TPA_CFG_REQ_MAX_AGGS_MAX; 4963 struct hwrm_vnic_tpa_cfg_input req = {0}; 4964 4965 if (vnic->fw_vnic_id == INVALID_HW_RING_ID) 4966 return 0; 4967 4968 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_TPA_CFG, -1, -1); 4969 4970 if (tpa_flags) { 4971 u16 mss = bp->dev->mtu - 40; 4972 u32 nsegs, n, segs = 0, flags; 4973 4974 flags = VNIC_TPA_CFG_REQ_FLAGS_TPA | 4975 VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA | 4976 VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE | 4977 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN | 4978 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ; 4979 if (tpa_flags & BNXT_FLAG_GRO) 4980 flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO; 4981 4982 req.flags = cpu_to_le32(flags); 4983 4984 req.enables = 4985 cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS | 4986 VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS | 4987 VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN); 4988 4989 /* Number of segs are log2 units, and first packet is not 4990 * included as part of this units. 4991 */ 4992 if (mss <= BNXT_RX_PAGE_SIZE) { 4993 n = BNXT_RX_PAGE_SIZE / mss; 4994 nsegs = (MAX_SKB_FRAGS - 1) * n; 4995 } else { 4996 n = mss / BNXT_RX_PAGE_SIZE; 4997 if (mss & (BNXT_RX_PAGE_SIZE - 1)) 4998 n++; 4999 nsegs = (MAX_SKB_FRAGS - n) / n; 5000 } 5001 5002 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5003 segs = MAX_TPA_SEGS_P5; 5004 max_aggs = bp->max_tpa; 5005 } else { 5006 segs = ilog2(nsegs); 5007 } 5008 req.max_agg_segs = cpu_to_le16(segs); 5009 req.max_aggs = cpu_to_le16(max_aggs); 5010 5011 req.min_agg_len = cpu_to_le32(512); 5012 } 5013 req.vnic_id = cpu_to_le16(vnic->fw_vnic_id); 5014 5015 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5016 } 5017 5018 static u16 bnxt_cp_ring_from_grp(struct bnxt *bp, struct bnxt_ring_struct *ring) 5019 { 5020 struct bnxt_ring_grp_info *grp_info; 5021 5022 grp_info = &bp->grp_info[ring->grp_idx]; 5023 return grp_info->cp_fw_ring_id; 5024 } 5025 5026 static u16 bnxt_cp_ring_for_rx(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) 5027 { 5028 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5029 struct bnxt_napi *bnapi = rxr->bnapi; 5030 struct bnxt_cp_ring_info *cpr; 5031 5032 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_RX_HDL]; 5033 return cpr->cp_ring_struct.fw_ring_id; 5034 } else { 5035 return bnxt_cp_ring_from_grp(bp, &rxr->rx_ring_struct); 5036 } 5037 } 5038 5039 static u16 bnxt_cp_ring_for_tx(struct bnxt *bp, struct bnxt_tx_ring_info *txr) 5040 { 5041 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5042 struct bnxt_napi *bnapi = txr->bnapi; 5043 struct bnxt_cp_ring_info *cpr; 5044 5045 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_TX_HDL]; 5046 return cpr->cp_ring_struct.fw_ring_id; 5047 } else { 5048 return bnxt_cp_ring_from_grp(bp, &txr->tx_ring_struct); 5049 } 5050 } 5051 5052 static int bnxt_alloc_rss_indir_tbl(struct bnxt *bp) 5053 { 5054 int entries; 5055 5056 if (bp->flags & BNXT_FLAG_CHIP_P5) 5057 entries = BNXT_MAX_RSS_TABLE_ENTRIES_P5; 5058 else 5059 entries = HW_HASH_INDEX_SIZE; 5060 5061 bp->rss_indir_tbl_entries = entries; 5062 bp->rss_indir_tbl = kmalloc_array(entries, sizeof(*bp->rss_indir_tbl), 5063 GFP_KERNEL); 5064 if (!bp->rss_indir_tbl) 5065 return -ENOMEM; 5066 return 0; 5067 } 5068 5069 static void bnxt_set_dflt_rss_indir_tbl(struct bnxt *bp) 5070 { 5071 u16 max_rings, max_entries, pad, i; 5072 5073 if (!bp->rx_nr_rings) 5074 return; 5075 5076 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 5077 max_rings = bp->rx_nr_rings - 1; 5078 else 5079 max_rings = bp->rx_nr_rings; 5080 5081 max_entries = bnxt_get_rxfh_indir_size(bp->dev); 5082 5083 for (i = 0; i < max_entries; i++) 5084 bp->rss_indir_tbl[i] = ethtool_rxfh_indir_default(i, max_rings); 5085 5086 pad = bp->rss_indir_tbl_entries - max_entries; 5087 if (pad) 5088 memset(&bp->rss_indir_tbl[i], 0, pad * sizeof(u16)); 5089 } 5090 5091 static u16 bnxt_get_max_rss_ring(struct bnxt *bp) 5092 { 5093 u16 i, tbl_size, max_ring = 0; 5094 5095 if (!bp->rss_indir_tbl) 5096 return 0; 5097 5098 tbl_size = bnxt_get_rxfh_indir_size(bp->dev); 5099 for (i = 0; i < tbl_size; i++) 5100 max_ring = max(max_ring, bp->rss_indir_tbl[i]); 5101 return max_ring; 5102 } 5103 5104 int bnxt_get_nr_rss_ctxs(struct bnxt *bp, int rx_rings) 5105 { 5106 if (bp->flags & BNXT_FLAG_CHIP_P5) 5107 return DIV_ROUND_UP(rx_rings, BNXT_RSS_TABLE_ENTRIES_P5); 5108 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 5109 return 2; 5110 return 1; 5111 } 5112 5113 static void __bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic) 5114 { 5115 bool no_rss = !(vnic->flags & BNXT_VNIC_RSS_FLAG); 5116 u16 i, j; 5117 5118 /* Fill the RSS indirection table with ring group ids */ 5119 for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++) { 5120 if (!no_rss) 5121 j = bp->rss_indir_tbl[i]; 5122 vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]); 5123 } 5124 } 5125 5126 static void __bnxt_fill_hw_rss_tbl_p5(struct bnxt *bp, 5127 struct bnxt_vnic_info *vnic) 5128 { 5129 __le16 *ring_tbl = vnic->rss_table; 5130 struct bnxt_rx_ring_info *rxr; 5131 u16 tbl_size, i; 5132 5133 tbl_size = bnxt_get_rxfh_indir_size(bp->dev); 5134 5135 for (i = 0; i < tbl_size; i++) { 5136 u16 ring_id, j; 5137 5138 j = bp->rss_indir_tbl[i]; 5139 rxr = &bp->rx_ring[j]; 5140 5141 ring_id = rxr->rx_ring_struct.fw_ring_id; 5142 *ring_tbl++ = cpu_to_le16(ring_id); 5143 ring_id = bnxt_cp_ring_for_rx(bp, rxr); 5144 *ring_tbl++ = cpu_to_le16(ring_id); 5145 } 5146 } 5147 5148 static void bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic) 5149 { 5150 if (bp->flags & BNXT_FLAG_CHIP_P5) 5151 __bnxt_fill_hw_rss_tbl_p5(bp, vnic); 5152 else 5153 __bnxt_fill_hw_rss_tbl(bp, vnic); 5154 } 5155 5156 static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss) 5157 { 5158 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5159 struct hwrm_vnic_rss_cfg_input req = {0}; 5160 5161 if ((bp->flags & BNXT_FLAG_CHIP_P5) || 5162 vnic->fw_rss_cos_lb_ctx[0] == INVALID_HW_RING_ID) 5163 return 0; 5164 5165 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1); 5166 if (set_rss) { 5167 bnxt_fill_hw_rss_tbl(bp, vnic); 5168 req.hash_type = cpu_to_le32(bp->rss_hash_cfg); 5169 req.hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT; 5170 req.ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr); 5171 req.hash_key_tbl_addr = 5172 cpu_to_le64(vnic->rss_hash_key_dma_addr); 5173 } 5174 req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]); 5175 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5176 } 5177 5178 static int bnxt_hwrm_vnic_set_rss_p5(struct bnxt *bp, u16 vnic_id, bool set_rss) 5179 { 5180 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5181 struct hwrm_vnic_rss_cfg_input req = {0}; 5182 dma_addr_t ring_tbl_map; 5183 u32 i, nr_ctxs; 5184 5185 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1); 5186 req.vnic_id = cpu_to_le16(vnic->fw_vnic_id); 5187 if (!set_rss) { 5188 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5189 return 0; 5190 } 5191 bnxt_fill_hw_rss_tbl(bp, vnic); 5192 req.hash_type = cpu_to_le32(bp->rss_hash_cfg); 5193 req.hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT; 5194 req.hash_key_tbl_addr = cpu_to_le64(vnic->rss_hash_key_dma_addr); 5195 ring_tbl_map = vnic->rss_table_dma_addr; 5196 nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings); 5197 for (i = 0; i < nr_ctxs; ring_tbl_map += BNXT_RSS_TABLE_SIZE_P5, i++) { 5198 int rc; 5199 5200 req.ring_grp_tbl_addr = cpu_to_le64(ring_tbl_map); 5201 req.ring_table_pair_index = i; 5202 req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[i]); 5203 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5204 if (rc) 5205 return rc; 5206 } 5207 return 0; 5208 } 5209 5210 static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id) 5211 { 5212 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5213 struct hwrm_vnic_plcmodes_cfg_input req = {0}; 5214 5215 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_PLCMODES_CFG, -1, -1); 5216 req.flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT | 5217 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 | 5218 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6); 5219 req.enables = 5220 cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID | 5221 VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID); 5222 /* thresholds not implemented in firmware yet */ 5223 req.jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh); 5224 req.hds_threshold = cpu_to_le16(bp->rx_copy_thresh); 5225 req.vnic_id = cpu_to_le32(vnic->fw_vnic_id); 5226 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5227 } 5228 5229 static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id, 5230 u16 ctx_idx) 5231 { 5232 struct hwrm_vnic_rss_cos_lb_ctx_free_input req = {0}; 5233 5234 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_FREE, -1, -1); 5235 req.rss_cos_lb_ctx_id = 5236 cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx]); 5237 5238 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5239 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID; 5240 } 5241 5242 static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp) 5243 { 5244 int i, j; 5245 5246 for (i = 0; i < bp->nr_vnics; i++) { 5247 struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; 5248 5249 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) { 5250 if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID) 5251 bnxt_hwrm_vnic_ctx_free_one(bp, i, j); 5252 } 5253 } 5254 bp->rsscos_nr_ctxs = 0; 5255 } 5256 5257 static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id, u16 ctx_idx) 5258 { 5259 int rc; 5260 struct hwrm_vnic_rss_cos_lb_ctx_alloc_input req = {0}; 5261 struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp = 5262 bp->hwrm_cmd_resp_addr; 5263 5264 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC, -1, 5265 -1); 5266 5267 mutex_lock(&bp->hwrm_cmd_lock); 5268 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5269 if (!rc) 5270 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = 5271 le16_to_cpu(resp->rss_cos_lb_ctx_id); 5272 mutex_unlock(&bp->hwrm_cmd_lock); 5273 5274 return rc; 5275 } 5276 5277 static u32 bnxt_get_roce_vnic_mode(struct bnxt *bp) 5278 { 5279 if (bp->flags & BNXT_FLAG_ROCE_MIRROR_CAP) 5280 return VNIC_CFG_REQ_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE; 5281 return VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE; 5282 } 5283 5284 int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id) 5285 { 5286 unsigned int ring = 0, grp_idx; 5287 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5288 struct hwrm_vnic_cfg_input req = {0}; 5289 u16 def_vlan = 0; 5290 5291 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_CFG, -1, -1); 5292 5293 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5294 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0]; 5295 5296 req.default_rx_ring_id = 5297 cpu_to_le16(rxr->rx_ring_struct.fw_ring_id); 5298 req.default_cmpl_ring_id = 5299 cpu_to_le16(bnxt_cp_ring_for_rx(bp, rxr)); 5300 req.enables = 5301 cpu_to_le32(VNIC_CFG_REQ_ENABLES_DEFAULT_RX_RING_ID | 5302 VNIC_CFG_REQ_ENABLES_DEFAULT_CMPL_RING_ID); 5303 goto vnic_mru; 5304 } 5305 req.enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP); 5306 /* Only RSS support for now TBD: COS & LB */ 5307 if (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID) { 5308 req.rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]); 5309 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE | 5310 VNIC_CFG_REQ_ENABLES_MRU); 5311 } else if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) { 5312 req.rss_rule = 5313 cpu_to_le16(bp->vnic_info[0].fw_rss_cos_lb_ctx[0]); 5314 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE | 5315 VNIC_CFG_REQ_ENABLES_MRU); 5316 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE); 5317 } else { 5318 req.rss_rule = cpu_to_le16(0xffff); 5319 } 5320 5321 if (BNXT_CHIP_TYPE_NITRO_A0(bp) && 5322 (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID)) { 5323 req.cos_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[1]); 5324 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_COS_RULE); 5325 } else { 5326 req.cos_rule = cpu_to_le16(0xffff); 5327 } 5328 5329 if (vnic->flags & BNXT_VNIC_RSS_FLAG) 5330 ring = 0; 5331 else if (vnic->flags & BNXT_VNIC_RFS_FLAG) 5332 ring = vnic_id - 1; 5333 else if ((vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp)) 5334 ring = bp->rx_nr_rings - 1; 5335 5336 grp_idx = bp->rx_ring[ring].bnapi->index; 5337 req.dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id); 5338 req.lb_rule = cpu_to_le16(0xffff); 5339 vnic_mru: 5340 req.mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + VLAN_HLEN); 5341 5342 req.vnic_id = cpu_to_le16(vnic->fw_vnic_id); 5343 #ifdef CONFIG_BNXT_SRIOV 5344 if (BNXT_VF(bp)) 5345 def_vlan = bp->vf.vlan; 5346 #endif 5347 if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan) 5348 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE); 5349 if (!vnic_id && bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP)) 5350 req.flags |= cpu_to_le32(bnxt_get_roce_vnic_mode(bp)); 5351 5352 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5353 } 5354 5355 static void bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id) 5356 { 5357 if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) { 5358 struct hwrm_vnic_free_input req = {0}; 5359 5360 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_FREE, -1, -1); 5361 req.vnic_id = 5362 cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id); 5363 5364 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5365 bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID; 5366 } 5367 } 5368 5369 static void bnxt_hwrm_vnic_free(struct bnxt *bp) 5370 { 5371 u16 i; 5372 5373 for (i = 0; i < bp->nr_vnics; i++) 5374 bnxt_hwrm_vnic_free_one(bp, i); 5375 } 5376 5377 static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id, 5378 unsigned int start_rx_ring_idx, 5379 unsigned int nr_rings) 5380 { 5381 int rc = 0; 5382 unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings; 5383 struct hwrm_vnic_alloc_input req = {0}; 5384 struct hwrm_vnic_alloc_output *resp = bp->hwrm_cmd_resp_addr; 5385 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5386 5387 if (bp->flags & BNXT_FLAG_CHIP_P5) 5388 goto vnic_no_ring_grps; 5389 5390 /* map ring groups to this vnic */ 5391 for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) { 5392 grp_idx = bp->rx_ring[i].bnapi->index; 5393 if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) { 5394 netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n", 5395 j, nr_rings); 5396 break; 5397 } 5398 vnic->fw_grp_ids[j] = bp->grp_info[grp_idx].fw_grp_id; 5399 } 5400 5401 vnic_no_ring_grps: 5402 for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++) 5403 vnic->fw_rss_cos_lb_ctx[i] = INVALID_HW_RING_ID; 5404 if (vnic_id == 0) 5405 req.flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT); 5406 5407 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_ALLOC, -1, -1); 5408 5409 mutex_lock(&bp->hwrm_cmd_lock); 5410 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5411 if (!rc) 5412 vnic->fw_vnic_id = le32_to_cpu(resp->vnic_id); 5413 mutex_unlock(&bp->hwrm_cmd_lock); 5414 return rc; 5415 } 5416 5417 static int bnxt_hwrm_vnic_qcaps(struct bnxt *bp) 5418 { 5419 struct hwrm_vnic_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 5420 struct hwrm_vnic_qcaps_input req = {0}; 5421 int rc; 5422 5423 bp->hw_ring_stats_size = sizeof(struct ctx_hw_stats); 5424 bp->flags &= ~(BNXT_FLAG_NEW_RSS_CAP | BNXT_FLAG_ROCE_MIRROR_CAP); 5425 if (bp->hwrm_spec_code < 0x10600) 5426 return 0; 5427 5428 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_QCAPS, -1, -1); 5429 mutex_lock(&bp->hwrm_cmd_lock); 5430 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5431 if (!rc) { 5432 u32 flags = le32_to_cpu(resp->flags); 5433 5434 if (!(bp->flags & BNXT_FLAG_CHIP_P5) && 5435 (flags & VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP)) 5436 bp->flags |= BNXT_FLAG_NEW_RSS_CAP; 5437 if (flags & 5438 VNIC_QCAPS_RESP_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_CAP) 5439 bp->flags |= BNXT_FLAG_ROCE_MIRROR_CAP; 5440 5441 /* Older P5 fw before EXT_HW_STATS support did not set 5442 * VLAN_STRIP_CAP properly. 5443 */ 5444 if ((flags & VNIC_QCAPS_RESP_FLAGS_VLAN_STRIP_CAP) || 5445 (BNXT_CHIP_P5_THOR(bp) && 5446 !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))) 5447 bp->fw_cap |= BNXT_FW_CAP_VLAN_RX_STRIP; 5448 bp->max_tpa_v2 = le16_to_cpu(resp->max_aggs_supported); 5449 if (bp->max_tpa_v2) { 5450 if (BNXT_CHIP_P5_THOR(bp)) 5451 bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P5; 5452 else 5453 bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P5_SR2; 5454 } 5455 } 5456 mutex_unlock(&bp->hwrm_cmd_lock); 5457 return rc; 5458 } 5459 5460 static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp) 5461 { 5462 u16 i; 5463 u32 rc = 0; 5464 5465 if (bp->flags & BNXT_FLAG_CHIP_P5) 5466 return 0; 5467 5468 mutex_lock(&bp->hwrm_cmd_lock); 5469 for (i = 0; i < bp->rx_nr_rings; i++) { 5470 struct hwrm_ring_grp_alloc_input req = {0}; 5471 struct hwrm_ring_grp_alloc_output *resp = 5472 bp->hwrm_cmd_resp_addr; 5473 unsigned int grp_idx = bp->rx_ring[i].bnapi->index; 5474 5475 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_ALLOC, -1, -1); 5476 5477 req.cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id); 5478 req.rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id); 5479 req.ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id); 5480 req.sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx); 5481 5482 rc = _hwrm_send_message(bp, &req, sizeof(req), 5483 HWRM_CMD_TIMEOUT); 5484 if (rc) 5485 break; 5486 5487 bp->grp_info[grp_idx].fw_grp_id = 5488 le32_to_cpu(resp->ring_group_id); 5489 } 5490 mutex_unlock(&bp->hwrm_cmd_lock); 5491 return rc; 5492 } 5493 5494 static void bnxt_hwrm_ring_grp_free(struct bnxt *bp) 5495 { 5496 u16 i; 5497 struct hwrm_ring_grp_free_input req = {0}; 5498 5499 if (!bp->grp_info || (bp->flags & BNXT_FLAG_CHIP_P5)) 5500 return; 5501 5502 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_FREE, -1, -1); 5503 5504 mutex_lock(&bp->hwrm_cmd_lock); 5505 for (i = 0; i < bp->cp_nr_rings; i++) { 5506 if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID) 5507 continue; 5508 req.ring_group_id = 5509 cpu_to_le32(bp->grp_info[i].fw_grp_id); 5510 5511 _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5512 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID; 5513 } 5514 mutex_unlock(&bp->hwrm_cmd_lock); 5515 } 5516 5517 static int hwrm_ring_alloc_send_msg(struct bnxt *bp, 5518 struct bnxt_ring_struct *ring, 5519 u32 ring_type, u32 map_index) 5520 { 5521 int rc = 0, err = 0; 5522 struct hwrm_ring_alloc_input req = {0}; 5523 struct hwrm_ring_alloc_output *resp = bp->hwrm_cmd_resp_addr; 5524 struct bnxt_ring_mem_info *rmem = &ring->ring_mem; 5525 struct bnxt_ring_grp_info *grp_info; 5526 u16 ring_id; 5527 5528 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_ALLOC, -1, -1); 5529 5530 req.enables = 0; 5531 if (rmem->nr_pages > 1) { 5532 req.page_tbl_addr = cpu_to_le64(rmem->pg_tbl_map); 5533 /* Page size is in log2 units */ 5534 req.page_size = BNXT_PAGE_SHIFT; 5535 req.page_tbl_depth = 1; 5536 } else { 5537 req.page_tbl_addr = cpu_to_le64(rmem->dma_arr[0]); 5538 } 5539 req.fbo = 0; 5540 /* Association of ring index with doorbell index and MSIX number */ 5541 req.logical_id = cpu_to_le16(map_index); 5542 5543 switch (ring_type) { 5544 case HWRM_RING_ALLOC_TX: { 5545 struct bnxt_tx_ring_info *txr; 5546 5547 txr = container_of(ring, struct bnxt_tx_ring_info, 5548 tx_ring_struct); 5549 req.ring_type = RING_ALLOC_REQ_RING_TYPE_TX; 5550 /* Association of transmit ring with completion ring */ 5551 grp_info = &bp->grp_info[ring->grp_idx]; 5552 req.cmpl_ring_id = cpu_to_le16(bnxt_cp_ring_for_tx(bp, txr)); 5553 req.length = cpu_to_le32(bp->tx_ring_mask + 1); 5554 req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx); 5555 req.queue_id = cpu_to_le16(ring->queue_id); 5556 break; 5557 } 5558 case HWRM_RING_ALLOC_RX: 5559 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX; 5560 req.length = cpu_to_le32(bp->rx_ring_mask + 1); 5561 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5562 u16 flags = 0; 5563 5564 /* Association of rx ring with stats context */ 5565 grp_info = &bp->grp_info[ring->grp_idx]; 5566 req.rx_buf_size = cpu_to_le16(bp->rx_buf_use_size); 5567 req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx); 5568 req.enables |= cpu_to_le32( 5569 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID); 5570 if (NET_IP_ALIGN == 2) 5571 flags = RING_ALLOC_REQ_FLAGS_RX_SOP_PAD; 5572 req.flags = cpu_to_le16(flags); 5573 } 5574 break; 5575 case HWRM_RING_ALLOC_AGG: 5576 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5577 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX_AGG; 5578 /* Association of agg ring with rx ring */ 5579 grp_info = &bp->grp_info[ring->grp_idx]; 5580 req.rx_ring_id = cpu_to_le16(grp_info->rx_fw_ring_id); 5581 req.rx_buf_size = cpu_to_le16(BNXT_RX_PAGE_SIZE); 5582 req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx); 5583 req.enables |= cpu_to_le32( 5584 RING_ALLOC_REQ_ENABLES_RX_RING_ID_VALID | 5585 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID); 5586 } else { 5587 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX; 5588 } 5589 req.length = cpu_to_le32(bp->rx_agg_ring_mask + 1); 5590 break; 5591 case HWRM_RING_ALLOC_CMPL: 5592 req.ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL; 5593 req.length = cpu_to_le32(bp->cp_ring_mask + 1); 5594 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5595 /* Association of cp ring with nq */ 5596 grp_info = &bp->grp_info[map_index]; 5597 req.nq_ring_id = cpu_to_le16(grp_info->cp_fw_ring_id); 5598 req.cq_handle = cpu_to_le64(ring->handle); 5599 req.enables |= cpu_to_le32( 5600 RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID); 5601 } else if (bp->flags & BNXT_FLAG_USING_MSIX) { 5602 req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX; 5603 } 5604 break; 5605 case HWRM_RING_ALLOC_NQ: 5606 req.ring_type = RING_ALLOC_REQ_RING_TYPE_NQ; 5607 req.length = cpu_to_le32(bp->cp_ring_mask + 1); 5608 if (bp->flags & BNXT_FLAG_USING_MSIX) 5609 req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX; 5610 break; 5611 default: 5612 netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n", 5613 ring_type); 5614 return -1; 5615 } 5616 5617 mutex_lock(&bp->hwrm_cmd_lock); 5618 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5619 err = le16_to_cpu(resp->error_code); 5620 ring_id = le16_to_cpu(resp->ring_id); 5621 mutex_unlock(&bp->hwrm_cmd_lock); 5622 5623 if (rc || err) { 5624 netdev_err(bp->dev, "hwrm_ring_alloc type %d failed. rc:%x err:%x\n", 5625 ring_type, rc, err); 5626 return -EIO; 5627 } 5628 ring->fw_ring_id = ring_id; 5629 return rc; 5630 } 5631 5632 static int bnxt_hwrm_set_async_event_cr(struct bnxt *bp, int idx) 5633 { 5634 int rc; 5635 5636 if (BNXT_PF(bp)) { 5637 struct hwrm_func_cfg_input req = {0}; 5638 5639 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1); 5640 req.fid = cpu_to_le16(0xffff); 5641 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR); 5642 req.async_event_cr = cpu_to_le16(idx); 5643 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5644 } else { 5645 struct hwrm_func_vf_cfg_input req = {0}; 5646 5647 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_VF_CFG, -1, -1); 5648 req.enables = 5649 cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR); 5650 req.async_event_cr = cpu_to_le16(idx); 5651 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5652 } 5653 return rc; 5654 } 5655 5656 static void bnxt_set_db(struct bnxt *bp, struct bnxt_db_info *db, u32 ring_type, 5657 u32 map_idx, u32 xid) 5658 { 5659 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5660 if (BNXT_PF(bp)) 5661 db->doorbell = bp->bar1 + DB_PF_OFFSET_P5; 5662 else 5663 db->doorbell = bp->bar1 + DB_VF_OFFSET_P5; 5664 switch (ring_type) { 5665 case HWRM_RING_ALLOC_TX: 5666 db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SQ; 5667 break; 5668 case HWRM_RING_ALLOC_RX: 5669 case HWRM_RING_ALLOC_AGG: 5670 db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SRQ; 5671 break; 5672 case HWRM_RING_ALLOC_CMPL: 5673 db->db_key64 = DBR_PATH_L2; 5674 break; 5675 case HWRM_RING_ALLOC_NQ: 5676 db->db_key64 = DBR_PATH_L2; 5677 break; 5678 } 5679 db->db_key64 |= (u64)xid << DBR_XID_SFT; 5680 } else { 5681 db->doorbell = bp->bar1 + map_idx * 0x80; 5682 switch (ring_type) { 5683 case HWRM_RING_ALLOC_TX: 5684 db->db_key32 = DB_KEY_TX; 5685 break; 5686 case HWRM_RING_ALLOC_RX: 5687 case HWRM_RING_ALLOC_AGG: 5688 db->db_key32 = DB_KEY_RX; 5689 break; 5690 case HWRM_RING_ALLOC_CMPL: 5691 db->db_key32 = DB_KEY_CP; 5692 break; 5693 } 5694 } 5695 } 5696 5697 static int bnxt_hwrm_ring_alloc(struct bnxt *bp) 5698 { 5699 bool agg_rings = !!(bp->flags & BNXT_FLAG_AGG_RINGS); 5700 int i, rc = 0; 5701 u32 type; 5702 5703 if (bp->flags & BNXT_FLAG_CHIP_P5) 5704 type = HWRM_RING_ALLOC_NQ; 5705 else 5706 type = HWRM_RING_ALLOC_CMPL; 5707 for (i = 0; i < bp->cp_nr_rings; i++) { 5708 struct bnxt_napi *bnapi = bp->bnapi[i]; 5709 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 5710 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 5711 u32 map_idx = ring->map_idx; 5712 unsigned int vector; 5713 5714 vector = bp->irq_tbl[map_idx].vector; 5715 disable_irq_nosync(vector); 5716 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 5717 if (rc) { 5718 enable_irq(vector); 5719 goto err_out; 5720 } 5721 bnxt_set_db(bp, &cpr->cp_db, type, map_idx, ring->fw_ring_id); 5722 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons); 5723 enable_irq(vector); 5724 bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id; 5725 5726 if (!i) { 5727 rc = bnxt_hwrm_set_async_event_cr(bp, ring->fw_ring_id); 5728 if (rc) 5729 netdev_warn(bp->dev, "Failed to set async event completion ring.\n"); 5730 } 5731 } 5732 5733 type = HWRM_RING_ALLOC_TX; 5734 for (i = 0; i < bp->tx_nr_rings; i++) { 5735 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 5736 struct bnxt_ring_struct *ring; 5737 u32 map_idx; 5738 5739 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5740 struct bnxt_napi *bnapi = txr->bnapi; 5741 struct bnxt_cp_ring_info *cpr, *cpr2; 5742 u32 type2 = HWRM_RING_ALLOC_CMPL; 5743 5744 cpr = &bnapi->cp_ring; 5745 cpr2 = cpr->cp_ring_arr[BNXT_TX_HDL]; 5746 ring = &cpr2->cp_ring_struct; 5747 ring->handle = BNXT_TX_HDL; 5748 map_idx = bnapi->index; 5749 rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx); 5750 if (rc) 5751 goto err_out; 5752 bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx, 5753 ring->fw_ring_id); 5754 bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons); 5755 } 5756 ring = &txr->tx_ring_struct; 5757 map_idx = i; 5758 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 5759 if (rc) 5760 goto err_out; 5761 bnxt_set_db(bp, &txr->tx_db, type, map_idx, ring->fw_ring_id); 5762 } 5763 5764 type = HWRM_RING_ALLOC_RX; 5765 for (i = 0; i < bp->rx_nr_rings; i++) { 5766 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 5767 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct; 5768 struct bnxt_napi *bnapi = rxr->bnapi; 5769 u32 map_idx = bnapi->index; 5770 5771 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 5772 if (rc) 5773 goto err_out; 5774 bnxt_set_db(bp, &rxr->rx_db, type, map_idx, ring->fw_ring_id); 5775 /* If we have agg rings, post agg buffers first. */ 5776 if (!agg_rings) 5777 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 5778 bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id; 5779 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5780 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 5781 u32 type2 = HWRM_RING_ALLOC_CMPL; 5782 struct bnxt_cp_ring_info *cpr2; 5783 5784 cpr2 = cpr->cp_ring_arr[BNXT_RX_HDL]; 5785 ring = &cpr2->cp_ring_struct; 5786 ring->handle = BNXT_RX_HDL; 5787 rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx); 5788 if (rc) 5789 goto err_out; 5790 bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx, 5791 ring->fw_ring_id); 5792 bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons); 5793 } 5794 } 5795 5796 if (agg_rings) { 5797 type = HWRM_RING_ALLOC_AGG; 5798 for (i = 0; i < bp->rx_nr_rings; i++) { 5799 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 5800 struct bnxt_ring_struct *ring = 5801 &rxr->rx_agg_ring_struct; 5802 u32 grp_idx = ring->grp_idx; 5803 u32 map_idx = grp_idx + bp->rx_nr_rings; 5804 5805 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 5806 if (rc) 5807 goto err_out; 5808 5809 bnxt_set_db(bp, &rxr->rx_agg_db, type, map_idx, 5810 ring->fw_ring_id); 5811 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); 5812 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 5813 bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id; 5814 } 5815 } 5816 err_out: 5817 return rc; 5818 } 5819 5820 static int hwrm_ring_free_send_msg(struct bnxt *bp, 5821 struct bnxt_ring_struct *ring, 5822 u32 ring_type, int cmpl_ring_id) 5823 { 5824 int rc; 5825 struct hwrm_ring_free_input req = {0}; 5826 struct hwrm_ring_free_output *resp = bp->hwrm_cmd_resp_addr; 5827 u16 error_code; 5828 5829 if (BNXT_NO_FW_ACCESS(bp)) 5830 return 0; 5831 5832 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_FREE, cmpl_ring_id, -1); 5833 req.ring_type = ring_type; 5834 req.ring_id = cpu_to_le16(ring->fw_ring_id); 5835 5836 mutex_lock(&bp->hwrm_cmd_lock); 5837 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5838 error_code = le16_to_cpu(resp->error_code); 5839 mutex_unlock(&bp->hwrm_cmd_lock); 5840 5841 if (rc || error_code) { 5842 netdev_err(bp->dev, "hwrm_ring_free type %d failed. rc:%x err:%x\n", 5843 ring_type, rc, error_code); 5844 return -EIO; 5845 } 5846 return 0; 5847 } 5848 5849 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path) 5850 { 5851 u32 type; 5852 int i; 5853 5854 if (!bp->bnapi) 5855 return; 5856 5857 for (i = 0; i < bp->tx_nr_rings; i++) { 5858 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 5859 struct bnxt_ring_struct *ring = &txr->tx_ring_struct; 5860 5861 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 5862 u32 cmpl_ring_id = bnxt_cp_ring_for_tx(bp, txr); 5863 5864 hwrm_ring_free_send_msg(bp, ring, 5865 RING_FREE_REQ_RING_TYPE_TX, 5866 close_path ? cmpl_ring_id : 5867 INVALID_HW_RING_ID); 5868 ring->fw_ring_id = INVALID_HW_RING_ID; 5869 } 5870 } 5871 5872 for (i = 0; i < bp->rx_nr_rings; i++) { 5873 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 5874 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct; 5875 u32 grp_idx = rxr->bnapi->index; 5876 5877 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 5878 u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr); 5879 5880 hwrm_ring_free_send_msg(bp, ring, 5881 RING_FREE_REQ_RING_TYPE_RX, 5882 close_path ? cmpl_ring_id : 5883 INVALID_HW_RING_ID); 5884 ring->fw_ring_id = INVALID_HW_RING_ID; 5885 bp->grp_info[grp_idx].rx_fw_ring_id = 5886 INVALID_HW_RING_ID; 5887 } 5888 } 5889 5890 if (bp->flags & BNXT_FLAG_CHIP_P5) 5891 type = RING_FREE_REQ_RING_TYPE_RX_AGG; 5892 else 5893 type = RING_FREE_REQ_RING_TYPE_RX; 5894 for (i = 0; i < bp->rx_nr_rings; i++) { 5895 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 5896 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct; 5897 u32 grp_idx = rxr->bnapi->index; 5898 5899 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 5900 u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr); 5901 5902 hwrm_ring_free_send_msg(bp, ring, type, 5903 close_path ? cmpl_ring_id : 5904 INVALID_HW_RING_ID); 5905 ring->fw_ring_id = INVALID_HW_RING_ID; 5906 bp->grp_info[grp_idx].agg_fw_ring_id = 5907 INVALID_HW_RING_ID; 5908 } 5909 } 5910 5911 /* The completion rings are about to be freed. After that the 5912 * IRQ doorbell will not work anymore. So we need to disable 5913 * IRQ here. 5914 */ 5915 bnxt_disable_int_sync(bp); 5916 5917 if (bp->flags & BNXT_FLAG_CHIP_P5) 5918 type = RING_FREE_REQ_RING_TYPE_NQ; 5919 else 5920 type = RING_FREE_REQ_RING_TYPE_L2_CMPL; 5921 for (i = 0; i < bp->cp_nr_rings; i++) { 5922 struct bnxt_napi *bnapi = bp->bnapi[i]; 5923 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 5924 struct bnxt_ring_struct *ring; 5925 int j; 5926 5927 for (j = 0; j < 2; j++) { 5928 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j]; 5929 5930 if (cpr2) { 5931 ring = &cpr2->cp_ring_struct; 5932 if (ring->fw_ring_id == INVALID_HW_RING_ID) 5933 continue; 5934 hwrm_ring_free_send_msg(bp, ring, 5935 RING_FREE_REQ_RING_TYPE_L2_CMPL, 5936 INVALID_HW_RING_ID); 5937 ring->fw_ring_id = INVALID_HW_RING_ID; 5938 } 5939 } 5940 ring = &cpr->cp_ring_struct; 5941 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 5942 hwrm_ring_free_send_msg(bp, ring, type, 5943 INVALID_HW_RING_ID); 5944 ring->fw_ring_id = INVALID_HW_RING_ID; 5945 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID; 5946 } 5947 } 5948 } 5949 5950 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max, 5951 bool shared); 5952 5953 static int bnxt_hwrm_get_rings(struct bnxt *bp) 5954 { 5955 struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 5956 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 5957 struct hwrm_func_qcfg_input req = {0}; 5958 int rc; 5959 5960 if (bp->hwrm_spec_code < 0x10601) 5961 return 0; 5962 5963 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1); 5964 req.fid = cpu_to_le16(0xffff); 5965 mutex_lock(&bp->hwrm_cmd_lock); 5966 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5967 if (rc) { 5968 mutex_unlock(&bp->hwrm_cmd_lock); 5969 return rc; 5970 } 5971 5972 hw_resc->resv_tx_rings = le16_to_cpu(resp->alloc_tx_rings); 5973 if (BNXT_NEW_RM(bp)) { 5974 u16 cp, stats; 5975 5976 hw_resc->resv_rx_rings = le16_to_cpu(resp->alloc_rx_rings); 5977 hw_resc->resv_hw_ring_grps = 5978 le32_to_cpu(resp->alloc_hw_ring_grps); 5979 hw_resc->resv_vnics = le16_to_cpu(resp->alloc_vnics); 5980 cp = le16_to_cpu(resp->alloc_cmpl_rings); 5981 stats = le16_to_cpu(resp->alloc_stat_ctx); 5982 hw_resc->resv_irqs = cp; 5983 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5984 int rx = hw_resc->resv_rx_rings; 5985 int tx = hw_resc->resv_tx_rings; 5986 5987 if (bp->flags & BNXT_FLAG_AGG_RINGS) 5988 rx >>= 1; 5989 if (cp < (rx + tx)) { 5990 bnxt_trim_rings(bp, &rx, &tx, cp, false); 5991 if (bp->flags & BNXT_FLAG_AGG_RINGS) 5992 rx <<= 1; 5993 hw_resc->resv_rx_rings = rx; 5994 hw_resc->resv_tx_rings = tx; 5995 } 5996 hw_resc->resv_irqs = le16_to_cpu(resp->alloc_msix); 5997 hw_resc->resv_hw_ring_grps = rx; 5998 } 5999 hw_resc->resv_cp_rings = cp; 6000 hw_resc->resv_stat_ctxs = stats; 6001 } 6002 mutex_unlock(&bp->hwrm_cmd_lock); 6003 return 0; 6004 } 6005 6006 /* Caller must hold bp->hwrm_cmd_lock */ 6007 int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings) 6008 { 6009 struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 6010 struct hwrm_func_qcfg_input req = {0}; 6011 int rc; 6012 6013 if (bp->hwrm_spec_code < 0x10601) 6014 return 0; 6015 6016 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1); 6017 req.fid = cpu_to_le16(fid); 6018 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6019 if (!rc) 6020 *tx_rings = le16_to_cpu(resp->alloc_tx_rings); 6021 6022 return rc; 6023 } 6024 6025 static bool bnxt_rfs_supported(struct bnxt *bp); 6026 6027 static void 6028 __bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, struct hwrm_func_cfg_input *req, 6029 int tx_rings, int rx_rings, int ring_grps, 6030 int cp_rings, int stats, int vnics) 6031 { 6032 u32 enables = 0; 6033 6034 bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_CFG, -1, -1); 6035 req->fid = cpu_to_le16(0xffff); 6036 enables |= tx_rings ? FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS : 0; 6037 req->num_tx_rings = cpu_to_le16(tx_rings); 6038 if (BNXT_NEW_RM(bp)) { 6039 enables |= rx_rings ? FUNC_CFG_REQ_ENABLES_NUM_RX_RINGS : 0; 6040 enables |= stats ? FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0; 6041 if (bp->flags & BNXT_FLAG_CHIP_P5) { 6042 enables |= cp_rings ? FUNC_CFG_REQ_ENABLES_NUM_MSIX : 0; 6043 enables |= tx_rings + ring_grps ? 6044 FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0; 6045 enables |= rx_rings ? 6046 FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0; 6047 } else { 6048 enables |= cp_rings ? 6049 FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0; 6050 enables |= ring_grps ? 6051 FUNC_CFG_REQ_ENABLES_NUM_HW_RING_GRPS | 6052 FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0; 6053 } 6054 enables |= vnics ? FUNC_CFG_REQ_ENABLES_NUM_VNICS : 0; 6055 6056 req->num_rx_rings = cpu_to_le16(rx_rings); 6057 if (bp->flags & BNXT_FLAG_CHIP_P5) { 6058 req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps); 6059 req->num_msix = cpu_to_le16(cp_rings); 6060 req->num_rsscos_ctxs = 6061 cpu_to_le16(DIV_ROUND_UP(ring_grps, 64)); 6062 } else { 6063 req->num_cmpl_rings = cpu_to_le16(cp_rings); 6064 req->num_hw_ring_grps = cpu_to_le16(ring_grps); 6065 req->num_rsscos_ctxs = cpu_to_le16(1); 6066 if (!(bp->flags & BNXT_FLAG_NEW_RSS_CAP) && 6067 bnxt_rfs_supported(bp)) 6068 req->num_rsscos_ctxs = 6069 cpu_to_le16(ring_grps + 1); 6070 } 6071 req->num_stat_ctxs = cpu_to_le16(stats); 6072 req->num_vnics = cpu_to_le16(vnics); 6073 } 6074 req->enables = cpu_to_le32(enables); 6075 } 6076 6077 static void 6078 __bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, 6079 struct hwrm_func_vf_cfg_input *req, int tx_rings, 6080 int rx_rings, int ring_grps, int cp_rings, 6081 int stats, int vnics) 6082 { 6083 u32 enables = 0; 6084 6085 bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_VF_CFG, -1, -1); 6086 enables |= tx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_TX_RINGS : 0; 6087 enables |= rx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_RX_RINGS | 6088 FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0; 6089 enables |= stats ? FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0; 6090 if (bp->flags & BNXT_FLAG_CHIP_P5) { 6091 enables |= tx_rings + ring_grps ? 6092 FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0; 6093 } else { 6094 enables |= cp_rings ? 6095 FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0; 6096 enables |= ring_grps ? 6097 FUNC_VF_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0; 6098 } 6099 enables |= vnics ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0; 6100 enables |= FUNC_VF_CFG_REQ_ENABLES_NUM_L2_CTXS; 6101 6102 req->num_l2_ctxs = cpu_to_le16(BNXT_VF_MAX_L2_CTX); 6103 req->num_tx_rings = cpu_to_le16(tx_rings); 6104 req->num_rx_rings = cpu_to_le16(rx_rings); 6105 if (bp->flags & BNXT_FLAG_CHIP_P5) { 6106 req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps); 6107 req->num_rsscos_ctxs = cpu_to_le16(DIV_ROUND_UP(ring_grps, 64)); 6108 } else { 6109 req->num_cmpl_rings = cpu_to_le16(cp_rings); 6110 req->num_hw_ring_grps = cpu_to_le16(ring_grps); 6111 req->num_rsscos_ctxs = cpu_to_le16(BNXT_VF_MAX_RSS_CTX); 6112 } 6113 req->num_stat_ctxs = cpu_to_le16(stats); 6114 req->num_vnics = cpu_to_le16(vnics); 6115 6116 req->enables = cpu_to_le32(enables); 6117 } 6118 6119 static int 6120 bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6121 int ring_grps, int cp_rings, int stats, int vnics) 6122 { 6123 struct hwrm_func_cfg_input req = {0}; 6124 int rc; 6125 6126 __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps, 6127 cp_rings, stats, vnics); 6128 if (!req.enables) 6129 return 0; 6130 6131 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6132 if (rc) 6133 return rc; 6134 6135 if (bp->hwrm_spec_code < 0x10601) 6136 bp->hw_resc.resv_tx_rings = tx_rings; 6137 6138 return bnxt_hwrm_get_rings(bp); 6139 } 6140 6141 static int 6142 bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6143 int ring_grps, int cp_rings, int stats, int vnics) 6144 { 6145 struct hwrm_func_vf_cfg_input req = {0}; 6146 int rc; 6147 6148 if (!BNXT_NEW_RM(bp)) { 6149 bp->hw_resc.resv_tx_rings = tx_rings; 6150 return 0; 6151 } 6152 6153 __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps, 6154 cp_rings, stats, vnics); 6155 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6156 if (rc) 6157 return rc; 6158 6159 return bnxt_hwrm_get_rings(bp); 6160 } 6161 6162 static int bnxt_hwrm_reserve_rings(struct bnxt *bp, int tx, int rx, int grp, 6163 int cp, int stat, int vnic) 6164 { 6165 if (BNXT_PF(bp)) 6166 return bnxt_hwrm_reserve_pf_rings(bp, tx, rx, grp, cp, stat, 6167 vnic); 6168 else 6169 return bnxt_hwrm_reserve_vf_rings(bp, tx, rx, grp, cp, stat, 6170 vnic); 6171 } 6172 6173 int bnxt_nq_rings_in_use(struct bnxt *bp) 6174 { 6175 int cp = bp->cp_nr_rings; 6176 int ulp_msix, ulp_base; 6177 6178 ulp_msix = bnxt_get_ulp_msix_num(bp); 6179 if (ulp_msix) { 6180 ulp_base = bnxt_get_ulp_msix_base(bp); 6181 cp += ulp_msix; 6182 if ((ulp_base + ulp_msix) > cp) 6183 cp = ulp_base + ulp_msix; 6184 } 6185 return cp; 6186 } 6187 6188 static int bnxt_cp_rings_in_use(struct bnxt *bp) 6189 { 6190 int cp; 6191 6192 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 6193 return bnxt_nq_rings_in_use(bp); 6194 6195 cp = bp->tx_nr_rings + bp->rx_nr_rings; 6196 return cp; 6197 } 6198 6199 static int bnxt_get_func_stat_ctxs(struct bnxt *bp) 6200 { 6201 int ulp_stat = bnxt_get_ulp_stat_ctxs(bp); 6202 int cp = bp->cp_nr_rings; 6203 6204 if (!ulp_stat) 6205 return cp; 6206 6207 if (bnxt_nq_rings_in_use(bp) > cp + bnxt_get_ulp_msix_num(bp)) 6208 return bnxt_get_ulp_msix_base(bp) + ulp_stat; 6209 6210 return cp + ulp_stat; 6211 } 6212 6213 /* Check if a default RSS map needs to be setup. This function is only 6214 * used on older firmware that does not require reserving RX rings. 6215 */ 6216 static void bnxt_check_rss_tbl_no_rmgr(struct bnxt *bp) 6217 { 6218 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 6219 6220 /* The RSS map is valid for RX rings set to resv_rx_rings */ 6221 if (hw_resc->resv_rx_rings != bp->rx_nr_rings) { 6222 hw_resc->resv_rx_rings = bp->rx_nr_rings; 6223 if (!netif_is_rxfh_configured(bp->dev)) 6224 bnxt_set_dflt_rss_indir_tbl(bp); 6225 } 6226 } 6227 6228 static bool bnxt_need_reserve_rings(struct bnxt *bp) 6229 { 6230 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 6231 int cp = bnxt_cp_rings_in_use(bp); 6232 int nq = bnxt_nq_rings_in_use(bp); 6233 int rx = bp->rx_nr_rings, stat; 6234 int vnic = 1, grp = rx; 6235 6236 if (hw_resc->resv_tx_rings != bp->tx_nr_rings && 6237 bp->hwrm_spec_code >= 0x10601) 6238 return true; 6239 6240 /* Old firmware does not need RX ring reservations but we still 6241 * need to setup a default RSS map when needed. With new firmware 6242 * we go through RX ring reservations first and then set up the 6243 * RSS map for the successfully reserved RX rings when needed. 6244 */ 6245 if (!BNXT_NEW_RM(bp)) { 6246 bnxt_check_rss_tbl_no_rmgr(bp); 6247 return false; 6248 } 6249 if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5)) 6250 vnic = rx + 1; 6251 if (bp->flags & BNXT_FLAG_AGG_RINGS) 6252 rx <<= 1; 6253 stat = bnxt_get_func_stat_ctxs(bp); 6254 if (hw_resc->resv_rx_rings != rx || hw_resc->resv_cp_rings != cp || 6255 hw_resc->resv_vnics != vnic || hw_resc->resv_stat_ctxs != stat || 6256 (hw_resc->resv_hw_ring_grps != grp && 6257 !(bp->flags & BNXT_FLAG_CHIP_P5))) 6258 return true; 6259 if ((bp->flags & BNXT_FLAG_CHIP_P5) && BNXT_PF(bp) && 6260 hw_resc->resv_irqs != nq) 6261 return true; 6262 return false; 6263 } 6264 6265 static int __bnxt_reserve_rings(struct bnxt *bp) 6266 { 6267 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 6268 int cp = bnxt_nq_rings_in_use(bp); 6269 int tx = bp->tx_nr_rings; 6270 int rx = bp->rx_nr_rings; 6271 int grp, rx_rings, rc; 6272 int vnic = 1, stat; 6273 bool sh = false; 6274 6275 if (!bnxt_need_reserve_rings(bp)) 6276 return 0; 6277 6278 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 6279 sh = true; 6280 if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5)) 6281 vnic = rx + 1; 6282 if (bp->flags & BNXT_FLAG_AGG_RINGS) 6283 rx <<= 1; 6284 grp = bp->rx_nr_rings; 6285 stat = bnxt_get_func_stat_ctxs(bp); 6286 6287 rc = bnxt_hwrm_reserve_rings(bp, tx, rx, grp, cp, stat, vnic); 6288 if (rc) 6289 return rc; 6290 6291 tx = hw_resc->resv_tx_rings; 6292 if (BNXT_NEW_RM(bp)) { 6293 rx = hw_resc->resv_rx_rings; 6294 cp = hw_resc->resv_irqs; 6295 grp = hw_resc->resv_hw_ring_grps; 6296 vnic = hw_resc->resv_vnics; 6297 stat = hw_resc->resv_stat_ctxs; 6298 } 6299 6300 rx_rings = rx; 6301 if (bp->flags & BNXT_FLAG_AGG_RINGS) { 6302 if (rx >= 2) { 6303 rx_rings = rx >> 1; 6304 } else { 6305 if (netif_running(bp->dev)) 6306 return -ENOMEM; 6307 6308 bp->flags &= ~BNXT_FLAG_AGG_RINGS; 6309 bp->flags |= BNXT_FLAG_NO_AGG_RINGS; 6310 bp->dev->hw_features &= ~NETIF_F_LRO; 6311 bp->dev->features &= ~NETIF_F_LRO; 6312 bnxt_set_ring_params(bp); 6313 } 6314 } 6315 rx_rings = min_t(int, rx_rings, grp); 6316 cp = min_t(int, cp, bp->cp_nr_rings); 6317 if (stat > bnxt_get_ulp_stat_ctxs(bp)) 6318 stat -= bnxt_get_ulp_stat_ctxs(bp); 6319 cp = min_t(int, cp, stat); 6320 rc = bnxt_trim_rings(bp, &rx_rings, &tx, cp, sh); 6321 if (bp->flags & BNXT_FLAG_AGG_RINGS) 6322 rx = rx_rings << 1; 6323 cp = sh ? max_t(int, tx, rx_rings) : tx + rx_rings; 6324 bp->tx_nr_rings = tx; 6325 6326 /* If we cannot reserve all the RX rings, reset the RSS map only 6327 * if absolutely necessary 6328 */ 6329 if (rx_rings != bp->rx_nr_rings) { 6330 netdev_warn(bp->dev, "Able to reserve only %d out of %d requested RX rings\n", 6331 rx_rings, bp->rx_nr_rings); 6332 if ((bp->dev->priv_flags & IFF_RXFH_CONFIGURED) && 6333 (bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings) != 6334 bnxt_get_nr_rss_ctxs(bp, rx_rings) || 6335 bnxt_get_max_rss_ring(bp) >= rx_rings)) { 6336 netdev_warn(bp->dev, "RSS table entries reverting to default\n"); 6337 bp->dev->priv_flags &= ~IFF_RXFH_CONFIGURED; 6338 } 6339 } 6340 bp->rx_nr_rings = rx_rings; 6341 bp->cp_nr_rings = cp; 6342 6343 if (!tx || !rx || !cp || !grp || !vnic || !stat) 6344 return -ENOMEM; 6345 6346 if (!netif_is_rxfh_configured(bp->dev)) 6347 bnxt_set_dflt_rss_indir_tbl(bp); 6348 6349 return rc; 6350 } 6351 6352 static int bnxt_hwrm_check_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6353 int ring_grps, int cp_rings, int stats, 6354 int vnics) 6355 { 6356 struct hwrm_func_vf_cfg_input req = {0}; 6357 u32 flags; 6358 6359 if (!BNXT_NEW_RM(bp)) 6360 return 0; 6361 6362 __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps, 6363 cp_rings, stats, vnics); 6364 flags = FUNC_VF_CFG_REQ_FLAGS_TX_ASSETS_TEST | 6365 FUNC_VF_CFG_REQ_FLAGS_RX_ASSETS_TEST | 6366 FUNC_VF_CFG_REQ_FLAGS_CMPL_ASSETS_TEST | 6367 FUNC_VF_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST | 6368 FUNC_VF_CFG_REQ_FLAGS_VNIC_ASSETS_TEST | 6369 FUNC_VF_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST; 6370 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 6371 flags |= FUNC_VF_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST; 6372 6373 req.flags = cpu_to_le32(flags); 6374 return hwrm_send_message_silent(bp, &req, sizeof(req), 6375 HWRM_CMD_TIMEOUT); 6376 } 6377 6378 static int bnxt_hwrm_check_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6379 int ring_grps, int cp_rings, int stats, 6380 int vnics) 6381 { 6382 struct hwrm_func_cfg_input req = {0}; 6383 u32 flags; 6384 6385 __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps, 6386 cp_rings, stats, vnics); 6387 flags = FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST; 6388 if (BNXT_NEW_RM(bp)) { 6389 flags |= FUNC_CFG_REQ_FLAGS_RX_ASSETS_TEST | 6390 FUNC_CFG_REQ_FLAGS_CMPL_ASSETS_TEST | 6391 FUNC_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST | 6392 FUNC_CFG_REQ_FLAGS_VNIC_ASSETS_TEST; 6393 if (bp->flags & BNXT_FLAG_CHIP_P5) 6394 flags |= FUNC_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST | 6395 FUNC_CFG_REQ_FLAGS_NQ_ASSETS_TEST; 6396 else 6397 flags |= FUNC_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST; 6398 } 6399 6400 req.flags = cpu_to_le32(flags); 6401 return hwrm_send_message_silent(bp, &req, sizeof(req), 6402 HWRM_CMD_TIMEOUT); 6403 } 6404 6405 static int bnxt_hwrm_check_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6406 int ring_grps, int cp_rings, int stats, 6407 int vnics) 6408 { 6409 if (bp->hwrm_spec_code < 0x10801) 6410 return 0; 6411 6412 if (BNXT_PF(bp)) 6413 return bnxt_hwrm_check_pf_rings(bp, tx_rings, rx_rings, 6414 ring_grps, cp_rings, stats, 6415 vnics); 6416 6417 return bnxt_hwrm_check_vf_rings(bp, tx_rings, rx_rings, ring_grps, 6418 cp_rings, stats, vnics); 6419 } 6420 6421 static void bnxt_hwrm_coal_params_qcaps(struct bnxt *bp) 6422 { 6423 struct hwrm_ring_aggint_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 6424 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 6425 struct hwrm_ring_aggint_qcaps_input req = {0}; 6426 int rc; 6427 6428 coal_cap->cmpl_params = BNXT_LEGACY_COAL_CMPL_PARAMS; 6429 coal_cap->num_cmpl_dma_aggr_max = 63; 6430 coal_cap->num_cmpl_dma_aggr_during_int_max = 63; 6431 coal_cap->cmpl_aggr_dma_tmr_max = 65535; 6432 coal_cap->cmpl_aggr_dma_tmr_during_int_max = 65535; 6433 coal_cap->int_lat_tmr_min_max = 65535; 6434 coal_cap->int_lat_tmr_max_max = 65535; 6435 coal_cap->num_cmpl_aggr_int_max = 65535; 6436 coal_cap->timer_units = 80; 6437 6438 if (bp->hwrm_spec_code < 0x10902) 6439 return; 6440 6441 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_AGGINT_QCAPS, -1, -1); 6442 mutex_lock(&bp->hwrm_cmd_lock); 6443 rc = _hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6444 if (!rc) { 6445 coal_cap->cmpl_params = le32_to_cpu(resp->cmpl_params); 6446 coal_cap->nq_params = le32_to_cpu(resp->nq_params); 6447 coal_cap->num_cmpl_dma_aggr_max = 6448 le16_to_cpu(resp->num_cmpl_dma_aggr_max); 6449 coal_cap->num_cmpl_dma_aggr_during_int_max = 6450 le16_to_cpu(resp->num_cmpl_dma_aggr_during_int_max); 6451 coal_cap->cmpl_aggr_dma_tmr_max = 6452 le16_to_cpu(resp->cmpl_aggr_dma_tmr_max); 6453 coal_cap->cmpl_aggr_dma_tmr_during_int_max = 6454 le16_to_cpu(resp->cmpl_aggr_dma_tmr_during_int_max); 6455 coal_cap->int_lat_tmr_min_max = 6456 le16_to_cpu(resp->int_lat_tmr_min_max); 6457 coal_cap->int_lat_tmr_max_max = 6458 le16_to_cpu(resp->int_lat_tmr_max_max); 6459 coal_cap->num_cmpl_aggr_int_max = 6460 le16_to_cpu(resp->num_cmpl_aggr_int_max); 6461 coal_cap->timer_units = le16_to_cpu(resp->timer_units); 6462 } 6463 mutex_unlock(&bp->hwrm_cmd_lock); 6464 } 6465 6466 static u16 bnxt_usec_to_coal_tmr(struct bnxt *bp, u16 usec) 6467 { 6468 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 6469 6470 return usec * 1000 / coal_cap->timer_units; 6471 } 6472 6473 static void bnxt_hwrm_set_coal_params(struct bnxt *bp, 6474 struct bnxt_coal *hw_coal, 6475 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req) 6476 { 6477 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 6478 u32 cmpl_params = coal_cap->cmpl_params; 6479 u16 val, tmr, max, flags = 0; 6480 6481 max = hw_coal->bufs_per_record * 128; 6482 if (hw_coal->budget) 6483 max = hw_coal->bufs_per_record * hw_coal->budget; 6484 max = min_t(u16, max, coal_cap->num_cmpl_aggr_int_max); 6485 6486 val = clamp_t(u16, hw_coal->coal_bufs, 1, max); 6487 req->num_cmpl_aggr_int = cpu_to_le16(val); 6488 6489 val = min_t(u16, val, coal_cap->num_cmpl_dma_aggr_max); 6490 req->num_cmpl_dma_aggr = cpu_to_le16(val); 6491 6492 val = clamp_t(u16, hw_coal->coal_bufs_irq, 1, 6493 coal_cap->num_cmpl_dma_aggr_during_int_max); 6494 req->num_cmpl_dma_aggr_during_int = cpu_to_le16(val); 6495 6496 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks); 6497 tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_max_max); 6498 req->int_lat_tmr_max = cpu_to_le16(tmr); 6499 6500 /* min timer set to 1/2 of interrupt timer */ 6501 if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_INT_LAT_TMR_MIN) { 6502 val = tmr / 2; 6503 val = clamp_t(u16, val, 1, coal_cap->int_lat_tmr_min_max); 6504 req->int_lat_tmr_min = cpu_to_le16(val); 6505 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE); 6506 } 6507 6508 /* buf timer set to 1/4 of interrupt timer */ 6509 val = clamp_t(u16, tmr / 4, 1, coal_cap->cmpl_aggr_dma_tmr_max); 6510 req->cmpl_aggr_dma_tmr = cpu_to_le16(val); 6511 6512 if (cmpl_params & 6513 RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_NUM_CMPL_DMA_AGGR_DURING_INT) { 6514 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks_irq); 6515 val = clamp_t(u16, tmr, 1, 6516 coal_cap->cmpl_aggr_dma_tmr_during_int_max); 6517 req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(val); 6518 req->enables |= 6519 cpu_to_le16(BNXT_COAL_CMPL_AGGR_TMR_DURING_INT_ENABLE); 6520 } 6521 6522 if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_TIMER_RESET) 6523 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET; 6524 if ((cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_RING_IDLE) && 6525 hw_coal->idle_thresh && hw_coal->coal_ticks < hw_coal->idle_thresh) 6526 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE; 6527 req->flags = cpu_to_le16(flags); 6528 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_ENABLES); 6529 } 6530 6531 /* Caller holds bp->hwrm_cmd_lock */ 6532 static int __bnxt_hwrm_set_coal_nq(struct bnxt *bp, struct bnxt_napi *bnapi, 6533 struct bnxt_coal *hw_coal) 6534 { 6535 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req = {0}; 6536 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 6537 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 6538 u32 nq_params = coal_cap->nq_params; 6539 u16 tmr; 6540 6541 if (!(nq_params & RING_AGGINT_QCAPS_RESP_NQ_PARAMS_INT_LAT_TMR_MIN)) 6542 return 0; 6543 6544 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, 6545 -1, -1); 6546 req.ring_id = cpu_to_le16(cpr->cp_ring_struct.fw_ring_id); 6547 req.flags = 6548 cpu_to_le16(RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_IS_NQ); 6549 6550 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks) / 2; 6551 tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_min_max); 6552 req.int_lat_tmr_min = cpu_to_le16(tmr); 6553 req.enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE); 6554 return _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6555 } 6556 6557 int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi) 6558 { 6559 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0}; 6560 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 6561 struct bnxt_coal coal; 6562 6563 /* Tick values in micro seconds. 6564 * 1 coal_buf x bufs_per_record = 1 completion record. 6565 */ 6566 memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal)); 6567 6568 coal.coal_ticks = cpr->rx_ring_coal.coal_ticks; 6569 coal.coal_bufs = cpr->rx_ring_coal.coal_bufs; 6570 6571 if (!bnapi->rx_ring) 6572 return -ENODEV; 6573 6574 bnxt_hwrm_cmd_hdr_init(bp, &req_rx, 6575 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1); 6576 6577 bnxt_hwrm_set_coal_params(bp, &coal, &req_rx); 6578 6579 req_rx.ring_id = cpu_to_le16(bnxt_cp_ring_for_rx(bp, bnapi->rx_ring)); 6580 6581 return hwrm_send_message(bp, &req_rx, sizeof(req_rx), 6582 HWRM_CMD_TIMEOUT); 6583 } 6584 6585 int bnxt_hwrm_set_coal(struct bnxt *bp) 6586 { 6587 int i, rc = 0; 6588 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0}, 6589 req_tx = {0}, *req; 6590 6591 bnxt_hwrm_cmd_hdr_init(bp, &req_rx, 6592 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1); 6593 bnxt_hwrm_cmd_hdr_init(bp, &req_tx, 6594 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1); 6595 6596 bnxt_hwrm_set_coal_params(bp, &bp->rx_coal, &req_rx); 6597 bnxt_hwrm_set_coal_params(bp, &bp->tx_coal, &req_tx); 6598 6599 mutex_lock(&bp->hwrm_cmd_lock); 6600 for (i = 0; i < bp->cp_nr_rings; i++) { 6601 struct bnxt_napi *bnapi = bp->bnapi[i]; 6602 struct bnxt_coal *hw_coal; 6603 u16 ring_id; 6604 6605 req = &req_rx; 6606 if (!bnapi->rx_ring) { 6607 ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring); 6608 req = &req_tx; 6609 } else { 6610 ring_id = bnxt_cp_ring_for_rx(bp, bnapi->rx_ring); 6611 } 6612 req->ring_id = cpu_to_le16(ring_id); 6613 6614 rc = _hwrm_send_message(bp, req, sizeof(*req), 6615 HWRM_CMD_TIMEOUT); 6616 if (rc) 6617 break; 6618 6619 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 6620 continue; 6621 6622 if (bnapi->rx_ring && bnapi->tx_ring) { 6623 req = &req_tx; 6624 ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring); 6625 req->ring_id = cpu_to_le16(ring_id); 6626 rc = _hwrm_send_message(bp, req, sizeof(*req), 6627 HWRM_CMD_TIMEOUT); 6628 if (rc) 6629 break; 6630 } 6631 if (bnapi->rx_ring) 6632 hw_coal = &bp->rx_coal; 6633 else 6634 hw_coal = &bp->tx_coal; 6635 __bnxt_hwrm_set_coal_nq(bp, bnapi, hw_coal); 6636 } 6637 mutex_unlock(&bp->hwrm_cmd_lock); 6638 return rc; 6639 } 6640 6641 static void bnxt_hwrm_stat_ctx_free(struct bnxt *bp) 6642 { 6643 struct hwrm_stat_ctx_clr_stats_input req0 = {0}; 6644 struct hwrm_stat_ctx_free_input req = {0}; 6645 int i; 6646 6647 if (!bp->bnapi) 6648 return; 6649 6650 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 6651 return; 6652 6653 bnxt_hwrm_cmd_hdr_init(bp, &req0, HWRM_STAT_CTX_CLR_STATS, -1, -1); 6654 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_FREE, -1, -1); 6655 6656 mutex_lock(&bp->hwrm_cmd_lock); 6657 for (i = 0; i < bp->cp_nr_rings; i++) { 6658 struct bnxt_napi *bnapi = bp->bnapi[i]; 6659 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 6660 6661 if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) { 6662 req.stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id); 6663 if (BNXT_FW_MAJ(bp) <= 20) { 6664 req0.stat_ctx_id = req.stat_ctx_id; 6665 _hwrm_send_message(bp, &req0, sizeof(req0), 6666 HWRM_CMD_TIMEOUT); 6667 } 6668 _hwrm_send_message(bp, &req, sizeof(req), 6669 HWRM_CMD_TIMEOUT); 6670 6671 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID; 6672 } 6673 } 6674 mutex_unlock(&bp->hwrm_cmd_lock); 6675 } 6676 6677 static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp) 6678 { 6679 int rc = 0, i; 6680 struct hwrm_stat_ctx_alloc_input req = {0}; 6681 struct hwrm_stat_ctx_alloc_output *resp = bp->hwrm_cmd_resp_addr; 6682 6683 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 6684 return 0; 6685 6686 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_ALLOC, -1, -1); 6687 6688 req.stats_dma_length = cpu_to_le16(bp->hw_ring_stats_size); 6689 req.update_period_ms = cpu_to_le32(bp->stats_coal_ticks / 1000); 6690 6691 mutex_lock(&bp->hwrm_cmd_lock); 6692 for (i = 0; i < bp->cp_nr_rings; i++) { 6693 struct bnxt_napi *bnapi = bp->bnapi[i]; 6694 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 6695 6696 req.stats_dma_addr = cpu_to_le64(cpr->stats.hw_stats_map); 6697 6698 rc = _hwrm_send_message(bp, &req, sizeof(req), 6699 HWRM_CMD_TIMEOUT); 6700 if (rc) 6701 break; 6702 6703 cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id); 6704 6705 bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id; 6706 } 6707 mutex_unlock(&bp->hwrm_cmd_lock); 6708 return rc; 6709 } 6710 6711 static int bnxt_hwrm_func_qcfg(struct bnxt *bp) 6712 { 6713 struct hwrm_func_qcfg_input req = {0}; 6714 struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 6715 u32 min_db_offset = 0; 6716 u16 flags; 6717 int rc; 6718 6719 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1); 6720 req.fid = cpu_to_le16(0xffff); 6721 mutex_lock(&bp->hwrm_cmd_lock); 6722 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6723 if (rc) 6724 goto func_qcfg_exit; 6725 6726 #ifdef CONFIG_BNXT_SRIOV 6727 if (BNXT_VF(bp)) { 6728 struct bnxt_vf_info *vf = &bp->vf; 6729 6730 vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK; 6731 } else { 6732 bp->pf.registered_vfs = le16_to_cpu(resp->registered_vfs); 6733 } 6734 #endif 6735 flags = le16_to_cpu(resp->flags); 6736 if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED | 6737 FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED)) { 6738 bp->fw_cap |= BNXT_FW_CAP_LLDP_AGENT; 6739 if (flags & FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED) 6740 bp->fw_cap |= BNXT_FW_CAP_DCBX_AGENT; 6741 } 6742 if (BNXT_PF(bp) && (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST)) 6743 bp->flags |= BNXT_FLAG_MULTI_HOST; 6744 if (flags & FUNC_QCFG_RESP_FLAGS_RING_MONITOR_ENABLED) 6745 bp->fw_cap |= BNXT_FW_CAP_RING_MONITOR; 6746 6747 switch (resp->port_partition_type) { 6748 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0: 6749 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5: 6750 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0: 6751 bp->port_partition_type = resp->port_partition_type; 6752 break; 6753 } 6754 if (bp->hwrm_spec_code < 0x10707 || 6755 resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEB) 6756 bp->br_mode = BRIDGE_MODE_VEB; 6757 else if (resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEPA) 6758 bp->br_mode = BRIDGE_MODE_VEPA; 6759 else 6760 bp->br_mode = BRIDGE_MODE_UNDEF; 6761 6762 bp->max_mtu = le16_to_cpu(resp->max_mtu_configured); 6763 if (!bp->max_mtu) 6764 bp->max_mtu = BNXT_MAX_MTU; 6765 6766 if (bp->db_size) 6767 goto func_qcfg_exit; 6768 6769 if (bp->flags & BNXT_FLAG_CHIP_P5) { 6770 if (BNXT_PF(bp)) 6771 min_db_offset = DB_PF_OFFSET_P5; 6772 else 6773 min_db_offset = DB_VF_OFFSET_P5; 6774 } 6775 bp->db_size = PAGE_ALIGN(le16_to_cpu(resp->l2_doorbell_bar_size_kb) * 6776 1024); 6777 if (!bp->db_size || bp->db_size > pci_resource_len(bp->pdev, 2) || 6778 bp->db_size <= min_db_offset) 6779 bp->db_size = pci_resource_len(bp->pdev, 2); 6780 6781 func_qcfg_exit: 6782 mutex_unlock(&bp->hwrm_cmd_lock); 6783 return rc; 6784 } 6785 6786 static void bnxt_init_ctx_initializer(struct bnxt_ctx_mem_info *ctx, 6787 struct hwrm_func_backing_store_qcaps_output *resp) 6788 { 6789 struct bnxt_mem_init *mem_init; 6790 u16 init_mask; 6791 u8 init_val; 6792 u8 *offset; 6793 int i; 6794 6795 init_val = resp->ctx_kind_initializer; 6796 init_mask = le16_to_cpu(resp->ctx_init_mask); 6797 offset = &resp->qp_init_offset; 6798 mem_init = &ctx->mem_init[BNXT_CTX_MEM_INIT_QP]; 6799 for (i = 0; i < BNXT_CTX_MEM_INIT_MAX; i++, mem_init++, offset++) { 6800 mem_init->init_val = init_val; 6801 mem_init->offset = BNXT_MEM_INVALID_OFFSET; 6802 if (!init_mask) 6803 continue; 6804 if (i == BNXT_CTX_MEM_INIT_STAT) 6805 offset = &resp->stat_init_offset; 6806 if (init_mask & (1 << i)) 6807 mem_init->offset = *offset * 4; 6808 else 6809 mem_init->init_val = 0; 6810 } 6811 ctx->mem_init[BNXT_CTX_MEM_INIT_QP].size = ctx->qp_entry_size; 6812 ctx->mem_init[BNXT_CTX_MEM_INIT_SRQ].size = ctx->srq_entry_size; 6813 ctx->mem_init[BNXT_CTX_MEM_INIT_CQ].size = ctx->cq_entry_size; 6814 ctx->mem_init[BNXT_CTX_MEM_INIT_VNIC].size = ctx->vnic_entry_size; 6815 ctx->mem_init[BNXT_CTX_MEM_INIT_STAT].size = ctx->stat_entry_size; 6816 ctx->mem_init[BNXT_CTX_MEM_INIT_MRAV].size = ctx->mrav_entry_size; 6817 } 6818 6819 static int bnxt_hwrm_func_backing_store_qcaps(struct bnxt *bp) 6820 { 6821 struct hwrm_func_backing_store_qcaps_input req = {0}; 6822 struct hwrm_func_backing_store_qcaps_output *resp = 6823 bp->hwrm_cmd_resp_addr; 6824 int rc; 6825 6826 if (bp->hwrm_spec_code < 0x10902 || BNXT_VF(bp) || bp->ctx) 6827 return 0; 6828 6829 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_BACKING_STORE_QCAPS, -1, -1); 6830 mutex_lock(&bp->hwrm_cmd_lock); 6831 rc = _hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6832 if (!rc) { 6833 struct bnxt_ctx_pg_info *ctx_pg; 6834 struct bnxt_ctx_mem_info *ctx; 6835 int i, tqm_rings; 6836 6837 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 6838 if (!ctx) { 6839 rc = -ENOMEM; 6840 goto ctx_err; 6841 } 6842 ctx->qp_max_entries = le32_to_cpu(resp->qp_max_entries); 6843 ctx->qp_min_qp1_entries = le16_to_cpu(resp->qp_min_qp1_entries); 6844 ctx->qp_max_l2_entries = le16_to_cpu(resp->qp_max_l2_entries); 6845 ctx->qp_entry_size = le16_to_cpu(resp->qp_entry_size); 6846 ctx->srq_max_l2_entries = le16_to_cpu(resp->srq_max_l2_entries); 6847 ctx->srq_max_entries = le32_to_cpu(resp->srq_max_entries); 6848 ctx->srq_entry_size = le16_to_cpu(resp->srq_entry_size); 6849 ctx->cq_max_l2_entries = le16_to_cpu(resp->cq_max_l2_entries); 6850 ctx->cq_max_entries = le32_to_cpu(resp->cq_max_entries); 6851 ctx->cq_entry_size = le16_to_cpu(resp->cq_entry_size); 6852 ctx->vnic_max_vnic_entries = 6853 le16_to_cpu(resp->vnic_max_vnic_entries); 6854 ctx->vnic_max_ring_table_entries = 6855 le16_to_cpu(resp->vnic_max_ring_table_entries); 6856 ctx->vnic_entry_size = le16_to_cpu(resp->vnic_entry_size); 6857 ctx->stat_max_entries = le32_to_cpu(resp->stat_max_entries); 6858 ctx->stat_entry_size = le16_to_cpu(resp->stat_entry_size); 6859 ctx->tqm_entry_size = le16_to_cpu(resp->tqm_entry_size); 6860 ctx->tqm_min_entries_per_ring = 6861 le32_to_cpu(resp->tqm_min_entries_per_ring); 6862 ctx->tqm_max_entries_per_ring = 6863 le32_to_cpu(resp->tqm_max_entries_per_ring); 6864 ctx->tqm_entries_multiple = resp->tqm_entries_multiple; 6865 if (!ctx->tqm_entries_multiple) 6866 ctx->tqm_entries_multiple = 1; 6867 ctx->mrav_max_entries = le32_to_cpu(resp->mrav_max_entries); 6868 ctx->mrav_entry_size = le16_to_cpu(resp->mrav_entry_size); 6869 ctx->mrav_num_entries_units = 6870 le16_to_cpu(resp->mrav_num_entries_units); 6871 ctx->tim_entry_size = le16_to_cpu(resp->tim_entry_size); 6872 ctx->tim_max_entries = le32_to_cpu(resp->tim_max_entries); 6873 6874 bnxt_init_ctx_initializer(ctx, resp); 6875 6876 ctx->tqm_fp_rings_count = resp->tqm_fp_rings_count; 6877 if (!ctx->tqm_fp_rings_count) 6878 ctx->tqm_fp_rings_count = bp->max_q; 6879 else if (ctx->tqm_fp_rings_count > BNXT_MAX_TQM_FP_RINGS) 6880 ctx->tqm_fp_rings_count = BNXT_MAX_TQM_FP_RINGS; 6881 6882 tqm_rings = ctx->tqm_fp_rings_count + BNXT_MAX_TQM_SP_RINGS; 6883 ctx_pg = kcalloc(tqm_rings, sizeof(*ctx_pg), GFP_KERNEL); 6884 if (!ctx_pg) { 6885 kfree(ctx); 6886 rc = -ENOMEM; 6887 goto ctx_err; 6888 } 6889 for (i = 0; i < tqm_rings; i++, ctx_pg++) 6890 ctx->tqm_mem[i] = ctx_pg; 6891 bp->ctx = ctx; 6892 } else { 6893 rc = 0; 6894 } 6895 ctx_err: 6896 mutex_unlock(&bp->hwrm_cmd_lock); 6897 return rc; 6898 } 6899 6900 static void bnxt_hwrm_set_pg_attr(struct bnxt_ring_mem_info *rmem, u8 *pg_attr, 6901 __le64 *pg_dir) 6902 { 6903 u8 pg_size = 0; 6904 6905 if (!rmem->nr_pages) 6906 return; 6907 6908 if (BNXT_PAGE_SHIFT == 13) 6909 pg_size = 1 << 4; 6910 else if (BNXT_PAGE_SIZE == 16) 6911 pg_size = 2 << 4; 6912 6913 *pg_attr = pg_size; 6914 if (rmem->depth >= 1) { 6915 if (rmem->depth == 2) 6916 *pg_attr |= 2; 6917 else 6918 *pg_attr |= 1; 6919 *pg_dir = cpu_to_le64(rmem->pg_tbl_map); 6920 } else { 6921 *pg_dir = cpu_to_le64(rmem->dma_arr[0]); 6922 } 6923 } 6924 6925 #define FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES \ 6926 (FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP | \ 6927 FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ | \ 6928 FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ | \ 6929 FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC | \ 6930 FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) 6931 6932 static int bnxt_hwrm_func_backing_store_cfg(struct bnxt *bp, u32 enables) 6933 { 6934 struct hwrm_func_backing_store_cfg_input req = {0}; 6935 struct bnxt_ctx_mem_info *ctx = bp->ctx; 6936 struct bnxt_ctx_pg_info *ctx_pg; 6937 u32 req_len = sizeof(req); 6938 __le32 *num_entries; 6939 __le64 *pg_dir; 6940 u32 flags = 0; 6941 u8 *pg_attr; 6942 u32 ena; 6943 int i; 6944 6945 if (!ctx) 6946 return 0; 6947 6948 if (req_len > bp->hwrm_max_ext_req_len) 6949 req_len = BNXT_BACKING_STORE_CFG_LEGACY_LEN; 6950 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_BACKING_STORE_CFG, -1, -1); 6951 req.enables = cpu_to_le32(enables); 6952 6953 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP) { 6954 ctx_pg = &ctx->qp_mem; 6955 req.qp_num_entries = cpu_to_le32(ctx_pg->entries); 6956 req.qp_num_qp1_entries = cpu_to_le16(ctx->qp_min_qp1_entries); 6957 req.qp_num_l2_entries = cpu_to_le16(ctx->qp_max_l2_entries); 6958 req.qp_entry_size = cpu_to_le16(ctx->qp_entry_size); 6959 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 6960 &req.qpc_pg_size_qpc_lvl, 6961 &req.qpc_page_dir); 6962 } 6963 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ) { 6964 ctx_pg = &ctx->srq_mem; 6965 req.srq_num_entries = cpu_to_le32(ctx_pg->entries); 6966 req.srq_num_l2_entries = cpu_to_le16(ctx->srq_max_l2_entries); 6967 req.srq_entry_size = cpu_to_le16(ctx->srq_entry_size); 6968 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 6969 &req.srq_pg_size_srq_lvl, 6970 &req.srq_page_dir); 6971 } 6972 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ) { 6973 ctx_pg = &ctx->cq_mem; 6974 req.cq_num_entries = cpu_to_le32(ctx_pg->entries); 6975 req.cq_num_l2_entries = cpu_to_le16(ctx->cq_max_l2_entries); 6976 req.cq_entry_size = cpu_to_le16(ctx->cq_entry_size); 6977 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, &req.cq_pg_size_cq_lvl, 6978 &req.cq_page_dir); 6979 } 6980 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC) { 6981 ctx_pg = &ctx->vnic_mem; 6982 req.vnic_num_vnic_entries = 6983 cpu_to_le16(ctx->vnic_max_vnic_entries); 6984 req.vnic_num_ring_table_entries = 6985 cpu_to_le16(ctx->vnic_max_ring_table_entries); 6986 req.vnic_entry_size = cpu_to_le16(ctx->vnic_entry_size); 6987 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 6988 &req.vnic_pg_size_vnic_lvl, 6989 &req.vnic_page_dir); 6990 } 6991 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) { 6992 ctx_pg = &ctx->stat_mem; 6993 req.stat_num_entries = cpu_to_le32(ctx->stat_max_entries); 6994 req.stat_entry_size = cpu_to_le16(ctx->stat_entry_size); 6995 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 6996 &req.stat_pg_size_stat_lvl, 6997 &req.stat_page_dir); 6998 } 6999 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV) { 7000 ctx_pg = &ctx->mrav_mem; 7001 req.mrav_num_entries = cpu_to_le32(ctx_pg->entries); 7002 if (ctx->mrav_num_entries_units) 7003 flags |= 7004 FUNC_BACKING_STORE_CFG_REQ_FLAGS_MRAV_RESERVATION_SPLIT; 7005 req.mrav_entry_size = cpu_to_le16(ctx->mrav_entry_size); 7006 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 7007 &req.mrav_pg_size_mrav_lvl, 7008 &req.mrav_page_dir); 7009 } 7010 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM) { 7011 ctx_pg = &ctx->tim_mem; 7012 req.tim_num_entries = cpu_to_le32(ctx_pg->entries); 7013 req.tim_entry_size = cpu_to_le16(ctx->tim_entry_size); 7014 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 7015 &req.tim_pg_size_tim_lvl, 7016 &req.tim_page_dir); 7017 } 7018 for (i = 0, num_entries = &req.tqm_sp_num_entries, 7019 pg_attr = &req.tqm_sp_pg_size_tqm_sp_lvl, 7020 pg_dir = &req.tqm_sp_page_dir, 7021 ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP; 7022 i < BNXT_MAX_TQM_RINGS; 7023 i++, num_entries++, pg_attr++, pg_dir++, ena <<= 1) { 7024 if (!(enables & ena)) 7025 continue; 7026 7027 req.tqm_entry_size = cpu_to_le16(ctx->tqm_entry_size); 7028 ctx_pg = ctx->tqm_mem[i]; 7029 *num_entries = cpu_to_le32(ctx_pg->entries); 7030 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, pg_attr, pg_dir); 7031 } 7032 req.flags = cpu_to_le32(flags); 7033 return hwrm_send_message(bp, &req, req_len, HWRM_CMD_TIMEOUT); 7034 } 7035 7036 static int bnxt_alloc_ctx_mem_blk(struct bnxt *bp, 7037 struct bnxt_ctx_pg_info *ctx_pg) 7038 { 7039 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem; 7040 7041 rmem->page_size = BNXT_PAGE_SIZE; 7042 rmem->pg_arr = ctx_pg->ctx_pg_arr; 7043 rmem->dma_arr = ctx_pg->ctx_dma_arr; 7044 rmem->flags = BNXT_RMEM_VALID_PTE_FLAG; 7045 if (rmem->depth >= 1) 7046 rmem->flags |= BNXT_RMEM_USE_FULL_PAGE_FLAG; 7047 return bnxt_alloc_ring(bp, rmem); 7048 } 7049 7050 static int bnxt_alloc_ctx_pg_tbls(struct bnxt *bp, 7051 struct bnxt_ctx_pg_info *ctx_pg, u32 mem_size, 7052 u8 depth, struct bnxt_mem_init *mem_init) 7053 { 7054 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem; 7055 int rc; 7056 7057 if (!mem_size) 7058 return -EINVAL; 7059 7060 ctx_pg->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE); 7061 if (ctx_pg->nr_pages > MAX_CTX_TOTAL_PAGES) { 7062 ctx_pg->nr_pages = 0; 7063 return -EINVAL; 7064 } 7065 if (ctx_pg->nr_pages > MAX_CTX_PAGES || depth > 1) { 7066 int nr_tbls, i; 7067 7068 rmem->depth = 2; 7069 ctx_pg->ctx_pg_tbl = kcalloc(MAX_CTX_PAGES, sizeof(ctx_pg), 7070 GFP_KERNEL); 7071 if (!ctx_pg->ctx_pg_tbl) 7072 return -ENOMEM; 7073 nr_tbls = DIV_ROUND_UP(ctx_pg->nr_pages, MAX_CTX_PAGES); 7074 rmem->nr_pages = nr_tbls; 7075 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg); 7076 if (rc) 7077 return rc; 7078 for (i = 0; i < nr_tbls; i++) { 7079 struct bnxt_ctx_pg_info *pg_tbl; 7080 7081 pg_tbl = kzalloc(sizeof(*pg_tbl), GFP_KERNEL); 7082 if (!pg_tbl) 7083 return -ENOMEM; 7084 ctx_pg->ctx_pg_tbl[i] = pg_tbl; 7085 rmem = &pg_tbl->ring_mem; 7086 rmem->pg_tbl = ctx_pg->ctx_pg_arr[i]; 7087 rmem->pg_tbl_map = ctx_pg->ctx_dma_arr[i]; 7088 rmem->depth = 1; 7089 rmem->nr_pages = MAX_CTX_PAGES; 7090 rmem->mem_init = mem_init; 7091 if (i == (nr_tbls - 1)) { 7092 int rem = ctx_pg->nr_pages % MAX_CTX_PAGES; 7093 7094 if (rem) 7095 rmem->nr_pages = rem; 7096 } 7097 rc = bnxt_alloc_ctx_mem_blk(bp, pg_tbl); 7098 if (rc) 7099 break; 7100 } 7101 } else { 7102 rmem->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE); 7103 if (rmem->nr_pages > 1 || depth) 7104 rmem->depth = 1; 7105 rmem->mem_init = mem_init; 7106 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg); 7107 } 7108 return rc; 7109 } 7110 7111 static void bnxt_free_ctx_pg_tbls(struct bnxt *bp, 7112 struct bnxt_ctx_pg_info *ctx_pg) 7113 { 7114 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem; 7115 7116 if (rmem->depth > 1 || ctx_pg->nr_pages > MAX_CTX_PAGES || 7117 ctx_pg->ctx_pg_tbl) { 7118 int i, nr_tbls = rmem->nr_pages; 7119 7120 for (i = 0; i < nr_tbls; i++) { 7121 struct bnxt_ctx_pg_info *pg_tbl; 7122 struct bnxt_ring_mem_info *rmem2; 7123 7124 pg_tbl = ctx_pg->ctx_pg_tbl[i]; 7125 if (!pg_tbl) 7126 continue; 7127 rmem2 = &pg_tbl->ring_mem; 7128 bnxt_free_ring(bp, rmem2); 7129 ctx_pg->ctx_pg_arr[i] = NULL; 7130 kfree(pg_tbl); 7131 ctx_pg->ctx_pg_tbl[i] = NULL; 7132 } 7133 kfree(ctx_pg->ctx_pg_tbl); 7134 ctx_pg->ctx_pg_tbl = NULL; 7135 } 7136 bnxt_free_ring(bp, rmem); 7137 ctx_pg->nr_pages = 0; 7138 } 7139 7140 static void bnxt_free_ctx_mem(struct bnxt *bp) 7141 { 7142 struct bnxt_ctx_mem_info *ctx = bp->ctx; 7143 int i; 7144 7145 if (!ctx) 7146 return; 7147 7148 if (ctx->tqm_mem[0]) { 7149 for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++) 7150 bnxt_free_ctx_pg_tbls(bp, ctx->tqm_mem[i]); 7151 kfree(ctx->tqm_mem[0]); 7152 ctx->tqm_mem[0] = NULL; 7153 } 7154 7155 bnxt_free_ctx_pg_tbls(bp, &ctx->tim_mem); 7156 bnxt_free_ctx_pg_tbls(bp, &ctx->mrav_mem); 7157 bnxt_free_ctx_pg_tbls(bp, &ctx->stat_mem); 7158 bnxt_free_ctx_pg_tbls(bp, &ctx->vnic_mem); 7159 bnxt_free_ctx_pg_tbls(bp, &ctx->cq_mem); 7160 bnxt_free_ctx_pg_tbls(bp, &ctx->srq_mem); 7161 bnxt_free_ctx_pg_tbls(bp, &ctx->qp_mem); 7162 ctx->flags &= ~BNXT_CTX_FLAG_INITED; 7163 } 7164 7165 static int bnxt_alloc_ctx_mem(struct bnxt *bp) 7166 { 7167 struct bnxt_ctx_pg_info *ctx_pg; 7168 struct bnxt_ctx_mem_info *ctx; 7169 struct bnxt_mem_init *init; 7170 u32 mem_size, ena, entries; 7171 u32 entries_sp, min; 7172 u32 num_mr, num_ah; 7173 u32 extra_srqs = 0; 7174 u32 extra_qps = 0; 7175 u8 pg_lvl = 1; 7176 int i, rc; 7177 7178 rc = bnxt_hwrm_func_backing_store_qcaps(bp); 7179 if (rc) { 7180 netdev_err(bp->dev, "Failed querying context mem capability, rc = %d.\n", 7181 rc); 7182 return rc; 7183 } 7184 ctx = bp->ctx; 7185 if (!ctx || (ctx->flags & BNXT_CTX_FLAG_INITED)) 7186 return 0; 7187 7188 if ((bp->flags & BNXT_FLAG_ROCE_CAP) && !is_kdump_kernel()) { 7189 pg_lvl = 2; 7190 extra_qps = 65536; 7191 extra_srqs = 8192; 7192 } 7193 7194 ctx_pg = &ctx->qp_mem; 7195 ctx_pg->entries = ctx->qp_min_qp1_entries + ctx->qp_max_l2_entries + 7196 extra_qps; 7197 if (ctx->qp_entry_size) { 7198 mem_size = ctx->qp_entry_size * ctx_pg->entries; 7199 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_QP]; 7200 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, init); 7201 if (rc) 7202 return rc; 7203 } 7204 7205 ctx_pg = &ctx->srq_mem; 7206 ctx_pg->entries = ctx->srq_max_l2_entries + extra_srqs; 7207 if (ctx->srq_entry_size) { 7208 mem_size = ctx->srq_entry_size * ctx_pg->entries; 7209 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_SRQ]; 7210 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, init); 7211 if (rc) 7212 return rc; 7213 } 7214 7215 ctx_pg = &ctx->cq_mem; 7216 ctx_pg->entries = ctx->cq_max_l2_entries + extra_qps * 2; 7217 if (ctx->cq_entry_size) { 7218 mem_size = ctx->cq_entry_size * ctx_pg->entries; 7219 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_CQ]; 7220 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, init); 7221 if (rc) 7222 return rc; 7223 } 7224 7225 ctx_pg = &ctx->vnic_mem; 7226 ctx_pg->entries = ctx->vnic_max_vnic_entries + 7227 ctx->vnic_max_ring_table_entries; 7228 if (ctx->vnic_entry_size) { 7229 mem_size = ctx->vnic_entry_size * ctx_pg->entries; 7230 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_VNIC]; 7231 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, init); 7232 if (rc) 7233 return rc; 7234 } 7235 7236 ctx_pg = &ctx->stat_mem; 7237 ctx_pg->entries = ctx->stat_max_entries; 7238 if (ctx->stat_entry_size) { 7239 mem_size = ctx->stat_entry_size * ctx_pg->entries; 7240 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_STAT]; 7241 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, init); 7242 if (rc) 7243 return rc; 7244 } 7245 7246 ena = 0; 7247 if (!(bp->flags & BNXT_FLAG_ROCE_CAP)) 7248 goto skip_rdma; 7249 7250 ctx_pg = &ctx->mrav_mem; 7251 /* 128K extra is needed to accommodate static AH context 7252 * allocation by f/w. 7253 */ 7254 num_mr = 1024 * 256; 7255 num_ah = 1024 * 128; 7256 ctx_pg->entries = num_mr + num_ah; 7257 if (ctx->mrav_entry_size) { 7258 mem_size = ctx->mrav_entry_size * ctx_pg->entries; 7259 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_MRAV]; 7260 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 2, init); 7261 if (rc) 7262 return rc; 7263 } 7264 ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV; 7265 if (ctx->mrav_num_entries_units) 7266 ctx_pg->entries = 7267 ((num_mr / ctx->mrav_num_entries_units) << 16) | 7268 (num_ah / ctx->mrav_num_entries_units); 7269 7270 ctx_pg = &ctx->tim_mem; 7271 ctx_pg->entries = ctx->qp_mem.entries; 7272 if (ctx->tim_entry_size) { 7273 mem_size = ctx->tim_entry_size * ctx_pg->entries; 7274 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, NULL); 7275 if (rc) 7276 return rc; 7277 } 7278 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM; 7279 7280 skip_rdma: 7281 min = ctx->tqm_min_entries_per_ring; 7282 entries_sp = ctx->vnic_max_vnic_entries + ctx->qp_max_l2_entries + 7283 2 * (extra_qps + ctx->qp_min_qp1_entries) + min; 7284 entries_sp = roundup(entries_sp, ctx->tqm_entries_multiple); 7285 entries = ctx->qp_max_l2_entries + extra_qps + ctx->qp_min_qp1_entries; 7286 entries = roundup(entries, ctx->tqm_entries_multiple); 7287 entries = clamp_t(u32, entries, min, ctx->tqm_max_entries_per_ring); 7288 for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++) { 7289 ctx_pg = ctx->tqm_mem[i]; 7290 ctx_pg->entries = i ? entries : entries_sp; 7291 if (ctx->tqm_entry_size) { 7292 mem_size = ctx->tqm_entry_size * ctx_pg->entries; 7293 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, 7294 NULL); 7295 if (rc) 7296 return rc; 7297 } 7298 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP << i; 7299 } 7300 ena |= FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES; 7301 rc = bnxt_hwrm_func_backing_store_cfg(bp, ena); 7302 if (rc) { 7303 netdev_err(bp->dev, "Failed configuring context mem, rc = %d.\n", 7304 rc); 7305 return rc; 7306 } 7307 ctx->flags |= BNXT_CTX_FLAG_INITED; 7308 return 0; 7309 } 7310 7311 int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp, bool all) 7312 { 7313 struct hwrm_func_resource_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 7314 struct hwrm_func_resource_qcaps_input req = {0}; 7315 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 7316 int rc; 7317 7318 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESOURCE_QCAPS, -1, -1); 7319 req.fid = cpu_to_le16(0xffff); 7320 7321 mutex_lock(&bp->hwrm_cmd_lock); 7322 rc = _hwrm_send_message_silent(bp, &req, sizeof(req), 7323 HWRM_CMD_TIMEOUT); 7324 if (rc) 7325 goto hwrm_func_resc_qcaps_exit; 7326 7327 hw_resc->max_tx_sch_inputs = le16_to_cpu(resp->max_tx_scheduler_inputs); 7328 if (!all) 7329 goto hwrm_func_resc_qcaps_exit; 7330 7331 hw_resc->min_rsscos_ctxs = le16_to_cpu(resp->min_rsscos_ctx); 7332 hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx); 7333 hw_resc->min_cp_rings = le16_to_cpu(resp->min_cmpl_rings); 7334 hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings); 7335 hw_resc->min_tx_rings = le16_to_cpu(resp->min_tx_rings); 7336 hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings); 7337 hw_resc->min_rx_rings = le16_to_cpu(resp->min_rx_rings); 7338 hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings); 7339 hw_resc->min_hw_ring_grps = le16_to_cpu(resp->min_hw_ring_grps); 7340 hw_resc->max_hw_ring_grps = le16_to_cpu(resp->max_hw_ring_grps); 7341 hw_resc->min_l2_ctxs = le16_to_cpu(resp->min_l2_ctxs); 7342 hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs); 7343 hw_resc->min_vnics = le16_to_cpu(resp->min_vnics); 7344 hw_resc->max_vnics = le16_to_cpu(resp->max_vnics); 7345 hw_resc->min_stat_ctxs = le16_to_cpu(resp->min_stat_ctx); 7346 hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx); 7347 7348 if (bp->flags & BNXT_FLAG_CHIP_P5) { 7349 u16 max_msix = le16_to_cpu(resp->max_msix); 7350 7351 hw_resc->max_nqs = max_msix; 7352 hw_resc->max_hw_ring_grps = hw_resc->max_rx_rings; 7353 } 7354 7355 if (BNXT_PF(bp)) { 7356 struct bnxt_pf_info *pf = &bp->pf; 7357 7358 pf->vf_resv_strategy = 7359 le16_to_cpu(resp->vf_reservation_strategy); 7360 if (pf->vf_resv_strategy > BNXT_VF_RESV_STRATEGY_MINIMAL_STATIC) 7361 pf->vf_resv_strategy = BNXT_VF_RESV_STRATEGY_MAXIMAL; 7362 } 7363 hwrm_func_resc_qcaps_exit: 7364 mutex_unlock(&bp->hwrm_cmd_lock); 7365 return rc; 7366 } 7367 7368 static int __bnxt_hwrm_func_qcaps(struct bnxt *bp) 7369 { 7370 int rc = 0; 7371 struct hwrm_func_qcaps_input req = {0}; 7372 struct hwrm_func_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 7373 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 7374 u32 flags, flags_ext; 7375 7376 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCAPS, -1, -1); 7377 req.fid = cpu_to_le16(0xffff); 7378 7379 mutex_lock(&bp->hwrm_cmd_lock); 7380 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 7381 if (rc) 7382 goto hwrm_func_qcaps_exit; 7383 7384 flags = le32_to_cpu(resp->flags); 7385 if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED) 7386 bp->flags |= BNXT_FLAG_ROCEV1_CAP; 7387 if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED) 7388 bp->flags |= BNXT_FLAG_ROCEV2_CAP; 7389 if (flags & FUNC_QCAPS_RESP_FLAGS_PCIE_STATS_SUPPORTED) 7390 bp->fw_cap |= BNXT_FW_CAP_PCIE_STATS_SUPPORTED; 7391 if (flags & FUNC_QCAPS_RESP_FLAGS_HOT_RESET_CAPABLE) 7392 bp->fw_cap |= BNXT_FW_CAP_HOT_RESET; 7393 if (flags & FUNC_QCAPS_RESP_FLAGS_EXT_STATS_SUPPORTED) 7394 bp->fw_cap |= BNXT_FW_CAP_EXT_STATS_SUPPORTED; 7395 if (flags & FUNC_QCAPS_RESP_FLAGS_ERROR_RECOVERY_CAPABLE) 7396 bp->fw_cap |= BNXT_FW_CAP_ERROR_RECOVERY; 7397 if (flags & FUNC_QCAPS_RESP_FLAGS_ERR_RECOVER_RELOAD) 7398 bp->fw_cap |= BNXT_FW_CAP_ERR_RECOVER_RELOAD; 7399 if (!(flags & FUNC_QCAPS_RESP_FLAGS_VLAN_ACCELERATION_TX_DISABLED)) 7400 bp->fw_cap |= BNXT_FW_CAP_VLAN_TX_INSERT; 7401 7402 flags_ext = le32_to_cpu(resp->flags_ext); 7403 if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_EXT_HW_STATS_SUPPORTED) 7404 bp->fw_cap |= BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED; 7405 7406 bp->tx_push_thresh = 0; 7407 if ((flags & FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED) && 7408 BNXT_FW_MAJ(bp) > 217) 7409 bp->tx_push_thresh = BNXT_TX_PUSH_THRESH; 7410 7411 hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx); 7412 hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings); 7413 hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings); 7414 hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings); 7415 hw_resc->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps); 7416 if (!hw_resc->max_hw_ring_grps) 7417 hw_resc->max_hw_ring_grps = hw_resc->max_tx_rings; 7418 hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs); 7419 hw_resc->max_vnics = le16_to_cpu(resp->max_vnics); 7420 hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx); 7421 7422 if (BNXT_PF(bp)) { 7423 struct bnxt_pf_info *pf = &bp->pf; 7424 7425 pf->fw_fid = le16_to_cpu(resp->fid); 7426 pf->port_id = le16_to_cpu(resp->port_id); 7427 memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN); 7428 pf->first_vf_id = le16_to_cpu(resp->first_vf_id); 7429 pf->max_vfs = le16_to_cpu(resp->max_vfs); 7430 pf->max_encap_records = le32_to_cpu(resp->max_encap_records); 7431 pf->max_decap_records = le32_to_cpu(resp->max_decap_records); 7432 pf->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows); 7433 pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows); 7434 pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows); 7435 pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows); 7436 bp->flags &= ~BNXT_FLAG_WOL_CAP; 7437 if (flags & FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED) 7438 bp->flags |= BNXT_FLAG_WOL_CAP; 7439 } else { 7440 #ifdef CONFIG_BNXT_SRIOV 7441 struct bnxt_vf_info *vf = &bp->vf; 7442 7443 vf->fw_fid = le16_to_cpu(resp->fid); 7444 memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN); 7445 #endif 7446 } 7447 7448 hwrm_func_qcaps_exit: 7449 mutex_unlock(&bp->hwrm_cmd_lock); 7450 return rc; 7451 } 7452 7453 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp); 7454 7455 static int bnxt_hwrm_func_qcaps(struct bnxt *bp) 7456 { 7457 int rc; 7458 7459 rc = __bnxt_hwrm_func_qcaps(bp); 7460 if (rc) 7461 return rc; 7462 rc = bnxt_hwrm_queue_qportcfg(bp); 7463 if (rc) { 7464 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %d\n", rc); 7465 return rc; 7466 } 7467 if (bp->hwrm_spec_code >= 0x10803) { 7468 rc = bnxt_alloc_ctx_mem(bp); 7469 if (rc) 7470 return rc; 7471 rc = bnxt_hwrm_func_resc_qcaps(bp, true); 7472 if (!rc) 7473 bp->fw_cap |= BNXT_FW_CAP_NEW_RM; 7474 } 7475 return 0; 7476 } 7477 7478 static int bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(struct bnxt *bp) 7479 { 7480 struct hwrm_cfa_adv_flow_mgnt_qcaps_input req = {0}; 7481 struct hwrm_cfa_adv_flow_mgnt_qcaps_output *resp; 7482 int rc = 0; 7483 u32 flags; 7484 7485 if (!(bp->fw_cap & BNXT_FW_CAP_CFA_ADV_FLOW)) 7486 return 0; 7487 7488 resp = bp->hwrm_cmd_resp_addr; 7489 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_ADV_FLOW_MGNT_QCAPS, -1, -1); 7490 7491 mutex_lock(&bp->hwrm_cmd_lock); 7492 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 7493 if (rc) 7494 goto hwrm_cfa_adv_qcaps_exit; 7495 7496 flags = le32_to_cpu(resp->flags); 7497 if (flags & 7498 CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_V2_SUPPORTED) 7499 bp->fw_cap |= BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2; 7500 7501 hwrm_cfa_adv_qcaps_exit: 7502 mutex_unlock(&bp->hwrm_cmd_lock); 7503 return rc; 7504 } 7505 7506 static int __bnxt_alloc_fw_health(struct bnxt *bp) 7507 { 7508 if (bp->fw_health) 7509 return 0; 7510 7511 bp->fw_health = kzalloc(sizeof(*bp->fw_health), GFP_KERNEL); 7512 if (!bp->fw_health) 7513 return -ENOMEM; 7514 7515 return 0; 7516 } 7517 7518 static int bnxt_alloc_fw_health(struct bnxt *bp) 7519 { 7520 int rc; 7521 7522 if (!(bp->fw_cap & BNXT_FW_CAP_HOT_RESET) && 7523 !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) 7524 return 0; 7525 7526 rc = __bnxt_alloc_fw_health(bp); 7527 if (rc) { 7528 bp->fw_cap &= ~BNXT_FW_CAP_HOT_RESET; 7529 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY; 7530 return rc; 7531 } 7532 7533 return 0; 7534 } 7535 7536 static void __bnxt_map_fw_health_reg(struct bnxt *bp, u32 reg) 7537 { 7538 writel(reg & BNXT_GRC_BASE_MASK, bp->bar0 + 7539 BNXT_GRCPF_REG_WINDOW_BASE_OUT + 7540 BNXT_FW_HEALTH_WIN_MAP_OFF); 7541 } 7542 7543 static void bnxt_try_map_fw_health_reg(struct bnxt *bp) 7544 { 7545 void __iomem *hs; 7546 u32 status_loc; 7547 u32 reg_type; 7548 u32 sig; 7549 7550 __bnxt_map_fw_health_reg(bp, HCOMM_STATUS_STRUCT_LOC); 7551 hs = bp->bar0 + BNXT_FW_HEALTH_WIN_OFF(HCOMM_STATUS_STRUCT_LOC); 7552 7553 sig = readl(hs + offsetof(struct hcomm_status, sig_ver)); 7554 if ((sig & HCOMM_STATUS_SIGNATURE_MASK) != HCOMM_STATUS_SIGNATURE_VAL) { 7555 if (!bp->chip_num) { 7556 __bnxt_map_fw_health_reg(bp, BNXT_GRC_REG_BASE); 7557 bp->chip_num = readl(bp->bar0 + 7558 BNXT_FW_HEALTH_WIN_BASE + 7559 BNXT_GRC_REG_CHIP_NUM); 7560 } 7561 if (!BNXT_CHIP_P5(bp)) { 7562 if (bp->fw_health) 7563 bp->fw_health->status_reliable = false; 7564 return; 7565 } 7566 status_loc = BNXT_GRC_REG_STATUS_P5 | 7567 BNXT_FW_HEALTH_REG_TYPE_BAR0; 7568 } else { 7569 status_loc = readl(hs + offsetof(struct hcomm_status, 7570 fw_status_loc)); 7571 } 7572 7573 if (__bnxt_alloc_fw_health(bp)) { 7574 netdev_warn(bp->dev, "no memory for firmware status checks\n"); 7575 return; 7576 } 7577 7578 bp->fw_health->regs[BNXT_FW_HEALTH_REG] = status_loc; 7579 reg_type = BNXT_FW_HEALTH_REG_TYPE(status_loc); 7580 if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC) { 7581 __bnxt_map_fw_health_reg(bp, status_loc); 7582 bp->fw_health->mapped_regs[BNXT_FW_HEALTH_REG] = 7583 BNXT_FW_HEALTH_WIN_OFF(status_loc); 7584 } 7585 7586 bp->fw_health->status_reliable = true; 7587 } 7588 7589 static int bnxt_map_fw_health_regs(struct bnxt *bp) 7590 { 7591 struct bnxt_fw_health *fw_health = bp->fw_health; 7592 u32 reg_base = 0xffffffff; 7593 int i; 7594 7595 /* Only pre-map the monitoring GRC registers using window 3 */ 7596 for (i = 0; i < 4; i++) { 7597 u32 reg = fw_health->regs[i]; 7598 7599 if (BNXT_FW_HEALTH_REG_TYPE(reg) != BNXT_FW_HEALTH_REG_TYPE_GRC) 7600 continue; 7601 if (reg_base == 0xffffffff) 7602 reg_base = reg & BNXT_GRC_BASE_MASK; 7603 if ((reg & BNXT_GRC_BASE_MASK) != reg_base) 7604 return -ERANGE; 7605 fw_health->mapped_regs[i] = BNXT_FW_HEALTH_WIN_OFF(reg); 7606 } 7607 if (reg_base == 0xffffffff) 7608 return 0; 7609 7610 __bnxt_map_fw_health_reg(bp, reg_base); 7611 return 0; 7612 } 7613 7614 static int bnxt_hwrm_error_recovery_qcfg(struct bnxt *bp) 7615 { 7616 struct hwrm_error_recovery_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 7617 struct bnxt_fw_health *fw_health = bp->fw_health; 7618 struct hwrm_error_recovery_qcfg_input req = {0}; 7619 int rc, i; 7620 7621 if (!(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) 7622 return 0; 7623 7624 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_ERROR_RECOVERY_QCFG, -1, -1); 7625 mutex_lock(&bp->hwrm_cmd_lock); 7626 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 7627 if (rc) 7628 goto err_recovery_out; 7629 fw_health->flags = le32_to_cpu(resp->flags); 7630 if ((fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) && 7631 !(bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL)) { 7632 rc = -EINVAL; 7633 goto err_recovery_out; 7634 } 7635 fw_health->polling_dsecs = le32_to_cpu(resp->driver_polling_freq); 7636 fw_health->master_func_wait_dsecs = 7637 le32_to_cpu(resp->master_func_wait_period); 7638 fw_health->normal_func_wait_dsecs = 7639 le32_to_cpu(resp->normal_func_wait_period); 7640 fw_health->post_reset_wait_dsecs = 7641 le32_to_cpu(resp->master_func_wait_period_after_reset); 7642 fw_health->post_reset_max_wait_dsecs = 7643 le32_to_cpu(resp->max_bailout_time_after_reset); 7644 fw_health->regs[BNXT_FW_HEALTH_REG] = 7645 le32_to_cpu(resp->fw_health_status_reg); 7646 fw_health->regs[BNXT_FW_HEARTBEAT_REG] = 7647 le32_to_cpu(resp->fw_heartbeat_reg); 7648 fw_health->regs[BNXT_FW_RESET_CNT_REG] = 7649 le32_to_cpu(resp->fw_reset_cnt_reg); 7650 fw_health->regs[BNXT_FW_RESET_INPROG_REG] = 7651 le32_to_cpu(resp->reset_inprogress_reg); 7652 fw_health->fw_reset_inprog_reg_mask = 7653 le32_to_cpu(resp->reset_inprogress_reg_mask); 7654 fw_health->fw_reset_seq_cnt = resp->reg_array_cnt; 7655 if (fw_health->fw_reset_seq_cnt >= 16) { 7656 rc = -EINVAL; 7657 goto err_recovery_out; 7658 } 7659 for (i = 0; i < fw_health->fw_reset_seq_cnt; i++) { 7660 fw_health->fw_reset_seq_regs[i] = 7661 le32_to_cpu(resp->reset_reg[i]); 7662 fw_health->fw_reset_seq_vals[i] = 7663 le32_to_cpu(resp->reset_reg_val[i]); 7664 fw_health->fw_reset_seq_delay_msec[i] = 7665 resp->delay_after_reset[i]; 7666 } 7667 err_recovery_out: 7668 mutex_unlock(&bp->hwrm_cmd_lock); 7669 if (!rc) 7670 rc = bnxt_map_fw_health_regs(bp); 7671 if (rc) 7672 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY; 7673 return rc; 7674 } 7675 7676 static int bnxt_hwrm_func_reset(struct bnxt *bp) 7677 { 7678 struct hwrm_func_reset_input req = {0}; 7679 7680 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESET, -1, -1); 7681 req.enables = 0; 7682 7683 return hwrm_send_message(bp, &req, sizeof(req), HWRM_RESET_TIMEOUT); 7684 } 7685 7686 static void bnxt_nvm_cfg_ver_get(struct bnxt *bp) 7687 { 7688 struct hwrm_nvm_get_dev_info_output nvm_info; 7689 7690 if (!bnxt_hwrm_nvm_get_dev_info(bp, &nvm_info)) 7691 snprintf(bp->nvm_cfg_ver, FW_VER_STR_LEN, "%d.%d.%d", 7692 nvm_info.nvm_cfg_ver_maj, nvm_info.nvm_cfg_ver_min, 7693 nvm_info.nvm_cfg_ver_upd); 7694 } 7695 7696 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp) 7697 { 7698 int rc = 0; 7699 struct hwrm_queue_qportcfg_input req = {0}; 7700 struct hwrm_queue_qportcfg_output *resp = bp->hwrm_cmd_resp_addr; 7701 u8 i, j, *qptr; 7702 bool no_rdma; 7703 7704 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_QUEUE_QPORTCFG, -1, -1); 7705 7706 mutex_lock(&bp->hwrm_cmd_lock); 7707 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 7708 if (rc) 7709 goto qportcfg_exit; 7710 7711 if (!resp->max_configurable_queues) { 7712 rc = -EINVAL; 7713 goto qportcfg_exit; 7714 } 7715 bp->max_tc = resp->max_configurable_queues; 7716 bp->max_lltc = resp->max_configurable_lossless_queues; 7717 if (bp->max_tc > BNXT_MAX_QUEUE) 7718 bp->max_tc = BNXT_MAX_QUEUE; 7719 7720 no_rdma = !(bp->flags & BNXT_FLAG_ROCE_CAP); 7721 qptr = &resp->queue_id0; 7722 for (i = 0, j = 0; i < bp->max_tc; i++) { 7723 bp->q_info[j].queue_id = *qptr; 7724 bp->q_ids[i] = *qptr++; 7725 bp->q_info[j].queue_profile = *qptr++; 7726 bp->tc_to_qidx[j] = j; 7727 if (!BNXT_CNPQ(bp->q_info[j].queue_profile) || 7728 (no_rdma && BNXT_PF(bp))) 7729 j++; 7730 } 7731 bp->max_q = bp->max_tc; 7732 bp->max_tc = max_t(u8, j, 1); 7733 7734 if (resp->queue_cfg_info & QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG) 7735 bp->max_tc = 1; 7736 7737 if (bp->max_lltc > bp->max_tc) 7738 bp->max_lltc = bp->max_tc; 7739 7740 qportcfg_exit: 7741 mutex_unlock(&bp->hwrm_cmd_lock); 7742 return rc; 7743 } 7744 7745 static int __bnxt_hwrm_ver_get(struct bnxt *bp, bool silent) 7746 { 7747 struct hwrm_ver_get_input req = {0}; 7748 int rc; 7749 7750 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VER_GET, -1, -1); 7751 req.hwrm_intf_maj = HWRM_VERSION_MAJOR; 7752 req.hwrm_intf_min = HWRM_VERSION_MINOR; 7753 req.hwrm_intf_upd = HWRM_VERSION_UPDATE; 7754 7755 rc = bnxt_hwrm_do_send_msg(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT, 7756 silent); 7757 return rc; 7758 } 7759 7760 static int bnxt_hwrm_ver_get(struct bnxt *bp) 7761 { 7762 struct hwrm_ver_get_output *resp = bp->hwrm_cmd_resp_addr; 7763 u16 fw_maj, fw_min, fw_bld, fw_rsv; 7764 u32 dev_caps_cfg, hwrm_ver; 7765 int rc, len; 7766 7767 bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN; 7768 mutex_lock(&bp->hwrm_cmd_lock); 7769 rc = __bnxt_hwrm_ver_get(bp, false); 7770 if (rc) 7771 goto hwrm_ver_get_exit; 7772 7773 memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output)); 7774 7775 bp->hwrm_spec_code = resp->hwrm_intf_maj_8b << 16 | 7776 resp->hwrm_intf_min_8b << 8 | 7777 resp->hwrm_intf_upd_8b; 7778 if (resp->hwrm_intf_maj_8b < 1) { 7779 netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n", 7780 resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b, 7781 resp->hwrm_intf_upd_8b); 7782 netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n"); 7783 } 7784 7785 hwrm_ver = HWRM_VERSION_MAJOR << 16 | HWRM_VERSION_MINOR << 8 | 7786 HWRM_VERSION_UPDATE; 7787 7788 if (bp->hwrm_spec_code > hwrm_ver) 7789 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d", 7790 HWRM_VERSION_MAJOR, HWRM_VERSION_MINOR, 7791 HWRM_VERSION_UPDATE); 7792 else 7793 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d", 7794 resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b, 7795 resp->hwrm_intf_upd_8b); 7796 7797 fw_maj = le16_to_cpu(resp->hwrm_fw_major); 7798 if (bp->hwrm_spec_code > 0x10803 && fw_maj) { 7799 fw_min = le16_to_cpu(resp->hwrm_fw_minor); 7800 fw_bld = le16_to_cpu(resp->hwrm_fw_build); 7801 fw_rsv = le16_to_cpu(resp->hwrm_fw_patch); 7802 len = FW_VER_STR_LEN; 7803 } else { 7804 fw_maj = resp->hwrm_fw_maj_8b; 7805 fw_min = resp->hwrm_fw_min_8b; 7806 fw_bld = resp->hwrm_fw_bld_8b; 7807 fw_rsv = resp->hwrm_fw_rsvd_8b; 7808 len = BC_HWRM_STR_LEN; 7809 } 7810 bp->fw_ver_code = BNXT_FW_VER_CODE(fw_maj, fw_min, fw_bld, fw_rsv); 7811 snprintf(bp->fw_ver_str, len, "%d.%d.%d.%d", fw_maj, fw_min, fw_bld, 7812 fw_rsv); 7813 7814 if (strlen(resp->active_pkg_name)) { 7815 int fw_ver_len = strlen(bp->fw_ver_str); 7816 7817 snprintf(bp->fw_ver_str + fw_ver_len, 7818 FW_VER_STR_LEN - fw_ver_len - 1, "/pkg %s", 7819 resp->active_pkg_name); 7820 bp->fw_cap |= BNXT_FW_CAP_PKG_VER; 7821 } 7822 7823 bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout); 7824 if (!bp->hwrm_cmd_timeout) 7825 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT; 7826 7827 if (resp->hwrm_intf_maj_8b >= 1) { 7828 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len); 7829 bp->hwrm_max_ext_req_len = le16_to_cpu(resp->max_ext_req_len); 7830 } 7831 if (bp->hwrm_max_ext_req_len < HWRM_MAX_REQ_LEN) 7832 bp->hwrm_max_ext_req_len = HWRM_MAX_REQ_LEN; 7833 7834 bp->chip_num = le16_to_cpu(resp->chip_num); 7835 bp->chip_rev = resp->chip_rev; 7836 if (bp->chip_num == CHIP_NUM_58700 && !resp->chip_rev && 7837 !resp->chip_metal) 7838 bp->flags |= BNXT_FLAG_CHIP_NITRO_A0; 7839 7840 dev_caps_cfg = le32_to_cpu(resp->dev_caps_cfg); 7841 if ((dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) && 7842 (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED)) 7843 bp->fw_cap |= BNXT_FW_CAP_SHORT_CMD; 7844 7845 if (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_KONG_MB_CHNL_SUPPORTED) 7846 bp->fw_cap |= BNXT_FW_CAP_KONG_MB_CHNL; 7847 7848 if (dev_caps_cfg & 7849 VER_GET_RESP_DEV_CAPS_CFG_FLOW_HANDLE_64BIT_SUPPORTED) 7850 bp->fw_cap |= BNXT_FW_CAP_OVS_64BIT_HANDLE; 7851 7852 if (dev_caps_cfg & 7853 VER_GET_RESP_DEV_CAPS_CFG_TRUSTED_VF_SUPPORTED) 7854 bp->fw_cap |= BNXT_FW_CAP_TRUSTED_VF; 7855 7856 if (dev_caps_cfg & 7857 VER_GET_RESP_DEV_CAPS_CFG_CFA_ADV_FLOW_MGNT_SUPPORTED) 7858 bp->fw_cap |= BNXT_FW_CAP_CFA_ADV_FLOW; 7859 7860 hwrm_ver_get_exit: 7861 mutex_unlock(&bp->hwrm_cmd_lock); 7862 return rc; 7863 } 7864 7865 int bnxt_hwrm_fw_set_time(struct bnxt *bp) 7866 { 7867 struct hwrm_fw_set_time_input req = {0}; 7868 struct tm tm; 7869 time64_t now = ktime_get_real_seconds(); 7870 7871 if ((BNXT_VF(bp) && bp->hwrm_spec_code < 0x10901) || 7872 bp->hwrm_spec_code < 0x10400) 7873 return -EOPNOTSUPP; 7874 7875 time64_to_tm(now, 0, &tm); 7876 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FW_SET_TIME, -1, -1); 7877 req.year = cpu_to_le16(1900 + tm.tm_year); 7878 req.month = 1 + tm.tm_mon; 7879 req.day = tm.tm_mday; 7880 req.hour = tm.tm_hour; 7881 req.minute = tm.tm_min; 7882 req.second = tm.tm_sec; 7883 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 7884 } 7885 7886 static void bnxt_add_one_ctr(u64 hw, u64 *sw, u64 mask) 7887 { 7888 u64 sw_tmp; 7889 7890 hw &= mask; 7891 sw_tmp = (*sw & ~mask) | hw; 7892 if (hw < (*sw & mask)) 7893 sw_tmp += mask + 1; 7894 WRITE_ONCE(*sw, sw_tmp); 7895 } 7896 7897 static void __bnxt_accumulate_stats(__le64 *hw_stats, u64 *sw_stats, u64 *masks, 7898 int count, bool ignore_zero) 7899 { 7900 int i; 7901 7902 for (i = 0; i < count; i++) { 7903 u64 hw = le64_to_cpu(READ_ONCE(hw_stats[i])); 7904 7905 if (ignore_zero && !hw) 7906 continue; 7907 7908 if (masks[i] == -1ULL) 7909 sw_stats[i] = hw; 7910 else 7911 bnxt_add_one_ctr(hw, &sw_stats[i], masks[i]); 7912 } 7913 } 7914 7915 static void bnxt_accumulate_stats(struct bnxt_stats_mem *stats) 7916 { 7917 if (!stats->hw_stats) 7918 return; 7919 7920 __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats, 7921 stats->hw_masks, stats->len / 8, false); 7922 } 7923 7924 static void bnxt_accumulate_all_stats(struct bnxt *bp) 7925 { 7926 struct bnxt_stats_mem *ring0_stats; 7927 bool ignore_zero = false; 7928 int i; 7929 7930 /* Chip bug. Counter intermittently becomes 0. */ 7931 if (bp->flags & BNXT_FLAG_CHIP_P5) 7932 ignore_zero = true; 7933 7934 for (i = 0; i < bp->cp_nr_rings; i++) { 7935 struct bnxt_napi *bnapi = bp->bnapi[i]; 7936 struct bnxt_cp_ring_info *cpr; 7937 struct bnxt_stats_mem *stats; 7938 7939 cpr = &bnapi->cp_ring; 7940 stats = &cpr->stats; 7941 if (!i) 7942 ring0_stats = stats; 7943 __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats, 7944 ring0_stats->hw_masks, 7945 ring0_stats->len / 8, ignore_zero); 7946 } 7947 if (bp->flags & BNXT_FLAG_PORT_STATS) { 7948 struct bnxt_stats_mem *stats = &bp->port_stats; 7949 __le64 *hw_stats = stats->hw_stats; 7950 u64 *sw_stats = stats->sw_stats; 7951 u64 *masks = stats->hw_masks; 7952 int cnt; 7953 7954 cnt = sizeof(struct rx_port_stats) / 8; 7955 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false); 7956 7957 hw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 7958 sw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 7959 masks += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 7960 cnt = sizeof(struct tx_port_stats) / 8; 7961 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false); 7962 } 7963 if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) { 7964 bnxt_accumulate_stats(&bp->rx_port_stats_ext); 7965 bnxt_accumulate_stats(&bp->tx_port_stats_ext); 7966 } 7967 } 7968 7969 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags) 7970 { 7971 struct bnxt_pf_info *pf = &bp->pf; 7972 struct hwrm_port_qstats_input req = {0}; 7973 7974 if (!(bp->flags & BNXT_FLAG_PORT_STATS)) 7975 return 0; 7976 7977 if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED)) 7978 return -EOPNOTSUPP; 7979 7980 req.flags = flags; 7981 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS, -1, -1); 7982 req.port_id = cpu_to_le16(pf->port_id); 7983 req.tx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map + 7984 BNXT_TX_PORT_STATS_BYTE_OFFSET); 7985 req.rx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map); 7986 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 7987 } 7988 7989 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags) 7990 { 7991 struct hwrm_port_qstats_ext_output *resp = bp->hwrm_cmd_resp_addr; 7992 struct hwrm_queue_pri2cos_qcfg_input req2 = {0}; 7993 struct hwrm_port_qstats_ext_input req = {0}; 7994 struct bnxt_pf_info *pf = &bp->pf; 7995 u32 tx_stat_size; 7996 int rc; 7997 7998 if (!(bp->flags & BNXT_FLAG_PORT_STATS_EXT)) 7999 return 0; 8000 8001 if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED)) 8002 return -EOPNOTSUPP; 8003 8004 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS_EXT, -1, -1); 8005 req.flags = flags; 8006 req.port_id = cpu_to_le16(pf->port_id); 8007 req.rx_stat_size = cpu_to_le16(sizeof(struct rx_port_stats_ext)); 8008 req.rx_stat_host_addr = cpu_to_le64(bp->rx_port_stats_ext.hw_stats_map); 8009 tx_stat_size = bp->tx_port_stats_ext.hw_stats ? 8010 sizeof(struct tx_port_stats_ext) : 0; 8011 req.tx_stat_size = cpu_to_le16(tx_stat_size); 8012 req.tx_stat_host_addr = cpu_to_le64(bp->tx_port_stats_ext.hw_stats_map); 8013 mutex_lock(&bp->hwrm_cmd_lock); 8014 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 8015 if (!rc) { 8016 bp->fw_rx_stats_ext_size = le16_to_cpu(resp->rx_stat_size) / 8; 8017 bp->fw_tx_stats_ext_size = tx_stat_size ? 8018 le16_to_cpu(resp->tx_stat_size) / 8 : 0; 8019 } else { 8020 bp->fw_rx_stats_ext_size = 0; 8021 bp->fw_tx_stats_ext_size = 0; 8022 } 8023 if (flags) 8024 goto qstats_done; 8025 8026 if (bp->fw_tx_stats_ext_size <= 8027 offsetof(struct tx_port_stats_ext, pfc_pri0_tx_duration_us) / 8) { 8028 mutex_unlock(&bp->hwrm_cmd_lock); 8029 bp->pri2cos_valid = 0; 8030 return rc; 8031 } 8032 8033 bnxt_hwrm_cmd_hdr_init(bp, &req2, HWRM_QUEUE_PRI2COS_QCFG, -1, -1); 8034 req2.flags = cpu_to_le32(QUEUE_PRI2COS_QCFG_REQ_FLAGS_IVLAN); 8035 8036 rc = _hwrm_send_message(bp, &req2, sizeof(req2), HWRM_CMD_TIMEOUT); 8037 if (!rc) { 8038 struct hwrm_queue_pri2cos_qcfg_output *resp2; 8039 u8 *pri2cos; 8040 int i, j; 8041 8042 resp2 = bp->hwrm_cmd_resp_addr; 8043 pri2cos = &resp2->pri0_cos_queue_id; 8044 for (i = 0; i < 8; i++) { 8045 u8 queue_id = pri2cos[i]; 8046 u8 queue_idx; 8047 8048 /* Per port queue IDs start from 0, 10, 20, etc */ 8049 queue_idx = queue_id % 10; 8050 if (queue_idx > BNXT_MAX_QUEUE) { 8051 bp->pri2cos_valid = false; 8052 goto qstats_done; 8053 } 8054 for (j = 0; j < bp->max_q; j++) { 8055 if (bp->q_ids[j] == queue_id) 8056 bp->pri2cos_idx[i] = queue_idx; 8057 } 8058 } 8059 bp->pri2cos_valid = 1; 8060 } 8061 qstats_done: 8062 mutex_unlock(&bp->hwrm_cmd_lock); 8063 return rc; 8064 } 8065 8066 static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp) 8067 { 8068 if (bp->vxlan_fw_dst_port_id != INVALID_HW_RING_ID) 8069 bnxt_hwrm_tunnel_dst_port_free( 8070 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN); 8071 if (bp->nge_fw_dst_port_id != INVALID_HW_RING_ID) 8072 bnxt_hwrm_tunnel_dst_port_free( 8073 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE); 8074 } 8075 8076 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa) 8077 { 8078 int rc, i; 8079 u32 tpa_flags = 0; 8080 8081 if (set_tpa) 8082 tpa_flags = bp->flags & BNXT_FLAG_TPA; 8083 else if (BNXT_NO_FW_ACCESS(bp)) 8084 return 0; 8085 for (i = 0; i < bp->nr_vnics; i++) { 8086 rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags); 8087 if (rc) { 8088 netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n", 8089 i, rc); 8090 return rc; 8091 } 8092 } 8093 return 0; 8094 } 8095 8096 static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp) 8097 { 8098 int i; 8099 8100 for (i = 0; i < bp->nr_vnics; i++) 8101 bnxt_hwrm_vnic_set_rss(bp, i, false); 8102 } 8103 8104 static void bnxt_clear_vnic(struct bnxt *bp) 8105 { 8106 if (!bp->vnic_info) 8107 return; 8108 8109 bnxt_hwrm_clear_vnic_filter(bp); 8110 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) { 8111 /* clear all RSS setting before free vnic ctx */ 8112 bnxt_hwrm_clear_vnic_rss(bp); 8113 bnxt_hwrm_vnic_ctx_free(bp); 8114 } 8115 /* before free the vnic, undo the vnic tpa settings */ 8116 if (bp->flags & BNXT_FLAG_TPA) 8117 bnxt_set_tpa(bp, false); 8118 bnxt_hwrm_vnic_free(bp); 8119 if (bp->flags & BNXT_FLAG_CHIP_P5) 8120 bnxt_hwrm_vnic_ctx_free(bp); 8121 } 8122 8123 static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path, 8124 bool irq_re_init) 8125 { 8126 bnxt_clear_vnic(bp); 8127 bnxt_hwrm_ring_free(bp, close_path); 8128 bnxt_hwrm_ring_grp_free(bp); 8129 if (irq_re_init) { 8130 bnxt_hwrm_stat_ctx_free(bp); 8131 bnxt_hwrm_free_tunnel_ports(bp); 8132 } 8133 } 8134 8135 static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode) 8136 { 8137 struct hwrm_func_cfg_input req = {0}; 8138 8139 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1); 8140 req.fid = cpu_to_le16(0xffff); 8141 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_EVB_MODE); 8142 if (br_mode == BRIDGE_MODE_VEB) 8143 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEB; 8144 else if (br_mode == BRIDGE_MODE_VEPA) 8145 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEPA; 8146 else 8147 return -EINVAL; 8148 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 8149 } 8150 8151 static int bnxt_hwrm_set_cache_line_size(struct bnxt *bp, int size) 8152 { 8153 struct hwrm_func_cfg_input req = {0}; 8154 8155 if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10803) 8156 return 0; 8157 8158 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1); 8159 req.fid = cpu_to_le16(0xffff); 8160 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_CACHE_LINESIZE); 8161 req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_64; 8162 if (size == 128) 8163 req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_128; 8164 8165 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 8166 } 8167 8168 static int __bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id) 8169 { 8170 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 8171 int rc; 8172 8173 if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) 8174 goto skip_rss_ctx; 8175 8176 /* allocate context for vnic */ 8177 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 0); 8178 if (rc) { 8179 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n", 8180 vnic_id, rc); 8181 goto vnic_setup_err; 8182 } 8183 bp->rsscos_nr_ctxs++; 8184 8185 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 8186 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 1); 8187 if (rc) { 8188 netdev_err(bp->dev, "hwrm vnic %d cos ctx alloc failure rc: %x\n", 8189 vnic_id, rc); 8190 goto vnic_setup_err; 8191 } 8192 bp->rsscos_nr_ctxs++; 8193 } 8194 8195 skip_rss_ctx: 8196 /* configure default vnic, ring grp */ 8197 rc = bnxt_hwrm_vnic_cfg(bp, vnic_id); 8198 if (rc) { 8199 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n", 8200 vnic_id, rc); 8201 goto vnic_setup_err; 8202 } 8203 8204 /* Enable RSS hashing on vnic */ 8205 rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true); 8206 if (rc) { 8207 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n", 8208 vnic_id, rc); 8209 goto vnic_setup_err; 8210 } 8211 8212 if (bp->flags & BNXT_FLAG_AGG_RINGS) { 8213 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id); 8214 if (rc) { 8215 netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n", 8216 vnic_id, rc); 8217 } 8218 } 8219 8220 vnic_setup_err: 8221 return rc; 8222 } 8223 8224 static int __bnxt_setup_vnic_p5(struct bnxt *bp, u16 vnic_id) 8225 { 8226 int rc, i, nr_ctxs; 8227 8228 nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings); 8229 for (i = 0; i < nr_ctxs; i++) { 8230 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, i); 8231 if (rc) { 8232 netdev_err(bp->dev, "hwrm vnic %d ctx %d alloc failure rc: %x\n", 8233 vnic_id, i, rc); 8234 break; 8235 } 8236 bp->rsscos_nr_ctxs++; 8237 } 8238 if (i < nr_ctxs) 8239 return -ENOMEM; 8240 8241 rc = bnxt_hwrm_vnic_set_rss_p5(bp, vnic_id, true); 8242 if (rc) { 8243 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %d\n", 8244 vnic_id, rc); 8245 return rc; 8246 } 8247 rc = bnxt_hwrm_vnic_cfg(bp, vnic_id); 8248 if (rc) { 8249 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n", 8250 vnic_id, rc); 8251 return rc; 8252 } 8253 if (bp->flags & BNXT_FLAG_AGG_RINGS) { 8254 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id); 8255 if (rc) { 8256 netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n", 8257 vnic_id, rc); 8258 } 8259 } 8260 return rc; 8261 } 8262 8263 static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id) 8264 { 8265 if (bp->flags & BNXT_FLAG_CHIP_P5) 8266 return __bnxt_setup_vnic_p5(bp, vnic_id); 8267 else 8268 return __bnxt_setup_vnic(bp, vnic_id); 8269 } 8270 8271 static int bnxt_alloc_rfs_vnics(struct bnxt *bp) 8272 { 8273 #ifdef CONFIG_RFS_ACCEL 8274 int i, rc = 0; 8275 8276 if (bp->flags & BNXT_FLAG_CHIP_P5) 8277 return 0; 8278 8279 for (i = 0; i < bp->rx_nr_rings; i++) { 8280 struct bnxt_vnic_info *vnic; 8281 u16 vnic_id = i + 1; 8282 u16 ring_id = i; 8283 8284 if (vnic_id >= bp->nr_vnics) 8285 break; 8286 8287 vnic = &bp->vnic_info[vnic_id]; 8288 vnic->flags |= BNXT_VNIC_RFS_FLAG; 8289 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP) 8290 vnic->flags |= BNXT_VNIC_RFS_NEW_RSS_FLAG; 8291 rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, ring_id, 1); 8292 if (rc) { 8293 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n", 8294 vnic_id, rc); 8295 break; 8296 } 8297 rc = bnxt_setup_vnic(bp, vnic_id); 8298 if (rc) 8299 break; 8300 } 8301 return rc; 8302 #else 8303 return 0; 8304 #endif 8305 } 8306 8307 /* Allow PF and VF with default VLAN to be in promiscuous mode */ 8308 static bool bnxt_promisc_ok(struct bnxt *bp) 8309 { 8310 #ifdef CONFIG_BNXT_SRIOV 8311 if (BNXT_VF(bp) && !bp->vf.vlan) 8312 return false; 8313 #endif 8314 return true; 8315 } 8316 8317 static int bnxt_setup_nitroa0_vnic(struct bnxt *bp) 8318 { 8319 unsigned int rc = 0; 8320 8321 rc = bnxt_hwrm_vnic_alloc(bp, 1, bp->rx_nr_rings - 1, 1); 8322 if (rc) { 8323 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n", 8324 rc); 8325 return rc; 8326 } 8327 8328 rc = bnxt_hwrm_vnic_cfg(bp, 1); 8329 if (rc) { 8330 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n", 8331 rc); 8332 return rc; 8333 } 8334 return rc; 8335 } 8336 8337 static int bnxt_cfg_rx_mode(struct bnxt *); 8338 static bool bnxt_mc_list_updated(struct bnxt *, u32 *); 8339 8340 static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init) 8341 { 8342 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 8343 int rc = 0; 8344 unsigned int rx_nr_rings = bp->rx_nr_rings; 8345 8346 if (irq_re_init) { 8347 rc = bnxt_hwrm_stat_ctx_alloc(bp); 8348 if (rc) { 8349 netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n", 8350 rc); 8351 goto err_out; 8352 } 8353 } 8354 8355 rc = bnxt_hwrm_ring_alloc(bp); 8356 if (rc) { 8357 netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc); 8358 goto err_out; 8359 } 8360 8361 rc = bnxt_hwrm_ring_grp_alloc(bp); 8362 if (rc) { 8363 netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc); 8364 goto err_out; 8365 } 8366 8367 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 8368 rx_nr_rings--; 8369 8370 /* default vnic 0 */ 8371 rc = bnxt_hwrm_vnic_alloc(bp, 0, 0, rx_nr_rings); 8372 if (rc) { 8373 netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc); 8374 goto err_out; 8375 } 8376 8377 rc = bnxt_setup_vnic(bp, 0); 8378 if (rc) 8379 goto err_out; 8380 8381 if (bp->flags & BNXT_FLAG_RFS) { 8382 rc = bnxt_alloc_rfs_vnics(bp); 8383 if (rc) 8384 goto err_out; 8385 } 8386 8387 if (bp->flags & BNXT_FLAG_TPA) { 8388 rc = bnxt_set_tpa(bp, true); 8389 if (rc) 8390 goto err_out; 8391 } 8392 8393 if (BNXT_VF(bp)) 8394 bnxt_update_vf_mac(bp); 8395 8396 /* Filter for default vnic 0 */ 8397 rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr); 8398 if (rc) { 8399 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc); 8400 goto err_out; 8401 } 8402 vnic->uc_filter_count = 1; 8403 8404 vnic->rx_mask = 0; 8405 if (bp->dev->flags & IFF_BROADCAST) 8406 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST; 8407 8408 if ((bp->dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp)) 8409 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 8410 8411 if (bp->dev->flags & IFF_ALLMULTI) { 8412 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 8413 vnic->mc_list_count = 0; 8414 } else { 8415 u32 mask = 0; 8416 8417 bnxt_mc_list_updated(bp, &mask); 8418 vnic->rx_mask |= mask; 8419 } 8420 8421 rc = bnxt_cfg_rx_mode(bp); 8422 if (rc) 8423 goto err_out; 8424 8425 rc = bnxt_hwrm_set_coal(bp); 8426 if (rc) 8427 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n", 8428 rc); 8429 8430 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 8431 rc = bnxt_setup_nitroa0_vnic(bp); 8432 if (rc) 8433 netdev_err(bp->dev, "Special vnic setup failure for NS2 A0 rc: %x\n", 8434 rc); 8435 } 8436 8437 if (BNXT_VF(bp)) { 8438 bnxt_hwrm_func_qcfg(bp); 8439 netdev_update_features(bp->dev); 8440 } 8441 8442 return 0; 8443 8444 err_out: 8445 bnxt_hwrm_resource_free(bp, 0, true); 8446 8447 return rc; 8448 } 8449 8450 static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init) 8451 { 8452 bnxt_hwrm_resource_free(bp, 1, irq_re_init); 8453 return 0; 8454 } 8455 8456 static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init) 8457 { 8458 bnxt_init_cp_rings(bp); 8459 bnxt_init_rx_rings(bp); 8460 bnxt_init_tx_rings(bp); 8461 bnxt_init_ring_grps(bp, irq_re_init); 8462 bnxt_init_vnics(bp); 8463 8464 return bnxt_init_chip(bp, irq_re_init); 8465 } 8466 8467 static int bnxt_set_real_num_queues(struct bnxt *bp) 8468 { 8469 int rc; 8470 struct net_device *dev = bp->dev; 8471 8472 rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings - 8473 bp->tx_nr_rings_xdp); 8474 if (rc) 8475 return rc; 8476 8477 rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings); 8478 if (rc) 8479 return rc; 8480 8481 #ifdef CONFIG_RFS_ACCEL 8482 if (bp->flags & BNXT_FLAG_RFS) 8483 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings); 8484 #endif 8485 8486 return rc; 8487 } 8488 8489 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max, 8490 bool shared) 8491 { 8492 int _rx = *rx, _tx = *tx; 8493 8494 if (shared) { 8495 *rx = min_t(int, _rx, max); 8496 *tx = min_t(int, _tx, max); 8497 } else { 8498 if (max < 2) 8499 return -ENOMEM; 8500 8501 while (_rx + _tx > max) { 8502 if (_rx > _tx && _rx > 1) 8503 _rx--; 8504 else if (_tx > 1) 8505 _tx--; 8506 } 8507 *rx = _rx; 8508 *tx = _tx; 8509 } 8510 return 0; 8511 } 8512 8513 static void bnxt_setup_msix(struct bnxt *bp) 8514 { 8515 const int len = sizeof(bp->irq_tbl[0].name); 8516 struct net_device *dev = bp->dev; 8517 int tcs, i; 8518 8519 tcs = netdev_get_num_tc(dev); 8520 if (tcs) { 8521 int i, off, count; 8522 8523 for (i = 0; i < tcs; i++) { 8524 count = bp->tx_nr_rings_per_tc; 8525 off = i * count; 8526 netdev_set_tc_queue(dev, i, count, off); 8527 } 8528 } 8529 8530 for (i = 0; i < bp->cp_nr_rings; i++) { 8531 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 8532 char *attr; 8533 8534 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 8535 attr = "TxRx"; 8536 else if (i < bp->rx_nr_rings) 8537 attr = "rx"; 8538 else 8539 attr = "tx"; 8540 8541 snprintf(bp->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name, 8542 attr, i); 8543 bp->irq_tbl[map_idx].handler = bnxt_msix; 8544 } 8545 } 8546 8547 static void bnxt_setup_inta(struct bnxt *bp) 8548 { 8549 const int len = sizeof(bp->irq_tbl[0].name); 8550 8551 if (netdev_get_num_tc(bp->dev)) 8552 netdev_reset_tc(bp->dev); 8553 8554 snprintf(bp->irq_tbl[0].name, len, "%s-%s-%d", bp->dev->name, "TxRx", 8555 0); 8556 bp->irq_tbl[0].handler = bnxt_inta; 8557 } 8558 8559 static int bnxt_setup_int_mode(struct bnxt *bp) 8560 { 8561 int rc; 8562 8563 if (bp->flags & BNXT_FLAG_USING_MSIX) 8564 bnxt_setup_msix(bp); 8565 else 8566 bnxt_setup_inta(bp); 8567 8568 rc = bnxt_set_real_num_queues(bp); 8569 return rc; 8570 } 8571 8572 #ifdef CONFIG_RFS_ACCEL 8573 static unsigned int bnxt_get_max_func_rss_ctxs(struct bnxt *bp) 8574 { 8575 return bp->hw_resc.max_rsscos_ctxs; 8576 } 8577 8578 static unsigned int bnxt_get_max_func_vnics(struct bnxt *bp) 8579 { 8580 return bp->hw_resc.max_vnics; 8581 } 8582 #endif 8583 8584 unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp) 8585 { 8586 return bp->hw_resc.max_stat_ctxs; 8587 } 8588 8589 unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp) 8590 { 8591 return bp->hw_resc.max_cp_rings; 8592 } 8593 8594 static unsigned int bnxt_get_max_func_cp_rings_for_en(struct bnxt *bp) 8595 { 8596 unsigned int cp = bp->hw_resc.max_cp_rings; 8597 8598 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 8599 cp -= bnxt_get_ulp_msix_num(bp); 8600 8601 return cp; 8602 } 8603 8604 static unsigned int bnxt_get_max_func_irqs(struct bnxt *bp) 8605 { 8606 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 8607 8608 if (bp->flags & BNXT_FLAG_CHIP_P5) 8609 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_nqs); 8610 8611 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_cp_rings); 8612 } 8613 8614 static void bnxt_set_max_func_irqs(struct bnxt *bp, unsigned int max_irqs) 8615 { 8616 bp->hw_resc.max_irqs = max_irqs; 8617 } 8618 8619 unsigned int bnxt_get_avail_cp_rings_for_en(struct bnxt *bp) 8620 { 8621 unsigned int cp; 8622 8623 cp = bnxt_get_max_func_cp_rings_for_en(bp); 8624 if (bp->flags & BNXT_FLAG_CHIP_P5) 8625 return cp - bp->rx_nr_rings - bp->tx_nr_rings; 8626 else 8627 return cp - bp->cp_nr_rings; 8628 } 8629 8630 unsigned int bnxt_get_avail_stat_ctxs_for_en(struct bnxt *bp) 8631 { 8632 return bnxt_get_max_func_stat_ctxs(bp) - bnxt_get_func_stat_ctxs(bp); 8633 } 8634 8635 int bnxt_get_avail_msix(struct bnxt *bp, int num) 8636 { 8637 int max_cp = bnxt_get_max_func_cp_rings(bp); 8638 int max_irq = bnxt_get_max_func_irqs(bp); 8639 int total_req = bp->cp_nr_rings + num; 8640 int max_idx, avail_msix; 8641 8642 max_idx = bp->total_irqs; 8643 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 8644 max_idx = min_t(int, bp->total_irqs, max_cp); 8645 avail_msix = max_idx - bp->cp_nr_rings; 8646 if (!BNXT_NEW_RM(bp) || avail_msix >= num) 8647 return avail_msix; 8648 8649 if (max_irq < total_req) { 8650 num = max_irq - bp->cp_nr_rings; 8651 if (num <= 0) 8652 return 0; 8653 } 8654 return num; 8655 } 8656 8657 static int bnxt_get_num_msix(struct bnxt *bp) 8658 { 8659 if (!BNXT_NEW_RM(bp)) 8660 return bnxt_get_max_func_irqs(bp); 8661 8662 return bnxt_nq_rings_in_use(bp); 8663 } 8664 8665 static int bnxt_init_msix(struct bnxt *bp) 8666 { 8667 int i, total_vecs, max, rc = 0, min = 1, ulp_msix; 8668 struct msix_entry *msix_ent; 8669 8670 total_vecs = bnxt_get_num_msix(bp); 8671 max = bnxt_get_max_func_irqs(bp); 8672 if (total_vecs > max) 8673 total_vecs = max; 8674 8675 if (!total_vecs) 8676 return 0; 8677 8678 msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL); 8679 if (!msix_ent) 8680 return -ENOMEM; 8681 8682 for (i = 0; i < total_vecs; i++) { 8683 msix_ent[i].entry = i; 8684 msix_ent[i].vector = 0; 8685 } 8686 8687 if (!(bp->flags & BNXT_FLAG_SHARED_RINGS)) 8688 min = 2; 8689 8690 total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs); 8691 ulp_msix = bnxt_get_ulp_msix_num(bp); 8692 if (total_vecs < 0 || total_vecs < ulp_msix) { 8693 rc = -ENODEV; 8694 goto msix_setup_exit; 8695 } 8696 8697 bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL); 8698 if (bp->irq_tbl) { 8699 for (i = 0; i < total_vecs; i++) 8700 bp->irq_tbl[i].vector = msix_ent[i].vector; 8701 8702 bp->total_irqs = total_vecs; 8703 /* Trim rings based upon num of vectors allocated */ 8704 rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings, 8705 total_vecs - ulp_msix, min == 1); 8706 if (rc) 8707 goto msix_setup_exit; 8708 8709 bp->cp_nr_rings = (min == 1) ? 8710 max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) : 8711 bp->tx_nr_rings + bp->rx_nr_rings; 8712 8713 } else { 8714 rc = -ENOMEM; 8715 goto msix_setup_exit; 8716 } 8717 bp->flags |= BNXT_FLAG_USING_MSIX; 8718 kfree(msix_ent); 8719 return 0; 8720 8721 msix_setup_exit: 8722 netdev_err(bp->dev, "bnxt_init_msix err: %x\n", rc); 8723 kfree(bp->irq_tbl); 8724 bp->irq_tbl = NULL; 8725 pci_disable_msix(bp->pdev); 8726 kfree(msix_ent); 8727 return rc; 8728 } 8729 8730 static int bnxt_init_inta(struct bnxt *bp) 8731 { 8732 bp->irq_tbl = kzalloc(sizeof(struct bnxt_irq), GFP_KERNEL); 8733 if (!bp->irq_tbl) 8734 return -ENOMEM; 8735 8736 bp->total_irqs = 1; 8737 bp->rx_nr_rings = 1; 8738 bp->tx_nr_rings = 1; 8739 bp->cp_nr_rings = 1; 8740 bp->flags |= BNXT_FLAG_SHARED_RINGS; 8741 bp->irq_tbl[0].vector = bp->pdev->irq; 8742 return 0; 8743 } 8744 8745 static int bnxt_init_int_mode(struct bnxt *bp) 8746 { 8747 int rc = 0; 8748 8749 if (bp->flags & BNXT_FLAG_MSIX_CAP) 8750 rc = bnxt_init_msix(bp); 8751 8752 if (!(bp->flags & BNXT_FLAG_USING_MSIX) && BNXT_PF(bp)) { 8753 /* fallback to INTA */ 8754 rc = bnxt_init_inta(bp); 8755 } 8756 return rc; 8757 } 8758 8759 static void bnxt_clear_int_mode(struct bnxt *bp) 8760 { 8761 if (bp->flags & BNXT_FLAG_USING_MSIX) 8762 pci_disable_msix(bp->pdev); 8763 8764 kfree(bp->irq_tbl); 8765 bp->irq_tbl = NULL; 8766 bp->flags &= ~BNXT_FLAG_USING_MSIX; 8767 } 8768 8769 int bnxt_reserve_rings(struct bnxt *bp, bool irq_re_init) 8770 { 8771 int tcs = netdev_get_num_tc(bp->dev); 8772 bool irq_cleared = false; 8773 int rc; 8774 8775 if (!bnxt_need_reserve_rings(bp)) 8776 return 0; 8777 8778 if (irq_re_init && BNXT_NEW_RM(bp) && 8779 bnxt_get_num_msix(bp) != bp->total_irqs) { 8780 bnxt_ulp_irq_stop(bp); 8781 bnxt_clear_int_mode(bp); 8782 irq_cleared = true; 8783 } 8784 rc = __bnxt_reserve_rings(bp); 8785 if (irq_cleared) { 8786 if (!rc) 8787 rc = bnxt_init_int_mode(bp); 8788 bnxt_ulp_irq_restart(bp, rc); 8789 } 8790 if (rc) { 8791 netdev_err(bp->dev, "ring reservation/IRQ init failure rc: %d\n", rc); 8792 return rc; 8793 } 8794 if (tcs && (bp->tx_nr_rings_per_tc * tcs != bp->tx_nr_rings)) { 8795 netdev_err(bp->dev, "tx ring reservation failure\n"); 8796 netdev_reset_tc(bp->dev); 8797 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 8798 return -ENOMEM; 8799 } 8800 return 0; 8801 } 8802 8803 static void bnxt_free_irq(struct bnxt *bp) 8804 { 8805 struct bnxt_irq *irq; 8806 int i; 8807 8808 #ifdef CONFIG_RFS_ACCEL 8809 free_irq_cpu_rmap(bp->dev->rx_cpu_rmap); 8810 bp->dev->rx_cpu_rmap = NULL; 8811 #endif 8812 if (!bp->irq_tbl || !bp->bnapi) 8813 return; 8814 8815 for (i = 0; i < bp->cp_nr_rings; i++) { 8816 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 8817 8818 irq = &bp->irq_tbl[map_idx]; 8819 if (irq->requested) { 8820 if (irq->have_cpumask) { 8821 irq_set_affinity_hint(irq->vector, NULL); 8822 free_cpumask_var(irq->cpu_mask); 8823 irq->have_cpumask = 0; 8824 } 8825 free_irq(irq->vector, bp->bnapi[i]); 8826 } 8827 8828 irq->requested = 0; 8829 } 8830 } 8831 8832 static int bnxt_request_irq(struct bnxt *bp) 8833 { 8834 int i, j, rc = 0; 8835 unsigned long flags = 0; 8836 #ifdef CONFIG_RFS_ACCEL 8837 struct cpu_rmap *rmap; 8838 #endif 8839 8840 rc = bnxt_setup_int_mode(bp); 8841 if (rc) { 8842 netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n", 8843 rc); 8844 return rc; 8845 } 8846 #ifdef CONFIG_RFS_ACCEL 8847 rmap = bp->dev->rx_cpu_rmap; 8848 #endif 8849 if (!(bp->flags & BNXT_FLAG_USING_MSIX)) 8850 flags = IRQF_SHARED; 8851 8852 for (i = 0, j = 0; i < bp->cp_nr_rings; i++) { 8853 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 8854 struct bnxt_irq *irq = &bp->irq_tbl[map_idx]; 8855 8856 #ifdef CONFIG_RFS_ACCEL 8857 if (rmap && bp->bnapi[i]->rx_ring) { 8858 rc = irq_cpu_rmap_add(rmap, irq->vector); 8859 if (rc) 8860 netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n", 8861 j); 8862 j++; 8863 } 8864 #endif 8865 rc = request_irq(irq->vector, irq->handler, flags, irq->name, 8866 bp->bnapi[i]); 8867 if (rc) 8868 break; 8869 8870 irq->requested = 1; 8871 8872 if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) { 8873 int numa_node = dev_to_node(&bp->pdev->dev); 8874 8875 irq->have_cpumask = 1; 8876 cpumask_set_cpu(cpumask_local_spread(i, numa_node), 8877 irq->cpu_mask); 8878 rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask); 8879 if (rc) { 8880 netdev_warn(bp->dev, 8881 "Set affinity failed, IRQ = %d\n", 8882 irq->vector); 8883 break; 8884 } 8885 } 8886 } 8887 return rc; 8888 } 8889 8890 static void bnxt_del_napi(struct bnxt *bp) 8891 { 8892 int i; 8893 8894 if (!bp->bnapi) 8895 return; 8896 8897 for (i = 0; i < bp->cp_nr_rings; i++) { 8898 struct bnxt_napi *bnapi = bp->bnapi[i]; 8899 8900 __netif_napi_del(&bnapi->napi); 8901 } 8902 /* We called __netif_napi_del(), we need 8903 * to respect an RCU grace period before freeing napi structures. 8904 */ 8905 synchronize_net(); 8906 } 8907 8908 static void bnxt_init_napi(struct bnxt *bp) 8909 { 8910 int i; 8911 unsigned int cp_nr_rings = bp->cp_nr_rings; 8912 struct bnxt_napi *bnapi; 8913 8914 if (bp->flags & BNXT_FLAG_USING_MSIX) { 8915 int (*poll_fn)(struct napi_struct *, int) = bnxt_poll; 8916 8917 if (bp->flags & BNXT_FLAG_CHIP_P5) 8918 poll_fn = bnxt_poll_p5; 8919 else if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 8920 cp_nr_rings--; 8921 for (i = 0; i < cp_nr_rings; i++) { 8922 bnapi = bp->bnapi[i]; 8923 netif_napi_add(bp->dev, &bnapi->napi, poll_fn, 64); 8924 } 8925 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 8926 bnapi = bp->bnapi[cp_nr_rings]; 8927 netif_napi_add(bp->dev, &bnapi->napi, 8928 bnxt_poll_nitroa0, 64); 8929 } 8930 } else { 8931 bnapi = bp->bnapi[0]; 8932 netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll, 64); 8933 } 8934 } 8935 8936 static void bnxt_disable_napi(struct bnxt *bp) 8937 { 8938 int i; 8939 8940 if (!bp->bnapi || 8941 test_and_set_bit(BNXT_STATE_NAPI_DISABLED, &bp->state)) 8942 return; 8943 8944 for (i = 0; i < bp->cp_nr_rings; i++) { 8945 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring; 8946 8947 if (bp->bnapi[i]->rx_ring) 8948 cancel_work_sync(&cpr->dim.work); 8949 8950 napi_disable(&bp->bnapi[i]->napi); 8951 } 8952 } 8953 8954 static void bnxt_enable_napi(struct bnxt *bp) 8955 { 8956 int i; 8957 8958 clear_bit(BNXT_STATE_NAPI_DISABLED, &bp->state); 8959 for (i = 0; i < bp->cp_nr_rings; i++) { 8960 struct bnxt_napi *bnapi = bp->bnapi[i]; 8961 struct bnxt_cp_ring_info *cpr; 8962 8963 cpr = &bnapi->cp_ring; 8964 if (bnapi->in_reset) 8965 cpr->sw_stats.rx.rx_resets++; 8966 bnapi->in_reset = false; 8967 8968 if (bnapi->rx_ring) { 8969 INIT_WORK(&cpr->dim.work, bnxt_dim_work); 8970 cpr->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; 8971 } 8972 napi_enable(&bnapi->napi); 8973 } 8974 } 8975 8976 void bnxt_tx_disable(struct bnxt *bp) 8977 { 8978 int i; 8979 struct bnxt_tx_ring_info *txr; 8980 8981 if (bp->tx_ring) { 8982 for (i = 0; i < bp->tx_nr_rings; i++) { 8983 txr = &bp->tx_ring[i]; 8984 txr->dev_state = BNXT_DEV_STATE_CLOSING; 8985 } 8986 } 8987 /* Drop carrier first to prevent TX timeout */ 8988 netif_carrier_off(bp->dev); 8989 /* Stop all TX queues */ 8990 netif_tx_disable(bp->dev); 8991 } 8992 8993 void bnxt_tx_enable(struct bnxt *bp) 8994 { 8995 int i; 8996 struct bnxt_tx_ring_info *txr; 8997 8998 for (i = 0; i < bp->tx_nr_rings; i++) { 8999 txr = &bp->tx_ring[i]; 9000 txr->dev_state = 0; 9001 } 9002 netif_tx_wake_all_queues(bp->dev); 9003 if (bp->link_info.link_up) 9004 netif_carrier_on(bp->dev); 9005 } 9006 9007 static char *bnxt_report_fec(struct bnxt_link_info *link_info) 9008 { 9009 u8 active_fec = link_info->active_fec_sig_mode & 9010 PORT_PHY_QCFG_RESP_ACTIVE_FEC_MASK; 9011 9012 switch (active_fec) { 9013 default: 9014 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_NONE_ACTIVE: 9015 return "None"; 9016 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE74_ACTIVE: 9017 return "Clause 74 BaseR"; 9018 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE91_ACTIVE: 9019 return "Clause 91 RS(528,514)"; 9020 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_1XN_ACTIVE: 9021 return "Clause 91 RS544_1XN"; 9022 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_IEEE_ACTIVE: 9023 return "Clause 91 RS(544,514)"; 9024 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_1XN_ACTIVE: 9025 return "Clause 91 RS272_1XN"; 9026 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_IEEE_ACTIVE: 9027 return "Clause 91 RS(272,257)"; 9028 } 9029 } 9030 9031 static void bnxt_report_link(struct bnxt *bp) 9032 { 9033 if (bp->link_info.link_up) { 9034 const char *duplex; 9035 const char *flow_ctrl; 9036 u32 speed; 9037 u16 fec; 9038 9039 netif_carrier_on(bp->dev); 9040 speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed); 9041 if (speed == SPEED_UNKNOWN) { 9042 netdev_info(bp->dev, "NIC Link is Up, speed unknown\n"); 9043 return; 9044 } 9045 if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL) 9046 duplex = "full"; 9047 else 9048 duplex = "half"; 9049 if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH) 9050 flow_ctrl = "ON - receive & transmit"; 9051 else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX) 9052 flow_ctrl = "ON - transmit"; 9053 else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX) 9054 flow_ctrl = "ON - receive"; 9055 else 9056 flow_ctrl = "none"; 9057 netdev_info(bp->dev, "NIC Link is Up, %u Mbps %s duplex, Flow control: %s\n", 9058 speed, duplex, flow_ctrl); 9059 if (bp->flags & BNXT_FLAG_EEE_CAP) 9060 netdev_info(bp->dev, "EEE is %s\n", 9061 bp->eee.eee_active ? "active" : 9062 "not active"); 9063 fec = bp->link_info.fec_cfg; 9064 if (!(fec & PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED)) 9065 netdev_info(bp->dev, "FEC autoneg %s encoding: %s\n", 9066 (fec & BNXT_FEC_AUTONEG) ? "on" : "off", 9067 bnxt_report_fec(&bp->link_info)); 9068 } else { 9069 netif_carrier_off(bp->dev); 9070 netdev_err(bp->dev, "NIC Link is Down\n"); 9071 } 9072 } 9073 9074 static bool bnxt_phy_qcaps_no_speed(struct hwrm_port_phy_qcaps_output *resp) 9075 { 9076 if (!resp->supported_speeds_auto_mode && 9077 !resp->supported_speeds_force_mode && 9078 !resp->supported_pam4_speeds_auto_mode && 9079 !resp->supported_pam4_speeds_force_mode) 9080 return true; 9081 return false; 9082 } 9083 9084 static int bnxt_hwrm_phy_qcaps(struct bnxt *bp) 9085 { 9086 int rc = 0; 9087 struct hwrm_port_phy_qcaps_input req = {0}; 9088 struct hwrm_port_phy_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 9089 struct bnxt_link_info *link_info = &bp->link_info; 9090 9091 bp->flags &= ~BNXT_FLAG_EEE_CAP; 9092 if (bp->test_info) 9093 bp->test_info->flags &= ~(BNXT_TEST_FL_EXT_LPBK | 9094 BNXT_TEST_FL_AN_PHY_LPBK); 9095 if (bp->hwrm_spec_code < 0x10201) 9096 return 0; 9097 9098 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCAPS, -1, -1); 9099 9100 mutex_lock(&bp->hwrm_cmd_lock); 9101 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 9102 if (rc) 9103 goto hwrm_phy_qcaps_exit; 9104 9105 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) { 9106 struct ethtool_eee *eee = &bp->eee; 9107 u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode); 9108 9109 bp->flags |= BNXT_FLAG_EEE_CAP; 9110 eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0); 9111 bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) & 9112 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK; 9113 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) & 9114 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK; 9115 } 9116 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EXTERNAL_LPBK_SUPPORTED) { 9117 if (bp->test_info) 9118 bp->test_info->flags |= BNXT_TEST_FL_EXT_LPBK; 9119 } 9120 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_AUTONEG_LPBK_SUPPORTED) { 9121 if (bp->test_info) 9122 bp->test_info->flags |= BNXT_TEST_FL_AN_PHY_LPBK; 9123 } 9124 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_SHARED_PHY_CFG_SUPPORTED) { 9125 if (BNXT_PF(bp)) 9126 bp->fw_cap |= BNXT_FW_CAP_SHARED_PORT_CFG; 9127 } 9128 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_CUMULATIVE_COUNTERS_ON_RESET) 9129 bp->fw_cap |= BNXT_FW_CAP_PORT_STATS_NO_RESET; 9130 9131 if (bp->hwrm_spec_code >= 0x10a01) { 9132 if (bnxt_phy_qcaps_no_speed(resp)) { 9133 link_info->phy_state = BNXT_PHY_STATE_DISABLED; 9134 netdev_warn(bp->dev, "Ethernet link disabled\n"); 9135 } else if (link_info->phy_state == BNXT_PHY_STATE_DISABLED) { 9136 link_info->phy_state = BNXT_PHY_STATE_ENABLED; 9137 netdev_info(bp->dev, "Ethernet link enabled\n"); 9138 /* Phy re-enabled, reprobe the speeds */ 9139 link_info->support_auto_speeds = 0; 9140 link_info->support_pam4_auto_speeds = 0; 9141 } 9142 } 9143 if (resp->supported_speeds_auto_mode) 9144 link_info->support_auto_speeds = 9145 le16_to_cpu(resp->supported_speeds_auto_mode); 9146 if (resp->supported_pam4_speeds_auto_mode) 9147 link_info->support_pam4_auto_speeds = 9148 le16_to_cpu(resp->supported_pam4_speeds_auto_mode); 9149 9150 bp->port_count = resp->port_cnt; 9151 9152 hwrm_phy_qcaps_exit: 9153 mutex_unlock(&bp->hwrm_cmd_lock); 9154 return rc; 9155 } 9156 9157 static bool bnxt_support_dropped(u16 advertising, u16 supported) 9158 { 9159 u16 diff = advertising ^ supported; 9160 9161 return ((supported | diff) != supported); 9162 } 9163 9164 int bnxt_update_link(struct bnxt *bp, bool chng_link_state) 9165 { 9166 int rc = 0; 9167 struct bnxt_link_info *link_info = &bp->link_info; 9168 struct hwrm_port_phy_qcfg_input req = {0}; 9169 struct hwrm_port_phy_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 9170 u8 link_up = link_info->link_up; 9171 bool support_changed = false; 9172 9173 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCFG, -1, -1); 9174 9175 mutex_lock(&bp->hwrm_cmd_lock); 9176 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 9177 if (rc) { 9178 mutex_unlock(&bp->hwrm_cmd_lock); 9179 return rc; 9180 } 9181 9182 memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp)); 9183 link_info->phy_link_status = resp->link; 9184 link_info->duplex = resp->duplex_cfg; 9185 if (bp->hwrm_spec_code >= 0x10800) 9186 link_info->duplex = resp->duplex_state; 9187 link_info->pause = resp->pause; 9188 link_info->auto_mode = resp->auto_mode; 9189 link_info->auto_pause_setting = resp->auto_pause; 9190 link_info->lp_pause = resp->link_partner_adv_pause; 9191 link_info->force_pause_setting = resp->force_pause; 9192 link_info->duplex_setting = resp->duplex_cfg; 9193 if (link_info->phy_link_status == BNXT_LINK_LINK) 9194 link_info->link_speed = le16_to_cpu(resp->link_speed); 9195 else 9196 link_info->link_speed = 0; 9197 link_info->force_link_speed = le16_to_cpu(resp->force_link_speed); 9198 link_info->force_pam4_link_speed = 9199 le16_to_cpu(resp->force_pam4_link_speed); 9200 link_info->support_speeds = le16_to_cpu(resp->support_speeds); 9201 link_info->support_pam4_speeds = le16_to_cpu(resp->support_pam4_speeds); 9202 link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask); 9203 link_info->auto_pam4_link_speeds = 9204 le16_to_cpu(resp->auto_pam4_link_speed_mask); 9205 link_info->lp_auto_link_speeds = 9206 le16_to_cpu(resp->link_partner_adv_speeds); 9207 link_info->lp_auto_pam4_link_speeds = 9208 resp->link_partner_pam4_adv_speeds; 9209 link_info->preemphasis = le32_to_cpu(resp->preemphasis); 9210 link_info->phy_ver[0] = resp->phy_maj; 9211 link_info->phy_ver[1] = resp->phy_min; 9212 link_info->phy_ver[2] = resp->phy_bld; 9213 link_info->media_type = resp->media_type; 9214 link_info->phy_type = resp->phy_type; 9215 link_info->transceiver = resp->xcvr_pkg_type; 9216 link_info->phy_addr = resp->eee_config_phy_addr & 9217 PORT_PHY_QCFG_RESP_PHY_ADDR_MASK; 9218 link_info->module_status = resp->module_status; 9219 9220 if (bp->flags & BNXT_FLAG_EEE_CAP) { 9221 struct ethtool_eee *eee = &bp->eee; 9222 u16 fw_speeds; 9223 9224 eee->eee_active = 0; 9225 if (resp->eee_config_phy_addr & 9226 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) { 9227 eee->eee_active = 1; 9228 fw_speeds = le16_to_cpu( 9229 resp->link_partner_adv_eee_link_speed_mask); 9230 eee->lp_advertised = 9231 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0); 9232 } 9233 9234 /* Pull initial EEE config */ 9235 if (!chng_link_state) { 9236 if (resp->eee_config_phy_addr & 9237 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED) 9238 eee->eee_enabled = 1; 9239 9240 fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask); 9241 eee->advertised = 9242 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0); 9243 9244 if (resp->eee_config_phy_addr & 9245 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) { 9246 __le32 tmr; 9247 9248 eee->tx_lpi_enabled = 1; 9249 tmr = resp->xcvr_identifier_type_tx_lpi_timer; 9250 eee->tx_lpi_timer = le32_to_cpu(tmr) & 9251 PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK; 9252 } 9253 } 9254 } 9255 9256 link_info->fec_cfg = PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED; 9257 if (bp->hwrm_spec_code >= 0x10504) { 9258 link_info->fec_cfg = le16_to_cpu(resp->fec_cfg); 9259 link_info->active_fec_sig_mode = resp->active_fec_signal_mode; 9260 } 9261 /* TODO: need to add more logic to report VF link */ 9262 if (chng_link_state) { 9263 if (link_info->phy_link_status == BNXT_LINK_LINK) 9264 link_info->link_up = 1; 9265 else 9266 link_info->link_up = 0; 9267 if (link_up != link_info->link_up) 9268 bnxt_report_link(bp); 9269 } else { 9270 /* alwasy link down if not require to update link state */ 9271 link_info->link_up = 0; 9272 } 9273 mutex_unlock(&bp->hwrm_cmd_lock); 9274 9275 if (!BNXT_PHY_CFG_ABLE(bp)) 9276 return 0; 9277 9278 /* Check if any advertised speeds are no longer supported. The caller 9279 * holds the link_lock mutex, so we can modify link_info settings. 9280 */ 9281 if (bnxt_support_dropped(link_info->advertising, 9282 link_info->support_auto_speeds)) { 9283 link_info->advertising = link_info->support_auto_speeds; 9284 support_changed = true; 9285 } 9286 if (bnxt_support_dropped(link_info->advertising_pam4, 9287 link_info->support_pam4_auto_speeds)) { 9288 link_info->advertising_pam4 = link_info->support_pam4_auto_speeds; 9289 support_changed = true; 9290 } 9291 if (support_changed && (link_info->autoneg & BNXT_AUTONEG_SPEED)) 9292 bnxt_hwrm_set_link_setting(bp, true, false); 9293 return 0; 9294 } 9295 9296 static void bnxt_get_port_module_status(struct bnxt *bp) 9297 { 9298 struct bnxt_link_info *link_info = &bp->link_info; 9299 struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp; 9300 u8 module_status; 9301 9302 if (bnxt_update_link(bp, true)) 9303 return; 9304 9305 module_status = link_info->module_status; 9306 switch (module_status) { 9307 case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX: 9308 case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN: 9309 case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG: 9310 netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n", 9311 bp->pf.port_id); 9312 if (bp->hwrm_spec_code >= 0x10201) { 9313 netdev_warn(bp->dev, "Module part number %s\n", 9314 resp->phy_vendor_partnumber); 9315 } 9316 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX) 9317 netdev_warn(bp->dev, "TX is disabled\n"); 9318 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN) 9319 netdev_warn(bp->dev, "SFP+ module is shutdown\n"); 9320 } 9321 } 9322 9323 static void 9324 bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req) 9325 { 9326 if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) { 9327 if (bp->hwrm_spec_code >= 0x10201) 9328 req->auto_pause = 9329 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE; 9330 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX) 9331 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX; 9332 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX) 9333 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX; 9334 req->enables |= 9335 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE); 9336 } else { 9337 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX) 9338 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX; 9339 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX) 9340 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX; 9341 req->enables |= 9342 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE); 9343 if (bp->hwrm_spec_code >= 0x10201) { 9344 req->auto_pause = req->force_pause; 9345 req->enables |= cpu_to_le32( 9346 PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE); 9347 } 9348 } 9349 } 9350 9351 static void bnxt_hwrm_set_link_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req) 9352 { 9353 if (bp->link_info.autoneg & BNXT_AUTONEG_SPEED) { 9354 req->auto_mode |= PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK; 9355 if (bp->link_info.advertising) { 9356 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK); 9357 req->auto_link_speed_mask = cpu_to_le16(bp->link_info.advertising); 9358 } 9359 if (bp->link_info.advertising_pam4) { 9360 req->enables |= 9361 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAM4_LINK_SPEED_MASK); 9362 req->auto_link_pam4_speed_mask = 9363 cpu_to_le16(bp->link_info.advertising_pam4); 9364 } 9365 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE); 9366 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG); 9367 } else { 9368 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE); 9369 if (bp->link_info.req_signal_mode == BNXT_SIG_MODE_PAM4) { 9370 req->force_pam4_link_speed = cpu_to_le16(bp->link_info.req_link_speed); 9371 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAM4_LINK_SPEED); 9372 } else { 9373 req->force_link_speed = cpu_to_le16(bp->link_info.req_link_speed); 9374 } 9375 } 9376 9377 /* tell chimp that the setting takes effect immediately */ 9378 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY); 9379 } 9380 9381 int bnxt_hwrm_set_pause(struct bnxt *bp) 9382 { 9383 struct hwrm_port_phy_cfg_input req = {0}; 9384 int rc; 9385 9386 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1); 9387 bnxt_hwrm_set_pause_common(bp, &req); 9388 9389 if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) || 9390 bp->link_info.force_link_chng) 9391 bnxt_hwrm_set_link_common(bp, &req); 9392 9393 mutex_lock(&bp->hwrm_cmd_lock); 9394 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 9395 if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) { 9396 /* since changing of pause setting doesn't trigger any link 9397 * change event, the driver needs to update the current pause 9398 * result upon successfully return of the phy_cfg command 9399 */ 9400 bp->link_info.pause = 9401 bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl; 9402 bp->link_info.auto_pause_setting = 0; 9403 if (!bp->link_info.force_link_chng) 9404 bnxt_report_link(bp); 9405 } 9406 bp->link_info.force_link_chng = false; 9407 mutex_unlock(&bp->hwrm_cmd_lock); 9408 return rc; 9409 } 9410 9411 static void bnxt_hwrm_set_eee(struct bnxt *bp, 9412 struct hwrm_port_phy_cfg_input *req) 9413 { 9414 struct ethtool_eee *eee = &bp->eee; 9415 9416 if (eee->eee_enabled) { 9417 u16 eee_speeds; 9418 u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE; 9419 9420 if (eee->tx_lpi_enabled) 9421 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE; 9422 else 9423 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE; 9424 9425 req->flags |= cpu_to_le32(flags); 9426 eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised); 9427 req->eee_link_speed_mask = cpu_to_le16(eee_speeds); 9428 req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer); 9429 } else { 9430 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE); 9431 } 9432 } 9433 9434 int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee) 9435 { 9436 struct hwrm_port_phy_cfg_input req = {0}; 9437 9438 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1); 9439 if (set_pause) 9440 bnxt_hwrm_set_pause_common(bp, &req); 9441 9442 bnxt_hwrm_set_link_common(bp, &req); 9443 9444 if (set_eee) 9445 bnxt_hwrm_set_eee(bp, &req); 9446 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 9447 } 9448 9449 static int bnxt_hwrm_shutdown_link(struct bnxt *bp) 9450 { 9451 struct hwrm_port_phy_cfg_input req = {0}; 9452 9453 if (!BNXT_SINGLE_PF(bp)) 9454 return 0; 9455 9456 if (pci_num_vf(bp->pdev)) 9457 return 0; 9458 9459 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1); 9460 req.flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN); 9461 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 9462 } 9463 9464 static int bnxt_fw_init_one(struct bnxt *bp); 9465 9466 static int bnxt_fw_reset_via_optee(struct bnxt *bp) 9467 { 9468 #ifdef CONFIG_TEE_BNXT_FW 9469 int rc = tee_bnxt_fw_load(); 9470 9471 if (rc) 9472 netdev_err(bp->dev, "Failed FW reset via OP-TEE, rc=%d\n", rc); 9473 9474 return rc; 9475 #else 9476 netdev_err(bp->dev, "OP-TEE not supported\n"); 9477 return -ENODEV; 9478 #endif 9479 } 9480 9481 static int bnxt_try_recover_fw(struct bnxt *bp) 9482 { 9483 if (bp->fw_health && bp->fw_health->status_reliable) { 9484 int retry = 0, rc; 9485 u32 sts; 9486 9487 mutex_lock(&bp->hwrm_cmd_lock); 9488 do { 9489 rc = __bnxt_hwrm_ver_get(bp, true); 9490 sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG); 9491 if (!sts || !BNXT_FW_IS_BOOTING(sts)) 9492 break; 9493 retry++; 9494 } while (rc == -EBUSY && retry < BNXT_FW_RETRY); 9495 mutex_unlock(&bp->hwrm_cmd_lock); 9496 9497 if (!BNXT_FW_IS_HEALTHY(sts)) { 9498 netdev_err(bp->dev, 9499 "Firmware not responding, status: 0x%x\n", 9500 sts); 9501 rc = -ENODEV; 9502 } 9503 if (sts & FW_STATUS_REG_CRASHED_NO_MASTER) { 9504 netdev_warn(bp->dev, "Firmware recover via OP-TEE requested\n"); 9505 return bnxt_fw_reset_via_optee(bp); 9506 } 9507 return rc; 9508 } 9509 9510 return -ENODEV; 9511 } 9512 9513 static int bnxt_hwrm_if_change(struct bnxt *bp, bool up) 9514 { 9515 struct hwrm_func_drv_if_change_output *resp = bp->hwrm_cmd_resp_addr; 9516 struct hwrm_func_drv_if_change_input req = {0}; 9517 bool resc_reinit = false, fw_reset = false; 9518 int rc, retry = 0; 9519 u32 flags = 0; 9520 9521 if (!(bp->fw_cap & BNXT_FW_CAP_IF_CHANGE)) 9522 return 0; 9523 9524 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_IF_CHANGE, -1, -1); 9525 if (up) 9526 req.flags = cpu_to_le32(FUNC_DRV_IF_CHANGE_REQ_FLAGS_UP); 9527 mutex_lock(&bp->hwrm_cmd_lock); 9528 while (retry < BNXT_FW_IF_RETRY) { 9529 rc = _hwrm_send_message(bp, &req, sizeof(req), 9530 HWRM_CMD_TIMEOUT); 9531 if (rc != -EAGAIN) 9532 break; 9533 9534 msleep(50); 9535 retry++; 9536 } 9537 if (!rc) 9538 flags = le32_to_cpu(resp->flags); 9539 mutex_unlock(&bp->hwrm_cmd_lock); 9540 9541 if (rc == -EAGAIN) 9542 return rc; 9543 if (rc && up) { 9544 rc = bnxt_try_recover_fw(bp); 9545 fw_reset = true; 9546 } 9547 if (rc) 9548 return rc; 9549 9550 if (!up) 9551 return 0; 9552 9553 if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_RESC_CHANGE) 9554 resc_reinit = true; 9555 if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_HOT_FW_RESET_DONE) 9556 fw_reset = true; 9557 9558 if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state) && !fw_reset) { 9559 netdev_err(bp->dev, "RESET_DONE not set during FW reset.\n"); 9560 return -ENODEV; 9561 } 9562 if (resc_reinit || fw_reset) { 9563 if (fw_reset) { 9564 if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 9565 bnxt_ulp_stop(bp); 9566 bnxt_free_ctx_mem(bp); 9567 kfree(bp->ctx); 9568 bp->ctx = NULL; 9569 bnxt_dcb_free(bp); 9570 rc = bnxt_fw_init_one(bp); 9571 if (rc) { 9572 set_bit(BNXT_STATE_ABORT_ERR, &bp->state); 9573 return rc; 9574 } 9575 bnxt_clear_int_mode(bp); 9576 rc = bnxt_init_int_mode(bp); 9577 if (rc) { 9578 netdev_err(bp->dev, "init int mode failed\n"); 9579 return rc; 9580 } 9581 set_bit(BNXT_STATE_FW_RESET_DET, &bp->state); 9582 } 9583 if (BNXT_NEW_RM(bp)) { 9584 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 9585 9586 rc = bnxt_hwrm_func_resc_qcaps(bp, true); 9587 hw_resc->resv_cp_rings = 0; 9588 hw_resc->resv_stat_ctxs = 0; 9589 hw_resc->resv_irqs = 0; 9590 hw_resc->resv_tx_rings = 0; 9591 hw_resc->resv_rx_rings = 0; 9592 hw_resc->resv_hw_ring_grps = 0; 9593 hw_resc->resv_vnics = 0; 9594 if (!fw_reset) { 9595 bp->tx_nr_rings = 0; 9596 bp->rx_nr_rings = 0; 9597 } 9598 } 9599 } 9600 return 0; 9601 } 9602 9603 static int bnxt_hwrm_port_led_qcaps(struct bnxt *bp) 9604 { 9605 struct hwrm_port_led_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 9606 struct hwrm_port_led_qcaps_input req = {0}; 9607 struct bnxt_pf_info *pf = &bp->pf; 9608 int rc; 9609 9610 bp->num_leds = 0; 9611 if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10601) 9612 return 0; 9613 9614 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_LED_QCAPS, -1, -1); 9615 req.port_id = cpu_to_le16(pf->port_id); 9616 mutex_lock(&bp->hwrm_cmd_lock); 9617 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 9618 if (rc) { 9619 mutex_unlock(&bp->hwrm_cmd_lock); 9620 return rc; 9621 } 9622 if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) { 9623 int i; 9624 9625 bp->num_leds = resp->num_leds; 9626 memcpy(bp->leds, &resp->led0_id, sizeof(bp->leds[0]) * 9627 bp->num_leds); 9628 for (i = 0; i < bp->num_leds; i++) { 9629 struct bnxt_led_info *led = &bp->leds[i]; 9630 __le16 caps = led->led_state_caps; 9631 9632 if (!led->led_group_id || 9633 !BNXT_LED_ALT_BLINK_CAP(caps)) { 9634 bp->num_leds = 0; 9635 break; 9636 } 9637 } 9638 } 9639 mutex_unlock(&bp->hwrm_cmd_lock); 9640 return 0; 9641 } 9642 9643 int bnxt_hwrm_alloc_wol_fltr(struct bnxt *bp) 9644 { 9645 struct hwrm_wol_filter_alloc_input req = {0}; 9646 struct hwrm_wol_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr; 9647 int rc; 9648 9649 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_ALLOC, -1, -1); 9650 req.port_id = cpu_to_le16(bp->pf.port_id); 9651 req.wol_type = WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT; 9652 req.enables = cpu_to_le32(WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS); 9653 memcpy(req.mac_address, bp->dev->dev_addr, ETH_ALEN); 9654 mutex_lock(&bp->hwrm_cmd_lock); 9655 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 9656 if (!rc) 9657 bp->wol_filter_id = resp->wol_filter_id; 9658 mutex_unlock(&bp->hwrm_cmd_lock); 9659 return rc; 9660 } 9661 9662 int bnxt_hwrm_free_wol_fltr(struct bnxt *bp) 9663 { 9664 struct hwrm_wol_filter_free_input req = {0}; 9665 9666 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_FREE, -1, -1); 9667 req.port_id = cpu_to_le16(bp->pf.port_id); 9668 req.enables = cpu_to_le32(WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID); 9669 req.wol_filter_id = bp->wol_filter_id; 9670 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 9671 } 9672 9673 static u16 bnxt_hwrm_get_wol_fltrs(struct bnxt *bp, u16 handle) 9674 { 9675 struct hwrm_wol_filter_qcfg_input req = {0}; 9676 struct hwrm_wol_filter_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 9677 u16 next_handle = 0; 9678 int rc; 9679 9680 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_QCFG, -1, -1); 9681 req.port_id = cpu_to_le16(bp->pf.port_id); 9682 req.handle = cpu_to_le16(handle); 9683 mutex_lock(&bp->hwrm_cmd_lock); 9684 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 9685 if (!rc) { 9686 next_handle = le16_to_cpu(resp->next_handle); 9687 if (next_handle != 0) { 9688 if (resp->wol_type == 9689 WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT) { 9690 bp->wol = 1; 9691 bp->wol_filter_id = resp->wol_filter_id; 9692 } 9693 } 9694 } 9695 mutex_unlock(&bp->hwrm_cmd_lock); 9696 return next_handle; 9697 } 9698 9699 static void bnxt_get_wol_settings(struct bnxt *bp) 9700 { 9701 u16 handle = 0; 9702 9703 bp->wol = 0; 9704 if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_WOL_CAP)) 9705 return; 9706 9707 do { 9708 handle = bnxt_hwrm_get_wol_fltrs(bp, handle); 9709 } while (handle && handle != 0xffff); 9710 } 9711 9712 #ifdef CONFIG_BNXT_HWMON 9713 static ssize_t bnxt_show_temp(struct device *dev, 9714 struct device_attribute *devattr, char *buf) 9715 { 9716 struct hwrm_temp_monitor_query_input req = {0}; 9717 struct hwrm_temp_monitor_query_output *resp; 9718 struct bnxt *bp = dev_get_drvdata(dev); 9719 u32 len = 0; 9720 int rc; 9721 9722 resp = bp->hwrm_cmd_resp_addr; 9723 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TEMP_MONITOR_QUERY, -1, -1); 9724 mutex_lock(&bp->hwrm_cmd_lock); 9725 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 9726 if (!rc) 9727 len = sprintf(buf, "%u\n", resp->temp * 1000); /* display millidegree */ 9728 mutex_unlock(&bp->hwrm_cmd_lock); 9729 return rc ?: len; 9730 } 9731 static SENSOR_DEVICE_ATTR(temp1_input, 0444, bnxt_show_temp, NULL, 0); 9732 9733 static struct attribute *bnxt_attrs[] = { 9734 &sensor_dev_attr_temp1_input.dev_attr.attr, 9735 NULL 9736 }; 9737 ATTRIBUTE_GROUPS(bnxt); 9738 9739 static void bnxt_hwmon_close(struct bnxt *bp) 9740 { 9741 if (bp->hwmon_dev) { 9742 hwmon_device_unregister(bp->hwmon_dev); 9743 bp->hwmon_dev = NULL; 9744 } 9745 } 9746 9747 static void bnxt_hwmon_open(struct bnxt *bp) 9748 { 9749 struct hwrm_temp_monitor_query_input req = {0}; 9750 struct pci_dev *pdev = bp->pdev; 9751 int rc; 9752 9753 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TEMP_MONITOR_QUERY, -1, -1); 9754 rc = hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 9755 if (rc == -EACCES || rc == -EOPNOTSUPP) { 9756 bnxt_hwmon_close(bp); 9757 return; 9758 } 9759 9760 if (bp->hwmon_dev) 9761 return; 9762 9763 bp->hwmon_dev = hwmon_device_register_with_groups(&pdev->dev, 9764 DRV_MODULE_NAME, bp, 9765 bnxt_groups); 9766 if (IS_ERR(bp->hwmon_dev)) { 9767 bp->hwmon_dev = NULL; 9768 dev_warn(&pdev->dev, "Cannot register hwmon device\n"); 9769 } 9770 } 9771 #else 9772 static void bnxt_hwmon_close(struct bnxt *bp) 9773 { 9774 } 9775 9776 static void bnxt_hwmon_open(struct bnxt *bp) 9777 { 9778 } 9779 #endif 9780 9781 static bool bnxt_eee_config_ok(struct bnxt *bp) 9782 { 9783 struct ethtool_eee *eee = &bp->eee; 9784 struct bnxt_link_info *link_info = &bp->link_info; 9785 9786 if (!(bp->flags & BNXT_FLAG_EEE_CAP)) 9787 return true; 9788 9789 if (eee->eee_enabled) { 9790 u32 advertising = 9791 _bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0); 9792 9793 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) { 9794 eee->eee_enabled = 0; 9795 return false; 9796 } 9797 if (eee->advertised & ~advertising) { 9798 eee->advertised = advertising & eee->supported; 9799 return false; 9800 } 9801 } 9802 return true; 9803 } 9804 9805 static int bnxt_update_phy_setting(struct bnxt *bp) 9806 { 9807 int rc; 9808 bool update_link = false; 9809 bool update_pause = false; 9810 bool update_eee = false; 9811 struct bnxt_link_info *link_info = &bp->link_info; 9812 9813 rc = bnxt_update_link(bp, true); 9814 if (rc) { 9815 netdev_err(bp->dev, "failed to update link (rc: %x)\n", 9816 rc); 9817 return rc; 9818 } 9819 if (!BNXT_SINGLE_PF(bp)) 9820 return 0; 9821 9822 if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) && 9823 (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) != 9824 link_info->req_flow_ctrl) 9825 update_pause = true; 9826 if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) && 9827 link_info->force_pause_setting != link_info->req_flow_ctrl) 9828 update_pause = true; 9829 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) { 9830 if (BNXT_AUTO_MODE(link_info->auto_mode)) 9831 update_link = true; 9832 if (link_info->req_signal_mode == BNXT_SIG_MODE_NRZ && 9833 link_info->req_link_speed != link_info->force_link_speed) 9834 update_link = true; 9835 else if (link_info->req_signal_mode == BNXT_SIG_MODE_PAM4 && 9836 link_info->req_link_speed != link_info->force_pam4_link_speed) 9837 update_link = true; 9838 if (link_info->req_duplex != link_info->duplex_setting) 9839 update_link = true; 9840 } else { 9841 if (link_info->auto_mode == BNXT_LINK_AUTO_NONE) 9842 update_link = true; 9843 if (link_info->advertising != link_info->auto_link_speeds || 9844 link_info->advertising_pam4 != link_info->auto_pam4_link_speeds) 9845 update_link = true; 9846 } 9847 9848 /* The last close may have shutdown the link, so need to call 9849 * PHY_CFG to bring it back up. 9850 */ 9851 if (!bp->link_info.link_up) 9852 update_link = true; 9853 9854 if (!bnxt_eee_config_ok(bp)) 9855 update_eee = true; 9856 9857 if (update_link) 9858 rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee); 9859 else if (update_pause) 9860 rc = bnxt_hwrm_set_pause(bp); 9861 if (rc) { 9862 netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n", 9863 rc); 9864 return rc; 9865 } 9866 9867 return rc; 9868 } 9869 9870 /* Common routine to pre-map certain register block to different GRC window. 9871 * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows 9872 * in PF and 3 windows in VF that can be customized to map in different 9873 * register blocks. 9874 */ 9875 static void bnxt_preset_reg_win(struct bnxt *bp) 9876 { 9877 if (BNXT_PF(bp)) { 9878 /* CAG registers map to GRC window #4 */ 9879 writel(BNXT_CAG_REG_BASE, 9880 bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12); 9881 } 9882 } 9883 9884 static int bnxt_init_dflt_ring_mode(struct bnxt *bp); 9885 9886 static int bnxt_reinit_after_abort(struct bnxt *bp) 9887 { 9888 int rc; 9889 9890 if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 9891 return -EBUSY; 9892 9893 rc = bnxt_fw_init_one(bp); 9894 if (!rc) { 9895 bnxt_clear_int_mode(bp); 9896 rc = bnxt_init_int_mode(bp); 9897 if (!rc) { 9898 clear_bit(BNXT_STATE_ABORT_ERR, &bp->state); 9899 set_bit(BNXT_STATE_FW_RESET_DET, &bp->state); 9900 } 9901 } 9902 return rc; 9903 } 9904 9905 static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) 9906 { 9907 int rc = 0; 9908 9909 bnxt_preset_reg_win(bp); 9910 netif_carrier_off(bp->dev); 9911 if (irq_re_init) { 9912 /* Reserve rings now if none were reserved at driver probe. */ 9913 rc = bnxt_init_dflt_ring_mode(bp); 9914 if (rc) { 9915 netdev_err(bp->dev, "Failed to reserve default rings at open\n"); 9916 return rc; 9917 } 9918 } 9919 rc = bnxt_reserve_rings(bp, irq_re_init); 9920 if (rc) 9921 return rc; 9922 if ((bp->flags & BNXT_FLAG_RFS) && 9923 !(bp->flags & BNXT_FLAG_USING_MSIX)) { 9924 /* disable RFS if falling back to INTA */ 9925 bp->dev->hw_features &= ~NETIF_F_NTUPLE; 9926 bp->flags &= ~BNXT_FLAG_RFS; 9927 } 9928 9929 rc = bnxt_alloc_mem(bp, irq_re_init); 9930 if (rc) { 9931 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc); 9932 goto open_err_free_mem; 9933 } 9934 9935 if (irq_re_init) { 9936 bnxt_init_napi(bp); 9937 rc = bnxt_request_irq(bp); 9938 if (rc) { 9939 netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc); 9940 goto open_err_irq; 9941 } 9942 } 9943 9944 rc = bnxt_init_nic(bp, irq_re_init); 9945 if (rc) { 9946 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc); 9947 goto open_err_irq; 9948 } 9949 9950 bnxt_enable_napi(bp); 9951 bnxt_debug_dev_init(bp); 9952 9953 if (link_re_init) { 9954 mutex_lock(&bp->link_lock); 9955 rc = bnxt_update_phy_setting(bp); 9956 mutex_unlock(&bp->link_lock); 9957 if (rc) { 9958 netdev_warn(bp->dev, "failed to update phy settings\n"); 9959 if (BNXT_SINGLE_PF(bp)) { 9960 bp->link_info.phy_retry = true; 9961 bp->link_info.phy_retry_expires = 9962 jiffies + 5 * HZ; 9963 } 9964 } 9965 } 9966 9967 if (irq_re_init) 9968 udp_tunnel_nic_reset_ntf(bp->dev); 9969 9970 set_bit(BNXT_STATE_OPEN, &bp->state); 9971 bnxt_enable_int(bp); 9972 /* Enable TX queues */ 9973 bnxt_tx_enable(bp); 9974 mod_timer(&bp->timer, jiffies + bp->current_interval); 9975 /* Poll link status and check for SFP+ module status */ 9976 bnxt_get_port_module_status(bp); 9977 9978 /* VF-reps may need to be re-opened after the PF is re-opened */ 9979 if (BNXT_PF(bp)) 9980 bnxt_vf_reps_open(bp); 9981 return 0; 9982 9983 open_err_irq: 9984 bnxt_del_napi(bp); 9985 9986 open_err_free_mem: 9987 bnxt_free_skbs(bp); 9988 bnxt_free_irq(bp); 9989 bnxt_free_mem(bp, true); 9990 return rc; 9991 } 9992 9993 /* rtnl_lock held */ 9994 int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) 9995 { 9996 int rc = 0; 9997 9998 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) 9999 rc = -EIO; 10000 if (!rc) 10001 rc = __bnxt_open_nic(bp, irq_re_init, link_re_init); 10002 if (rc) { 10003 netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc); 10004 dev_close(bp->dev); 10005 } 10006 return rc; 10007 } 10008 10009 /* rtnl_lock held, open the NIC half way by allocating all resources, but 10010 * NAPI, IRQ, and TX are not enabled. This is mainly used for offline 10011 * self tests. 10012 */ 10013 int bnxt_half_open_nic(struct bnxt *bp) 10014 { 10015 int rc = 0; 10016 10017 rc = bnxt_alloc_mem(bp, false); 10018 if (rc) { 10019 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc); 10020 goto half_open_err; 10021 } 10022 rc = bnxt_init_nic(bp, false); 10023 if (rc) { 10024 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc); 10025 goto half_open_err; 10026 } 10027 return 0; 10028 10029 half_open_err: 10030 bnxt_free_skbs(bp); 10031 bnxt_free_mem(bp, false); 10032 dev_close(bp->dev); 10033 return rc; 10034 } 10035 10036 /* rtnl_lock held, this call can only be made after a previous successful 10037 * call to bnxt_half_open_nic(). 10038 */ 10039 void bnxt_half_close_nic(struct bnxt *bp) 10040 { 10041 bnxt_hwrm_resource_free(bp, false, false); 10042 bnxt_free_skbs(bp); 10043 bnxt_free_mem(bp, false); 10044 } 10045 10046 static void bnxt_reenable_sriov(struct bnxt *bp) 10047 { 10048 if (BNXT_PF(bp)) { 10049 struct bnxt_pf_info *pf = &bp->pf; 10050 int n = pf->active_vfs; 10051 10052 if (n) 10053 bnxt_cfg_hw_sriov(bp, &n, true); 10054 } 10055 } 10056 10057 static int bnxt_open(struct net_device *dev) 10058 { 10059 struct bnxt *bp = netdev_priv(dev); 10060 int rc; 10061 10062 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) { 10063 rc = bnxt_reinit_after_abort(bp); 10064 if (rc) { 10065 if (rc == -EBUSY) 10066 netdev_err(bp->dev, "A previous firmware reset has not completed, aborting\n"); 10067 else 10068 netdev_err(bp->dev, "Failed to reinitialize after aborted firmware reset\n"); 10069 return -ENODEV; 10070 } 10071 } 10072 10073 rc = bnxt_hwrm_if_change(bp, true); 10074 if (rc) 10075 return rc; 10076 rc = __bnxt_open_nic(bp, true, true); 10077 if (rc) { 10078 bnxt_hwrm_if_change(bp, false); 10079 } else { 10080 if (test_and_clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state)) { 10081 if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) { 10082 bnxt_ulp_start(bp, 0); 10083 bnxt_reenable_sriov(bp); 10084 } 10085 } 10086 bnxt_hwmon_open(bp); 10087 } 10088 10089 return rc; 10090 } 10091 10092 static bool bnxt_drv_busy(struct bnxt *bp) 10093 { 10094 return (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state) || 10095 test_bit(BNXT_STATE_READ_STATS, &bp->state)); 10096 } 10097 10098 static void bnxt_get_ring_stats(struct bnxt *bp, 10099 struct rtnl_link_stats64 *stats); 10100 10101 static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init, 10102 bool link_re_init) 10103 { 10104 /* Close the VF-reps before closing PF */ 10105 if (BNXT_PF(bp)) 10106 bnxt_vf_reps_close(bp); 10107 10108 /* Change device state to avoid TX queue wake up's */ 10109 bnxt_tx_disable(bp); 10110 10111 clear_bit(BNXT_STATE_OPEN, &bp->state); 10112 smp_mb__after_atomic(); 10113 while (bnxt_drv_busy(bp)) 10114 msleep(20); 10115 10116 /* Flush rings and and disable interrupts */ 10117 bnxt_shutdown_nic(bp, irq_re_init); 10118 10119 /* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */ 10120 10121 bnxt_debug_dev_exit(bp); 10122 bnxt_disable_napi(bp); 10123 del_timer_sync(&bp->timer); 10124 bnxt_free_skbs(bp); 10125 10126 /* Save ring stats before shutdown */ 10127 if (bp->bnapi && irq_re_init) 10128 bnxt_get_ring_stats(bp, &bp->net_stats_prev); 10129 if (irq_re_init) { 10130 bnxt_free_irq(bp); 10131 bnxt_del_napi(bp); 10132 } 10133 bnxt_free_mem(bp, irq_re_init); 10134 } 10135 10136 int bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) 10137 { 10138 int rc = 0; 10139 10140 if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) { 10141 /* If we get here, it means firmware reset is in progress 10142 * while we are trying to close. We can safely proceed with 10143 * the close because we are holding rtnl_lock(). Some firmware 10144 * messages may fail as we proceed to close. We set the 10145 * ABORT_ERR flag here so that the FW reset thread will later 10146 * abort when it gets the rtnl_lock() and sees the flag. 10147 */ 10148 netdev_warn(bp->dev, "FW reset in progress during close, FW reset will be aborted\n"); 10149 set_bit(BNXT_STATE_ABORT_ERR, &bp->state); 10150 } 10151 10152 #ifdef CONFIG_BNXT_SRIOV 10153 if (bp->sriov_cfg) { 10154 rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait, 10155 !bp->sriov_cfg, 10156 BNXT_SRIOV_CFG_WAIT_TMO); 10157 if (rc) 10158 netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete!\n"); 10159 } 10160 #endif 10161 __bnxt_close_nic(bp, irq_re_init, link_re_init); 10162 return rc; 10163 } 10164 10165 static int bnxt_close(struct net_device *dev) 10166 { 10167 struct bnxt *bp = netdev_priv(dev); 10168 10169 bnxt_hwmon_close(bp); 10170 bnxt_close_nic(bp, true, true); 10171 bnxt_hwrm_shutdown_link(bp); 10172 bnxt_hwrm_if_change(bp, false); 10173 return 0; 10174 } 10175 10176 static int bnxt_hwrm_port_phy_read(struct bnxt *bp, u16 phy_addr, u16 reg, 10177 u16 *val) 10178 { 10179 struct hwrm_port_phy_mdio_read_output *resp = bp->hwrm_cmd_resp_addr; 10180 struct hwrm_port_phy_mdio_read_input req = {0}; 10181 int rc; 10182 10183 if (bp->hwrm_spec_code < 0x10a00) 10184 return -EOPNOTSUPP; 10185 10186 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_MDIO_READ, -1, -1); 10187 req.port_id = cpu_to_le16(bp->pf.port_id); 10188 req.phy_addr = phy_addr; 10189 req.reg_addr = cpu_to_le16(reg & 0x1f); 10190 if (mdio_phy_id_is_c45(phy_addr)) { 10191 req.cl45_mdio = 1; 10192 req.phy_addr = mdio_phy_id_prtad(phy_addr); 10193 req.dev_addr = mdio_phy_id_devad(phy_addr); 10194 req.reg_addr = cpu_to_le16(reg); 10195 } 10196 10197 mutex_lock(&bp->hwrm_cmd_lock); 10198 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 10199 if (!rc) 10200 *val = le16_to_cpu(resp->reg_data); 10201 mutex_unlock(&bp->hwrm_cmd_lock); 10202 return rc; 10203 } 10204 10205 static int bnxt_hwrm_port_phy_write(struct bnxt *bp, u16 phy_addr, u16 reg, 10206 u16 val) 10207 { 10208 struct hwrm_port_phy_mdio_write_input req = {0}; 10209 10210 if (bp->hwrm_spec_code < 0x10a00) 10211 return -EOPNOTSUPP; 10212 10213 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_MDIO_WRITE, -1, -1); 10214 req.port_id = cpu_to_le16(bp->pf.port_id); 10215 req.phy_addr = phy_addr; 10216 req.reg_addr = cpu_to_le16(reg & 0x1f); 10217 if (mdio_phy_id_is_c45(phy_addr)) { 10218 req.cl45_mdio = 1; 10219 req.phy_addr = mdio_phy_id_prtad(phy_addr); 10220 req.dev_addr = mdio_phy_id_devad(phy_addr); 10221 req.reg_addr = cpu_to_le16(reg); 10222 } 10223 req.reg_data = cpu_to_le16(val); 10224 10225 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 10226 } 10227 10228 /* rtnl_lock held */ 10229 static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) 10230 { 10231 struct mii_ioctl_data *mdio = if_mii(ifr); 10232 struct bnxt *bp = netdev_priv(dev); 10233 int rc; 10234 10235 switch (cmd) { 10236 case SIOCGMIIPHY: 10237 mdio->phy_id = bp->link_info.phy_addr; 10238 10239 fallthrough; 10240 case SIOCGMIIREG: { 10241 u16 mii_regval = 0; 10242 10243 if (!netif_running(dev)) 10244 return -EAGAIN; 10245 10246 rc = bnxt_hwrm_port_phy_read(bp, mdio->phy_id, mdio->reg_num, 10247 &mii_regval); 10248 mdio->val_out = mii_regval; 10249 return rc; 10250 } 10251 10252 case SIOCSMIIREG: 10253 if (!netif_running(dev)) 10254 return -EAGAIN; 10255 10256 return bnxt_hwrm_port_phy_write(bp, mdio->phy_id, mdio->reg_num, 10257 mdio->val_in); 10258 10259 default: 10260 /* do nothing */ 10261 break; 10262 } 10263 return -EOPNOTSUPP; 10264 } 10265 10266 static void bnxt_get_ring_stats(struct bnxt *bp, 10267 struct rtnl_link_stats64 *stats) 10268 { 10269 int i; 10270 10271 for (i = 0; i < bp->cp_nr_rings; i++) { 10272 struct bnxt_napi *bnapi = bp->bnapi[i]; 10273 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 10274 u64 *sw = cpr->stats.sw_stats; 10275 10276 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_ucast_pkts); 10277 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts); 10278 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_bcast_pkts); 10279 10280 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_ucast_pkts); 10281 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_mcast_pkts); 10282 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_bcast_pkts); 10283 10284 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_ucast_bytes); 10285 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_mcast_bytes); 10286 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_bcast_bytes); 10287 10288 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_ucast_bytes); 10289 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_mcast_bytes); 10290 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_bcast_bytes); 10291 10292 stats->rx_missed_errors += 10293 BNXT_GET_RING_STATS64(sw, rx_discard_pkts); 10294 10295 stats->multicast += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts); 10296 10297 stats->tx_dropped += BNXT_GET_RING_STATS64(sw, tx_error_pkts); 10298 } 10299 } 10300 10301 static void bnxt_add_prev_stats(struct bnxt *bp, 10302 struct rtnl_link_stats64 *stats) 10303 { 10304 struct rtnl_link_stats64 *prev_stats = &bp->net_stats_prev; 10305 10306 stats->rx_packets += prev_stats->rx_packets; 10307 stats->tx_packets += prev_stats->tx_packets; 10308 stats->rx_bytes += prev_stats->rx_bytes; 10309 stats->tx_bytes += prev_stats->tx_bytes; 10310 stats->rx_missed_errors += prev_stats->rx_missed_errors; 10311 stats->multicast += prev_stats->multicast; 10312 stats->tx_dropped += prev_stats->tx_dropped; 10313 } 10314 10315 static void 10316 bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) 10317 { 10318 struct bnxt *bp = netdev_priv(dev); 10319 10320 set_bit(BNXT_STATE_READ_STATS, &bp->state); 10321 /* Make sure bnxt_close_nic() sees that we are reading stats before 10322 * we check the BNXT_STATE_OPEN flag. 10323 */ 10324 smp_mb__after_atomic(); 10325 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 10326 clear_bit(BNXT_STATE_READ_STATS, &bp->state); 10327 *stats = bp->net_stats_prev; 10328 return; 10329 } 10330 10331 bnxt_get_ring_stats(bp, stats); 10332 bnxt_add_prev_stats(bp, stats); 10333 10334 if (bp->flags & BNXT_FLAG_PORT_STATS) { 10335 u64 *rx = bp->port_stats.sw_stats; 10336 u64 *tx = bp->port_stats.sw_stats + 10337 BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 10338 10339 stats->rx_crc_errors = 10340 BNXT_GET_RX_PORT_STATS64(rx, rx_fcs_err_frames); 10341 stats->rx_frame_errors = 10342 BNXT_GET_RX_PORT_STATS64(rx, rx_align_err_frames); 10343 stats->rx_length_errors = 10344 BNXT_GET_RX_PORT_STATS64(rx, rx_undrsz_frames) + 10345 BNXT_GET_RX_PORT_STATS64(rx, rx_ovrsz_frames) + 10346 BNXT_GET_RX_PORT_STATS64(rx, rx_runt_frames); 10347 stats->rx_errors = 10348 BNXT_GET_RX_PORT_STATS64(rx, rx_false_carrier_frames) + 10349 BNXT_GET_RX_PORT_STATS64(rx, rx_jbr_frames); 10350 stats->collisions = 10351 BNXT_GET_TX_PORT_STATS64(tx, tx_total_collisions); 10352 stats->tx_fifo_errors = 10353 BNXT_GET_TX_PORT_STATS64(tx, tx_fifo_underruns); 10354 stats->tx_errors = BNXT_GET_TX_PORT_STATS64(tx, tx_err); 10355 } 10356 clear_bit(BNXT_STATE_READ_STATS, &bp->state); 10357 } 10358 10359 static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask) 10360 { 10361 struct net_device *dev = bp->dev; 10362 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 10363 struct netdev_hw_addr *ha; 10364 u8 *haddr; 10365 int mc_count = 0; 10366 bool update = false; 10367 int off = 0; 10368 10369 netdev_for_each_mc_addr(ha, dev) { 10370 if (mc_count >= BNXT_MAX_MC_ADDRS) { 10371 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 10372 vnic->mc_list_count = 0; 10373 return false; 10374 } 10375 haddr = ha->addr; 10376 if (!ether_addr_equal(haddr, vnic->mc_list + off)) { 10377 memcpy(vnic->mc_list + off, haddr, ETH_ALEN); 10378 update = true; 10379 } 10380 off += ETH_ALEN; 10381 mc_count++; 10382 } 10383 if (mc_count) 10384 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST; 10385 10386 if (mc_count != vnic->mc_list_count) { 10387 vnic->mc_list_count = mc_count; 10388 update = true; 10389 } 10390 return update; 10391 } 10392 10393 static bool bnxt_uc_list_updated(struct bnxt *bp) 10394 { 10395 struct net_device *dev = bp->dev; 10396 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 10397 struct netdev_hw_addr *ha; 10398 int off = 0; 10399 10400 if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1)) 10401 return true; 10402 10403 netdev_for_each_uc_addr(ha, dev) { 10404 if (!ether_addr_equal(ha->addr, vnic->uc_list + off)) 10405 return true; 10406 10407 off += ETH_ALEN; 10408 } 10409 return false; 10410 } 10411 10412 static void bnxt_set_rx_mode(struct net_device *dev) 10413 { 10414 struct bnxt *bp = netdev_priv(dev); 10415 struct bnxt_vnic_info *vnic; 10416 bool mc_update = false; 10417 bool uc_update; 10418 u32 mask; 10419 10420 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) 10421 return; 10422 10423 vnic = &bp->vnic_info[0]; 10424 mask = vnic->rx_mask; 10425 mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS | 10426 CFA_L2_SET_RX_MASK_REQ_MASK_MCAST | 10427 CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST | 10428 CFA_L2_SET_RX_MASK_REQ_MASK_BCAST); 10429 10430 if ((dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp)) 10431 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 10432 10433 uc_update = bnxt_uc_list_updated(bp); 10434 10435 if (dev->flags & IFF_BROADCAST) 10436 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST; 10437 if (dev->flags & IFF_ALLMULTI) { 10438 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 10439 vnic->mc_list_count = 0; 10440 } else { 10441 mc_update = bnxt_mc_list_updated(bp, &mask); 10442 } 10443 10444 if (mask != vnic->rx_mask || uc_update || mc_update) { 10445 vnic->rx_mask = mask; 10446 10447 set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event); 10448 bnxt_queue_sp_work(bp); 10449 } 10450 } 10451 10452 static int bnxt_cfg_rx_mode(struct bnxt *bp) 10453 { 10454 struct net_device *dev = bp->dev; 10455 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 10456 struct netdev_hw_addr *ha; 10457 int i, off = 0, rc; 10458 bool uc_update; 10459 10460 netif_addr_lock_bh(dev); 10461 uc_update = bnxt_uc_list_updated(bp); 10462 netif_addr_unlock_bh(dev); 10463 10464 if (!uc_update) 10465 goto skip_uc; 10466 10467 mutex_lock(&bp->hwrm_cmd_lock); 10468 for (i = 1; i < vnic->uc_filter_count; i++) { 10469 struct hwrm_cfa_l2_filter_free_input req = {0}; 10470 10471 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_FREE, -1, 10472 -1); 10473 10474 req.l2_filter_id = vnic->fw_l2_filter_id[i]; 10475 10476 rc = _hwrm_send_message(bp, &req, sizeof(req), 10477 HWRM_CMD_TIMEOUT); 10478 } 10479 mutex_unlock(&bp->hwrm_cmd_lock); 10480 10481 vnic->uc_filter_count = 1; 10482 10483 netif_addr_lock_bh(dev); 10484 if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) { 10485 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 10486 } else { 10487 netdev_for_each_uc_addr(ha, dev) { 10488 memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN); 10489 off += ETH_ALEN; 10490 vnic->uc_filter_count++; 10491 } 10492 } 10493 netif_addr_unlock_bh(dev); 10494 10495 for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) { 10496 rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off); 10497 if (rc) { 10498 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", 10499 rc); 10500 vnic->uc_filter_count = i; 10501 return rc; 10502 } 10503 } 10504 10505 skip_uc: 10506 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0); 10507 if (rc && vnic->mc_list_count) { 10508 netdev_info(bp->dev, "Failed setting MC filters rc: %d, turning on ALL_MCAST mode\n", 10509 rc); 10510 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 10511 vnic->mc_list_count = 0; 10512 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0); 10513 } 10514 if (rc) 10515 netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %d\n", 10516 rc); 10517 10518 return rc; 10519 } 10520 10521 static bool bnxt_can_reserve_rings(struct bnxt *bp) 10522 { 10523 #ifdef CONFIG_BNXT_SRIOV 10524 if (BNXT_NEW_RM(bp) && BNXT_VF(bp)) { 10525 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 10526 10527 /* No minimum rings were provisioned by the PF. Don't 10528 * reserve rings by default when device is down. 10529 */ 10530 if (hw_resc->min_tx_rings || hw_resc->resv_tx_rings) 10531 return true; 10532 10533 if (!netif_running(bp->dev)) 10534 return false; 10535 } 10536 #endif 10537 return true; 10538 } 10539 10540 /* If the chip and firmware supports RFS */ 10541 static bool bnxt_rfs_supported(struct bnxt *bp) 10542 { 10543 if (bp->flags & BNXT_FLAG_CHIP_P5) { 10544 if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2) 10545 return true; 10546 return false; 10547 } 10548 if (BNXT_PF(bp) && !BNXT_CHIP_TYPE_NITRO_A0(bp)) 10549 return true; 10550 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP) 10551 return true; 10552 return false; 10553 } 10554 10555 /* If runtime conditions support RFS */ 10556 static bool bnxt_rfs_capable(struct bnxt *bp) 10557 { 10558 #ifdef CONFIG_RFS_ACCEL 10559 int vnics, max_vnics, max_rss_ctxs; 10560 10561 if (bp->flags & BNXT_FLAG_CHIP_P5) 10562 return bnxt_rfs_supported(bp); 10563 if (!(bp->flags & BNXT_FLAG_MSIX_CAP) || !bnxt_can_reserve_rings(bp)) 10564 return false; 10565 10566 vnics = 1 + bp->rx_nr_rings; 10567 max_vnics = bnxt_get_max_func_vnics(bp); 10568 max_rss_ctxs = bnxt_get_max_func_rss_ctxs(bp); 10569 10570 /* RSS contexts not a limiting factor */ 10571 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP) 10572 max_rss_ctxs = max_vnics; 10573 if (vnics > max_vnics || vnics > max_rss_ctxs) { 10574 if (bp->rx_nr_rings > 1) 10575 netdev_warn(bp->dev, 10576 "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n", 10577 min(max_rss_ctxs - 1, max_vnics - 1)); 10578 return false; 10579 } 10580 10581 if (!BNXT_NEW_RM(bp)) 10582 return true; 10583 10584 if (vnics == bp->hw_resc.resv_vnics) 10585 return true; 10586 10587 bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, vnics); 10588 if (vnics <= bp->hw_resc.resv_vnics) 10589 return true; 10590 10591 netdev_warn(bp->dev, "Unable to reserve resources to support NTUPLE filters.\n"); 10592 bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, 1); 10593 return false; 10594 #else 10595 return false; 10596 #endif 10597 } 10598 10599 static netdev_features_t bnxt_fix_features(struct net_device *dev, 10600 netdev_features_t features) 10601 { 10602 struct bnxt *bp = netdev_priv(dev); 10603 netdev_features_t vlan_features; 10604 10605 if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp)) 10606 features &= ~NETIF_F_NTUPLE; 10607 10608 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS) 10609 features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); 10610 10611 if (!(features & NETIF_F_GRO)) 10612 features &= ~NETIF_F_GRO_HW; 10613 10614 if (features & NETIF_F_GRO_HW) 10615 features &= ~NETIF_F_LRO; 10616 10617 /* Both CTAG and STAG VLAN accelaration on the RX side have to be 10618 * turned on or off together. 10619 */ 10620 vlan_features = features & BNXT_HW_FEATURE_VLAN_ALL_RX; 10621 if (vlan_features != BNXT_HW_FEATURE_VLAN_ALL_RX) { 10622 if (dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX) 10623 features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX; 10624 else if (vlan_features) 10625 features |= BNXT_HW_FEATURE_VLAN_ALL_RX; 10626 } 10627 #ifdef CONFIG_BNXT_SRIOV 10628 if (BNXT_VF(bp) && bp->vf.vlan) 10629 features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX; 10630 #endif 10631 return features; 10632 } 10633 10634 static int bnxt_set_features(struct net_device *dev, netdev_features_t features) 10635 { 10636 struct bnxt *bp = netdev_priv(dev); 10637 u32 flags = bp->flags; 10638 u32 changes; 10639 int rc = 0; 10640 bool re_init = false; 10641 bool update_tpa = false; 10642 10643 flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS; 10644 if (features & NETIF_F_GRO_HW) 10645 flags |= BNXT_FLAG_GRO; 10646 else if (features & NETIF_F_LRO) 10647 flags |= BNXT_FLAG_LRO; 10648 10649 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS) 10650 flags &= ~BNXT_FLAG_TPA; 10651 10652 if (features & BNXT_HW_FEATURE_VLAN_ALL_RX) 10653 flags |= BNXT_FLAG_STRIP_VLAN; 10654 10655 if (features & NETIF_F_NTUPLE) 10656 flags |= BNXT_FLAG_RFS; 10657 10658 changes = flags ^ bp->flags; 10659 if (changes & BNXT_FLAG_TPA) { 10660 update_tpa = true; 10661 if ((bp->flags & BNXT_FLAG_TPA) == 0 || 10662 (flags & BNXT_FLAG_TPA) == 0 || 10663 (bp->flags & BNXT_FLAG_CHIP_P5)) 10664 re_init = true; 10665 } 10666 10667 if (changes & ~BNXT_FLAG_TPA) 10668 re_init = true; 10669 10670 if (flags != bp->flags) { 10671 u32 old_flags = bp->flags; 10672 10673 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 10674 bp->flags = flags; 10675 if (update_tpa) 10676 bnxt_set_ring_params(bp); 10677 return rc; 10678 } 10679 10680 if (re_init) { 10681 bnxt_close_nic(bp, false, false); 10682 bp->flags = flags; 10683 if (update_tpa) 10684 bnxt_set_ring_params(bp); 10685 10686 return bnxt_open_nic(bp, false, false); 10687 } 10688 if (update_tpa) { 10689 bp->flags = flags; 10690 rc = bnxt_set_tpa(bp, 10691 (flags & BNXT_FLAG_TPA) ? 10692 true : false); 10693 if (rc) 10694 bp->flags = old_flags; 10695 } 10696 } 10697 return rc; 10698 } 10699 10700 int bnxt_dbg_hwrm_rd_reg(struct bnxt *bp, u32 reg_off, u16 num_words, 10701 u32 *reg_buf) 10702 { 10703 struct hwrm_dbg_read_direct_output *resp = bp->hwrm_cmd_resp_addr; 10704 struct hwrm_dbg_read_direct_input req = {0}; 10705 __le32 *dbg_reg_buf; 10706 dma_addr_t mapping; 10707 int rc, i; 10708 10709 dbg_reg_buf = dma_alloc_coherent(&bp->pdev->dev, num_words * 4, 10710 &mapping, GFP_KERNEL); 10711 if (!dbg_reg_buf) 10712 return -ENOMEM; 10713 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_DBG_READ_DIRECT, -1, -1); 10714 req.host_dest_addr = cpu_to_le64(mapping); 10715 req.read_addr = cpu_to_le32(reg_off + CHIMP_REG_VIEW_ADDR); 10716 req.read_len32 = cpu_to_le32(num_words); 10717 mutex_lock(&bp->hwrm_cmd_lock); 10718 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 10719 if (rc || resp->error_code) { 10720 rc = -EIO; 10721 goto dbg_rd_reg_exit; 10722 } 10723 for (i = 0; i < num_words; i++) 10724 reg_buf[i] = le32_to_cpu(dbg_reg_buf[i]); 10725 10726 dbg_rd_reg_exit: 10727 mutex_unlock(&bp->hwrm_cmd_lock); 10728 dma_free_coherent(&bp->pdev->dev, num_words * 4, dbg_reg_buf, mapping); 10729 return rc; 10730 } 10731 10732 static int bnxt_dbg_hwrm_ring_info_get(struct bnxt *bp, u8 ring_type, 10733 u32 ring_id, u32 *prod, u32 *cons) 10734 { 10735 struct hwrm_dbg_ring_info_get_output *resp = bp->hwrm_cmd_resp_addr; 10736 struct hwrm_dbg_ring_info_get_input req = {0}; 10737 int rc; 10738 10739 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_DBG_RING_INFO_GET, -1, -1); 10740 req.ring_type = ring_type; 10741 req.fw_ring_id = cpu_to_le32(ring_id); 10742 mutex_lock(&bp->hwrm_cmd_lock); 10743 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 10744 if (!rc) { 10745 *prod = le32_to_cpu(resp->producer_index); 10746 *cons = le32_to_cpu(resp->consumer_index); 10747 } 10748 mutex_unlock(&bp->hwrm_cmd_lock); 10749 return rc; 10750 } 10751 10752 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi) 10753 { 10754 struct bnxt_tx_ring_info *txr = bnapi->tx_ring; 10755 int i = bnapi->index; 10756 10757 if (!txr) 10758 return; 10759 10760 netdev_info(bnapi->bp->dev, "[%d]: tx{fw_ring: %d prod: %x cons: %x}\n", 10761 i, txr->tx_ring_struct.fw_ring_id, txr->tx_prod, 10762 txr->tx_cons); 10763 } 10764 10765 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi) 10766 { 10767 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 10768 int i = bnapi->index; 10769 10770 if (!rxr) 10771 return; 10772 10773 netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n", 10774 i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod, 10775 rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod, 10776 rxr->rx_sw_agg_prod); 10777 } 10778 10779 static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi) 10780 { 10781 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 10782 int i = bnapi->index; 10783 10784 netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n", 10785 i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons); 10786 } 10787 10788 static void bnxt_dbg_dump_states(struct bnxt *bp) 10789 { 10790 int i; 10791 struct bnxt_napi *bnapi; 10792 10793 for (i = 0; i < bp->cp_nr_rings; i++) { 10794 bnapi = bp->bnapi[i]; 10795 if (netif_msg_drv(bp)) { 10796 bnxt_dump_tx_sw_state(bnapi); 10797 bnxt_dump_rx_sw_state(bnapi); 10798 bnxt_dump_cp_sw_state(bnapi); 10799 } 10800 } 10801 } 10802 10803 static int bnxt_hwrm_rx_ring_reset(struct bnxt *bp, int ring_nr) 10804 { 10805 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr]; 10806 struct hwrm_ring_reset_input req = {0}; 10807 struct bnxt_napi *bnapi = rxr->bnapi; 10808 struct bnxt_cp_ring_info *cpr; 10809 u16 cp_ring_id; 10810 10811 cpr = &bnapi->cp_ring; 10812 cp_ring_id = cpr->cp_ring_struct.fw_ring_id; 10813 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_RESET, cp_ring_id, -1); 10814 req.ring_type = RING_RESET_REQ_RING_TYPE_RX_RING_GRP; 10815 req.ring_id = cpu_to_le16(bp->grp_info[bnapi->index].fw_grp_id); 10816 return hwrm_send_message_silent(bp, &req, sizeof(req), 10817 HWRM_CMD_TIMEOUT); 10818 } 10819 10820 static void bnxt_reset_task(struct bnxt *bp, bool silent) 10821 { 10822 if (!silent) 10823 bnxt_dbg_dump_states(bp); 10824 if (netif_running(bp->dev)) { 10825 int rc; 10826 10827 if (silent) { 10828 bnxt_close_nic(bp, false, false); 10829 bnxt_open_nic(bp, false, false); 10830 } else { 10831 bnxt_ulp_stop(bp); 10832 bnxt_close_nic(bp, true, false); 10833 rc = bnxt_open_nic(bp, true, false); 10834 bnxt_ulp_start(bp, rc); 10835 } 10836 } 10837 } 10838 10839 static void bnxt_tx_timeout(struct net_device *dev, unsigned int txqueue) 10840 { 10841 struct bnxt *bp = netdev_priv(dev); 10842 10843 netdev_err(bp->dev, "TX timeout detected, starting reset task!\n"); 10844 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event); 10845 bnxt_queue_sp_work(bp); 10846 } 10847 10848 static void bnxt_fw_health_check(struct bnxt *bp) 10849 { 10850 struct bnxt_fw_health *fw_health = bp->fw_health; 10851 u32 val; 10852 10853 if (!fw_health->enabled || test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 10854 return; 10855 10856 if (fw_health->tmr_counter) { 10857 fw_health->tmr_counter--; 10858 return; 10859 } 10860 10861 val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG); 10862 if (val == fw_health->last_fw_heartbeat) 10863 goto fw_reset; 10864 10865 fw_health->last_fw_heartbeat = val; 10866 10867 val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG); 10868 if (val != fw_health->last_fw_reset_cnt) 10869 goto fw_reset; 10870 10871 fw_health->tmr_counter = fw_health->tmr_multiplier; 10872 return; 10873 10874 fw_reset: 10875 set_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event); 10876 bnxt_queue_sp_work(bp); 10877 } 10878 10879 static void bnxt_timer(struct timer_list *t) 10880 { 10881 struct bnxt *bp = from_timer(bp, t, timer); 10882 struct net_device *dev = bp->dev; 10883 10884 if (!netif_running(dev) || !test_bit(BNXT_STATE_OPEN, &bp->state)) 10885 return; 10886 10887 if (atomic_read(&bp->intr_sem) != 0) 10888 goto bnxt_restart_timer; 10889 10890 if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) 10891 bnxt_fw_health_check(bp); 10892 10893 if (bp->link_info.link_up && bp->stats_coal_ticks) { 10894 set_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event); 10895 bnxt_queue_sp_work(bp); 10896 } 10897 10898 if (bnxt_tc_flower_enabled(bp)) { 10899 set_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event); 10900 bnxt_queue_sp_work(bp); 10901 } 10902 10903 #ifdef CONFIG_RFS_ACCEL 10904 if ((bp->flags & BNXT_FLAG_RFS) && bp->ntp_fltr_count) { 10905 set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event); 10906 bnxt_queue_sp_work(bp); 10907 } 10908 #endif /*CONFIG_RFS_ACCEL*/ 10909 10910 if (bp->link_info.phy_retry) { 10911 if (time_after(jiffies, bp->link_info.phy_retry_expires)) { 10912 bp->link_info.phy_retry = false; 10913 netdev_warn(bp->dev, "failed to update phy settings after maximum retries.\n"); 10914 } else { 10915 set_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event); 10916 bnxt_queue_sp_work(bp); 10917 } 10918 } 10919 10920 if ((bp->flags & BNXT_FLAG_CHIP_P5) && !bp->chip_rev && 10921 netif_carrier_ok(dev)) { 10922 set_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event); 10923 bnxt_queue_sp_work(bp); 10924 } 10925 bnxt_restart_timer: 10926 mod_timer(&bp->timer, jiffies + bp->current_interval); 10927 } 10928 10929 static void bnxt_rtnl_lock_sp(struct bnxt *bp) 10930 { 10931 /* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK 10932 * set. If the device is being closed, bnxt_close() may be holding 10933 * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear. So we 10934 * must clear BNXT_STATE_IN_SP_TASK before holding rtnl(). 10935 */ 10936 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 10937 rtnl_lock(); 10938 } 10939 10940 static void bnxt_rtnl_unlock_sp(struct bnxt *bp) 10941 { 10942 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 10943 rtnl_unlock(); 10944 } 10945 10946 /* Only called from bnxt_sp_task() */ 10947 static void bnxt_reset(struct bnxt *bp, bool silent) 10948 { 10949 bnxt_rtnl_lock_sp(bp); 10950 if (test_bit(BNXT_STATE_OPEN, &bp->state)) 10951 bnxt_reset_task(bp, silent); 10952 bnxt_rtnl_unlock_sp(bp); 10953 } 10954 10955 /* Only called from bnxt_sp_task() */ 10956 static void bnxt_rx_ring_reset(struct bnxt *bp) 10957 { 10958 int i; 10959 10960 bnxt_rtnl_lock_sp(bp); 10961 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 10962 bnxt_rtnl_unlock_sp(bp); 10963 return; 10964 } 10965 /* Disable and flush TPA before resetting the RX ring */ 10966 if (bp->flags & BNXT_FLAG_TPA) 10967 bnxt_set_tpa(bp, false); 10968 for (i = 0; i < bp->rx_nr_rings; i++) { 10969 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 10970 struct bnxt_cp_ring_info *cpr; 10971 int rc; 10972 10973 if (!rxr->bnapi->in_reset) 10974 continue; 10975 10976 rc = bnxt_hwrm_rx_ring_reset(bp, i); 10977 if (rc) { 10978 if (rc == -EINVAL || rc == -EOPNOTSUPP) 10979 netdev_info_once(bp->dev, "RX ring reset not supported by firmware, falling back to global reset\n"); 10980 else 10981 netdev_warn(bp->dev, "RX ring reset failed, rc = %d, falling back to global reset\n", 10982 rc); 10983 bnxt_reset_task(bp, true); 10984 break; 10985 } 10986 bnxt_free_one_rx_ring_skbs(bp, i); 10987 rxr->rx_prod = 0; 10988 rxr->rx_agg_prod = 0; 10989 rxr->rx_sw_agg_prod = 0; 10990 rxr->rx_next_cons = 0; 10991 rxr->bnapi->in_reset = false; 10992 bnxt_alloc_one_rx_ring(bp, i); 10993 cpr = &rxr->bnapi->cp_ring; 10994 cpr->sw_stats.rx.rx_resets++; 10995 if (bp->flags & BNXT_FLAG_AGG_RINGS) 10996 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); 10997 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 10998 } 10999 if (bp->flags & BNXT_FLAG_TPA) 11000 bnxt_set_tpa(bp, true); 11001 bnxt_rtnl_unlock_sp(bp); 11002 } 11003 11004 static void bnxt_fw_reset_close(struct bnxt *bp) 11005 { 11006 bnxt_ulp_stop(bp); 11007 /* When firmware is in fatal state, quiesce device and disable 11008 * bus master to prevent any potential bad DMAs before freeing 11009 * kernel memory. 11010 */ 11011 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) { 11012 u16 val = 0; 11013 11014 pci_read_config_word(bp->pdev, PCI_SUBSYSTEM_ID, &val); 11015 if (val == 0xffff) 11016 bp->fw_reset_min_dsecs = 0; 11017 bnxt_tx_disable(bp); 11018 bnxt_disable_napi(bp); 11019 bnxt_disable_int_sync(bp); 11020 bnxt_free_irq(bp); 11021 bnxt_clear_int_mode(bp); 11022 pci_disable_device(bp->pdev); 11023 } 11024 __bnxt_close_nic(bp, true, false); 11025 bnxt_clear_int_mode(bp); 11026 bnxt_hwrm_func_drv_unrgtr(bp); 11027 if (pci_is_enabled(bp->pdev)) 11028 pci_disable_device(bp->pdev); 11029 bnxt_free_ctx_mem(bp); 11030 kfree(bp->ctx); 11031 bp->ctx = NULL; 11032 } 11033 11034 static bool is_bnxt_fw_ok(struct bnxt *bp) 11035 { 11036 struct bnxt_fw_health *fw_health = bp->fw_health; 11037 bool no_heartbeat = false, has_reset = false; 11038 u32 val; 11039 11040 val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG); 11041 if (val == fw_health->last_fw_heartbeat) 11042 no_heartbeat = true; 11043 11044 val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG); 11045 if (val != fw_health->last_fw_reset_cnt) 11046 has_reset = true; 11047 11048 if (!no_heartbeat && has_reset) 11049 return true; 11050 11051 return false; 11052 } 11053 11054 /* rtnl_lock is acquired before calling this function */ 11055 static void bnxt_force_fw_reset(struct bnxt *bp) 11056 { 11057 struct bnxt_fw_health *fw_health = bp->fw_health; 11058 u32 wait_dsecs; 11059 11060 if (!test_bit(BNXT_STATE_OPEN, &bp->state) || 11061 test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 11062 return; 11063 11064 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 11065 bnxt_fw_reset_close(bp); 11066 wait_dsecs = fw_health->master_func_wait_dsecs; 11067 if (fw_health->master) { 11068 if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) 11069 wait_dsecs = 0; 11070 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW; 11071 } else { 11072 bp->fw_reset_timestamp = jiffies + wait_dsecs * HZ / 10; 11073 wait_dsecs = fw_health->normal_func_wait_dsecs; 11074 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 11075 } 11076 11077 bp->fw_reset_min_dsecs = fw_health->post_reset_wait_dsecs; 11078 bp->fw_reset_max_dsecs = fw_health->post_reset_max_wait_dsecs; 11079 bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10); 11080 } 11081 11082 void bnxt_fw_exception(struct bnxt *bp) 11083 { 11084 netdev_warn(bp->dev, "Detected firmware fatal condition, initiating reset\n"); 11085 set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state); 11086 bnxt_rtnl_lock_sp(bp); 11087 bnxt_force_fw_reset(bp); 11088 bnxt_rtnl_unlock_sp(bp); 11089 } 11090 11091 /* Returns the number of registered VFs, or 1 if VF configuration is pending, or 11092 * < 0 on error. 11093 */ 11094 static int bnxt_get_registered_vfs(struct bnxt *bp) 11095 { 11096 #ifdef CONFIG_BNXT_SRIOV 11097 int rc; 11098 11099 if (!BNXT_PF(bp)) 11100 return 0; 11101 11102 rc = bnxt_hwrm_func_qcfg(bp); 11103 if (rc) { 11104 netdev_err(bp->dev, "func_qcfg cmd failed, rc = %d\n", rc); 11105 return rc; 11106 } 11107 if (bp->pf.registered_vfs) 11108 return bp->pf.registered_vfs; 11109 if (bp->sriov_cfg) 11110 return 1; 11111 #endif 11112 return 0; 11113 } 11114 11115 void bnxt_fw_reset(struct bnxt *bp) 11116 { 11117 bnxt_rtnl_lock_sp(bp); 11118 if (test_bit(BNXT_STATE_OPEN, &bp->state) && 11119 !test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) { 11120 int n = 0, tmo; 11121 11122 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 11123 if (bp->pf.active_vfs && 11124 !test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) 11125 n = bnxt_get_registered_vfs(bp); 11126 if (n < 0) { 11127 netdev_err(bp->dev, "Firmware reset aborted, rc = %d\n", 11128 n); 11129 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 11130 dev_close(bp->dev); 11131 goto fw_reset_exit; 11132 } else if (n > 0) { 11133 u16 vf_tmo_dsecs = n * 10; 11134 11135 if (bp->fw_reset_max_dsecs < vf_tmo_dsecs) 11136 bp->fw_reset_max_dsecs = vf_tmo_dsecs; 11137 bp->fw_reset_state = 11138 BNXT_FW_RESET_STATE_POLL_VF; 11139 bnxt_queue_fw_reset_work(bp, HZ / 10); 11140 goto fw_reset_exit; 11141 } 11142 bnxt_fw_reset_close(bp); 11143 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) { 11144 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN; 11145 tmo = HZ / 10; 11146 } else { 11147 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 11148 tmo = bp->fw_reset_min_dsecs * HZ / 10; 11149 } 11150 bnxt_queue_fw_reset_work(bp, tmo); 11151 } 11152 fw_reset_exit: 11153 bnxt_rtnl_unlock_sp(bp); 11154 } 11155 11156 static void bnxt_chk_missed_irq(struct bnxt *bp) 11157 { 11158 int i; 11159 11160 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 11161 return; 11162 11163 for (i = 0; i < bp->cp_nr_rings; i++) { 11164 struct bnxt_napi *bnapi = bp->bnapi[i]; 11165 struct bnxt_cp_ring_info *cpr; 11166 u32 fw_ring_id; 11167 int j; 11168 11169 if (!bnapi) 11170 continue; 11171 11172 cpr = &bnapi->cp_ring; 11173 for (j = 0; j < 2; j++) { 11174 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j]; 11175 u32 val[2]; 11176 11177 if (!cpr2 || cpr2->has_more_work || 11178 !bnxt_has_work(bp, cpr2)) 11179 continue; 11180 11181 if (cpr2->cp_raw_cons != cpr2->last_cp_raw_cons) { 11182 cpr2->last_cp_raw_cons = cpr2->cp_raw_cons; 11183 continue; 11184 } 11185 fw_ring_id = cpr2->cp_ring_struct.fw_ring_id; 11186 bnxt_dbg_hwrm_ring_info_get(bp, 11187 DBG_RING_INFO_GET_REQ_RING_TYPE_L2_CMPL, 11188 fw_ring_id, &val[0], &val[1]); 11189 cpr->sw_stats.cmn.missed_irqs++; 11190 } 11191 } 11192 } 11193 11194 static void bnxt_cfg_ntp_filters(struct bnxt *); 11195 11196 static void bnxt_init_ethtool_link_settings(struct bnxt *bp) 11197 { 11198 struct bnxt_link_info *link_info = &bp->link_info; 11199 11200 if (BNXT_AUTO_MODE(link_info->auto_mode)) { 11201 link_info->autoneg = BNXT_AUTONEG_SPEED; 11202 if (bp->hwrm_spec_code >= 0x10201) { 11203 if (link_info->auto_pause_setting & 11204 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE) 11205 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL; 11206 } else { 11207 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL; 11208 } 11209 link_info->advertising = link_info->auto_link_speeds; 11210 link_info->advertising_pam4 = link_info->auto_pam4_link_speeds; 11211 } else { 11212 link_info->req_link_speed = link_info->force_link_speed; 11213 link_info->req_signal_mode = BNXT_SIG_MODE_NRZ; 11214 if (link_info->force_pam4_link_speed) { 11215 link_info->req_link_speed = 11216 link_info->force_pam4_link_speed; 11217 link_info->req_signal_mode = BNXT_SIG_MODE_PAM4; 11218 } 11219 link_info->req_duplex = link_info->duplex_setting; 11220 } 11221 if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) 11222 link_info->req_flow_ctrl = 11223 link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH; 11224 else 11225 link_info->req_flow_ctrl = link_info->force_pause_setting; 11226 } 11227 11228 static void bnxt_fw_echo_reply(struct bnxt *bp) 11229 { 11230 struct bnxt_fw_health *fw_health = bp->fw_health; 11231 struct hwrm_func_echo_response_input req = {0}; 11232 11233 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_ECHO_RESPONSE, -1, -1); 11234 req.event_data1 = cpu_to_le32(fw_health->echo_req_data1); 11235 req.event_data2 = cpu_to_le32(fw_health->echo_req_data2); 11236 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 11237 } 11238 11239 static void bnxt_sp_task(struct work_struct *work) 11240 { 11241 struct bnxt *bp = container_of(work, struct bnxt, sp_task); 11242 11243 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 11244 smp_mb__after_atomic(); 11245 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 11246 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 11247 return; 11248 } 11249 11250 if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event)) 11251 bnxt_cfg_rx_mode(bp); 11252 11253 if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event)) 11254 bnxt_cfg_ntp_filters(bp); 11255 if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event)) 11256 bnxt_hwrm_exec_fwd_req(bp); 11257 if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event)) { 11258 bnxt_hwrm_port_qstats(bp, 0); 11259 bnxt_hwrm_port_qstats_ext(bp, 0); 11260 bnxt_accumulate_all_stats(bp); 11261 } 11262 11263 if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) { 11264 int rc; 11265 11266 mutex_lock(&bp->link_lock); 11267 if (test_and_clear_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, 11268 &bp->sp_event)) 11269 bnxt_hwrm_phy_qcaps(bp); 11270 11271 rc = bnxt_update_link(bp, true); 11272 if (rc) 11273 netdev_err(bp->dev, "SP task can't update link (rc: %x)\n", 11274 rc); 11275 11276 if (test_and_clear_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT, 11277 &bp->sp_event)) 11278 bnxt_init_ethtool_link_settings(bp); 11279 mutex_unlock(&bp->link_lock); 11280 } 11281 if (test_and_clear_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event)) { 11282 int rc; 11283 11284 mutex_lock(&bp->link_lock); 11285 rc = bnxt_update_phy_setting(bp); 11286 mutex_unlock(&bp->link_lock); 11287 if (rc) { 11288 netdev_warn(bp->dev, "update phy settings retry failed\n"); 11289 } else { 11290 bp->link_info.phy_retry = false; 11291 netdev_info(bp->dev, "update phy settings retry succeeded\n"); 11292 } 11293 } 11294 if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event)) { 11295 mutex_lock(&bp->link_lock); 11296 bnxt_get_port_module_status(bp); 11297 mutex_unlock(&bp->link_lock); 11298 } 11299 11300 if (test_and_clear_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event)) 11301 bnxt_tc_flow_stats_work(bp); 11302 11303 if (test_and_clear_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event)) 11304 bnxt_chk_missed_irq(bp); 11305 11306 if (test_and_clear_bit(BNXT_FW_ECHO_REQUEST_SP_EVENT, &bp->sp_event)) 11307 bnxt_fw_echo_reply(bp); 11308 11309 /* These functions below will clear BNXT_STATE_IN_SP_TASK. They 11310 * must be the last functions to be called before exiting. 11311 */ 11312 if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event)) 11313 bnxt_reset(bp, false); 11314 11315 if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event)) 11316 bnxt_reset(bp, true); 11317 11318 if (test_and_clear_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event)) 11319 bnxt_rx_ring_reset(bp); 11320 11321 if (test_and_clear_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event)) 11322 bnxt_devlink_health_report(bp, BNXT_FW_RESET_NOTIFY_SP_EVENT); 11323 11324 if (test_and_clear_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event)) { 11325 if (!is_bnxt_fw_ok(bp)) 11326 bnxt_devlink_health_report(bp, 11327 BNXT_FW_EXCEPTION_SP_EVENT); 11328 } 11329 11330 smp_mb__before_atomic(); 11331 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 11332 } 11333 11334 /* Under rtnl_lock */ 11335 int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs, 11336 int tx_xdp) 11337 { 11338 int max_rx, max_tx, tx_sets = 1; 11339 int tx_rings_needed, stats; 11340 int rx_rings = rx; 11341 int cp, vnics, rc; 11342 11343 if (tcs) 11344 tx_sets = tcs; 11345 11346 rc = bnxt_get_max_rings(bp, &max_rx, &max_tx, sh); 11347 if (rc) 11348 return rc; 11349 11350 if (max_rx < rx) 11351 return -ENOMEM; 11352 11353 tx_rings_needed = tx * tx_sets + tx_xdp; 11354 if (max_tx < tx_rings_needed) 11355 return -ENOMEM; 11356 11357 vnics = 1; 11358 if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS) 11359 vnics += rx_rings; 11360 11361 if (bp->flags & BNXT_FLAG_AGG_RINGS) 11362 rx_rings <<= 1; 11363 cp = sh ? max_t(int, tx_rings_needed, rx) : tx_rings_needed + rx; 11364 stats = cp; 11365 if (BNXT_NEW_RM(bp)) { 11366 cp += bnxt_get_ulp_msix_num(bp); 11367 stats += bnxt_get_ulp_stat_ctxs(bp); 11368 } 11369 return bnxt_hwrm_check_rings(bp, tx_rings_needed, rx_rings, rx, cp, 11370 stats, vnics); 11371 } 11372 11373 static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev) 11374 { 11375 if (bp->bar2) { 11376 pci_iounmap(pdev, bp->bar2); 11377 bp->bar2 = NULL; 11378 } 11379 11380 if (bp->bar1) { 11381 pci_iounmap(pdev, bp->bar1); 11382 bp->bar1 = NULL; 11383 } 11384 11385 if (bp->bar0) { 11386 pci_iounmap(pdev, bp->bar0); 11387 bp->bar0 = NULL; 11388 } 11389 } 11390 11391 static void bnxt_cleanup_pci(struct bnxt *bp) 11392 { 11393 bnxt_unmap_bars(bp, bp->pdev); 11394 pci_release_regions(bp->pdev); 11395 if (pci_is_enabled(bp->pdev)) 11396 pci_disable_device(bp->pdev); 11397 } 11398 11399 static void bnxt_init_dflt_coal(struct bnxt *bp) 11400 { 11401 struct bnxt_coal *coal; 11402 11403 /* Tick values in micro seconds. 11404 * 1 coal_buf x bufs_per_record = 1 completion record. 11405 */ 11406 coal = &bp->rx_coal; 11407 coal->coal_ticks = 10; 11408 coal->coal_bufs = 30; 11409 coal->coal_ticks_irq = 1; 11410 coal->coal_bufs_irq = 2; 11411 coal->idle_thresh = 50; 11412 coal->bufs_per_record = 2; 11413 coal->budget = 64; /* NAPI budget */ 11414 11415 coal = &bp->tx_coal; 11416 coal->coal_ticks = 28; 11417 coal->coal_bufs = 30; 11418 coal->coal_ticks_irq = 2; 11419 coal->coal_bufs_irq = 2; 11420 coal->bufs_per_record = 1; 11421 11422 bp->stats_coal_ticks = BNXT_DEF_STATS_COAL_TICKS; 11423 } 11424 11425 static int bnxt_fw_init_one_p1(struct bnxt *bp) 11426 { 11427 int rc; 11428 11429 bp->fw_cap = 0; 11430 rc = bnxt_hwrm_ver_get(bp); 11431 bnxt_try_map_fw_health_reg(bp); 11432 if (rc) { 11433 rc = bnxt_try_recover_fw(bp); 11434 if (rc) 11435 return rc; 11436 rc = bnxt_hwrm_ver_get(bp); 11437 if (rc) 11438 return rc; 11439 } 11440 11441 if (bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL) { 11442 rc = bnxt_alloc_kong_hwrm_resources(bp); 11443 if (rc) 11444 bp->fw_cap &= ~BNXT_FW_CAP_KONG_MB_CHNL; 11445 } 11446 11447 if ((bp->fw_cap & BNXT_FW_CAP_SHORT_CMD) || 11448 bp->hwrm_max_ext_req_len > BNXT_HWRM_MAX_REQ_LEN) { 11449 rc = bnxt_alloc_hwrm_short_cmd_req(bp); 11450 if (rc) 11451 return rc; 11452 } 11453 bnxt_nvm_cfg_ver_get(bp); 11454 11455 rc = bnxt_hwrm_func_reset(bp); 11456 if (rc) 11457 return -ENODEV; 11458 11459 bnxt_hwrm_fw_set_time(bp); 11460 return 0; 11461 } 11462 11463 static int bnxt_fw_init_one_p2(struct bnxt *bp) 11464 { 11465 int rc; 11466 11467 /* Get the MAX capabilities for this function */ 11468 rc = bnxt_hwrm_func_qcaps(bp); 11469 if (rc) { 11470 netdev_err(bp->dev, "hwrm query capability failure rc: %x\n", 11471 rc); 11472 return -ENODEV; 11473 } 11474 11475 rc = bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(bp); 11476 if (rc) 11477 netdev_warn(bp->dev, "hwrm query adv flow mgnt failure rc: %d\n", 11478 rc); 11479 11480 if (bnxt_alloc_fw_health(bp)) { 11481 netdev_warn(bp->dev, "no memory for firmware error recovery\n"); 11482 } else { 11483 rc = bnxt_hwrm_error_recovery_qcfg(bp); 11484 if (rc) 11485 netdev_warn(bp->dev, "hwrm query error recovery failure rc: %d\n", 11486 rc); 11487 } 11488 11489 rc = bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false); 11490 if (rc) 11491 return -ENODEV; 11492 11493 bnxt_hwrm_func_qcfg(bp); 11494 bnxt_hwrm_vnic_qcaps(bp); 11495 bnxt_hwrm_port_led_qcaps(bp); 11496 bnxt_ethtool_init(bp); 11497 bnxt_dcb_init(bp); 11498 return 0; 11499 } 11500 11501 static void bnxt_set_dflt_rss_hash_type(struct bnxt *bp) 11502 { 11503 bp->flags &= ~BNXT_FLAG_UDP_RSS_CAP; 11504 bp->rss_hash_cfg = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 | 11505 VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 | 11506 VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 | 11507 VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6; 11508 if (BNXT_CHIP_P4_PLUS(bp) && bp->hwrm_spec_code >= 0x10501) { 11509 bp->flags |= BNXT_FLAG_UDP_RSS_CAP; 11510 bp->rss_hash_cfg |= VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 | 11511 VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6; 11512 } 11513 } 11514 11515 static void bnxt_set_dflt_rfs(struct bnxt *bp) 11516 { 11517 struct net_device *dev = bp->dev; 11518 11519 dev->hw_features &= ~NETIF_F_NTUPLE; 11520 dev->features &= ~NETIF_F_NTUPLE; 11521 bp->flags &= ~BNXT_FLAG_RFS; 11522 if (bnxt_rfs_supported(bp)) { 11523 dev->hw_features |= NETIF_F_NTUPLE; 11524 if (bnxt_rfs_capable(bp)) { 11525 bp->flags |= BNXT_FLAG_RFS; 11526 dev->features |= NETIF_F_NTUPLE; 11527 } 11528 } 11529 } 11530 11531 static void bnxt_fw_init_one_p3(struct bnxt *bp) 11532 { 11533 struct pci_dev *pdev = bp->pdev; 11534 11535 bnxt_set_dflt_rss_hash_type(bp); 11536 bnxt_set_dflt_rfs(bp); 11537 11538 bnxt_get_wol_settings(bp); 11539 if (bp->flags & BNXT_FLAG_WOL_CAP) 11540 device_set_wakeup_enable(&pdev->dev, bp->wol); 11541 else 11542 device_set_wakeup_capable(&pdev->dev, false); 11543 11544 bnxt_hwrm_set_cache_line_size(bp, cache_line_size()); 11545 bnxt_hwrm_coal_params_qcaps(bp); 11546 } 11547 11548 static int bnxt_fw_init_one(struct bnxt *bp) 11549 { 11550 int rc; 11551 11552 rc = bnxt_fw_init_one_p1(bp); 11553 if (rc) { 11554 netdev_err(bp->dev, "Firmware init phase 1 failed\n"); 11555 return rc; 11556 } 11557 rc = bnxt_fw_init_one_p2(bp); 11558 if (rc) { 11559 netdev_err(bp->dev, "Firmware init phase 2 failed\n"); 11560 return rc; 11561 } 11562 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, false); 11563 if (rc) 11564 return rc; 11565 11566 /* In case fw capabilities have changed, destroy the unneeded 11567 * reporters and create newly capable ones. 11568 */ 11569 bnxt_dl_fw_reporters_destroy(bp, false); 11570 bnxt_dl_fw_reporters_create(bp); 11571 bnxt_fw_init_one_p3(bp); 11572 return 0; 11573 } 11574 11575 static void bnxt_fw_reset_writel(struct bnxt *bp, int reg_idx) 11576 { 11577 struct bnxt_fw_health *fw_health = bp->fw_health; 11578 u32 reg = fw_health->fw_reset_seq_regs[reg_idx]; 11579 u32 val = fw_health->fw_reset_seq_vals[reg_idx]; 11580 u32 reg_type, reg_off, delay_msecs; 11581 11582 delay_msecs = fw_health->fw_reset_seq_delay_msec[reg_idx]; 11583 reg_type = BNXT_FW_HEALTH_REG_TYPE(reg); 11584 reg_off = BNXT_FW_HEALTH_REG_OFF(reg); 11585 switch (reg_type) { 11586 case BNXT_FW_HEALTH_REG_TYPE_CFG: 11587 pci_write_config_dword(bp->pdev, reg_off, val); 11588 break; 11589 case BNXT_FW_HEALTH_REG_TYPE_GRC: 11590 writel(reg_off & BNXT_GRC_BASE_MASK, 11591 bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4); 11592 reg_off = (reg_off & BNXT_GRC_OFFSET_MASK) + 0x2000; 11593 fallthrough; 11594 case BNXT_FW_HEALTH_REG_TYPE_BAR0: 11595 writel(val, bp->bar0 + reg_off); 11596 break; 11597 case BNXT_FW_HEALTH_REG_TYPE_BAR1: 11598 writel(val, bp->bar1 + reg_off); 11599 break; 11600 } 11601 if (delay_msecs) { 11602 pci_read_config_dword(bp->pdev, 0, &val); 11603 msleep(delay_msecs); 11604 } 11605 } 11606 11607 static void bnxt_reset_all(struct bnxt *bp) 11608 { 11609 struct bnxt_fw_health *fw_health = bp->fw_health; 11610 int i, rc; 11611 11612 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) { 11613 bnxt_fw_reset_via_optee(bp); 11614 bp->fw_reset_timestamp = jiffies; 11615 return; 11616 } 11617 11618 if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_HOST) { 11619 for (i = 0; i < fw_health->fw_reset_seq_cnt; i++) 11620 bnxt_fw_reset_writel(bp, i); 11621 } else if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) { 11622 struct hwrm_fw_reset_input req = {0}; 11623 11624 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FW_RESET, -1, -1); 11625 req.resp_addr = cpu_to_le64(bp->hwrm_cmd_kong_resp_dma_addr); 11626 req.embedded_proc_type = FW_RESET_REQ_EMBEDDED_PROC_TYPE_CHIP; 11627 req.selfrst_status = FW_RESET_REQ_SELFRST_STATUS_SELFRSTASAP; 11628 req.flags = FW_RESET_REQ_FLAGS_RESET_GRACEFUL; 11629 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 11630 if (rc) 11631 netdev_warn(bp->dev, "Unable to reset FW rc=%d\n", rc); 11632 } 11633 bp->fw_reset_timestamp = jiffies; 11634 } 11635 11636 static bool bnxt_fw_reset_timeout(struct bnxt *bp) 11637 { 11638 return time_after(jiffies, bp->fw_reset_timestamp + 11639 (bp->fw_reset_max_dsecs * HZ / 10)); 11640 } 11641 11642 static void bnxt_fw_reset_task(struct work_struct *work) 11643 { 11644 struct bnxt *bp = container_of(work, struct bnxt, fw_reset_task.work); 11645 int rc; 11646 11647 if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) { 11648 netdev_err(bp->dev, "bnxt_fw_reset_task() called when not in fw reset mode!\n"); 11649 return; 11650 } 11651 11652 switch (bp->fw_reset_state) { 11653 case BNXT_FW_RESET_STATE_POLL_VF: { 11654 int n = bnxt_get_registered_vfs(bp); 11655 int tmo; 11656 11657 if (n < 0) { 11658 netdev_err(bp->dev, "Firmware reset aborted, subsequent func_qcfg cmd failed, rc = %d, %d msecs since reset timestamp\n", 11659 n, jiffies_to_msecs(jiffies - 11660 bp->fw_reset_timestamp)); 11661 goto fw_reset_abort; 11662 } else if (n > 0) { 11663 if (bnxt_fw_reset_timeout(bp)) { 11664 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 11665 bp->fw_reset_state = 0; 11666 netdev_err(bp->dev, "Firmware reset aborted, bnxt_get_registered_vfs() returns %d\n", 11667 n); 11668 return; 11669 } 11670 bnxt_queue_fw_reset_work(bp, HZ / 10); 11671 return; 11672 } 11673 bp->fw_reset_timestamp = jiffies; 11674 rtnl_lock(); 11675 bnxt_fw_reset_close(bp); 11676 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) { 11677 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN; 11678 tmo = HZ / 10; 11679 } else { 11680 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 11681 tmo = bp->fw_reset_min_dsecs * HZ / 10; 11682 } 11683 rtnl_unlock(); 11684 bnxt_queue_fw_reset_work(bp, tmo); 11685 return; 11686 } 11687 case BNXT_FW_RESET_STATE_POLL_FW_DOWN: { 11688 u32 val; 11689 11690 val = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG); 11691 if (!(val & BNXT_FW_STATUS_SHUTDOWN) && 11692 !bnxt_fw_reset_timeout(bp)) { 11693 bnxt_queue_fw_reset_work(bp, HZ / 5); 11694 return; 11695 } 11696 11697 if (!bp->fw_health->master) { 11698 u32 wait_dsecs = bp->fw_health->normal_func_wait_dsecs; 11699 11700 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 11701 bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10); 11702 return; 11703 } 11704 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW; 11705 } 11706 fallthrough; 11707 case BNXT_FW_RESET_STATE_RESET_FW: 11708 bnxt_reset_all(bp); 11709 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 11710 bnxt_queue_fw_reset_work(bp, bp->fw_reset_min_dsecs * HZ / 10); 11711 return; 11712 case BNXT_FW_RESET_STATE_ENABLE_DEV: 11713 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) { 11714 u32 val; 11715 11716 if (!bp->fw_reset_min_dsecs) { 11717 u16 val; 11718 11719 pci_read_config_word(bp->pdev, PCI_SUBSYSTEM_ID, 11720 &val); 11721 if (val == 0xffff) { 11722 if (bnxt_fw_reset_timeout(bp)) { 11723 netdev_err(bp->dev, "Firmware reset aborted, PCI config space invalid\n"); 11724 goto fw_reset_abort; 11725 } 11726 bnxt_queue_fw_reset_work(bp, HZ / 1000); 11727 return; 11728 } 11729 } 11730 val = bnxt_fw_health_readl(bp, 11731 BNXT_FW_RESET_INPROG_REG); 11732 if (val) 11733 netdev_warn(bp->dev, "FW reset inprog %x after min wait time.\n", 11734 val); 11735 } 11736 clear_bit(BNXT_STATE_FW_FATAL_COND, &bp->state); 11737 if (pci_enable_device(bp->pdev)) { 11738 netdev_err(bp->dev, "Cannot re-enable PCI device\n"); 11739 goto fw_reset_abort; 11740 } 11741 pci_set_master(bp->pdev); 11742 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW; 11743 fallthrough; 11744 case BNXT_FW_RESET_STATE_POLL_FW: 11745 bp->hwrm_cmd_timeout = SHORT_HWRM_CMD_TIMEOUT; 11746 rc = __bnxt_hwrm_ver_get(bp, true); 11747 if (rc) { 11748 if (bnxt_fw_reset_timeout(bp)) { 11749 netdev_err(bp->dev, "Firmware reset aborted\n"); 11750 goto fw_reset_abort_status; 11751 } 11752 bnxt_queue_fw_reset_work(bp, HZ / 5); 11753 return; 11754 } 11755 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT; 11756 bp->fw_reset_state = BNXT_FW_RESET_STATE_OPENING; 11757 fallthrough; 11758 case BNXT_FW_RESET_STATE_OPENING: 11759 while (!rtnl_trylock()) { 11760 bnxt_queue_fw_reset_work(bp, HZ / 10); 11761 return; 11762 } 11763 rc = bnxt_open(bp->dev); 11764 if (rc) { 11765 netdev_err(bp->dev, "bnxt_open_nic() failed\n"); 11766 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 11767 dev_close(bp->dev); 11768 } 11769 11770 bp->fw_reset_state = 0; 11771 /* Make sure fw_reset_state is 0 before clearing the flag */ 11772 smp_mb__before_atomic(); 11773 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 11774 bnxt_ulp_start(bp, rc); 11775 if (!rc) 11776 bnxt_reenable_sriov(bp); 11777 bnxt_dl_health_recovery_done(bp); 11778 bnxt_dl_health_status_update(bp, true); 11779 rtnl_unlock(); 11780 break; 11781 } 11782 return; 11783 11784 fw_reset_abort_status: 11785 if (bp->fw_health->status_reliable || 11786 (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) { 11787 u32 sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG); 11788 11789 netdev_err(bp->dev, "fw_health_status 0x%x\n", sts); 11790 } 11791 fw_reset_abort: 11792 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 11793 if (bp->fw_reset_state != BNXT_FW_RESET_STATE_POLL_VF) 11794 bnxt_dl_health_status_update(bp, false); 11795 bp->fw_reset_state = 0; 11796 rtnl_lock(); 11797 dev_close(bp->dev); 11798 rtnl_unlock(); 11799 } 11800 11801 static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev) 11802 { 11803 int rc; 11804 struct bnxt *bp = netdev_priv(dev); 11805 11806 SET_NETDEV_DEV(dev, &pdev->dev); 11807 11808 /* enable device (incl. PCI PM wakeup), and bus-mastering */ 11809 rc = pci_enable_device(pdev); 11810 if (rc) { 11811 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n"); 11812 goto init_err; 11813 } 11814 11815 if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) { 11816 dev_err(&pdev->dev, 11817 "Cannot find PCI device base address, aborting\n"); 11818 rc = -ENODEV; 11819 goto init_err_disable; 11820 } 11821 11822 rc = pci_request_regions(pdev, DRV_MODULE_NAME); 11823 if (rc) { 11824 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n"); 11825 goto init_err_disable; 11826 } 11827 11828 if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 && 11829 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) { 11830 dev_err(&pdev->dev, "System does not support DMA, aborting\n"); 11831 rc = -EIO; 11832 goto init_err_release; 11833 } 11834 11835 pci_set_master(pdev); 11836 11837 bp->dev = dev; 11838 bp->pdev = pdev; 11839 11840 /* Doorbell BAR bp->bar1 is mapped after bnxt_fw_init_one_p2() 11841 * determines the BAR size. 11842 */ 11843 bp->bar0 = pci_ioremap_bar(pdev, 0); 11844 if (!bp->bar0) { 11845 dev_err(&pdev->dev, "Cannot map device registers, aborting\n"); 11846 rc = -ENOMEM; 11847 goto init_err_release; 11848 } 11849 11850 bp->bar2 = pci_ioremap_bar(pdev, 4); 11851 if (!bp->bar2) { 11852 dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n"); 11853 rc = -ENOMEM; 11854 goto init_err_release; 11855 } 11856 11857 pci_enable_pcie_error_reporting(pdev); 11858 11859 INIT_WORK(&bp->sp_task, bnxt_sp_task); 11860 INIT_DELAYED_WORK(&bp->fw_reset_task, bnxt_fw_reset_task); 11861 11862 spin_lock_init(&bp->ntp_fltr_lock); 11863 #if BITS_PER_LONG == 32 11864 spin_lock_init(&bp->db_lock); 11865 #endif 11866 11867 bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE; 11868 bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE; 11869 11870 bnxt_init_dflt_coal(bp); 11871 11872 timer_setup(&bp->timer, bnxt_timer, 0); 11873 bp->current_interval = BNXT_TIMER_INTERVAL; 11874 11875 bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID; 11876 bp->nge_fw_dst_port_id = INVALID_HW_RING_ID; 11877 11878 clear_bit(BNXT_STATE_OPEN, &bp->state); 11879 return 0; 11880 11881 init_err_release: 11882 bnxt_unmap_bars(bp, pdev); 11883 pci_release_regions(pdev); 11884 11885 init_err_disable: 11886 pci_disable_device(pdev); 11887 11888 init_err: 11889 return rc; 11890 } 11891 11892 /* rtnl_lock held */ 11893 static int bnxt_change_mac_addr(struct net_device *dev, void *p) 11894 { 11895 struct sockaddr *addr = p; 11896 struct bnxt *bp = netdev_priv(dev); 11897 int rc = 0; 11898 11899 if (!is_valid_ether_addr(addr->sa_data)) 11900 return -EADDRNOTAVAIL; 11901 11902 if (ether_addr_equal(addr->sa_data, dev->dev_addr)) 11903 return 0; 11904 11905 rc = bnxt_approve_mac(bp, addr->sa_data, true); 11906 if (rc) 11907 return rc; 11908 11909 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len); 11910 if (netif_running(dev)) { 11911 bnxt_close_nic(bp, false, false); 11912 rc = bnxt_open_nic(bp, false, false); 11913 } 11914 11915 return rc; 11916 } 11917 11918 /* rtnl_lock held */ 11919 static int bnxt_change_mtu(struct net_device *dev, int new_mtu) 11920 { 11921 struct bnxt *bp = netdev_priv(dev); 11922 11923 if (netif_running(dev)) 11924 bnxt_close_nic(bp, true, false); 11925 11926 dev->mtu = new_mtu; 11927 bnxt_set_ring_params(bp); 11928 11929 if (netif_running(dev)) 11930 return bnxt_open_nic(bp, true, false); 11931 11932 return 0; 11933 } 11934 11935 int bnxt_setup_mq_tc(struct net_device *dev, u8 tc) 11936 { 11937 struct bnxt *bp = netdev_priv(dev); 11938 bool sh = false; 11939 int rc; 11940 11941 if (tc > bp->max_tc) { 11942 netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n", 11943 tc, bp->max_tc); 11944 return -EINVAL; 11945 } 11946 11947 if (netdev_get_num_tc(dev) == tc) 11948 return 0; 11949 11950 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 11951 sh = true; 11952 11953 rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings, 11954 sh, tc, bp->tx_nr_rings_xdp); 11955 if (rc) 11956 return rc; 11957 11958 /* Needs to close the device and do hw resource re-allocations */ 11959 if (netif_running(bp->dev)) 11960 bnxt_close_nic(bp, true, false); 11961 11962 if (tc) { 11963 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc; 11964 netdev_set_num_tc(dev, tc); 11965 } else { 11966 bp->tx_nr_rings = bp->tx_nr_rings_per_tc; 11967 netdev_reset_tc(dev); 11968 } 11969 bp->tx_nr_rings += bp->tx_nr_rings_xdp; 11970 bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) : 11971 bp->tx_nr_rings + bp->rx_nr_rings; 11972 11973 if (netif_running(bp->dev)) 11974 return bnxt_open_nic(bp, true, false); 11975 11976 return 0; 11977 } 11978 11979 static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 11980 void *cb_priv) 11981 { 11982 struct bnxt *bp = cb_priv; 11983 11984 if (!bnxt_tc_flower_enabled(bp) || 11985 !tc_cls_can_offload_and_chain0(bp->dev, type_data)) 11986 return -EOPNOTSUPP; 11987 11988 switch (type) { 11989 case TC_SETUP_CLSFLOWER: 11990 return bnxt_tc_setup_flower(bp, bp->pf.fw_fid, type_data); 11991 default: 11992 return -EOPNOTSUPP; 11993 } 11994 } 11995 11996 LIST_HEAD(bnxt_block_cb_list); 11997 11998 static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type, 11999 void *type_data) 12000 { 12001 struct bnxt *bp = netdev_priv(dev); 12002 12003 switch (type) { 12004 case TC_SETUP_BLOCK: 12005 return flow_block_cb_setup_simple(type_data, 12006 &bnxt_block_cb_list, 12007 bnxt_setup_tc_block_cb, 12008 bp, bp, true); 12009 case TC_SETUP_QDISC_MQPRIO: { 12010 struct tc_mqprio_qopt *mqprio = type_data; 12011 12012 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 12013 12014 return bnxt_setup_mq_tc(dev, mqprio->num_tc); 12015 } 12016 default: 12017 return -EOPNOTSUPP; 12018 } 12019 } 12020 12021 #ifdef CONFIG_RFS_ACCEL 12022 static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1, 12023 struct bnxt_ntuple_filter *f2) 12024 { 12025 struct flow_keys *keys1 = &f1->fkeys; 12026 struct flow_keys *keys2 = &f2->fkeys; 12027 12028 if (keys1->basic.n_proto != keys2->basic.n_proto || 12029 keys1->basic.ip_proto != keys2->basic.ip_proto) 12030 return false; 12031 12032 if (keys1->basic.n_proto == htons(ETH_P_IP)) { 12033 if (keys1->addrs.v4addrs.src != keys2->addrs.v4addrs.src || 12034 keys1->addrs.v4addrs.dst != keys2->addrs.v4addrs.dst) 12035 return false; 12036 } else { 12037 if (memcmp(&keys1->addrs.v6addrs.src, &keys2->addrs.v6addrs.src, 12038 sizeof(keys1->addrs.v6addrs.src)) || 12039 memcmp(&keys1->addrs.v6addrs.dst, &keys2->addrs.v6addrs.dst, 12040 sizeof(keys1->addrs.v6addrs.dst))) 12041 return false; 12042 } 12043 12044 if (keys1->ports.ports == keys2->ports.ports && 12045 keys1->control.flags == keys2->control.flags && 12046 ether_addr_equal(f1->src_mac_addr, f2->src_mac_addr) && 12047 ether_addr_equal(f1->dst_mac_addr, f2->dst_mac_addr)) 12048 return true; 12049 12050 return false; 12051 } 12052 12053 static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb, 12054 u16 rxq_index, u32 flow_id) 12055 { 12056 struct bnxt *bp = netdev_priv(dev); 12057 struct bnxt_ntuple_filter *fltr, *new_fltr; 12058 struct flow_keys *fkeys; 12059 struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb); 12060 int rc = 0, idx, bit_id, l2_idx = 0; 12061 struct hlist_head *head; 12062 u32 flags; 12063 12064 if (!ether_addr_equal(dev->dev_addr, eth->h_dest)) { 12065 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 12066 int off = 0, j; 12067 12068 netif_addr_lock_bh(dev); 12069 for (j = 0; j < vnic->uc_filter_count; j++, off += ETH_ALEN) { 12070 if (ether_addr_equal(eth->h_dest, 12071 vnic->uc_list + off)) { 12072 l2_idx = j + 1; 12073 break; 12074 } 12075 } 12076 netif_addr_unlock_bh(dev); 12077 if (!l2_idx) 12078 return -EINVAL; 12079 } 12080 new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC); 12081 if (!new_fltr) 12082 return -ENOMEM; 12083 12084 fkeys = &new_fltr->fkeys; 12085 if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) { 12086 rc = -EPROTONOSUPPORT; 12087 goto err_free; 12088 } 12089 12090 if ((fkeys->basic.n_proto != htons(ETH_P_IP) && 12091 fkeys->basic.n_proto != htons(ETH_P_IPV6)) || 12092 ((fkeys->basic.ip_proto != IPPROTO_TCP) && 12093 (fkeys->basic.ip_proto != IPPROTO_UDP))) { 12094 rc = -EPROTONOSUPPORT; 12095 goto err_free; 12096 } 12097 if (fkeys->basic.n_proto == htons(ETH_P_IPV6) && 12098 bp->hwrm_spec_code < 0x10601) { 12099 rc = -EPROTONOSUPPORT; 12100 goto err_free; 12101 } 12102 flags = fkeys->control.flags; 12103 if (((flags & FLOW_DIS_ENCAPSULATION) && 12104 bp->hwrm_spec_code < 0x10601) || (flags & FLOW_DIS_IS_FRAGMENT)) { 12105 rc = -EPROTONOSUPPORT; 12106 goto err_free; 12107 } 12108 12109 memcpy(new_fltr->dst_mac_addr, eth->h_dest, ETH_ALEN); 12110 memcpy(new_fltr->src_mac_addr, eth->h_source, ETH_ALEN); 12111 12112 idx = skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK; 12113 head = &bp->ntp_fltr_hash_tbl[idx]; 12114 rcu_read_lock(); 12115 hlist_for_each_entry_rcu(fltr, head, hash) { 12116 if (bnxt_fltr_match(fltr, new_fltr)) { 12117 rcu_read_unlock(); 12118 rc = 0; 12119 goto err_free; 12120 } 12121 } 12122 rcu_read_unlock(); 12123 12124 spin_lock_bh(&bp->ntp_fltr_lock); 12125 bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap, 12126 BNXT_NTP_FLTR_MAX_FLTR, 0); 12127 if (bit_id < 0) { 12128 spin_unlock_bh(&bp->ntp_fltr_lock); 12129 rc = -ENOMEM; 12130 goto err_free; 12131 } 12132 12133 new_fltr->sw_id = (u16)bit_id; 12134 new_fltr->flow_id = flow_id; 12135 new_fltr->l2_fltr_idx = l2_idx; 12136 new_fltr->rxq = rxq_index; 12137 hlist_add_head_rcu(&new_fltr->hash, head); 12138 bp->ntp_fltr_count++; 12139 spin_unlock_bh(&bp->ntp_fltr_lock); 12140 12141 set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event); 12142 bnxt_queue_sp_work(bp); 12143 12144 return new_fltr->sw_id; 12145 12146 err_free: 12147 kfree(new_fltr); 12148 return rc; 12149 } 12150 12151 static void bnxt_cfg_ntp_filters(struct bnxt *bp) 12152 { 12153 int i; 12154 12155 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) { 12156 struct hlist_head *head; 12157 struct hlist_node *tmp; 12158 struct bnxt_ntuple_filter *fltr; 12159 int rc; 12160 12161 head = &bp->ntp_fltr_hash_tbl[i]; 12162 hlist_for_each_entry_safe(fltr, tmp, head, hash) { 12163 bool del = false; 12164 12165 if (test_bit(BNXT_FLTR_VALID, &fltr->state)) { 12166 if (rps_may_expire_flow(bp->dev, fltr->rxq, 12167 fltr->flow_id, 12168 fltr->sw_id)) { 12169 bnxt_hwrm_cfa_ntuple_filter_free(bp, 12170 fltr); 12171 del = true; 12172 } 12173 } else { 12174 rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp, 12175 fltr); 12176 if (rc) 12177 del = true; 12178 else 12179 set_bit(BNXT_FLTR_VALID, &fltr->state); 12180 } 12181 12182 if (del) { 12183 spin_lock_bh(&bp->ntp_fltr_lock); 12184 hlist_del_rcu(&fltr->hash); 12185 bp->ntp_fltr_count--; 12186 spin_unlock_bh(&bp->ntp_fltr_lock); 12187 synchronize_rcu(); 12188 clear_bit(fltr->sw_id, bp->ntp_fltr_bmap); 12189 kfree(fltr); 12190 } 12191 } 12192 } 12193 if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event)) 12194 netdev_info(bp->dev, "Receive PF driver unload event!\n"); 12195 } 12196 12197 #else 12198 12199 static void bnxt_cfg_ntp_filters(struct bnxt *bp) 12200 { 12201 } 12202 12203 #endif /* CONFIG_RFS_ACCEL */ 12204 12205 static int bnxt_udp_tunnel_sync(struct net_device *netdev, unsigned int table) 12206 { 12207 struct bnxt *bp = netdev_priv(netdev); 12208 struct udp_tunnel_info ti; 12209 unsigned int cmd; 12210 12211 udp_tunnel_nic_get_port(netdev, table, 0, &ti); 12212 if (ti.type == UDP_TUNNEL_TYPE_VXLAN) 12213 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN; 12214 else 12215 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE; 12216 12217 if (ti.port) 12218 return bnxt_hwrm_tunnel_dst_port_alloc(bp, ti.port, cmd); 12219 12220 return bnxt_hwrm_tunnel_dst_port_free(bp, cmd); 12221 } 12222 12223 static const struct udp_tunnel_nic_info bnxt_udp_tunnels = { 12224 .sync_table = bnxt_udp_tunnel_sync, 12225 .flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP | 12226 UDP_TUNNEL_NIC_INFO_OPEN_ONLY, 12227 .tables = { 12228 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN, }, 12229 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_GENEVE, }, 12230 }, 12231 }; 12232 12233 static int bnxt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 12234 struct net_device *dev, u32 filter_mask, 12235 int nlflags) 12236 { 12237 struct bnxt *bp = netdev_priv(dev); 12238 12239 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bp->br_mode, 0, 0, 12240 nlflags, filter_mask, NULL); 12241 } 12242 12243 static int bnxt_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh, 12244 u16 flags, struct netlink_ext_ack *extack) 12245 { 12246 struct bnxt *bp = netdev_priv(dev); 12247 struct nlattr *attr, *br_spec; 12248 int rem, rc = 0; 12249 12250 if (bp->hwrm_spec_code < 0x10708 || !BNXT_SINGLE_PF(bp)) 12251 return -EOPNOTSUPP; 12252 12253 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 12254 if (!br_spec) 12255 return -EINVAL; 12256 12257 nla_for_each_nested(attr, br_spec, rem) { 12258 u16 mode; 12259 12260 if (nla_type(attr) != IFLA_BRIDGE_MODE) 12261 continue; 12262 12263 if (nla_len(attr) < sizeof(mode)) 12264 return -EINVAL; 12265 12266 mode = nla_get_u16(attr); 12267 if (mode == bp->br_mode) 12268 break; 12269 12270 rc = bnxt_hwrm_set_br_mode(bp, mode); 12271 if (!rc) 12272 bp->br_mode = mode; 12273 break; 12274 } 12275 return rc; 12276 } 12277 12278 int bnxt_get_port_parent_id(struct net_device *dev, 12279 struct netdev_phys_item_id *ppid) 12280 { 12281 struct bnxt *bp = netdev_priv(dev); 12282 12283 if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV) 12284 return -EOPNOTSUPP; 12285 12286 /* The PF and it's VF-reps only support the switchdev framework */ 12287 if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_DSN_VALID)) 12288 return -EOPNOTSUPP; 12289 12290 ppid->id_len = sizeof(bp->dsn); 12291 memcpy(ppid->id, bp->dsn, ppid->id_len); 12292 12293 return 0; 12294 } 12295 12296 static struct devlink_port *bnxt_get_devlink_port(struct net_device *dev) 12297 { 12298 struct bnxt *bp = netdev_priv(dev); 12299 12300 return &bp->dl_port; 12301 } 12302 12303 static const struct net_device_ops bnxt_netdev_ops = { 12304 .ndo_open = bnxt_open, 12305 .ndo_start_xmit = bnxt_start_xmit, 12306 .ndo_stop = bnxt_close, 12307 .ndo_get_stats64 = bnxt_get_stats64, 12308 .ndo_set_rx_mode = bnxt_set_rx_mode, 12309 .ndo_do_ioctl = bnxt_ioctl, 12310 .ndo_validate_addr = eth_validate_addr, 12311 .ndo_set_mac_address = bnxt_change_mac_addr, 12312 .ndo_change_mtu = bnxt_change_mtu, 12313 .ndo_fix_features = bnxt_fix_features, 12314 .ndo_set_features = bnxt_set_features, 12315 .ndo_tx_timeout = bnxt_tx_timeout, 12316 #ifdef CONFIG_BNXT_SRIOV 12317 .ndo_get_vf_config = bnxt_get_vf_config, 12318 .ndo_set_vf_mac = bnxt_set_vf_mac, 12319 .ndo_set_vf_vlan = bnxt_set_vf_vlan, 12320 .ndo_set_vf_rate = bnxt_set_vf_bw, 12321 .ndo_set_vf_link_state = bnxt_set_vf_link_state, 12322 .ndo_set_vf_spoofchk = bnxt_set_vf_spoofchk, 12323 .ndo_set_vf_trust = bnxt_set_vf_trust, 12324 #endif 12325 .ndo_setup_tc = bnxt_setup_tc, 12326 #ifdef CONFIG_RFS_ACCEL 12327 .ndo_rx_flow_steer = bnxt_rx_flow_steer, 12328 #endif 12329 .ndo_bpf = bnxt_xdp, 12330 .ndo_xdp_xmit = bnxt_xdp_xmit, 12331 .ndo_bridge_getlink = bnxt_bridge_getlink, 12332 .ndo_bridge_setlink = bnxt_bridge_setlink, 12333 .ndo_get_devlink_port = bnxt_get_devlink_port, 12334 }; 12335 12336 static void bnxt_remove_one(struct pci_dev *pdev) 12337 { 12338 struct net_device *dev = pci_get_drvdata(pdev); 12339 struct bnxt *bp = netdev_priv(dev); 12340 12341 if (BNXT_PF(bp)) 12342 bnxt_sriov_disable(bp); 12343 12344 if (BNXT_PF(bp)) 12345 devlink_port_type_clear(&bp->dl_port); 12346 pci_disable_pcie_error_reporting(pdev); 12347 unregister_netdev(dev); 12348 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 12349 /* Flush any pending tasks */ 12350 cancel_work_sync(&bp->sp_task); 12351 cancel_delayed_work_sync(&bp->fw_reset_task); 12352 bp->sp_event = 0; 12353 12354 bnxt_dl_fw_reporters_destroy(bp, true); 12355 bnxt_dl_unregister(bp); 12356 bnxt_shutdown_tc(bp); 12357 12358 bnxt_clear_int_mode(bp); 12359 bnxt_hwrm_func_drv_unrgtr(bp); 12360 bnxt_free_hwrm_resources(bp); 12361 bnxt_free_hwrm_short_cmd_req(bp); 12362 bnxt_ethtool_free(bp); 12363 bnxt_dcb_free(bp); 12364 kfree(bp->edev); 12365 bp->edev = NULL; 12366 kfree(bp->fw_health); 12367 bp->fw_health = NULL; 12368 bnxt_cleanup_pci(bp); 12369 bnxt_free_ctx_mem(bp); 12370 kfree(bp->ctx); 12371 bp->ctx = NULL; 12372 kfree(bp->rss_indir_tbl); 12373 bp->rss_indir_tbl = NULL; 12374 bnxt_free_port_stats(bp); 12375 free_netdev(dev); 12376 } 12377 12378 static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt) 12379 { 12380 int rc = 0; 12381 struct bnxt_link_info *link_info = &bp->link_info; 12382 12383 rc = bnxt_hwrm_phy_qcaps(bp); 12384 if (rc) { 12385 netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n", 12386 rc); 12387 return rc; 12388 } 12389 if (!fw_dflt) 12390 return 0; 12391 12392 rc = bnxt_update_link(bp, false); 12393 if (rc) { 12394 netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n", 12395 rc); 12396 return rc; 12397 } 12398 12399 /* Older firmware does not have supported_auto_speeds, so assume 12400 * that all supported speeds can be autonegotiated. 12401 */ 12402 if (link_info->auto_link_speeds && !link_info->support_auto_speeds) 12403 link_info->support_auto_speeds = link_info->support_speeds; 12404 12405 bnxt_init_ethtool_link_settings(bp); 12406 return 0; 12407 } 12408 12409 static int bnxt_get_max_irq(struct pci_dev *pdev) 12410 { 12411 u16 ctrl; 12412 12413 if (!pdev->msix_cap) 12414 return 1; 12415 12416 pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl); 12417 return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1; 12418 } 12419 12420 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, 12421 int *max_cp) 12422 { 12423 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 12424 int max_ring_grps = 0, max_irq; 12425 12426 *max_tx = hw_resc->max_tx_rings; 12427 *max_rx = hw_resc->max_rx_rings; 12428 *max_cp = bnxt_get_max_func_cp_rings_for_en(bp); 12429 max_irq = min_t(int, bnxt_get_max_func_irqs(bp) - 12430 bnxt_get_ulp_msix_num(bp), 12431 hw_resc->max_stat_ctxs - bnxt_get_ulp_stat_ctxs(bp)); 12432 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 12433 *max_cp = min_t(int, *max_cp, max_irq); 12434 max_ring_grps = hw_resc->max_hw_ring_grps; 12435 if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) { 12436 *max_cp -= 1; 12437 *max_rx -= 2; 12438 } 12439 if (bp->flags & BNXT_FLAG_AGG_RINGS) 12440 *max_rx >>= 1; 12441 if (bp->flags & BNXT_FLAG_CHIP_P5) { 12442 bnxt_trim_rings(bp, max_rx, max_tx, *max_cp, false); 12443 /* On P5 chips, max_cp output param should be available NQs */ 12444 *max_cp = max_irq; 12445 } 12446 *max_rx = min_t(int, *max_rx, max_ring_grps); 12447 } 12448 12449 int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared) 12450 { 12451 int rx, tx, cp; 12452 12453 _bnxt_get_max_rings(bp, &rx, &tx, &cp); 12454 *max_rx = rx; 12455 *max_tx = tx; 12456 if (!rx || !tx || !cp) 12457 return -ENOMEM; 12458 12459 return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared); 12460 } 12461 12462 static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx, 12463 bool shared) 12464 { 12465 int rc; 12466 12467 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared); 12468 if (rc && (bp->flags & BNXT_FLAG_AGG_RINGS)) { 12469 /* Not enough rings, try disabling agg rings. */ 12470 bp->flags &= ~BNXT_FLAG_AGG_RINGS; 12471 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared); 12472 if (rc) { 12473 /* set BNXT_FLAG_AGG_RINGS back for consistency */ 12474 bp->flags |= BNXT_FLAG_AGG_RINGS; 12475 return rc; 12476 } 12477 bp->flags |= BNXT_FLAG_NO_AGG_RINGS; 12478 bp->dev->hw_features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); 12479 bp->dev->features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); 12480 bnxt_set_ring_params(bp); 12481 } 12482 12483 if (bp->flags & BNXT_FLAG_ROCE_CAP) { 12484 int max_cp, max_stat, max_irq; 12485 12486 /* Reserve minimum resources for RoCE */ 12487 max_cp = bnxt_get_max_func_cp_rings(bp); 12488 max_stat = bnxt_get_max_func_stat_ctxs(bp); 12489 max_irq = bnxt_get_max_func_irqs(bp); 12490 if (max_cp <= BNXT_MIN_ROCE_CP_RINGS || 12491 max_irq <= BNXT_MIN_ROCE_CP_RINGS || 12492 max_stat <= BNXT_MIN_ROCE_STAT_CTXS) 12493 return 0; 12494 12495 max_cp -= BNXT_MIN_ROCE_CP_RINGS; 12496 max_irq -= BNXT_MIN_ROCE_CP_RINGS; 12497 max_stat -= BNXT_MIN_ROCE_STAT_CTXS; 12498 max_cp = min_t(int, max_cp, max_irq); 12499 max_cp = min_t(int, max_cp, max_stat); 12500 rc = bnxt_trim_rings(bp, max_rx, max_tx, max_cp, shared); 12501 if (rc) 12502 rc = 0; 12503 } 12504 return rc; 12505 } 12506 12507 /* In initial default shared ring setting, each shared ring must have a 12508 * RX/TX ring pair. 12509 */ 12510 static void bnxt_trim_dflt_sh_rings(struct bnxt *bp) 12511 { 12512 bp->cp_nr_rings = min_t(int, bp->tx_nr_rings_per_tc, bp->rx_nr_rings); 12513 bp->rx_nr_rings = bp->cp_nr_rings; 12514 bp->tx_nr_rings_per_tc = bp->cp_nr_rings; 12515 bp->tx_nr_rings = bp->tx_nr_rings_per_tc; 12516 } 12517 12518 static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh) 12519 { 12520 int dflt_rings, max_rx_rings, max_tx_rings, rc; 12521 12522 if (!bnxt_can_reserve_rings(bp)) 12523 return 0; 12524 12525 if (sh) 12526 bp->flags |= BNXT_FLAG_SHARED_RINGS; 12527 dflt_rings = is_kdump_kernel() ? 1 : netif_get_num_default_rss_queues(); 12528 /* Reduce default rings on multi-port cards so that total default 12529 * rings do not exceed CPU count. 12530 */ 12531 if (bp->port_count > 1) { 12532 int max_rings = 12533 max_t(int, num_online_cpus() / bp->port_count, 1); 12534 12535 dflt_rings = min_t(int, dflt_rings, max_rings); 12536 } 12537 rc = bnxt_get_dflt_rings(bp, &max_rx_rings, &max_tx_rings, sh); 12538 if (rc) 12539 return rc; 12540 bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings); 12541 bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings); 12542 if (sh) 12543 bnxt_trim_dflt_sh_rings(bp); 12544 else 12545 bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings; 12546 bp->tx_nr_rings = bp->tx_nr_rings_per_tc; 12547 12548 rc = __bnxt_reserve_rings(bp); 12549 if (rc) 12550 netdev_warn(bp->dev, "Unable to reserve tx rings\n"); 12551 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 12552 if (sh) 12553 bnxt_trim_dflt_sh_rings(bp); 12554 12555 /* Rings may have been trimmed, re-reserve the trimmed rings. */ 12556 if (bnxt_need_reserve_rings(bp)) { 12557 rc = __bnxt_reserve_rings(bp); 12558 if (rc) 12559 netdev_warn(bp->dev, "2nd rings reservation failed.\n"); 12560 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 12561 } 12562 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 12563 bp->rx_nr_rings++; 12564 bp->cp_nr_rings++; 12565 } 12566 if (rc) { 12567 bp->tx_nr_rings = 0; 12568 bp->rx_nr_rings = 0; 12569 } 12570 return rc; 12571 } 12572 12573 static int bnxt_init_dflt_ring_mode(struct bnxt *bp) 12574 { 12575 int rc; 12576 12577 if (bp->tx_nr_rings) 12578 return 0; 12579 12580 bnxt_ulp_irq_stop(bp); 12581 bnxt_clear_int_mode(bp); 12582 rc = bnxt_set_dflt_rings(bp, true); 12583 if (rc) { 12584 netdev_err(bp->dev, "Not enough rings available.\n"); 12585 goto init_dflt_ring_err; 12586 } 12587 rc = bnxt_init_int_mode(bp); 12588 if (rc) 12589 goto init_dflt_ring_err; 12590 12591 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 12592 if (bnxt_rfs_supported(bp) && bnxt_rfs_capable(bp)) { 12593 bp->flags |= BNXT_FLAG_RFS; 12594 bp->dev->features |= NETIF_F_NTUPLE; 12595 } 12596 init_dflt_ring_err: 12597 bnxt_ulp_irq_restart(bp, rc); 12598 return rc; 12599 } 12600 12601 int bnxt_restore_pf_fw_resources(struct bnxt *bp) 12602 { 12603 int rc; 12604 12605 ASSERT_RTNL(); 12606 bnxt_hwrm_func_qcaps(bp); 12607 12608 if (netif_running(bp->dev)) 12609 __bnxt_close_nic(bp, true, false); 12610 12611 bnxt_ulp_irq_stop(bp); 12612 bnxt_clear_int_mode(bp); 12613 rc = bnxt_init_int_mode(bp); 12614 bnxt_ulp_irq_restart(bp, rc); 12615 12616 if (netif_running(bp->dev)) { 12617 if (rc) 12618 dev_close(bp->dev); 12619 else 12620 rc = bnxt_open_nic(bp, true, false); 12621 } 12622 12623 return rc; 12624 } 12625 12626 static int bnxt_init_mac_addr(struct bnxt *bp) 12627 { 12628 int rc = 0; 12629 12630 if (BNXT_PF(bp)) { 12631 memcpy(bp->dev->dev_addr, bp->pf.mac_addr, ETH_ALEN); 12632 } else { 12633 #ifdef CONFIG_BNXT_SRIOV 12634 struct bnxt_vf_info *vf = &bp->vf; 12635 bool strict_approval = true; 12636 12637 if (is_valid_ether_addr(vf->mac_addr)) { 12638 /* overwrite netdev dev_addr with admin VF MAC */ 12639 memcpy(bp->dev->dev_addr, vf->mac_addr, ETH_ALEN); 12640 /* Older PF driver or firmware may not approve this 12641 * correctly. 12642 */ 12643 strict_approval = false; 12644 } else { 12645 eth_hw_addr_random(bp->dev); 12646 } 12647 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, strict_approval); 12648 #endif 12649 } 12650 return rc; 12651 } 12652 12653 #define BNXT_VPD_LEN 512 12654 static void bnxt_vpd_read_info(struct bnxt *bp) 12655 { 12656 struct pci_dev *pdev = bp->pdev; 12657 int i, len, pos, ro_size, size; 12658 ssize_t vpd_size; 12659 u8 *vpd_data; 12660 12661 vpd_data = kmalloc(BNXT_VPD_LEN, GFP_KERNEL); 12662 if (!vpd_data) 12663 return; 12664 12665 vpd_size = pci_read_vpd(pdev, 0, BNXT_VPD_LEN, vpd_data); 12666 if (vpd_size <= 0) { 12667 netdev_err(bp->dev, "Unable to read VPD\n"); 12668 goto exit; 12669 } 12670 12671 i = pci_vpd_find_tag(vpd_data, 0, vpd_size, PCI_VPD_LRDT_RO_DATA); 12672 if (i < 0) { 12673 netdev_err(bp->dev, "VPD READ-Only not found\n"); 12674 goto exit; 12675 } 12676 12677 ro_size = pci_vpd_lrdt_size(&vpd_data[i]); 12678 i += PCI_VPD_LRDT_TAG_SIZE; 12679 if (i + ro_size > vpd_size) 12680 goto exit; 12681 12682 pos = pci_vpd_find_info_keyword(vpd_data, i, ro_size, 12683 PCI_VPD_RO_KEYWORD_PARTNO); 12684 if (pos < 0) 12685 goto read_sn; 12686 12687 len = pci_vpd_info_field_size(&vpd_data[pos]); 12688 pos += PCI_VPD_INFO_FLD_HDR_SIZE; 12689 if (len + pos > vpd_size) 12690 goto read_sn; 12691 12692 size = min(len, BNXT_VPD_FLD_LEN - 1); 12693 memcpy(bp->board_partno, &vpd_data[pos], size); 12694 12695 read_sn: 12696 pos = pci_vpd_find_info_keyword(vpd_data, i, ro_size, 12697 PCI_VPD_RO_KEYWORD_SERIALNO); 12698 if (pos < 0) 12699 goto exit; 12700 12701 len = pci_vpd_info_field_size(&vpd_data[pos]); 12702 pos += PCI_VPD_INFO_FLD_HDR_SIZE; 12703 if (len + pos > vpd_size) 12704 goto exit; 12705 12706 size = min(len, BNXT_VPD_FLD_LEN - 1); 12707 memcpy(bp->board_serialno, &vpd_data[pos], size); 12708 exit: 12709 kfree(vpd_data); 12710 } 12711 12712 static int bnxt_pcie_dsn_get(struct bnxt *bp, u8 dsn[]) 12713 { 12714 struct pci_dev *pdev = bp->pdev; 12715 u64 qword; 12716 12717 qword = pci_get_dsn(pdev); 12718 if (!qword) { 12719 netdev_info(bp->dev, "Unable to read adapter's DSN\n"); 12720 return -EOPNOTSUPP; 12721 } 12722 12723 put_unaligned_le64(qword, dsn); 12724 12725 bp->flags |= BNXT_FLAG_DSN_VALID; 12726 return 0; 12727 } 12728 12729 static int bnxt_map_db_bar(struct bnxt *bp) 12730 { 12731 if (!bp->db_size) 12732 return -ENODEV; 12733 bp->bar1 = pci_iomap(bp->pdev, 2, bp->db_size); 12734 if (!bp->bar1) 12735 return -ENOMEM; 12736 return 0; 12737 } 12738 12739 static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) 12740 { 12741 struct net_device *dev; 12742 struct bnxt *bp; 12743 int rc, max_irqs; 12744 12745 if (pci_is_bridge(pdev)) 12746 return -ENODEV; 12747 12748 /* Clear any pending DMA transactions from crash kernel 12749 * while loading driver in capture kernel. 12750 */ 12751 if (is_kdump_kernel()) { 12752 pci_clear_master(pdev); 12753 pcie_flr(pdev); 12754 } 12755 12756 max_irqs = bnxt_get_max_irq(pdev); 12757 dev = alloc_etherdev_mq(sizeof(*bp), max_irqs); 12758 if (!dev) 12759 return -ENOMEM; 12760 12761 bp = netdev_priv(dev); 12762 bp->msg_enable = BNXT_DEF_MSG_ENABLE; 12763 bnxt_set_max_func_irqs(bp, max_irqs); 12764 12765 if (bnxt_vf_pciid(ent->driver_data)) 12766 bp->flags |= BNXT_FLAG_VF; 12767 12768 if (pdev->msix_cap) 12769 bp->flags |= BNXT_FLAG_MSIX_CAP; 12770 12771 rc = bnxt_init_board(pdev, dev); 12772 if (rc < 0) 12773 goto init_err_free; 12774 12775 dev->netdev_ops = &bnxt_netdev_ops; 12776 dev->watchdog_timeo = BNXT_TX_TIMEOUT; 12777 dev->ethtool_ops = &bnxt_ethtool_ops; 12778 pci_set_drvdata(pdev, dev); 12779 12780 rc = bnxt_alloc_hwrm_resources(bp); 12781 if (rc) 12782 goto init_err_pci_clean; 12783 12784 mutex_init(&bp->hwrm_cmd_lock); 12785 mutex_init(&bp->link_lock); 12786 12787 rc = bnxt_fw_init_one_p1(bp); 12788 if (rc) 12789 goto init_err_pci_clean; 12790 12791 if (BNXT_PF(bp)) 12792 bnxt_vpd_read_info(bp); 12793 12794 if (BNXT_CHIP_P5(bp)) { 12795 bp->flags |= BNXT_FLAG_CHIP_P5; 12796 if (BNXT_CHIP_SR2(bp)) 12797 bp->flags |= BNXT_FLAG_CHIP_SR2; 12798 } 12799 12800 rc = bnxt_alloc_rss_indir_tbl(bp); 12801 if (rc) 12802 goto init_err_pci_clean; 12803 12804 rc = bnxt_fw_init_one_p2(bp); 12805 if (rc) 12806 goto init_err_pci_clean; 12807 12808 rc = bnxt_map_db_bar(bp); 12809 if (rc) { 12810 dev_err(&pdev->dev, "Cannot map doorbell BAR rc = %d, aborting\n", 12811 rc); 12812 goto init_err_pci_clean; 12813 } 12814 12815 dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG | 12816 NETIF_F_TSO | NETIF_F_TSO6 | 12817 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE | 12818 NETIF_F_GSO_IPXIP4 | 12819 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM | 12820 NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH | 12821 NETIF_F_RXCSUM | NETIF_F_GRO; 12822 12823 if (BNXT_SUPPORTS_TPA(bp)) 12824 dev->hw_features |= NETIF_F_LRO; 12825 12826 dev->hw_enc_features = 12827 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG | 12828 NETIF_F_TSO | NETIF_F_TSO6 | 12829 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE | 12830 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM | 12831 NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL; 12832 dev->udp_tunnel_nic_info = &bnxt_udp_tunnels; 12833 12834 dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM | 12835 NETIF_F_GSO_GRE_CSUM; 12836 dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA; 12837 if (bp->fw_cap & BNXT_FW_CAP_VLAN_RX_STRIP) 12838 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_RX; 12839 if (bp->fw_cap & BNXT_FW_CAP_VLAN_TX_INSERT) 12840 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_TX; 12841 if (BNXT_SUPPORTS_TPA(bp)) 12842 dev->hw_features |= NETIF_F_GRO_HW; 12843 dev->features |= dev->hw_features | NETIF_F_HIGHDMA; 12844 if (dev->features & NETIF_F_GRO_HW) 12845 dev->features &= ~NETIF_F_LRO; 12846 dev->priv_flags |= IFF_UNICAST_FLT; 12847 12848 #ifdef CONFIG_BNXT_SRIOV 12849 init_waitqueue_head(&bp->sriov_cfg_wait); 12850 mutex_init(&bp->sriov_lock); 12851 #endif 12852 if (BNXT_SUPPORTS_TPA(bp)) { 12853 bp->gro_func = bnxt_gro_func_5730x; 12854 if (BNXT_CHIP_P4(bp)) 12855 bp->gro_func = bnxt_gro_func_5731x; 12856 else if (BNXT_CHIP_P5(bp)) 12857 bp->gro_func = bnxt_gro_func_5750x; 12858 } 12859 if (!BNXT_CHIP_P4_PLUS(bp)) 12860 bp->flags |= BNXT_FLAG_DOUBLE_DB; 12861 12862 bp->ulp_probe = bnxt_ulp_probe; 12863 12864 rc = bnxt_init_mac_addr(bp); 12865 if (rc) { 12866 dev_err(&pdev->dev, "Unable to initialize mac address.\n"); 12867 rc = -EADDRNOTAVAIL; 12868 goto init_err_pci_clean; 12869 } 12870 12871 if (BNXT_PF(bp)) { 12872 /* Read the adapter's DSN to use as the eswitch switch_id */ 12873 rc = bnxt_pcie_dsn_get(bp, bp->dsn); 12874 } 12875 12876 /* MTU range: 60 - FW defined max */ 12877 dev->min_mtu = ETH_ZLEN; 12878 dev->max_mtu = bp->max_mtu; 12879 12880 rc = bnxt_probe_phy(bp, true); 12881 if (rc) 12882 goto init_err_pci_clean; 12883 12884 bnxt_set_rx_skb_mode(bp, false); 12885 bnxt_set_tpa_flags(bp); 12886 bnxt_set_ring_params(bp); 12887 rc = bnxt_set_dflt_rings(bp, true); 12888 if (rc) { 12889 netdev_err(bp->dev, "Not enough rings available.\n"); 12890 rc = -ENOMEM; 12891 goto init_err_pci_clean; 12892 } 12893 12894 bnxt_fw_init_one_p3(bp); 12895 12896 if (dev->hw_features & BNXT_HW_FEATURE_VLAN_ALL_RX) 12897 bp->flags |= BNXT_FLAG_STRIP_VLAN; 12898 12899 rc = bnxt_init_int_mode(bp); 12900 if (rc) 12901 goto init_err_pci_clean; 12902 12903 /* No TC has been set yet and rings may have been trimmed due to 12904 * limited MSIX, so we re-initialize the TX rings per TC. 12905 */ 12906 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 12907 12908 if (BNXT_PF(bp)) { 12909 if (!bnxt_pf_wq) { 12910 bnxt_pf_wq = 12911 create_singlethread_workqueue("bnxt_pf_wq"); 12912 if (!bnxt_pf_wq) { 12913 dev_err(&pdev->dev, "Unable to create workqueue.\n"); 12914 rc = -ENOMEM; 12915 goto init_err_pci_clean; 12916 } 12917 } 12918 rc = bnxt_init_tc(bp); 12919 if (rc) 12920 netdev_err(dev, "Failed to initialize TC flower offload, err = %d.\n", 12921 rc); 12922 } 12923 12924 bnxt_dl_register(bp); 12925 12926 rc = register_netdev(dev); 12927 if (rc) 12928 goto init_err_cleanup; 12929 12930 if (BNXT_PF(bp)) 12931 devlink_port_type_eth_set(&bp->dl_port, bp->dev); 12932 bnxt_dl_fw_reporters_create(bp); 12933 12934 netdev_info(dev, "%s found at mem %lx, node addr %pM\n", 12935 board_info[ent->driver_data].name, 12936 (long)pci_resource_start(pdev, 0), dev->dev_addr); 12937 pcie_print_link_status(pdev); 12938 12939 pci_save_state(pdev); 12940 return 0; 12941 12942 init_err_cleanup: 12943 bnxt_dl_unregister(bp); 12944 bnxt_shutdown_tc(bp); 12945 bnxt_clear_int_mode(bp); 12946 12947 init_err_pci_clean: 12948 bnxt_hwrm_func_drv_unrgtr(bp); 12949 bnxt_free_hwrm_short_cmd_req(bp); 12950 bnxt_free_hwrm_resources(bp); 12951 kfree(bp->fw_health); 12952 bp->fw_health = NULL; 12953 bnxt_cleanup_pci(bp); 12954 bnxt_free_ctx_mem(bp); 12955 kfree(bp->ctx); 12956 bp->ctx = NULL; 12957 kfree(bp->rss_indir_tbl); 12958 bp->rss_indir_tbl = NULL; 12959 12960 init_err_free: 12961 free_netdev(dev); 12962 return rc; 12963 } 12964 12965 static void bnxt_shutdown(struct pci_dev *pdev) 12966 { 12967 struct net_device *dev = pci_get_drvdata(pdev); 12968 struct bnxt *bp; 12969 12970 if (!dev) 12971 return; 12972 12973 rtnl_lock(); 12974 bp = netdev_priv(dev); 12975 if (!bp) 12976 goto shutdown_exit; 12977 12978 if (netif_running(dev)) 12979 dev_close(dev); 12980 12981 bnxt_ulp_shutdown(bp); 12982 bnxt_clear_int_mode(bp); 12983 pci_disable_device(pdev); 12984 12985 if (system_state == SYSTEM_POWER_OFF) { 12986 pci_wake_from_d3(pdev, bp->wol); 12987 pci_set_power_state(pdev, PCI_D3hot); 12988 } 12989 12990 shutdown_exit: 12991 rtnl_unlock(); 12992 } 12993 12994 #ifdef CONFIG_PM_SLEEP 12995 static int bnxt_suspend(struct device *device) 12996 { 12997 struct net_device *dev = dev_get_drvdata(device); 12998 struct bnxt *bp = netdev_priv(dev); 12999 int rc = 0; 13000 13001 rtnl_lock(); 13002 bnxt_ulp_stop(bp); 13003 if (netif_running(dev)) { 13004 netif_device_detach(dev); 13005 rc = bnxt_close(dev); 13006 } 13007 bnxt_hwrm_func_drv_unrgtr(bp); 13008 pci_disable_device(bp->pdev); 13009 bnxt_free_ctx_mem(bp); 13010 kfree(bp->ctx); 13011 bp->ctx = NULL; 13012 rtnl_unlock(); 13013 return rc; 13014 } 13015 13016 static int bnxt_resume(struct device *device) 13017 { 13018 struct net_device *dev = dev_get_drvdata(device); 13019 struct bnxt *bp = netdev_priv(dev); 13020 int rc = 0; 13021 13022 rtnl_lock(); 13023 rc = pci_enable_device(bp->pdev); 13024 if (rc) { 13025 netdev_err(dev, "Cannot re-enable PCI device during resume, err = %d\n", 13026 rc); 13027 goto resume_exit; 13028 } 13029 pci_set_master(bp->pdev); 13030 if (bnxt_hwrm_ver_get(bp)) { 13031 rc = -ENODEV; 13032 goto resume_exit; 13033 } 13034 rc = bnxt_hwrm_func_reset(bp); 13035 if (rc) { 13036 rc = -EBUSY; 13037 goto resume_exit; 13038 } 13039 13040 rc = bnxt_hwrm_func_qcaps(bp); 13041 if (rc) 13042 goto resume_exit; 13043 13044 if (bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false)) { 13045 rc = -ENODEV; 13046 goto resume_exit; 13047 } 13048 13049 bnxt_get_wol_settings(bp); 13050 if (netif_running(dev)) { 13051 rc = bnxt_open(dev); 13052 if (!rc) 13053 netif_device_attach(dev); 13054 } 13055 13056 resume_exit: 13057 bnxt_ulp_start(bp, rc); 13058 if (!rc) 13059 bnxt_reenable_sriov(bp); 13060 rtnl_unlock(); 13061 return rc; 13062 } 13063 13064 static SIMPLE_DEV_PM_OPS(bnxt_pm_ops, bnxt_suspend, bnxt_resume); 13065 #define BNXT_PM_OPS (&bnxt_pm_ops) 13066 13067 #else 13068 13069 #define BNXT_PM_OPS NULL 13070 13071 #endif /* CONFIG_PM_SLEEP */ 13072 13073 /** 13074 * bnxt_io_error_detected - called when PCI error is detected 13075 * @pdev: Pointer to PCI device 13076 * @state: The current pci connection state 13077 * 13078 * This function is called after a PCI bus error affecting 13079 * this device has been detected. 13080 */ 13081 static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev, 13082 pci_channel_state_t state) 13083 { 13084 struct net_device *netdev = pci_get_drvdata(pdev); 13085 struct bnxt *bp = netdev_priv(netdev); 13086 13087 netdev_info(netdev, "PCI I/O error detected\n"); 13088 13089 rtnl_lock(); 13090 netif_device_detach(netdev); 13091 13092 bnxt_ulp_stop(bp); 13093 13094 if (state == pci_channel_io_perm_failure) { 13095 rtnl_unlock(); 13096 return PCI_ERS_RESULT_DISCONNECT; 13097 } 13098 13099 if (state == pci_channel_io_frozen) 13100 set_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, &bp->state); 13101 13102 if (netif_running(netdev)) 13103 bnxt_close(netdev); 13104 13105 pci_disable_device(pdev); 13106 bnxt_free_ctx_mem(bp); 13107 kfree(bp->ctx); 13108 bp->ctx = NULL; 13109 rtnl_unlock(); 13110 13111 /* Request a slot slot reset. */ 13112 return PCI_ERS_RESULT_NEED_RESET; 13113 } 13114 13115 /** 13116 * bnxt_io_slot_reset - called after the pci bus has been reset. 13117 * @pdev: Pointer to PCI device 13118 * 13119 * Restart the card from scratch, as if from a cold-boot. 13120 * At this point, the card has exprienced a hard reset, 13121 * followed by fixups by BIOS, and has its config space 13122 * set up identically to what it was at cold boot. 13123 */ 13124 static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev) 13125 { 13126 pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT; 13127 struct net_device *netdev = pci_get_drvdata(pdev); 13128 struct bnxt *bp = netdev_priv(netdev); 13129 int err = 0, off; 13130 13131 netdev_info(bp->dev, "PCI Slot Reset\n"); 13132 13133 rtnl_lock(); 13134 13135 if (pci_enable_device(pdev)) { 13136 dev_err(&pdev->dev, 13137 "Cannot re-enable PCI device after reset.\n"); 13138 } else { 13139 pci_set_master(pdev); 13140 /* Upon fatal error, our device internal logic that latches to 13141 * BAR value is getting reset and will restore only upon 13142 * rewritting the BARs. 13143 * 13144 * As pci_restore_state() does not re-write the BARs if the 13145 * value is same as saved value earlier, driver needs to 13146 * write the BARs to 0 to force restore, in case of fatal error. 13147 */ 13148 if (test_and_clear_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, 13149 &bp->state)) { 13150 for (off = PCI_BASE_ADDRESS_0; 13151 off <= PCI_BASE_ADDRESS_5; off += 4) 13152 pci_write_config_dword(bp->pdev, off, 0); 13153 } 13154 pci_restore_state(pdev); 13155 pci_save_state(pdev); 13156 13157 err = bnxt_hwrm_func_reset(bp); 13158 if (!err) 13159 result = PCI_ERS_RESULT_RECOVERED; 13160 } 13161 13162 rtnl_unlock(); 13163 13164 return result; 13165 } 13166 13167 /** 13168 * bnxt_io_resume - called when traffic can start flowing again. 13169 * @pdev: Pointer to PCI device 13170 * 13171 * This callback is called when the error recovery driver tells 13172 * us that its OK to resume normal operation. 13173 */ 13174 static void bnxt_io_resume(struct pci_dev *pdev) 13175 { 13176 struct net_device *netdev = pci_get_drvdata(pdev); 13177 struct bnxt *bp = netdev_priv(netdev); 13178 int err; 13179 13180 netdev_info(bp->dev, "PCI Slot Resume\n"); 13181 rtnl_lock(); 13182 13183 err = bnxt_hwrm_func_qcaps(bp); 13184 if (!err && netif_running(netdev)) 13185 err = bnxt_open(netdev); 13186 13187 bnxt_ulp_start(bp, err); 13188 if (!err) { 13189 bnxt_reenable_sriov(bp); 13190 netif_device_attach(netdev); 13191 } 13192 13193 rtnl_unlock(); 13194 } 13195 13196 static const struct pci_error_handlers bnxt_err_handler = { 13197 .error_detected = bnxt_io_error_detected, 13198 .slot_reset = bnxt_io_slot_reset, 13199 .resume = bnxt_io_resume 13200 }; 13201 13202 static struct pci_driver bnxt_pci_driver = { 13203 .name = DRV_MODULE_NAME, 13204 .id_table = bnxt_pci_tbl, 13205 .probe = bnxt_init_one, 13206 .remove = bnxt_remove_one, 13207 .shutdown = bnxt_shutdown, 13208 .driver.pm = BNXT_PM_OPS, 13209 .err_handler = &bnxt_err_handler, 13210 #if defined(CONFIG_BNXT_SRIOV) 13211 .sriov_configure = bnxt_sriov_configure, 13212 #endif 13213 }; 13214 13215 static int __init bnxt_init(void) 13216 { 13217 bnxt_debug_init(); 13218 return pci_register_driver(&bnxt_pci_driver); 13219 } 13220 13221 static void __exit bnxt_exit(void) 13222 { 13223 pci_unregister_driver(&bnxt_pci_driver); 13224 if (bnxt_pf_wq) 13225 destroy_workqueue(bnxt_pf_wq); 13226 bnxt_debug_exit(); 13227 } 13228 13229 module_init(bnxt_init); 13230 module_exit(bnxt_exit); 13231