1 /* Broadcom NetXtreme-C/E network driver. 2 * 3 * Copyright (c) 2014-2016 Broadcom Corporation 4 * Copyright (c) 2016-2019 Broadcom Limited 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License as published by 8 * the Free Software Foundation. 9 */ 10 11 #include <linux/module.h> 12 13 #include <linux/stringify.h> 14 #include <linux/kernel.h> 15 #include <linux/timer.h> 16 #include <linux/errno.h> 17 #include <linux/ioport.h> 18 #include <linux/slab.h> 19 #include <linux/vmalloc.h> 20 #include <linux/interrupt.h> 21 #include <linux/pci.h> 22 #include <linux/netdevice.h> 23 #include <linux/etherdevice.h> 24 #include <linux/skbuff.h> 25 #include <linux/dma-mapping.h> 26 #include <linux/bitops.h> 27 #include <linux/io.h> 28 #include <linux/irq.h> 29 #include <linux/delay.h> 30 #include <asm/byteorder.h> 31 #include <asm/page.h> 32 #include <linux/time.h> 33 #include <linux/mii.h> 34 #include <linux/mdio.h> 35 #include <linux/if.h> 36 #include <linux/if_vlan.h> 37 #include <linux/if_bridge.h> 38 #include <linux/rtc.h> 39 #include <linux/bpf.h> 40 #include <net/gro.h> 41 #include <net/ip.h> 42 #include <net/tcp.h> 43 #include <net/udp.h> 44 #include <net/checksum.h> 45 #include <net/ip6_checksum.h> 46 #include <net/udp_tunnel.h> 47 #include <linux/workqueue.h> 48 #include <linux/prefetch.h> 49 #include <linux/cache.h> 50 #include <linux/log2.h> 51 #include <linux/aer.h> 52 #include <linux/bitmap.h> 53 #include <linux/cpu_rmap.h> 54 #include <linux/cpumask.h> 55 #include <net/pkt_cls.h> 56 #include <linux/hwmon.h> 57 #include <linux/hwmon-sysfs.h> 58 #include <net/page_pool.h> 59 60 #include "bnxt_hsi.h" 61 #include "bnxt.h" 62 #include "bnxt_hwrm.h" 63 #include "bnxt_ulp.h" 64 #include "bnxt_sriov.h" 65 #include "bnxt_ethtool.h" 66 #include "bnxt_dcb.h" 67 #include "bnxt_xdp.h" 68 #include "bnxt_ptp.h" 69 #include "bnxt_vfr.h" 70 #include "bnxt_tc.h" 71 #include "bnxt_devlink.h" 72 #include "bnxt_debugfs.h" 73 74 #define BNXT_TX_TIMEOUT (5 * HZ) 75 #define BNXT_DEF_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_HW | \ 76 NETIF_MSG_TX_ERR) 77 78 MODULE_LICENSE("GPL"); 79 MODULE_DESCRIPTION("Broadcom BCM573xx network driver"); 80 81 #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN) 82 #define BNXT_RX_DMA_OFFSET NET_SKB_PAD 83 #define BNXT_RX_COPY_THRESH 256 84 85 #define BNXT_TX_PUSH_THRESH 164 86 87 /* indexed by enum board_idx */ 88 static const struct { 89 char *name; 90 } board_info[] = { 91 [BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" }, 92 [BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" }, 93 [BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" }, 94 [BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" }, 95 [BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" }, 96 [BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" }, 97 [BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" }, 98 [BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" }, 99 [BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" }, 100 [BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" }, 101 [BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" }, 102 [BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" }, 103 [BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" }, 104 [BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" }, 105 [BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" }, 106 [BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" }, 107 [BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" }, 108 [BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" }, 109 [BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" }, 110 [BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" }, 111 [BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" }, 112 [BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" }, 113 [BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" }, 114 [BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" }, 115 [BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" }, 116 [BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" }, 117 [BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" }, 118 [BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" }, 119 [BCM5745x_NPAR] = { "Broadcom BCM5745x NetXtreme-E Ethernet Partition" }, 120 [BCM57508] = { "Broadcom BCM57508 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" }, 121 [BCM57504] = { "Broadcom BCM57504 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" }, 122 [BCM57502] = { "Broadcom BCM57502 NetXtreme-E 10Gb/25Gb/50Gb Ethernet" }, 123 [BCM57508_NPAR] = { "Broadcom BCM57508 NetXtreme-E Ethernet Partition" }, 124 [BCM57504_NPAR] = { "Broadcom BCM57504 NetXtreme-E Ethernet Partition" }, 125 [BCM57502_NPAR] = { "Broadcom BCM57502 NetXtreme-E Ethernet Partition" }, 126 [BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" }, 127 [BCM58804] = { "Broadcom BCM58804 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" }, 128 [BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" }, 129 [NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" }, 130 [NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" }, 131 [NETXTREME_S_VF] = { "Broadcom NetXtreme-S Ethernet Virtual Function" }, 132 [NETXTREME_C_VF_HV] = { "Broadcom NetXtreme-C Virtual Function for Hyper-V" }, 133 [NETXTREME_E_VF_HV] = { "Broadcom NetXtreme-E Virtual Function for Hyper-V" }, 134 [NETXTREME_E_P5_VF] = { "Broadcom BCM5750X NetXtreme-E Ethernet Virtual Function" }, 135 [NETXTREME_E_P5_VF_HV] = { "Broadcom BCM5750X NetXtreme-E Virtual Function for Hyper-V" }, 136 }; 137 138 static const struct pci_device_id bnxt_pci_tbl[] = { 139 { PCI_VDEVICE(BROADCOM, 0x1604), .driver_data = BCM5745x_NPAR }, 140 { PCI_VDEVICE(BROADCOM, 0x1605), .driver_data = BCM5745x_NPAR }, 141 { PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 }, 142 { PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR }, 143 { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 }, 144 { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 }, 145 { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 }, 146 { PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR }, 147 { PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 }, 148 { PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 }, 149 { PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 }, 150 { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 }, 151 { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 }, 152 { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 }, 153 { PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR }, 154 { PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 }, 155 { PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 }, 156 { PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 }, 157 { PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 }, 158 { PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 }, 159 { PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR }, 160 { PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 }, 161 { PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP }, 162 { PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP }, 163 { PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR }, 164 { PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR }, 165 { PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP }, 166 { PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR }, 167 { PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR }, 168 { PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR }, 169 { PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR }, 170 { PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR }, 171 { PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR }, 172 { PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 }, 173 { PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 }, 174 { PCI_VDEVICE(BROADCOM, 0x1750), .driver_data = BCM57508 }, 175 { PCI_VDEVICE(BROADCOM, 0x1751), .driver_data = BCM57504 }, 176 { PCI_VDEVICE(BROADCOM, 0x1752), .driver_data = BCM57502 }, 177 { PCI_VDEVICE(BROADCOM, 0x1800), .driver_data = BCM57508_NPAR }, 178 { PCI_VDEVICE(BROADCOM, 0x1801), .driver_data = BCM57504_NPAR }, 179 { PCI_VDEVICE(BROADCOM, 0x1802), .driver_data = BCM57502_NPAR }, 180 { PCI_VDEVICE(BROADCOM, 0x1803), .driver_data = BCM57508_NPAR }, 181 { PCI_VDEVICE(BROADCOM, 0x1804), .driver_data = BCM57504_NPAR }, 182 { PCI_VDEVICE(BROADCOM, 0x1805), .driver_data = BCM57502_NPAR }, 183 { PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 }, 184 { PCI_VDEVICE(BROADCOM, 0xd804), .driver_data = BCM58804 }, 185 #ifdef CONFIG_BNXT_SRIOV 186 { PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF }, 187 { PCI_VDEVICE(BROADCOM, 0x1607), .driver_data = NETXTREME_E_VF_HV }, 188 { PCI_VDEVICE(BROADCOM, 0x1608), .driver_data = NETXTREME_E_VF_HV }, 189 { PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF }, 190 { PCI_VDEVICE(BROADCOM, 0x16bd), .driver_data = NETXTREME_E_VF_HV }, 191 { PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF }, 192 { PCI_VDEVICE(BROADCOM, 0x16c2), .driver_data = NETXTREME_C_VF_HV }, 193 { PCI_VDEVICE(BROADCOM, 0x16c3), .driver_data = NETXTREME_C_VF_HV }, 194 { PCI_VDEVICE(BROADCOM, 0x16c4), .driver_data = NETXTREME_E_VF_HV }, 195 { PCI_VDEVICE(BROADCOM, 0x16c5), .driver_data = NETXTREME_E_VF_HV }, 196 { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF }, 197 { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF }, 198 { PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF }, 199 { PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF }, 200 { PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF }, 201 { PCI_VDEVICE(BROADCOM, 0x16e6), .driver_data = NETXTREME_C_VF_HV }, 202 { PCI_VDEVICE(BROADCOM, 0x1806), .driver_data = NETXTREME_E_P5_VF }, 203 { PCI_VDEVICE(BROADCOM, 0x1807), .driver_data = NETXTREME_E_P5_VF }, 204 { PCI_VDEVICE(BROADCOM, 0x1808), .driver_data = NETXTREME_E_P5_VF_HV }, 205 { PCI_VDEVICE(BROADCOM, 0x1809), .driver_data = NETXTREME_E_P5_VF_HV }, 206 { PCI_VDEVICE(BROADCOM, 0xd800), .driver_data = NETXTREME_S_VF }, 207 #endif 208 { 0 } 209 }; 210 211 MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl); 212 213 static const u16 bnxt_vf_req_snif[] = { 214 HWRM_FUNC_CFG, 215 HWRM_FUNC_VF_CFG, 216 HWRM_PORT_PHY_QCFG, 217 HWRM_CFA_L2_FILTER_ALLOC, 218 }; 219 220 static const u16 bnxt_async_events_arr[] = { 221 ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE, 222 ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE, 223 ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD, 224 ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED, 225 ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE, 226 ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE, 227 ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE, 228 ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY, 229 ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY, 230 ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION, 231 ASYNC_EVENT_CMPL_EVENT_ID_DEFERRED_RESPONSE, 232 ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG, 233 ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST, 234 ASYNC_EVENT_CMPL_EVENT_ID_PPS_TIMESTAMP, 235 ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT, 236 }; 237 238 static struct workqueue_struct *bnxt_pf_wq; 239 240 static bool bnxt_vf_pciid(enum board_idx idx) 241 { 242 return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF || 243 idx == NETXTREME_S_VF || idx == NETXTREME_C_VF_HV || 244 idx == NETXTREME_E_VF_HV || idx == NETXTREME_E_P5_VF || 245 idx == NETXTREME_E_P5_VF_HV); 246 } 247 248 #define DB_CP_REARM_FLAGS (DB_KEY_CP | DB_IDX_VALID) 249 #define DB_CP_FLAGS (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS) 250 #define DB_CP_IRQ_DIS_FLAGS (DB_KEY_CP | DB_IRQ_DIS) 251 252 #define BNXT_CP_DB_IRQ_DIS(db) \ 253 writel(DB_CP_IRQ_DIS_FLAGS, db) 254 255 #define BNXT_DB_CQ(db, idx) \ 256 writel(DB_CP_FLAGS | RING_CMP(idx), (db)->doorbell) 257 258 #define BNXT_DB_NQ_P5(db, idx) \ 259 bnxt_writeq(bp, (db)->db_key64 | DBR_TYPE_NQ | RING_CMP(idx), \ 260 (db)->doorbell) 261 262 #define BNXT_DB_CQ_ARM(db, idx) \ 263 writel(DB_CP_REARM_FLAGS | RING_CMP(idx), (db)->doorbell) 264 265 #define BNXT_DB_NQ_ARM_P5(db, idx) \ 266 bnxt_writeq(bp, (db)->db_key64 | DBR_TYPE_NQ_ARM | RING_CMP(idx),\ 267 (db)->doorbell) 268 269 static void bnxt_db_nq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx) 270 { 271 if (bp->flags & BNXT_FLAG_CHIP_P5) 272 BNXT_DB_NQ_P5(db, idx); 273 else 274 BNXT_DB_CQ(db, idx); 275 } 276 277 static void bnxt_db_nq_arm(struct bnxt *bp, struct bnxt_db_info *db, u32 idx) 278 { 279 if (bp->flags & BNXT_FLAG_CHIP_P5) 280 BNXT_DB_NQ_ARM_P5(db, idx); 281 else 282 BNXT_DB_CQ_ARM(db, idx); 283 } 284 285 static void bnxt_db_cq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx) 286 { 287 if (bp->flags & BNXT_FLAG_CHIP_P5) 288 bnxt_writeq(bp, db->db_key64 | DBR_TYPE_CQ_ARMALL | 289 RING_CMP(idx), db->doorbell); 290 else 291 BNXT_DB_CQ(db, idx); 292 } 293 294 const u16 bnxt_lhint_arr[] = { 295 TX_BD_FLAGS_LHINT_512_AND_SMALLER, 296 TX_BD_FLAGS_LHINT_512_TO_1023, 297 TX_BD_FLAGS_LHINT_1024_TO_2047, 298 TX_BD_FLAGS_LHINT_1024_TO_2047, 299 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 300 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 301 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 302 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 303 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 304 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 305 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 306 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 307 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 308 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 309 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 310 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 311 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 312 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 313 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 314 }; 315 316 static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb) 317 { 318 struct metadata_dst *md_dst = skb_metadata_dst(skb); 319 320 if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX) 321 return 0; 322 323 return md_dst->u.port_info.port_id; 324 } 325 326 static void bnxt_txr_db_kick(struct bnxt *bp, struct bnxt_tx_ring_info *txr, 327 u16 prod) 328 { 329 bnxt_db_write(bp, &txr->tx_db, prod); 330 txr->kick_pending = 0; 331 } 332 333 static bool bnxt_txr_netif_try_stop_queue(struct bnxt *bp, 334 struct bnxt_tx_ring_info *txr, 335 struct netdev_queue *txq) 336 { 337 netif_tx_stop_queue(txq); 338 339 /* netif_tx_stop_queue() must be done before checking 340 * tx index in bnxt_tx_avail() below, because in 341 * bnxt_tx_int(), we update tx index before checking for 342 * netif_tx_queue_stopped(). 343 */ 344 smp_mb(); 345 if (bnxt_tx_avail(bp, txr) >= bp->tx_wake_thresh) { 346 netif_tx_wake_queue(txq); 347 return false; 348 } 349 350 return true; 351 } 352 353 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev) 354 { 355 struct bnxt *bp = netdev_priv(dev); 356 struct tx_bd *txbd; 357 struct tx_bd_ext *txbd1; 358 struct netdev_queue *txq; 359 int i; 360 dma_addr_t mapping; 361 unsigned int length, pad = 0; 362 u32 len, free_size, vlan_tag_flags, cfa_action, flags; 363 u16 prod, last_frag; 364 struct pci_dev *pdev = bp->pdev; 365 struct bnxt_tx_ring_info *txr; 366 struct bnxt_sw_tx_bd *tx_buf; 367 __le32 lflags = 0; 368 369 i = skb_get_queue_mapping(skb); 370 if (unlikely(i >= bp->tx_nr_rings)) { 371 dev_kfree_skb_any(skb); 372 atomic_long_inc(&dev->tx_dropped); 373 return NETDEV_TX_OK; 374 } 375 376 txq = netdev_get_tx_queue(dev, i); 377 txr = &bp->tx_ring[bp->tx_ring_map[i]]; 378 prod = txr->tx_prod; 379 380 free_size = bnxt_tx_avail(bp, txr); 381 if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) { 382 /* We must have raced with NAPI cleanup */ 383 if (net_ratelimit() && txr->kick_pending) 384 netif_warn(bp, tx_err, dev, 385 "bnxt: ring busy w/ flush pending!\n"); 386 if (bnxt_txr_netif_try_stop_queue(bp, txr, txq)) 387 return NETDEV_TX_BUSY; 388 } 389 390 length = skb->len; 391 len = skb_headlen(skb); 392 last_frag = skb_shinfo(skb)->nr_frags; 393 394 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 395 396 txbd->tx_bd_opaque = prod; 397 398 tx_buf = &txr->tx_buf_ring[prod]; 399 tx_buf->skb = skb; 400 tx_buf->nr_frags = last_frag; 401 402 vlan_tag_flags = 0; 403 cfa_action = bnxt_xmit_get_cfa_action(skb); 404 if (skb_vlan_tag_present(skb)) { 405 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN | 406 skb_vlan_tag_get(skb); 407 /* Currently supports 8021Q, 8021AD vlan offloads 408 * QINQ1, QINQ2, QINQ3 vlan headers are deprecated 409 */ 410 if (skb->vlan_proto == htons(ETH_P_8021Q)) 411 vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT; 412 } 413 414 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) { 415 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; 416 417 if (ptp && ptp->tx_tstamp_en && !skb_is_gso(skb) && 418 atomic_dec_if_positive(&ptp->tx_avail) >= 0) { 419 if (!bnxt_ptp_parse(skb, &ptp->tx_seqid, 420 &ptp->tx_hdr_off)) { 421 if (vlan_tag_flags) 422 ptp->tx_hdr_off += VLAN_HLEN; 423 lflags |= cpu_to_le32(TX_BD_FLAGS_STAMP); 424 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 425 } else { 426 atomic_inc(&bp->ptp_cfg->tx_avail); 427 } 428 } 429 } 430 431 if (unlikely(skb->no_fcs)) 432 lflags |= cpu_to_le32(TX_BD_FLAGS_NO_CRC); 433 434 if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh && 435 !lflags) { 436 struct tx_push_buffer *tx_push_buf = txr->tx_push; 437 struct tx_push_bd *tx_push = &tx_push_buf->push_bd; 438 struct tx_bd_ext *tx_push1 = &tx_push->txbd2; 439 void __iomem *db = txr->tx_db.doorbell; 440 void *pdata = tx_push_buf->data; 441 u64 *end; 442 int j, push_len; 443 444 /* Set COAL_NOW to be ready quickly for the next push */ 445 tx_push->tx_bd_len_flags_type = 446 cpu_to_le32((length << TX_BD_LEN_SHIFT) | 447 TX_BD_TYPE_LONG_TX_BD | 448 TX_BD_FLAGS_LHINT_512_AND_SMALLER | 449 TX_BD_FLAGS_COAL_NOW | 450 TX_BD_FLAGS_PACKET_END | 451 (2 << TX_BD_FLAGS_BD_CNT_SHIFT)); 452 453 if (skb->ip_summed == CHECKSUM_PARTIAL) 454 tx_push1->tx_bd_hsize_lflags = 455 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM); 456 else 457 tx_push1->tx_bd_hsize_lflags = 0; 458 459 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags); 460 tx_push1->tx_bd_cfa_action = 461 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT); 462 463 end = pdata + length; 464 end = PTR_ALIGN(end, 8) - 1; 465 *end = 0; 466 467 skb_copy_from_linear_data(skb, pdata, len); 468 pdata += len; 469 for (j = 0; j < last_frag; j++) { 470 skb_frag_t *frag = &skb_shinfo(skb)->frags[j]; 471 void *fptr; 472 473 fptr = skb_frag_address_safe(frag); 474 if (!fptr) 475 goto normal_tx; 476 477 memcpy(pdata, fptr, skb_frag_size(frag)); 478 pdata += skb_frag_size(frag); 479 } 480 481 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type; 482 txbd->tx_bd_haddr = txr->data_mapping; 483 prod = NEXT_TX(prod); 484 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 485 memcpy(txbd, tx_push1, sizeof(*txbd)); 486 prod = NEXT_TX(prod); 487 tx_push->doorbell = 488 cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | prod); 489 txr->tx_prod = prod; 490 491 tx_buf->is_push = 1; 492 netdev_tx_sent_queue(txq, skb->len); 493 wmb(); /* Sync is_push and byte queue before pushing data */ 494 495 push_len = (length + sizeof(*tx_push) + 7) / 8; 496 if (push_len > 16) { 497 __iowrite64_copy(db, tx_push_buf, 16); 498 __iowrite32_copy(db + 4, tx_push_buf + 1, 499 (push_len - 16) << 1); 500 } else { 501 __iowrite64_copy(db, tx_push_buf, push_len); 502 } 503 504 goto tx_done; 505 } 506 507 normal_tx: 508 if (length < BNXT_MIN_PKT_SIZE) { 509 pad = BNXT_MIN_PKT_SIZE - length; 510 if (skb_pad(skb, pad)) 511 /* SKB already freed. */ 512 goto tx_kick_pending; 513 length = BNXT_MIN_PKT_SIZE; 514 } 515 516 mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE); 517 518 if (unlikely(dma_mapping_error(&pdev->dev, mapping))) 519 goto tx_free; 520 521 dma_unmap_addr_set(tx_buf, mapping, mapping); 522 flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD | 523 ((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT); 524 525 txbd->tx_bd_haddr = cpu_to_le64(mapping); 526 527 prod = NEXT_TX(prod); 528 txbd1 = (struct tx_bd_ext *) 529 &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 530 531 txbd1->tx_bd_hsize_lflags = lflags; 532 if (skb_is_gso(skb)) { 533 u32 hdr_len; 534 535 if (skb->encapsulation) 536 hdr_len = skb_inner_network_offset(skb) + 537 skb_inner_network_header_len(skb) + 538 inner_tcp_hdrlen(skb); 539 else 540 hdr_len = skb_transport_offset(skb) + 541 tcp_hdrlen(skb); 542 543 txbd1->tx_bd_hsize_lflags |= cpu_to_le32(TX_BD_FLAGS_LSO | 544 TX_BD_FLAGS_T_IPID | 545 (hdr_len << (TX_BD_HSIZE_SHIFT - 1))); 546 length = skb_shinfo(skb)->gso_size; 547 txbd1->tx_bd_mss = cpu_to_le32(length); 548 length += hdr_len; 549 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { 550 txbd1->tx_bd_hsize_lflags |= 551 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM); 552 txbd1->tx_bd_mss = 0; 553 } 554 555 length >>= 9; 556 if (unlikely(length >= ARRAY_SIZE(bnxt_lhint_arr))) { 557 dev_warn_ratelimited(&pdev->dev, "Dropped oversize %d bytes TX packet.\n", 558 skb->len); 559 i = 0; 560 goto tx_dma_error; 561 } 562 flags |= bnxt_lhint_arr[length]; 563 txbd->tx_bd_len_flags_type = cpu_to_le32(flags); 564 565 txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags); 566 txbd1->tx_bd_cfa_action = 567 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT); 568 for (i = 0; i < last_frag; i++) { 569 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 570 571 prod = NEXT_TX(prod); 572 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 573 574 len = skb_frag_size(frag); 575 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len, 576 DMA_TO_DEVICE); 577 578 if (unlikely(dma_mapping_error(&pdev->dev, mapping))) 579 goto tx_dma_error; 580 581 tx_buf = &txr->tx_buf_ring[prod]; 582 dma_unmap_addr_set(tx_buf, mapping, mapping); 583 584 txbd->tx_bd_haddr = cpu_to_le64(mapping); 585 586 flags = len << TX_BD_LEN_SHIFT; 587 txbd->tx_bd_len_flags_type = cpu_to_le32(flags); 588 } 589 590 flags &= ~TX_BD_LEN; 591 txbd->tx_bd_len_flags_type = 592 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags | 593 TX_BD_FLAGS_PACKET_END); 594 595 netdev_tx_sent_queue(txq, skb->len); 596 597 skb_tx_timestamp(skb); 598 599 /* Sync BD data before updating doorbell */ 600 wmb(); 601 602 prod = NEXT_TX(prod); 603 txr->tx_prod = prod; 604 605 if (!netdev_xmit_more() || netif_xmit_stopped(txq)) 606 bnxt_txr_db_kick(bp, txr, prod); 607 else 608 txr->kick_pending = 1; 609 610 tx_done: 611 612 if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) { 613 if (netdev_xmit_more() && !tx_buf->is_push) 614 bnxt_txr_db_kick(bp, txr, prod); 615 616 bnxt_txr_netif_try_stop_queue(bp, txr, txq); 617 } 618 return NETDEV_TX_OK; 619 620 tx_dma_error: 621 if (BNXT_TX_PTP_IS_SET(lflags)) 622 atomic_inc(&bp->ptp_cfg->tx_avail); 623 624 last_frag = i; 625 626 /* start back at beginning and unmap skb */ 627 prod = txr->tx_prod; 628 tx_buf = &txr->tx_buf_ring[prod]; 629 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping), 630 skb_headlen(skb), DMA_TO_DEVICE); 631 prod = NEXT_TX(prod); 632 633 /* unmap remaining mapped pages */ 634 for (i = 0; i < last_frag; i++) { 635 prod = NEXT_TX(prod); 636 tx_buf = &txr->tx_buf_ring[prod]; 637 dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping), 638 skb_frag_size(&skb_shinfo(skb)->frags[i]), 639 DMA_TO_DEVICE); 640 } 641 642 tx_free: 643 dev_kfree_skb_any(skb); 644 tx_kick_pending: 645 if (txr->kick_pending) 646 bnxt_txr_db_kick(bp, txr, txr->tx_prod); 647 txr->tx_buf_ring[txr->tx_prod].skb = NULL; 648 atomic_long_inc(&dev->tx_dropped); 649 return NETDEV_TX_OK; 650 } 651 652 static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts) 653 { 654 struct bnxt_tx_ring_info *txr = bnapi->tx_ring; 655 struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index); 656 u16 cons = txr->tx_cons; 657 struct pci_dev *pdev = bp->pdev; 658 int i; 659 unsigned int tx_bytes = 0; 660 661 for (i = 0; i < nr_pkts; i++) { 662 struct bnxt_sw_tx_bd *tx_buf; 663 bool compl_deferred = false; 664 struct sk_buff *skb; 665 int j, last; 666 667 tx_buf = &txr->tx_buf_ring[cons]; 668 cons = NEXT_TX(cons); 669 skb = tx_buf->skb; 670 tx_buf->skb = NULL; 671 672 if (tx_buf->is_push) { 673 tx_buf->is_push = 0; 674 goto next_tx_int; 675 } 676 677 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping), 678 skb_headlen(skb), DMA_TO_DEVICE); 679 last = tx_buf->nr_frags; 680 681 for (j = 0; j < last; j++) { 682 cons = NEXT_TX(cons); 683 tx_buf = &txr->tx_buf_ring[cons]; 684 dma_unmap_page( 685 &pdev->dev, 686 dma_unmap_addr(tx_buf, mapping), 687 skb_frag_size(&skb_shinfo(skb)->frags[j]), 688 DMA_TO_DEVICE); 689 } 690 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)) { 691 if (bp->flags & BNXT_FLAG_CHIP_P5) { 692 if (!bnxt_get_tx_ts_p5(bp, skb)) 693 compl_deferred = true; 694 else 695 atomic_inc(&bp->ptp_cfg->tx_avail); 696 } 697 } 698 699 next_tx_int: 700 cons = NEXT_TX(cons); 701 702 tx_bytes += skb->len; 703 if (!compl_deferred) 704 dev_kfree_skb_any(skb); 705 } 706 707 netdev_tx_completed_queue(txq, nr_pkts, tx_bytes); 708 txr->tx_cons = cons; 709 710 /* Need to make the tx_cons update visible to bnxt_start_xmit() 711 * before checking for netif_tx_queue_stopped(). Without the 712 * memory barrier, there is a small possibility that bnxt_start_xmit() 713 * will miss it and cause the queue to be stopped forever. 714 */ 715 smp_mb(); 716 717 if (unlikely(netif_tx_queue_stopped(txq)) && 718 bnxt_tx_avail(bp, txr) >= bp->tx_wake_thresh && 719 READ_ONCE(txr->dev_state) != BNXT_DEV_STATE_CLOSING) 720 netif_tx_wake_queue(txq); 721 } 722 723 static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping, 724 struct bnxt_rx_ring_info *rxr, 725 gfp_t gfp) 726 { 727 struct device *dev = &bp->pdev->dev; 728 struct page *page; 729 730 page = page_pool_dev_alloc_pages(rxr->page_pool); 731 if (!page) 732 return NULL; 733 734 *mapping = dma_map_page_attrs(dev, page, 0, PAGE_SIZE, bp->rx_dir, 735 DMA_ATTR_WEAK_ORDERING); 736 if (dma_mapping_error(dev, *mapping)) { 737 page_pool_recycle_direct(rxr->page_pool, page); 738 return NULL; 739 } 740 *mapping += bp->rx_dma_offset; 741 return page; 742 } 743 744 static inline u8 *__bnxt_alloc_rx_frag(struct bnxt *bp, dma_addr_t *mapping, 745 gfp_t gfp) 746 { 747 u8 *data; 748 struct pci_dev *pdev = bp->pdev; 749 750 if (gfp == GFP_ATOMIC) 751 data = napi_alloc_frag(bp->rx_buf_size); 752 else 753 data = netdev_alloc_frag(bp->rx_buf_size); 754 if (!data) 755 return NULL; 756 757 *mapping = dma_map_single_attrs(&pdev->dev, data + bp->rx_dma_offset, 758 bp->rx_buf_use_size, bp->rx_dir, 759 DMA_ATTR_WEAK_ORDERING); 760 761 if (dma_mapping_error(&pdev->dev, *mapping)) { 762 skb_free_frag(data); 763 data = NULL; 764 } 765 return data; 766 } 767 768 int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, 769 u16 prod, gfp_t gfp) 770 { 771 struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 772 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod]; 773 dma_addr_t mapping; 774 775 if (BNXT_RX_PAGE_MODE(bp)) { 776 struct page *page = 777 __bnxt_alloc_rx_page(bp, &mapping, rxr, gfp); 778 779 if (!page) 780 return -ENOMEM; 781 782 rx_buf->data = page; 783 rx_buf->data_ptr = page_address(page) + bp->rx_offset; 784 } else { 785 u8 *data = __bnxt_alloc_rx_frag(bp, &mapping, gfp); 786 787 if (!data) 788 return -ENOMEM; 789 790 rx_buf->data = data; 791 rx_buf->data_ptr = data + bp->rx_offset; 792 } 793 rx_buf->mapping = mapping; 794 795 rxbd->rx_bd_haddr = cpu_to_le64(mapping); 796 return 0; 797 } 798 799 void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data) 800 { 801 u16 prod = rxr->rx_prod; 802 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf; 803 struct rx_bd *cons_bd, *prod_bd; 804 805 prod_rx_buf = &rxr->rx_buf_ring[prod]; 806 cons_rx_buf = &rxr->rx_buf_ring[cons]; 807 808 prod_rx_buf->data = data; 809 prod_rx_buf->data_ptr = cons_rx_buf->data_ptr; 810 811 prod_rx_buf->mapping = cons_rx_buf->mapping; 812 813 prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 814 cons_bd = &rxr->rx_desc_ring[RX_RING(cons)][RX_IDX(cons)]; 815 816 prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr; 817 } 818 819 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx) 820 { 821 u16 next, max = rxr->rx_agg_bmap_size; 822 823 next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx); 824 if (next >= max) 825 next = find_first_zero_bit(rxr->rx_agg_bmap, max); 826 return next; 827 } 828 829 static inline int bnxt_alloc_rx_page(struct bnxt *bp, 830 struct bnxt_rx_ring_info *rxr, 831 u16 prod, gfp_t gfp) 832 { 833 struct rx_bd *rxbd = 834 &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 835 struct bnxt_sw_rx_agg_bd *rx_agg_buf; 836 struct pci_dev *pdev = bp->pdev; 837 struct page *page; 838 dma_addr_t mapping; 839 u16 sw_prod = rxr->rx_sw_agg_prod; 840 unsigned int offset = 0; 841 842 if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) { 843 page = rxr->rx_page; 844 if (!page) { 845 page = alloc_page(gfp); 846 if (!page) 847 return -ENOMEM; 848 rxr->rx_page = page; 849 rxr->rx_page_offset = 0; 850 } 851 offset = rxr->rx_page_offset; 852 rxr->rx_page_offset += BNXT_RX_PAGE_SIZE; 853 if (rxr->rx_page_offset == PAGE_SIZE) 854 rxr->rx_page = NULL; 855 else 856 get_page(page); 857 } else { 858 page = alloc_page(gfp); 859 if (!page) 860 return -ENOMEM; 861 } 862 863 mapping = dma_map_page_attrs(&pdev->dev, page, offset, 864 BNXT_RX_PAGE_SIZE, DMA_FROM_DEVICE, 865 DMA_ATTR_WEAK_ORDERING); 866 if (dma_mapping_error(&pdev->dev, mapping)) { 867 __free_page(page); 868 return -EIO; 869 } 870 871 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap))) 872 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod); 873 874 __set_bit(sw_prod, rxr->rx_agg_bmap); 875 rx_agg_buf = &rxr->rx_agg_ring[sw_prod]; 876 rxr->rx_sw_agg_prod = NEXT_RX_AGG(sw_prod); 877 878 rx_agg_buf->page = page; 879 rx_agg_buf->offset = offset; 880 rx_agg_buf->mapping = mapping; 881 rxbd->rx_bd_haddr = cpu_to_le64(mapping); 882 rxbd->rx_bd_opaque = sw_prod; 883 return 0; 884 } 885 886 static struct rx_agg_cmp *bnxt_get_agg(struct bnxt *bp, 887 struct bnxt_cp_ring_info *cpr, 888 u16 cp_cons, u16 curr) 889 { 890 struct rx_agg_cmp *agg; 891 892 cp_cons = RING_CMP(ADV_RAW_CMP(cp_cons, curr)); 893 agg = (struct rx_agg_cmp *) 894 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 895 return agg; 896 } 897 898 static struct rx_agg_cmp *bnxt_get_tpa_agg_p5(struct bnxt *bp, 899 struct bnxt_rx_ring_info *rxr, 900 u16 agg_id, u16 curr) 901 { 902 struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[agg_id]; 903 904 return &tpa_info->agg_arr[curr]; 905 } 906 907 static void bnxt_reuse_rx_agg_bufs(struct bnxt_cp_ring_info *cpr, u16 idx, 908 u16 start, u32 agg_bufs, bool tpa) 909 { 910 struct bnxt_napi *bnapi = cpr->bnapi; 911 struct bnxt *bp = bnapi->bp; 912 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 913 u16 prod = rxr->rx_agg_prod; 914 u16 sw_prod = rxr->rx_sw_agg_prod; 915 bool p5_tpa = false; 916 u32 i; 917 918 if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa) 919 p5_tpa = true; 920 921 for (i = 0; i < agg_bufs; i++) { 922 u16 cons; 923 struct rx_agg_cmp *agg; 924 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf; 925 struct rx_bd *prod_bd; 926 struct page *page; 927 928 if (p5_tpa) 929 agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, start + i); 930 else 931 agg = bnxt_get_agg(bp, cpr, idx, start + i); 932 cons = agg->rx_agg_cmp_opaque; 933 __clear_bit(cons, rxr->rx_agg_bmap); 934 935 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap))) 936 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod); 937 938 __set_bit(sw_prod, rxr->rx_agg_bmap); 939 prod_rx_buf = &rxr->rx_agg_ring[sw_prod]; 940 cons_rx_buf = &rxr->rx_agg_ring[cons]; 941 942 /* It is possible for sw_prod to be equal to cons, so 943 * set cons_rx_buf->page to NULL first. 944 */ 945 page = cons_rx_buf->page; 946 cons_rx_buf->page = NULL; 947 prod_rx_buf->page = page; 948 prod_rx_buf->offset = cons_rx_buf->offset; 949 950 prod_rx_buf->mapping = cons_rx_buf->mapping; 951 952 prod_bd = &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 953 954 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping); 955 prod_bd->rx_bd_opaque = sw_prod; 956 957 prod = NEXT_RX_AGG(prod); 958 sw_prod = NEXT_RX_AGG(sw_prod); 959 } 960 rxr->rx_agg_prod = prod; 961 rxr->rx_sw_agg_prod = sw_prod; 962 } 963 964 static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp, 965 struct bnxt_rx_ring_info *rxr, 966 u16 cons, void *data, u8 *data_ptr, 967 dma_addr_t dma_addr, 968 unsigned int offset_and_len) 969 { 970 unsigned int payload = offset_and_len >> 16; 971 unsigned int len = offset_and_len & 0xffff; 972 skb_frag_t *frag; 973 struct page *page = data; 974 u16 prod = rxr->rx_prod; 975 struct sk_buff *skb; 976 int off, err; 977 978 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC); 979 if (unlikely(err)) { 980 bnxt_reuse_rx_data(rxr, cons, data); 981 return NULL; 982 } 983 dma_addr -= bp->rx_dma_offset; 984 dma_unmap_page_attrs(&bp->pdev->dev, dma_addr, PAGE_SIZE, bp->rx_dir, 985 DMA_ATTR_WEAK_ORDERING); 986 page_pool_release_page(rxr->page_pool, page); 987 988 if (unlikely(!payload)) 989 payload = eth_get_headlen(bp->dev, data_ptr, len); 990 991 skb = napi_alloc_skb(&rxr->bnapi->napi, payload); 992 if (!skb) { 993 __free_page(page); 994 return NULL; 995 } 996 997 off = (void *)data_ptr - page_address(page); 998 skb_add_rx_frag(skb, 0, page, off, len, PAGE_SIZE); 999 memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN, 1000 payload + NET_IP_ALIGN); 1001 1002 frag = &skb_shinfo(skb)->frags[0]; 1003 skb_frag_size_sub(frag, payload); 1004 skb_frag_off_add(frag, payload); 1005 skb->data_len -= payload; 1006 skb->tail += payload; 1007 1008 return skb; 1009 } 1010 1011 static struct sk_buff *bnxt_rx_skb(struct bnxt *bp, 1012 struct bnxt_rx_ring_info *rxr, u16 cons, 1013 void *data, u8 *data_ptr, 1014 dma_addr_t dma_addr, 1015 unsigned int offset_and_len) 1016 { 1017 u16 prod = rxr->rx_prod; 1018 struct sk_buff *skb; 1019 int err; 1020 1021 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC); 1022 if (unlikely(err)) { 1023 bnxt_reuse_rx_data(rxr, cons, data); 1024 return NULL; 1025 } 1026 1027 skb = build_skb(data, bp->rx_buf_size); 1028 dma_unmap_single_attrs(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size, 1029 bp->rx_dir, DMA_ATTR_WEAK_ORDERING); 1030 if (!skb) { 1031 skb_free_frag(data); 1032 return NULL; 1033 } 1034 1035 skb_reserve(skb, bp->rx_offset); 1036 skb_put(skb, offset_and_len & 0xffff); 1037 return skb; 1038 } 1039 1040 static struct sk_buff *bnxt_rx_pages(struct bnxt *bp, 1041 struct bnxt_cp_ring_info *cpr, 1042 struct sk_buff *skb, u16 idx, 1043 u32 agg_bufs, bool tpa) 1044 { 1045 struct bnxt_napi *bnapi = cpr->bnapi; 1046 struct pci_dev *pdev = bp->pdev; 1047 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1048 u16 prod = rxr->rx_agg_prod; 1049 bool p5_tpa = false; 1050 u32 i; 1051 1052 if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa) 1053 p5_tpa = true; 1054 1055 for (i = 0; i < agg_bufs; i++) { 1056 u16 cons, frag_len; 1057 struct rx_agg_cmp *agg; 1058 struct bnxt_sw_rx_agg_bd *cons_rx_buf; 1059 struct page *page; 1060 dma_addr_t mapping; 1061 1062 if (p5_tpa) 1063 agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, i); 1064 else 1065 agg = bnxt_get_agg(bp, cpr, idx, i); 1066 cons = agg->rx_agg_cmp_opaque; 1067 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) & 1068 RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT; 1069 1070 cons_rx_buf = &rxr->rx_agg_ring[cons]; 1071 skb_fill_page_desc(skb, i, cons_rx_buf->page, 1072 cons_rx_buf->offset, frag_len); 1073 __clear_bit(cons, rxr->rx_agg_bmap); 1074 1075 /* It is possible for bnxt_alloc_rx_page() to allocate 1076 * a sw_prod index that equals the cons index, so we 1077 * need to clear the cons entry now. 1078 */ 1079 mapping = cons_rx_buf->mapping; 1080 page = cons_rx_buf->page; 1081 cons_rx_buf->page = NULL; 1082 1083 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) { 1084 struct skb_shared_info *shinfo; 1085 unsigned int nr_frags; 1086 1087 shinfo = skb_shinfo(skb); 1088 nr_frags = --shinfo->nr_frags; 1089 __skb_frag_set_page(&shinfo->frags[nr_frags], NULL); 1090 1091 dev_kfree_skb(skb); 1092 1093 cons_rx_buf->page = page; 1094 1095 /* Update prod since possibly some pages have been 1096 * allocated already. 1097 */ 1098 rxr->rx_agg_prod = prod; 1099 bnxt_reuse_rx_agg_bufs(cpr, idx, i, agg_bufs - i, tpa); 1100 return NULL; 1101 } 1102 1103 dma_unmap_page_attrs(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE, 1104 DMA_FROM_DEVICE, 1105 DMA_ATTR_WEAK_ORDERING); 1106 1107 skb->data_len += frag_len; 1108 skb->len += frag_len; 1109 skb->truesize += PAGE_SIZE; 1110 1111 prod = NEXT_RX_AGG(prod); 1112 } 1113 rxr->rx_agg_prod = prod; 1114 return skb; 1115 } 1116 1117 static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 1118 u8 agg_bufs, u32 *raw_cons) 1119 { 1120 u16 last; 1121 struct rx_agg_cmp *agg; 1122 1123 *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs); 1124 last = RING_CMP(*raw_cons); 1125 agg = (struct rx_agg_cmp *) 1126 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)]; 1127 return RX_AGG_CMP_VALID(agg, *raw_cons); 1128 } 1129 1130 static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data, 1131 unsigned int len, 1132 dma_addr_t mapping) 1133 { 1134 struct bnxt *bp = bnapi->bp; 1135 struct pci_dev *pdev = bp->pdev; 1136 struct sk_buff *skb; 1137 1138 skb = napi_alloc_skb(&bnapi->napi, len); 1139 if (!skb) 1140 return NULL; 1141 1142 dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copy_thresh, 1143 bp->rx_dir); 1144 1145 memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN, 1146 len + NET_IP_ALIGN); 1147 1148 dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copy_thresh, 1149 bp->rx_dir); 1150 1151 skb_put(skb, len); 1152 return skb; 1153 } 1154 1155 static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 1156 u32 *raw_cons, void *cmp) 1157 { 1158 struct rx_cmp *rxcmp = cmp; 1159 u32 tmp_raw_cons = *raw_cons; 1160 u8 cmp_type, agg_bufs = 0; 1161 1162 cmp_type = RX_CMP_TYPE(rxcmp); 1163 1164 if (cmp_type == CMP_TYPE_RX_L2_CMP) { 1165 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) & 1166 RX_CMP_AGG_BUFS) >> 1167 RX_CMP_AGG_BUFS_SHIFT; 1168 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) { 1169 struct rx_tpa_end_cmp *tpa_end = cmp; 1170 1171 if (bp->flags & BNXT_FLAG_CHIP_P5) 1172 return 0; 1173 1174 agg_bufs = TPA_END_AGG_BUFS(tpa_end); 1175 } 1176 1177 if (agg_bufs) { 1178 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons)) 1179 return -EBUSY; 1180 } 1181 *raw_cons = tmp_raw_cons; 1182 return 0; 1183 } 1184 1185 static void bnxt_queue_fw_reset_work(struct bnxt *bp, unsigned long delay) 1186 { 1187 if (!(test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))) 1188 return; 1189 1190 if (BNXT_PF(bp)) 1191 queue_delayed_work(bnxt_pf_wq, &bp->fw_reset_task, delay); 1192 else 1193 schedule_delayed_work(&bp->fw_reset_task, delay); 1194 } 1195 1196 static void bnxt_queue_sp_work(struct bnxt *bp) 1197 { 1198 if (BNXT_PF(bp)) 1199 queue_work(bnxt_pf_wq, &bp->sp_task); 1200 else 1201 schedule_work(&bp->sp_task); 1202 } 1203 1204 static void bnxt_sched_reset(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) 1205 { 1206 if (!rxr->bnapi->in_reset) { 1207 rxr->bnapi->in_reset = true; 1208 if (bp->flags & BNXT_FLAG_CHIP_P5) 1209 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event); 1210 else 1211 set_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event); 1212 bnxt_queue_sp_work(bp); 1213 } 1214 rxr->rx_next_cons = 0xffff; 1215 } 1216 1217 static u16 bnxt_alloc_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id) 1218 { 1219 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map; 1220 u16 idx = agg_id & MAX_TPA_P5_MASK; 1221 1222 if (test_bit(idx, map->agg_idx_bmap)) 1223 idx = find_first_zero_bit(map->agg_idx_bmap, 1224 BNXT_AGG_IDX_BMAP_SIZE); 1225 __set_bit(idx, map->agg_idx_bmap); 1226 map->agg_id_tbl[agg_id] = idx; 1227 return idx; 1228 } 1229 1230 static void bnxt_free_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx) 1231 { 1232 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map; 1233 1234 __clear_bit(idx, map->agg_idx_bmap); 1235 } 1236 1237 static u16 bnxt_lookup_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id) 1238 { 1239 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map; 1240 1241 return map->agg_id_tbl[agg_id]; 1242 } 1243 1244 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, 1245 struct rx_tpa_start_cmp *tpa_start, 1246 struct rx_tpa_start_cmp_ext *tpa_start1) 1247 { 1248 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf; 1249 struct bnxt_tpa_info *tpa_info; 1250 u16 cons, prod, agg_id; 1251 struct rx_bd *prod_bd; 1252 dma_addr_t mapping; 1253 1254 if (bp->flags & BNXT_FLAG_CHIP_P5) { 1255 agg_id = TPA_START_AGG_ID_P5(tpa_start); 1256 agg_id = bnxt_alloc_agg_idx(rxr, agg_id); 1257 } else { 1258 agg_id = TPA_START_AGG_ID(tpa_start); 1259 } 1260 cons = tpa_start->rx_tpa_start_cmp_opaque; 1261 prod = rxr->rx_prod; 1262 cons_rx_buf = &rxr->rx_buf_ring[cons]; 1263 prod_rx_buf = &rxr->rx_buf_ring[prod]; 1264 tpa_info = &rxr->rx_tpa[agg_id]; 1265 1266 if (unlikely(cons != rxr->rx_next_cons || 1267 TPA_START_ERROR(tpa_start))) { 1268 netdev_warn(bp->dev, "TPA cons %x, expected cons %x, error code %x\n", 1269 cons, rxr->rx_next_cons, 1270 TPA_START_ERROR_CODE(tpa_start1)); 1271 bnxt_sched_reset(bp, rxr); 1272 return; 1273 } 1274 /* Store cfa_code in tpa_info to use in tpa_end 1275 * completion processing. 1276 */ 1277 tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1); 1278 prod_rx_buf->data = tpa_info->data; 1279 prod_rx_buf->data_ptr = tpa_info->data_ptr; 1280 1281 mapping = tpa_info->mapping; 1282 prod_rx_buf->mapping = mapping; 1283 1284 prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 1285 1286 prod_bd->rx_bd_haddr = cpu_to_le64(mapping); 1287 1288 tpa_info->data = cons_rx_buf->data; 1289 tpa_info->data_ptr = cons_rx_buf->data_ptr; 1290 cons_rx_buf->data = NULL; 1291 tpa_info->mapping = cons_rx_buf->mapping; 1292 1293 tpa_info->len = 1294 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >> 1295 RX_TPA_START_CMP_LEN_SHIFT; 1296 if (likely(TPA_START_HASH_VALID(tpa_start))) { 1297 u32 hash_type = TPA_START_HASH_TYPE(tpa_start); 1298 1299 tpa_info->hash_type = PKT_HASH_TYPE_L4; 1300 tpa_info->gso_type = SKB_GSO_TCPV4; 1301 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */ 1302 if (hash_type == 3 || TPA_START_IS_IPV6(tpa_start1)) 1303 tpa_info->gso_type = SKB_GSO_TCPV6; 1304 tpa_info->rss_hash = 1305 le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash); 1306 } else { 1307 tpa_info->hash_type = PKT_HASH_TYPE_NONE; 1308 tpa_info->gso_type = 0; 1309 netif_warn(bp, rx_err, bp->dev, "TPA packet without valid hash\n"); 1310 } 1311 tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2); 1312 tpa_info->metadata = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata); 1313 tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info); 1314 tpa_info->agg_count = 0; 1315 1316 rxr->rx_prod = NEXT_RX(prod); 1317 cons = NEXT_RX(cons); 1318 rxr->rx_next_cons = NEXT_RX(cons); 1319 cons_rx_buf = &rxr->rx_buf_ring[cons]; 1320 1321 bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data); 1322 rxr->rx_prod = NEXT_RX(rxr->rx_prod); 1323 cons_rx_buf->data = NULL; 1324 } 1325 1326 static void bnxt_abort_tpa(struct bnxt_cp_ring_info *cpr, u16 idx, u32 agg_bufs) 1327 { 1328 if (agg_bufs) 1329 bnxt_reuse_rx_agg_bufs(cpr, idx, 0, agg_bufs, true); 1330 } 1331 1332 #ifdef CONFIG_INET 1333 static void bnxt_gro_tunnel(struct sk_buff *skb, __be16 ip_proto) 1334 { 1335 struct udphdr *uh = NULL; 1336 1337 if (ip_proto == htons(ETH_P_IP)) { 1338 struct iphdr *iph = (struct iphdr *)skb->data; 1339 1340 if (iph->protocol == IPPROTO_UDP) 1341 uh = (struct udphdr *)(iph + 1); 1342 } else { 1343 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data; 1344 1345 if (iph->nexthdr == IPPROTO_UDP) 1346 uh = (struct udphdr *)(iph + 1); 1347 } 1348 if (uh) { 1349 if (uh->check) 1350 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM; 1351 else 1352 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL; 1353 } 1354 } 1355 #endif 1356 1357 static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info, 1358 int payload_off, int tcp_ts, 1359 struct sk_buff *skb) 1360 { 1361 #ifdef CONFIG_INET 1362 struct tcphdr *th; 1363 int len, nw_off; 1364 u16 outer_ip_off, inner_ip_off, inner_mac_off; 1365 u32 hdr_info = tpa_info->hdr_info; 1366 bool loopback = false; 1367 1368 inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info); 1369 inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info); 1370 outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info); 1371 1372 /* If the packet is an internal loopback packet, the offsets will 1373 * have an extra 4 bytes. 1374 */ 1375 if (inner_mac_off == 4) { 1376 loopback = true; 1377 } else if (inner_mac_off > 4) { 1378 __be16 proto = *((__be16 *)(skb->data + inner_ip_off - 1379 ETH_HLEN - 2)); 1380 1381 /* We only support inner iPv4/ipv6. If we don't see the 1382 * correct protocol ID, it must be a loopback packet where 1383 * the offsets are off by 4. 1384 */ 1385 if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6)) 1386 loopback = true; 1387 } 1388 if (loopback) { 1389 /* internal loopback packet, subtract all offsets by 4 */ 1390 inner_ip_off -= 4; 1391 inner_mac_off -= 4; 1392 outer_ip_off -= 4; 1393 } 1394 1395 nw_off = inner_ip_off - ETH_HLEN; 1396 skb_set_network_header(skb, nw_off); 1397 if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) { 1398 struct ipv6hdr *iph = ipv6_hdr(skb); 1399 1400 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr)); 1401 len = skb->len - skb_transport_offset(skb); 1402 th = tcp_hdr(skb); 1403 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0); 1404 } else { 1405 struct iphdr *iph = ip_hdr(skb); 1406 1407 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr)); 1408 len = skb->len - skb_transport_offset(skb); 1409 th = tcp_hdr(skb); 1410 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0); 1411 } 1412 1413 if (inner_mac_off) { /* tunnel */ 1414 __be16 proto = *((__be16 *)(skb->data + outer_ip_off - 1415 ETH_HLEN - 2)); 1416 1417 bnxt_gro_tunnel(skb, proto); 1418 } 1419 #endif 1420 return skb; 1421 } 1422 1423 static struct sk_buff *bnxt_gro_func_5750x(struct bnxt_tpa_info *tpa_info, 1424 int payload_off, int tcp_ts, 1425 struct sk_buff *skb) 1426 { 1427 #ifdef CONFIG_INET 1428 u16 outer_ip_off, inner_ip_off, inner_mac_off; 1429 u32 hdr_info = tpa_info->hdr_info; 1430 int iphdr_len, nw_off; 1431 1432 inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info); 1433 inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info); 1434 outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info); 1435 1436 nw_off = inner_ip_off - ETH_HLEN; 1437 skb_set_network_header(skb, nw_off); 1438 iphdr_len = (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) ? 1439 sizeof(struct ipv6hdr) : sizeof(struct iphdr); 1440 skb_set_transport_header(skb, nw_off + iphdr_len); 1441 1442 if (inner_mac_off) { /* tunnel */ 1443 __be16 proto = *((__be16 *)(skb->data + outer_ip_off - 1444 ETH_HLEN - 2)); 1445 1446 bnxt_gro_tunnel(skb, proto); 1447 } 1448 #endif 1449 return skb; 1450 } 1451 1452 #define BNXT_IPV4_HDR_SIZE (sizeof(struct iphdr) + sizeof(struct tcphdr)) 1453 #define BNXT_IPV6_HDR_SIZE (sizeof(struct ipv6hdr) + sizeof(struct tcphdr)) 1454 1455 static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info, 1456 int payload_off, int tcp_ts, 1457 struct sk_buff *skb) 1458 { 1459 #ifdef CONFIG_INET 1460 struct tcphdr *th; 1461 int len, nw_off, tcp_opt_len = 0; 1462 1463 if (tcp_ts) 1464 tcp_opt_len = 12; 1465 1466 if (tpa_info->gso_type == SKB_GSO_TCPV4) { 1467 struct iphdr *iph; 1468 1469 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len - 1470 ETH_HLEN; 1471 skb_set_network_header(skb, nw_off); 1472 iph = ip_hdr(skb); 1473 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr)); 1474 len = skb->len - skb_transport_offset(skb); 1475 th = tcp_hdr(skb); 1476 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0); 1477 } else if (tpa_info->gso_type == SKB_GSO_TCPV6) { 1478 struct ipv6hdr *iph; 1479 1480 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len - 1481 ETH_HLEN; 1482 skb_set_network_header(skb, nw_off); 1483 iph = ipv6_hdr(skb); 1484 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr)); 1485 len = skb->len - skb_transport_offset(skb); 1486 th = tcp_hdr(skb); 1487 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0); 1488 } else { 1489 dev_kfree_skb_any(skb); 1490 return NULL; 1491 } 1492 1493 if (nw_off) /* tunnel */ 1494 bnxt_gro_tunnel(skb, skb->protocol); 1495 #endif 1496 return skb; 1497 } 1498 1499 static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp, 1500 struct bnxt_tpa_info *tpa_info, 1501 struct rx_tpa_end_cmp *tpa_end, 1502 struct rx_tpa_end_cmp_ext *tpa_end1, 1503 struct sk_buff *skb) 1504 { 1505 #ifdef CONFIG_INET 1506 int payload_off; 1507 u16 segs; 1508 1509 segs = TPA_END_TPA_SEGS(tpa_end); 1510 if (segs == 1) 1511 return skb; 1512 1513 NAPI_GRO_CB(skb)->count = segs; 1514 skb_shinfo(skb)->gso_size = 1515 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len); 1516 skb_shinfo(skb)->gso_type = tpa_info->gso_type; 1517 if (bp->flags & BNXT_FLAG_CHIP_P5) 1518 payload_off = TPA_END_PAYLOAD_OFF_P5(tpa_end1); 1519 else 1520 payload_off = TPA_END_PAYLOAD_OFF(tpa_end); 1521 skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb); 1522 if (likely(skb)) 1523 tcp_gro_complete(skb); 1524 #endif 1525 return skb; 1526 } 1527 1528 /* Given the cfa_code of a received packet determine which 1529 * netdev (vf-rep or PF) the packet is destined to. 1530 */ 1531 static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code) 1532 { 1533 struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code); 1534 1535 /* if vf-rep dev is NULL, the must belongs to the PF */ 1536 return dev ? dev : bp->dev; 1537 } 1538 1539 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp, 1540 struct bnxt_cp_ring_info *cpr, 1541 u32 *raw_cons, 1542 struct rx_tpa_end_cmp *tpa_end, 1543 struct rx_tpa_end_cmp_ext *tpa_end1, 1544 u8 *event) 1545 { 1546 struct bnxt_napi *bnapi = cpr->bnapi; 1547 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1548 u8 *data_ptr, agg_bufs; 1549 unsigned int len; 1550 struct bnxt_tpa_info *tpa_info; 1551 dma_addr_t mapping; 1552 struct sk_buff *skb; 1553 u16 idx = 0, agg_id; 1554 void *data; 1555 bool gro; 1556 1557 if (unlikely(bnapi->in_reset)) { 1558 int rc = bnxt_discard_rx(bp, cpr, raw_cons, tpa_end); 1559 1560 if (rc < 0) 1561 return ERR_PTR(-EBUSY); 1562 return NULL; 1563 } 1564 1565 if (bp->flags & BNXT_FLAG_CHIP_P5) { 1566 agg_id = TPA_END_AGG_ID_P5(tpa_end); 1567 agg_id = bnxt_lookup_agg_idx(rxr, agg_id); 1568 agg_bufs = TPA_END_AGG_BUFS_P5(tpa_end1); 1569 tpa_info = &rxr->rx_tpa[agg_id]; 1570 if (unlikely(agg_bufs != tpa_info->agg_count)) { 1571 netdev_warn(bp->dev, "TPA end agg_buf %d != expected agg_bufs %d\n", 1572 agg_bufs, tpa_info->agg_count); 1573 agg_bufs = tpa_info->agg_count; 1574 } 1575 tpa_info->agg_count = 0; 1576 *event |= BNXT_AGG_EVENT; 1577 bnxt_free_agg_idx(rxr, agg_id); 1578 idx = agg_id; 1579 gro = !!(bp->flags & BNXT_FLAG_GRO); 1580 } else { 1581 agg_id = TPA_END_AGG_ID(tpa_end); 1582 agg_bufs = TPA_END_AGG_BUFS(tpa_end); 1583 tpa_info = &rxr->rx_tpa[agg_id]; 1584 idx = RING_CMP(*raw_cons); 1585 if (agg_bufs) { 1586 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons)) 1587 return ERR_PTR(-EBUSY); 1588 1589 *event |= BNXT_AGG_EVENT; 1590 idx = NEXT_CMP(idx); 1591 } 1592 gro = !!TPA_END_GRO(tpa_end); 1593 } 1594 data = tpa_info->data; 1595 data_ptr = tpa_info->data_ptr; 1596 prefetch(data_ptr); 1597 len = tpa_info->len; 1598 mapping = tpa_info->mapping; 1599 1600 if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) { 1601 bnxt_abort_tpa(cpr, idx, agg_bufs); 1602 if (agg_bufs > MAX_SKB_FRAGS) 1603 netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n", 1604 agg_bufs, (int)MAX_SKB_FRAGS); 1605 return NULL; 1606 } 1607 1608 if (len <= bp->rx_copy_thresh) { 1609 skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping); 1610 if (!skb) { 1611 bnxt_abort_tpa(cpr, idx, agg_bufs); 1612 cpr->sw_stats.rx.rx_oom_discards += 1; 1613 return NULL; 1614 } 1615 } else { 1616 u8 *new_data; 1617 dma_addr_t new_mapping; 1618 1619 new_data = __bnxt_alloc_rx_frag(bp, &new_mapping, GFP_ATOMIC); 1620 if (!new_data) { 1621 bnxt_abort_tpa(cpr, idx, agg_bufs); 1622 cpr->sw_stats.rx.rx_oom_discards += 1; 1623 return NULL; 1624 } 1625 1626 tpa_info->data = new_data; 1627 tpa_info->data_ptr = new_data + bp->rx_offset; 1628 tpa_info->mapping = new_mapping; 1629 1630 skb = build_skb(data, bp->rx_buf_size); 1631 dma_unmap_single_attrs(&bp->pdev->dev, mapping, 1632 bp->rx_buf_use_size, bp->rx_dir, 1633 DMA_ATTR_WEAK_ORDERING); 1634 1635 if (!skb) { 1636 skb_free_frag(data); 1637 bnxt_abort_tpa(cpr, idx, agg_bufs); 1638 cpr->sw_stats.rx.rx_oom_discards += 1; 1639 return NULL; 1640 } 1641 skb_reserve(skb, bp->rx_offset); 1642 skb_put(skb, len); 1643 } 1644 1645 if (agg_bufs) { 1646 skb = bnxt_rx_pages(bp, cpr, skb, idx, agg_bufs, true); 1647 if (!skb) { 1648 /* Page reuse already handled by bnxt_rx_pages(). */ 1649 cpr->sw_stats.rx.rx_oom_discards += 1; 1650 return NULL; 1651 } 1652 } 1653 1654 skb->protocol = 1655 eth_type_trans(skb, bnxt_get_pkt_dev(bp, tpa_info->cfa_code)); 1656 1657 if (tpa_info->hash_type != PKT_HASH_TYPE_NONE) 1658 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type); 1659 1660 if ((tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) && 1661 (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) { 1662 __be16 vlan_proto = htons(tpa_info->metadata >> 1663 RX_CMP_FLAGS2_METADATA_TPID_SFT); 1664 u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_TCI_MASK; 1665 1666 if (eth_type_vlan(vlan_proto)) { 1667 __vlan_hwaccel_put_tag(skb, vlan_proto, vtag); 1668 } else { 1669 dev_kfree_skb(skb); 1670 return NULL; 1671 } 1672 } 1673 1674 skb_checksum_none_assert(skb); 1675 if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) { 1676 skb->ip_summed = CHECKSUM_UNNECESSARY; 1677 skb->csum_level = 1678 (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3; 1679 } 1680 1681 if (gro) 1682 skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb); 1683 1684 return skb; 1685 } 1686 1687 static void bnxt_tpa_agg(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, 1688 struct rx_agg_cmp *rx_agg) 1689 { 1690 u16 agg_id = TPA_AGG_AGG_ID(rx_agg); 1691 struct bnxt_tpa_info *tpa_info; 1692 1693 agg_id = bnxt_lookup_agg_idx(rxr, agg_id); 1694 tpa_info = &rxr->rx_tpa[agg_id]; 1695 BUG_ON(tpa_info->agg_count >= MAX_SKB_FRAGS); 1696 tpa_info->agg_arr[tpa_info->agg_count++] = *rx_agg; 1697 } 1698 1699 static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi, 1700 struct sk_buff *skb) 1701 { 1702 if (skb->dev != bp->dev) { 1703 /* this packet belongs to a vf-rep */ 1704 bnxt_vf_rep_rx(bp, skb); 1705 return; 1706 } 1707 skb_record_rx_queue(skb, bnapi->index); 1708 napi_gro_receive(&bnapi->napi, skb); 1709 } 1710 1711 /* returns the following: 1712 * 1 - 1 packet successfully received 1713 * 0 - successful TPA_START, packet not completed yet 1714 * -EBUSY - completion ring does not have all the agg buffers yet 1715 * -ENOMEM - packet aborted due to out of memory 1716 * -EIO - packet aborted due to hw error indicated in BD 1717 */ 1718 static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 1719 u32 *raw_cons, u8 *event) 1720 { 1721 struct bnxt_napi *bnapi = cpr->bnapi; 1722 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1723 struct net_device *dev = bp->dev; 1724 struct rx_cmp *rxcmp; 1725 struct rx_cmp_ext *rxcmp1; 1726 u32 tmp_raw_cons = *raw_cons; 1727 u16 cfa_code, cons, prod, cp_cons = RING_CMP(tmp_raw_cons); 1728 struct bnxt_sw_rx_bd *rx_buf; 1729 unsigned int len; 1730 u8 *data_ptr, agg_bufs, cmp_type; 1731 dma_addr_t dma_addr; 1732 struct sk_buff *skb; 1733 u32 flags, misc; 1734 void *data; 1735 int rc = 0; 1736 1737 rxcmp = (struct rx_cmp *) 1738 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1739 1740 cmp_type = RX_CMP_TYPE(rxcmp); 1741 1742 if (cmp_type == CMP_TYPE_RX_TPA_AGG_CMP) { 1743 bnxt_tpa_agg(bp, rxr, (struct rx_agg_cmp *)rxcmp); 1744 goto next_rx_no_prod_no_len; 1745 } 1746 1747 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons); 1748 cp_cons = RING_CMP(tmp_raw_cons); 1749 rxcmp1 = (struct rx_cmp_ext *) 1750 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1751 1752 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons)) 1753 return -EBUSY; 1754 1755 /* The valid test of the entry must be done first before 1756 * reading any further. 1757 */ 1758 dma_rmb(); 1759 prod = rxr->rx_prod; 1760 1761 if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) { 1762 bnxt_tpa_start(bp, rxr, (struct rx_tpa_start_cmp *)rxcmp, 1763 (struct rx_tpa_start_cmp_ext *)rxcmp1); 1764 1765 *event |= BNXT_RX_EVENT; 1766 goto next_rx_no_prod_no_len; 1767 1768 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) { 1769 skb = bnxt_tpa_end(bp, cpr, &tmp_raw_cons, 1770 (struct rx_tpa_end_cmp *)rxcmp, 1771 (struct rx_tpa_end_cmp_ext *)rxcmp1, event); 1772 1773 if (IS_ERR(skb)) 1774 return -EBUSY; 1775 1776 rc = -ENOMEM; 1777 if (likely(skb)) { 1778 bnxt_deliver_skb(bp, bnapi, skb); 1779 rc = 1; 1780 } 1781 *event |= BNXT_RX_EVENT; 1782 goto next_rx_no_prod_no_len; 1783 } 1784 1785 cons = rxcmp->rx_cmp_opaque; 1786 if (unlikely(cons != rxr->rx_next_cons)) { 1787 int rc1 = bnxt_discard_rx(bp, cpr, &tmp_raw_cons, rxcmp); 1788 1789 /* 0xffff is forced error, don't print it */ 1790 if (rxr->rx_next_cons != 0xffff) 1791 netdev_warn(bp->dev, "RX cons %x != expected cons %x\n", 1792 cons, rxr->rx_next_cons); 1793 bnxt_sched_reset(bp, rxr); 1794 if (rc1) 1795 return rc1; 1796 goto next_rx_no_prod_no_len; 1797 } 1798 rx_buf = &rxr->rx_buf_ring[cons]; 1799 data = rx_buf->data; 1800 data_ptr = rx_buf->data_ptr; 1801 prefetch(data_ptr); 1802 1803 misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1); 1804 agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT; 1805 1806 if (agg_bufs) { 1807 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons)) 1808 return -EBUSY; 1809 1810 cp_cons = NEXT_CMP(cp_cons); 1811 *event |= BNXT_AGG_EVENT; 1812 } 1813 *event |= BNXT_RX_EVENT; 1814 1815 rx_buf->data = NULL; 1816 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) { 1817 u32 rx_err = le32_to_cpu(rxcmp1->rx_cmp_cfa_code_errors_v2); 1818 1819 bnxt_reuse_rx_data(rxr, cons, data); 1820 if (agg_bufs) 1821 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, agg_bufs, 1822 false); 1823 1824 rc = -EIO; 1825 if (rx_err & RX_CMPL_ERRORS_BUFFER_ERROR_MASK) { 1826 bnapi->cp_ring.sw_stats.rx.rx_buf_errors++; 1827 if (!(bp->flags & BNXT_FLAG_CHIP_P5) && 1828 !(bp->fw_cap & BNXT_FW_CAP_RING_MONITOR)) { 1829 netdev_warn_once(bp->dev, "RX buffer error %x\n", 1830 rx_err); 1831 bnxt_sched_reset(bp, rxr); 1832 } 1833 } 1834 goto next_rx_no_len; 1835 } 1836 1837 flags = le32_to_cpu(rxcmp->rx_cmp_len_flags_type); 1838 len = flags >> RX_CMP_LEN_SHIFT; 1839 dma_addr = rx_buf->mapping; 1840 1841 if (bnxt_rx_xdp(bp, rxr, cons, data, &data_ptr, &len, event)) { 1842 rc = 1; 1843 goto next_rx; 1844 } 1845 1846 if (len <= bp->rx_copy_thresh) { 1847 skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr); 1848 bnxt_reuse_rx_data(rxr, cons, data); 1849 if (!skb) { 1850 if (agg_bufs) 1851 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, 1852 agg_bufs, false); 1853 cpr->sw_stats.rx.rx_oom_discards += 1; 1854 rc = -ENOMEM; 1855 goto next_rx; 1856 } 1857 } else { 1858 u32 payload; 1859 1860 if (rx_buf->data_ptr == data_ptr) 1861 payload = misc & RX_CMP_PAYLOAD_OFFSET; 1862 else 1863 payload = 0; 1864 skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr, 1865 payload | len); 1866 if (!skb) { 1867 cpr->sw_stats.rx.rx_oom_discards += 1; 1868 rc = -ENOMEM; 1869 goto next_rx; 1870 } 1871 } 1872 1873 if (agg_bufs) { 1874 skb = bnxt_rx_pages(bp, cpr, skb, cp_cons, agg_bufs, false); 1875 if (!skb) { 1876 cpr->sw_stats.rx.rx_oom_discards += 1; 1877 rc = -ENOMEM; 1878 goto next_rx; 1879 } 1880 } 1881 1882 if (RX_CMP_HASH_VALID(rxcmp)) { 1883 u32 hash_type = RX_CMP_HASH_TYPE(rxcmp); 1884 enum pkt_hash_types type = PKT_HASH_TYPE_L4; 1885 1886 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */ 1887 if (hash_type != 1 && hash_type != 3) 1888 type = PKT_HASH_TYPE_L3; 1889 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type); 1890 } 1891 1892 cfa_code = RX_CMP_CFA_CODE(rxcmp1); 1893 skb->protocol = eth_type_trans(skb, bnxt_get_pkt_dev(bp, cfa_code)); 1894 1895 if ((rxcmp1->rx_cmp_flags2 & 1896 cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) && 1897 (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) { 1898 u32 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data); 1899 u16 vtag = meta_data & RX_CMP_FLAGS2_METADATA_TCI_MASK; 1900 __be16 vlan_proto = htons(meta_data >> 1901 RX_CMP_FLAGS2_METADATA_TPID_SFT); 1902 1903 if (eth_type_vlan(vlan_proto)) { 1904 __vlan_hwaccel_put_tag(skb, vlan_proto, vtag); 1905 } else { 1906 dev_kfree_skb(skb); 1907 goto next_rx; 1908 } 1909 } 1910 1911 skb_checksum_none_assert(skb); 1912 if (RX_CMP_L4_CS_OK(rxcmp1)) { 1913 if (dev->features & NETIF_F_RXCSUM) { 1914 skb->ip_summed = CHECKSUM_UNNECESSARY; 1915 skb->csum_level = RX_CMP_ENCAP(rxcmp1); 1916 } 1917 } else { 1918 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) { 1919 if (dev->features & NETIF_F_RXCSUM) 1920 bnapi->cp_ring.sw_stats.rx.rx_l4_csum_errors++; 1921 } 1922 } 1923 1924 if (unlikely((flags & RX_CMP_FLAGS_ITYPES_MASK) == 1925 RX_CMP_FLAGS_ITYPE_PTP_W_TS)) { 1926 if (bp->flags & BNXT_FLAG_CHIP_P5) { 1927 u32 cmpl_ts = le32_to_cpu(rxcmp1->rx_cmp_timestamp); 1928 u64 ns, ts; 1929 1930 if (!bnxt_get_rx_ts_p5(bp, &ts, cmpl_ts)) { 1931 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; 1932 1933 spin_lock_bh(&ptp->ptp_lock); 1934 ns = timecounter_cyc2time(&ptp->tc, ts); 1935 spin_unlock_bh(&ptp->ptp_lock); 1936 memset(skb_hwtstamps(skb), 0, 1937 sizeof(*skb_hwtstamps(skb))); 1938 skb_hwtstamps(skb)->hwtstamp = ns_to_ktime(ns); 1939 } 1940 } 1941 } 1942 bnxt_deliver_skb(bp, bnapi, skb); 1943 rc = 1; 1944 1945 next_rx: 1946 cpr->rx_packets += 1; 1947 cpr->rx_bytes += len; 1948 1949 next_rx_no_len: 1950 rxr->rx_prod = NEXT_RX(prod); 1951 rxr->rx_next_cons = NEXT_RX(cons); 1952 1953 next_rx_no_prod_no_len: 1954 *raw_cons = tmp_raw_cons; 1955 1956 return rc; 1957 } 1958 1959 /* In netpoll mode, if we are using a combined completion ring, we need to 1960 * discard the rx packets and recycle the buffers. 1961 */ 1962 static int bnxt_force_rx_discard(struct bnxt *bp, 1963 struct bnxt_cp_ring_info *cpr, 1964 u32 *raw_cons, u8 *event) 1965 { 1966 u32 tmp_raw_cons = *raw_cons; 1967 struct rx_cmp_ext *rxcmp1; 1968 struct rx_cmp *rxcmp; 1969 u16 cp_cons; 1970 u8 cmp_type; 1971 int rc; 1972 1973 cp_cons = RING_CMP(tmp_raw_cons); 1974 rxcmp = (struct rx_cmp *) 1975 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1976 1977 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons); 1978 cp_cons = RING_CMP(tmp_raw_cons); 1979 rxcmp1 = (struct rx_cmp_ext *) 1980 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1981 1982 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons)) 1983 return -EBUSY; 1984 1985 /* The valid test of the entry must be done first before 1986 * reading any further. 1987 */ 1988 dma_rmb(); 1989 cmp_type = RX_CMP_TYPE(rxcmp); 1990 if (cmp_type == CMP_TYPE_RX_L2_CMP) { 1991 rxcmp1->rx_cmp_cfa_code_errors_v2 |= 1992 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR); 1993 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) { 1994 struct rx_tpa_end_cmp_ext *tpa_end1; 1995 1996 tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1; 1997 tpa_end1->rx_tpa_end_cmp_errors_v2 |= 1998 cpu_to_le32(RX_TPA_END_CMP_ERRORS); 1999 } 2000 rc = bnxt_rx_pkt(bp, cpr, raw_cons, event); 2001 if (rc && rc != -EBUSY) 2002 cpr->sw_stats.rx.rx_netpoll_discards += 1; 2003 return rc; 2004 } 2005 2006 u32 bnxt_fw_health_readl(struct bnxt *bp, int reg_idx) 2007 { 2008 struct bnxt_fw_health *fw_health = bp->fw_health; 2009 u32 reg = fw_health->regs[reg_idx]; 2010 u32 reg_type, reg_off, val = 0; 2011 2012 reg_type = BNXT_FW_HEALTH_REG_TYPE(reg); 2013 reg_off = BNXT_FW_HEALTH_REG_OFF(reg); 2014 switch (reg_type) { 2015 case BNXT_FW_HEALTH_REG_TYPE_CFG: 2016 pci_read_config_dword(bp->pdev, reg_off, &val); 2017 break; 2018 case BNXT_FW_HEALTH_REG_TYPE_GRC: 2019 reg_off = fw_health->mapped_regs[reg_idx]; 2020 fallthrough; 2021 case BNXT_FW_HEALTH_REG_TYPE_BAR0: 2022 val = readl(bp->bar0 + reg_off); 2023 break; 2024 case BNXT_FW_HEALTH_REG_TYPE_BAR1: 2025 val = readl(bp->bar1 + reg_off); 2026 break; 2027 } 2028 if (reg_idx == BNXT_FW_RESET_INPROG_REG) 2029 val &= fw_health->fw_reset_inprog_reg_mask; 2030 return val; 2031 } 2032 2033 static u16 bnxt_agg_ring_id_to_grp_idx(struct bnxt *bp, u16 ring_id) 2034 { 2035 int i; 2036 2037 for (i = 0; i < bp->rx_nr_rings; i++) { 2038 u16 grp_idx = bp->rx_ring[i].bnapi->index; 2039 struct bnxt_ring_grp_info *grp_info; 2040 2041 grp_info = &bp->grp_info[grp_idx]; 2042 if (grp_info->agg_fw_ring_id == ring_id) 2043 return grp_idx; 2044 } 2045 return INVALID_HW_RING_ID; 2046 } 2047 2048 static void bnxt_event_error_report(struct bnxt *bp, u32 data1, u32 data2) 2049 { 2050 u32 err_type = BNXT_EVENT_ERROR_REPORT_TYPE(data1); 2051 2052 switch (err_type) { 2053 case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_INVALID_SIGNAL: 2054 netdev_err(bp->dev, "1PPS: Received invalid signal on pin%lu from the external source. Please fix the signal and reconfigure the pin\n", 2055 BNXT_EVENT_INVALID_SIGNAL_DATA(data2)); 2056 break; 2057 case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_PAUSE_STORM: 2058 netdev_warn(bp->dev, "Pause Storm detected!\n"); 2059 break; 2060 case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_DOORBELL_DROP_THRESHOLD: 2061 netdev_warn(bp->dev, "One or more MMIO doorbells dropped by the device!\n"); 2062 break; 2063 default: 2064 netdev_err(bp->dev, "FW reported unknown error type %u\n", 2065 err_type); 2066 break; 2067 } 2068 } 2069 2070 #define BNXT_GET_EVENT_PORT(data) \ 2071 ((data) & \ 2072 ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK) 2073 2074 #define BNXT_EVENT_RING_TYPE(data2) \ 2075 ((data2) & \ 2076 ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_MASK) 2077 2078 #define BNXT_EVENT_RING_TYPE_RX(data2) \ 2079 (BNXT_EVENT_RING_TYPE(data2) == \ 2080 ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_RX) 2081 2082 static int bnxt_async_event_process(struct bnxt *bp, 2083 struct hwrm_async_event_cmpl *cmpl) 2084 { 2085 u16 event_id = le16_to_cpu(cmpl->event_id); 2086 u32 data1 = le32_to_cpu(cmpl->event_data1); 2087 u32 data2 = le32_to_cpu(cmpl->event_data2); 2088 2089 netdev_dbg(bp->dev, "hwrm event 0x%x {0x%x, 0x%x}\n", 2090 event_id, data1, data2); 2091 2092 /* TODO CHIMP_FW: Define event id's for link change, error etc */ 2093 switch (event_id) { 2094 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: { 2095 struct bnxt_link_info *link_info = &bp->link_info; 2096 2097 if (BNXT_VF(bp)) 2098 goto async_event_process_exit; 2099 2100 /* print unsupported speed warning in forced speed mode only */ 2101 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED) && 2102 (data1 & 0x20000)) { 2103 u16 fw_speed = link_info->force_link_speed; 2104 u32 speed = bnxt_fw_to_ethtool_speed(fw_speed); 2105 2106 if (speed != SPEED_UNKNOWN) 2107 netdev_warn(bp->dev, "Link speed %d no longer supported\n", 2108 speed); 2109 } 2110 set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event); 2111 } 2112 fallthrough; 2113 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE: 2114 case ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE: 2115 set_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT, &bp->sp_event); 2116 fallthrough; 2117 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE: 2118 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event); 2119 break; 2120 case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD: 2121 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event); 2122 break; 2123 case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: { 2124 u16 port_id = BNXT_GET_EVENT_PORT(data1); 2125 2126 if (BNXT_VF(bp)) 2127 break; 2128 2129 if (bp->pf.port_id != port_id) 2130 break; 2131 2132 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event); 2133 break; 2134 } 2135 case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE: 2136 if (BNXT_PF(bp)) 2137 goto async_event_process_exit; 2138 set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event); 2139 break; 2140 case ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY: { 2141 char *type_str = "Solicited"; 2142 2143 if (!bp->fw_health) 2144 goto async_event_process_exit; 2145 2146 bp->fw_reset_timestamp = jiffies; 2147 bp->fw_reset_min_dsecs = cmpl->timestamp_lo; 2148 if (!bp->fw_reset_min_dsecs) 2149 bp->fw_reset_min_dsecs = BNXT_DFLT_FW_RST_MIN_DSECS; 2150 bp->fw_reset_max_dsecs = le16_to_cpu(cmpl->timestamp_hi); 2151 if (!bp->fw_reset_max_dsecs) 2152 bp->fw_reset_max_dsecs = BNXT_DFLT_FW_RST_MAX_DSECS; 2153 if (EVENT_DATA1_RESET_NOTIFY_FW_ACTIVATION(data1)) { 2154 set_bit(BNXT_STATE_FW_ACTIVATE_RESET, &bp->state); 2155 } else if (EVENT_DATA1_RESET_NOTIFY_FATAL(data1)) { 2156 type_str = "Fatal"; 2157 bp->fw_health->fatalities++; 2158 set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state); 2159 } else if (data2 && BNXT_FW_STATUS_HEALTHY != 2160 EVENT_DATA2_RESET_NOTIFY_FW_STATUS_CODE(data2)) { 2161 type_str = "Non-fatal"; 2162 bp->fw_health->survivals++; 2163 set_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state); 2164 } 2165 netif_warn(bp, hw, bp->dev, 2166 "%s firmware reset event, data1: 0x%x, data2: 0x%x, min wait %u ms, max wait %u ms\n", 2167 type_str, data1, data2, 2168 bp->fw_reset_min_dsecs * 100, 2169 bp->fw_reset_max_dsecs * 100); 2170 set_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event); 2171 break; 2172 } 2173 case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY: { 2174 struct bnxt_fw_health *fw_health = bp->fw_health; 2175 char *status_desc = "healthy"; 2176 u32 status; 2177 2178 if (!fw_health) 2179 goto async_event_process_exit; 2180 2181 if (!EVENT_DATA1_RECOVERY_ENABLED(data1)) { 2182 fw_health->enabled = false; 2183 netif_info(bp, drv, bp->dev, "Driver recovery watchdog is disabled\n"); 2184 break; 2185 } 2186 fw_health->primary = EVENT_DATA1_RECOVERY_MASTER_FUNC(data1); 2187 fw_health->tmr_multiplier = 2188 DIV_ROUND_UP(fw_health->polling_dsecs * HZ, 2189 bp->current_interval * 10); 2190 fw_health->tmr_counter = fw_health->tmr_multiplier; 2191 if (!fw_health->enabled) 2192 fw_health->last_fw_heartbeat = 2193 bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG); 2194 fw_health->last_fw_reset_cnt = 2195 bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG); 2196 status = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG); 2197 if (status != BNXT_FW_STATUS_HEALTHY) 2198 status_desc = "unhealthy"; 2199 netif_info(bp, drv, bp->dev, 2200 "Driver recovery watchdog, role: %s, firmware status: 0x%x (%s), resets: %u\n", 2201 fw_health->primary ? "primary" : "backup", status, 2202 status_desc, fw_health->last_fw_reset_cnt); 2203 if (!fw_health->enabled) { 2204 /* Make sure tmr_counter is set and visible to 2205 * bnxt_health_check() before setting enabled to true. 2206 */ 2207 smp_wmb(); 2208 fw_health->enabled = true; 2209 } 2210 goto async_event_process_exit; 2211 } 2212 case ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION: 2213 netif_notice(bp, hw, bp->dev, 2214 "Received firmware debug notification, data1: 0x%x, data2: 0x%x\n", 2215 data1, data2); 2216 goto async_event_process_exit; 2217 case ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG: { 2218 struct bnxt_rx_ring_info *rxr; 2219 u16 grp_idx; 2220 2221 if (bp->flags & BNXT_FLAG_CHIP_P5) 2222 goto async_event_process_exit; 2223 2224 netdev_warn(bp->dev, "Ring monitor event, ring type %lu id 0x%x\n", 2225 BNXT_EVENT_RING_TYPE(data2), data1); 2226 if (!BNXT_EVENT_RING_TYPE_RX(data2)) 2227 goto async_event_process_exit; 2228 2229 grp_idx = bnxt_agg_ring_id_to_grp_idx(bp, data1); 2230 if (grp_idx == INVALID_HW_RING_ID) { 2231 netdev_warn(bp->dev, "Unknown RX agg ring id 0x%x\n", 2232 data1); 2233 goto async_event_process_exit; 2234 } 2235 rxr = bp->bnapi[grp_idx]->rx_ring; 2236 bnxt_sched_reset(bp, rxr); 2237 goto async_event_process_exit; 2238 } 2239 case ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST: { 2240 struct bnxt_fw_health *fw_health = bp->fw_health; 2241 2242 netif_notice(bp, hw, bp->dev, 2243 "Received firmware echo request, data1: 0x%x, data2: 0x%x\n", 2244 data1, data2); 2245 if (fw_health) { 2246 fw_health->echo_req_data1 = data1; 2247 fw_health->echo_req_data2 = data2; 2248 set_bit(BNXT_FW_ECHO_REQUEST_SP_EVENT, &bp->sp_event); 2249 break; 2250 } 2251 goto async_event_process_exit; 2252 } 2253 case ASYNC_EVENT_CMPL_EVENT_ID_PPS_TIMESTAMP: { 2254 bnxt_ptp_pps_event(bp, data1, data2); 2255 goto async_event_process_exit; 2256 } 2257 case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT: { 2258 bnxt_event_error_report(bp, data1, data2); 2259 goto async_event_process_exit; 2260 } 2261 case ASYNC_EVENT_CMPL_EVENT_ID_DEFERRED_RESPONSE: { 2262 u16 seq_id = le32_to_cpu(cmpl->event_data2) & 0xffff; 2263 2264 hwrm_update_token(bp, seq_id, BNXT_HWRM_DEFERRED); 2265 goto async_event_process_exit; 2266 } 2267 default: 2268 goto async_event_process_exit; 2269 } 2270 bnxt_queue_sp_work(bp); 2271 async_event_process_exit: 2272 bnxt_ulp_async_events(bp, cmpl); 2273 return 0; 2274 } 2275 2276 static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp) 2277 { 2278 u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id; 2279 struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp; 2280 struct hwrm_fwd_req_cmpl *fwd_req_cmpl = 2281 (struct hwrm_fwd_req_cmpl *)txcmp; 2282 2283 switch (cmpl_type) { 2284 case CMPL_BASE_TYPE_HWRM_DONE: 2285 seq_id = le16_to_cpu(h_cmpl->sequence_id); 2286 hwrm_update_token(bp, seq_id, BNXT_HWRM_COMPLETE); 2287 break; 2288 2289 case CMPL_BASE_TYPE_HWRM_FWD_REQ: 2290 vf_id = le16_to_cpu(fwd_req_cmpl->source_id); 2291 2292 if ((vf_id < bp->pf.first_vf_id) || 2293 (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) { 2294 netdev_err(bp->dev, "Msg contains invalid VF id %x\n", 2295 vf_id); 2296 return -EINVAL; 2297 } 2298 2299 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap); 2300 set_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event); 2301 bnxt_queue_sp_work(bp); 2302 break; 2303 2304 case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT: 2305 bnxt_async_event_process(bp, 2306 (struct hwrm_async_event_cmpl *)txcmp); 2307 break; 2308 2309 default: 2310 break; 2311 } 2312 2313 return 0; 2314 } 2315 2316 static irqreturn_t bnxt_msix(int irq, void *dev_instance) 2317 { 2318 struct bnxt_napi *bnapi = dev_instance; 2319 struct bnxt *bp = bnapi->bp; 2320 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2321 u32 cons = RING_CMP(cpr->cp_raw_cons); 2322 2323 cpr->event_ctr++; 2324 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]); 2325 napi_schedule(&bnapi->napi); 2326 return IRQ_HANDLED; 2327 } 2328 2329 static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr) 2330 { 2331 u32 raw_cons = cpr->cp_raw_cons; 2332 u16 cons = RING_CMP(raw_cons); 2333 struct tx_cmp *txcmp; 2334 2335 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]; 2336 2337 return TX_CMP_VALID(txcmp, raw_cons); 2338 } 2339 2340 static irqreturn_t bnxt_inta(int irq, void *dev_instance) 2341 { 2342 struct bnxt_napi *bnapi = dev_instance; 2343 struct bnxt *bp = bnapi->bp; 2344 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2345 u32 cons = RING_CMP(cpr->cp_raw_cons); 2346 u32 int_status; 2347 2348 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]); 2349 2350 if (!bnxt_has_work(bp, cpr)) { 2351 int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS); 2352 /* return if erroneous interrupt */ 2353 if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id))) 2354 return IRQ_NONE; 2355 } 2356 2357 /* disable ring IRQ */ 2358 BNXT_CP_DB_IRQ_DIS(cpr->cp_db.doorbell); 2359 2360 /* Return here if interrupt is shared and is disabled. */ 2361 if (unlikely(atomic_read(&bp->intr_sem) != 0)) 2362 return IRQ_HANDLED; 2363 2364 napi_schedule(&bnapi->napi); 2365 return IRQ_HANDLED; 2366 } 2367 2368 static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 2369 int budget) 2370 { 2371 struct bnxt_napi *bnapi = cpr->bnapi; 2372 u32 raw_cons = cpr->cp_raw_cons; 2373 u32 cons; 2374 int tx_pkts = 0; 2375 int rx_pkts = 0; 2376 u8 event = 0; 2377 struct tx_cmp *txcmp; 2378 2379 cpr->has_more_work = 0; 2380 cpr->had_work_done = 1; 2381 while (1) { 2382 int rc; 2383 2384 cons = RING_CMP(raw_cons); 2385 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]; 2386 2387 if (!TX_CMP_VALID(txcmp, raw_cons)) 2388 break; 2389 2390 /* The valid test of the entry must be done first before 2391 * reading any further. 2392 */ 2393 dma_rmb(); 2394 if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) { 2395 tx_pkts++; 2396 /* return full budget so NAPI will complete. */ 2397 if (unlikely(tx_pkts >= bp->tx_wake_thresh)) { 2398 rx_pkts = budget; 2399 raw_cons = NEXT_RAW_CMP(raw_cons); 2400 if (budget) 2401 cpr->has_more_work = 1; 2402 break; 2403 } 2404 } else if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) { 2405 if (likely(budget)) 2406 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event); 2407 else 2408 rc = bnxt_force_rx_discard(bp, cpr, &raw_cons, 2409 &event); 2410 if (likely(rc >= 0)) 2411 rx_pkts += rc; 2412 /* Increment rx_pkts when rc is -ENOMEM to count towards 2413 * the NAPI budget. Otherwise, we may potentially loop 2414 * here forever if we consistently cannot allocate 2415 * buffers. 2416 */ 2417 else if (rc == -ENOMEM && budget) 2418 rx_pkts++; 2419 else if (rc == -EBUSY) /* partial completion */ 2420 break; 2421 } else if (unlikely((TX_CMP_TYPE(txcmp) == 2422 CMPL_BASE_TYPE_HWRM_DONE) || 2423 (TX_CMP_TYPE(txcmp) == 2424 CMPL_BASE_TYPE_HWRM_FWD_REQ) || 2425 (TX_CMP_TYPE(txcmp) == 2426 CMPL_BASE_TYPE_HWRM_ASYNC_EVENT))) { 2427 bnxt_hwrm_handler(bp, txcmp); 2428 } 2429 raw_cons = NEXT_RAW_CMP(raw_cons); 2430 2431 if (rx_pkts && rx_pkts == budget) { 2432 cpr->has_more_work = 1; 2433 break; 2434 } 2435 } 2436 2437 if (event & BNXT_REDIRECT_EVENT) 2438 xdp_do_flush(); 2439 2440 if (event & BNXT_TX_EVENT) { 2441 struct bnxt_tx_ring_info *txr = bnapi->tx_ring; 2442 u16 prod = txr->tx_prod; 2443 2444 /* Sync BD data before updating doorbell */ 2445 wmb(); 2446 2447 bnxt_db_write_relaxed(bp, &txr->tx_db, prod); 2448 } 2449 2450 cpr->cp_raw_cons = raw_cons; 2451 bnapi->tx_pkts += tx_pkts; 2452 bnapi->events |= event; 2453 return rx_pkts; 2454 } 2455 2456 static void __bnxt_poll_work_done(struct bnxt *bp, struct bnxt_napi *bnapi) 2457 { 2458 if (bnapi->tx_pkts) { 2459 bnapi->tx_int(bp, bnapi, bnapi->tx_pkts); 2460 bnapi->tx_pkts = 0; 2461 } 2462 2463 if ((bnapi->events & BNXT_RX_EVENT) && !(bnapi->in_reset)) { 2464 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 2465 2466 if (bnapi->events & BNXT_AGG_EVENT) 2467 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); 2468 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 2469 } 2470 bnapi->events = 0; 2471 } 2472 2473 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 2474 int budget) 2475 { 2476 struct bnxt_napi *bnapi = cpr->bnapi; 2477 int rx_pkts; 2478 2479 rx_pkts = __bnxt_poll_work(bp, cpr, budget); 2480 2481 /* ACK completion ring before freeing tx ring and producing new 2482 * buffers in rx/agg rings to prevent overflowing the completion 2483 * ring. 2484 */ 2485 bnxt_db_cq(bp, &cpr->cp_db, cpr->cp_raw_cons); 2486 2487 __bnxt_poll_work_done(bp, bnapi); 2488 return rx_pkts; 2489 } 2490 2491 static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget) 2492 { 2493 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi); 2494 struct bnxt *bp = bnapi->bp; 2495 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2496 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 2497 struct tx_cmp *txcmp; 2498 struct rx_cmp_ext *rxcmp1; 2499 u32 cp_cons, tmp_raw_cons; 2500 u32 raw_cons = cpr->cp_raw_cons; 2501 u32 rx_pkts = 0; 2502 u8 event = 0; 2503 2504 while (1) { 2505 int rc; 2506 2507 cp_cons = RING_CMP(raw_cons); 2508 txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 2509 2510 if (!TX_CMP_VALID(txcmp, raw_cons)) 2511 break; 2512 2513 /* The valid test of the entry must be done first before 2514 * reading any further. 2515 */ 2516 dma_rmb(); 2517 if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) { 2518 tmp_raw_cons = NEXT_RAW_CMP(raw_cons); 2519 cp_cons = RING_CMP(tmp_raw_cons); 2520 rxcmp1 = (struct rx_cmp_ext *) 2521 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 2522 2523 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons)) 2524 break; 2525 2526 /* force an error to recycle the buffer */ 2527 rxcmp1->rx_cmp_cfa_code_errors_v2 |= 2528 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR); 2529 2530 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event); 2531 if (likely(rc == -EIO) && budget) 2532 rx_pkts++; 2533 else if (rc == -EBUSY) /* partial completion */ 2534 break; 2535 } else if (unlikely(TX_CMP_TYPE(txcmp) == 2536 CMPL_BASE_TYPE_HWRM_DONE)) { 2537 bnxt_hwrm_handler(bp, txcmp); 2538 } else { 2539 netdev_err(bp->dev, 2540 "Invalid completion received on special ring\n"); 2541 } 2542 raw_cons = NEXT_RAW_CMP(raw_cons); 2543 2544 if (rx_pkts == budget) 2545 break; 2546 } 2547 2548 cpr->cp_raw_cons = raw_cons; 2549 BNXT_DB_CQ(&cpr->cp_db, cpr->cp_raw_cons); 2550 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 2551 2552 if (event & BNXT_AGG_EVENT) 2553 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); 2554 2555 if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) { 2556 napi_complete_done(napi, rx_pkts); 2557 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons); 2558 } 2559 return rx_pkts; 2560 } 2561 2562 static int bnxt_poll(struct napi_struct *napi, int budget) 2563 { 2564 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi); 2565 struct bnxt *bp = bnapi->bp; 2566 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2567 int work_done = 0; 2568 2569 if (unlikely(test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))) { 2570 napi_complete(napi); 2571 return 0; 2572 } 2573 while (1) { 2574 work_done += bnxt_poll_work(bp, cpr, budget - work_done); 2575 2576 if (work_done >= budget) { 2577 if (!budget) 2578 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons); 2579 break; 2580 } 2581 2582 if (!bnxt_has_work(bp, cpr)) { 2583 if (napi_complete_done(napi, work_done)) 2584 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons); 2585 break; 2586 } 2587 } 2588 if (bp->flags & BNXT_FLAG_DIM) { 2589 struct dim_sample dim_sample = {}; 2590 2591 dim_update_sample(cpr->event_ctr, 2592 cpr->rx_packets, 2593 cpr->rx_bytes, 2594 &dim_sample); 2595 net_dim(&cpr->dim, dim_sample); 2596 } 2597 return work_done; 2598 } 2599 2600 static int __bnxt_poll_cqs(struct bnxt *bp, struct bnxt_napi *bnapi, int budget) 2601 { 2602 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2603 int i, work_done = 0; 2604 2605 for (i = 0; i < 2; i++) { 2606 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i]; 2607 2608 if (cpr2) { 2609 work_done += __bnxt_poll_work(bp, cpr2, 2610 budget - work_done); 2611 cpr->has_more_work |= cpr2->has_more_work; 2612 } 2613 } 2614 return work_done; 2615 } 2616 2617 static void __bnxt_poll_cqs_done(struct bnxt *bp, struct bnxt_napi *bnapi, 2618 u64 dbr_type) 2619 { 2620 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2621 int i; 2622 2623 for (i = 0; i < 2; i++) { 2624 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i]; 2625 struct bnxt_db_info *db; 2626 2627 if (cpr2 && cpr2->had_work_done) { 2628 db = &cpr2->cp_db; 2629 bnxt_writeq(bp, db->db_key64 | dbr_type | 2630 RING_CMP(cpr2->cp_raw_cons), db->doorbell); 2631 cpr2->had_work_done = 0; 2632 } 2633 } 2634 __bnxt_poll_work_done(bp, bnapi); 2635 } 2636 2637 static int bnxt_poll_p5(struct napi_struct *napi, int budget) 2638 { 2639 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi); 2640 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2641 struct bnxt_cp_ring_info *cpr_rx; 2642 u32 raw_cons = cpr->cp_raw_cons; 2643 struct bnxt *bp = bnapi->bp; 2644 struct nqe_cn *nqcmp; 2645 int work_done = 0; 2646 u32 cons; 2647 2648 if (unlikely(test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))) { 2649 napi_complete(napi); 2650 return 0; 2651 } 2652 if (cpr->has_more_work) { 2653 cpr->has_more_work = 0; 2654 work_done = __bnxt_poll_cqs(bp, bnapi, budget); 2655 } 2656 while (1) { 2657 cons = RING_CMP(raw_cons); 2658 nqcmp = &cpr->nq_desc_ring[CP_RING(cons)][CP_IDX(cons)]; 2659 2660 if (!NQ_CMP_VALID(nqcmp, raw_cons)) { 2661 if (cpr->has_more_work) 2662 break; 2663 2664 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ_ARMALL); 2665 cpr->cp_raw_cons = raw_cons; 2666 if (napi_complete_done(napi, work_done)) 2667 BNXT_DB_NQ_ARM_P5(&cpr->cp_db, 2668 cpr->cp_raw_cons); 2669 goto poll_done; 2670 } 2671 2672 /* The valid test of the entry must be done first before 2673 * reading any further. 2674 */ 2675 dma_rmb(); 2676 2677 if (nqcmp->type == cpu_to_le16(NQ_CN_TYPE_CQ_NOTIFICATION)) { 2678 u32 idx = le32_to_cpu(nqcmp->cq_handle_low); 2679 struct bnxt_cp_ring_info *cpr2; 2680 2681 cpr2 = cpr->cp_ring_arr[idx]; 2682 work_done += __bnxt_poll_work(bp, cpr2, 2683 budget - work_done); 2684 cpr->has_more_work |= cpr2->has_more_work; 2685 } else { 2686 bnxt_hwrm_handler(bp, (struct tx_cmp *)nqcmp); 2687 } 2688 raw_cons = NEXT_RAW_CMP(raw_cons); 2689 } 2690 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ); 2691 if (raw_cons != cpr->cp_raw_cons) { 2692 cpr->cp_raw_cons = raw_cons; 2693 BNXT_DB_NQ_P5(&cpr->cp_db, raw_cons); 2694 } 2695 poll_done: 2696 cpr_rx = cpr->cp_ring_arr[BNXT_RX_HDL]; 2697 if (cpr_rx && (bp->flags & BNXT_FLAG_DIM)) { 2698 struct dim_sample dim_sample = {}; 2699 2700 dim_update_sample(cpr->event_ctr, 2701 cpr_rx->rx_packets, 2702 cpr_rx->rx_bytes, 2703 &dim_sample); 2704 net_dim(&cpr->dim, dim_sample); 2705 } 2706 return work_done; 2707 } 2708 2709 static void bnxt_free_tx_skbs(struct bnxt *bp) 2710 { 2711 int i, max_idx; 2712 struct pci_dev *pdev = bp->pdev; 2713 2714 if (!bp->tx_ring) 2715 return; 2716 2717 max_idx = bp->tx_nr_pages * TX_DESC_CNT; 2718 for (i = 0; i < bp->tx_nr_rings; i++) { 2719 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 2720 int j; 2721 2722 if (!txr->tx_buf_ring) 2723 continue; 2724 2725 for (j = 0; j < max_idx;) { 2726 struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j]; 2727 struct sk_buff *skb; 2728 int k, last; 2729 2730 if (i < bp->tx_nr_rings_xdp && 2731 tx_buf->action == XDP_REDIRECT) { 2732 dma_unmap_single(&pdev->dev, 2733 dma_unmap_addr(tx_buf, mapping), 2734 dma_unmap_len(tx_buf, len), 2735 DMA_TO_DEVICE); 2736 xdp_return_frame(tx_buf->xdpf); 2737 tx_buf->action = 0; 2738 tx_buf->xdpf = NULL; 2739 j++; 2740 continue; 2741 } 2742 2743 skb = tx_buf->skb; 2744 if (!skb) { 2745 j++; 2746 continue; 2747 } 2748 2749 tx_buf->skb = NULL; 2750 2751 if (tx_buf->is_push) { 2752 dev_kfree_skb(skb); 2753 j += 2; 2754 continue; 2755 } 2756 2757 dma_unmap_single(&pdev->dev, 2758 dma_unmap_addr(tx_buf, mapping), 2759 skb_headlen(skb), 2760 DMA_TO_DEVICE); 2761 2762 last = tx_buf->nr_frags; 2763 j += 2; 2764 for (k = 0; k < last; k++, j++) { 2765 int ring_idx = j & bp->tx_ring_mask; 2766 skb_frag_t *frag = &skb_shinfo(skb)->frags[k]; 2767 2768 tx_buf = &txr->tx_buf_ring[ring_idx]; 2769 dma_unmap_page( 2770 &pdev->dev, 2771 dma_unmap_addr(tx_buf, mapping), 2772 skb_frag_size(frag), DMA_TO_DEVICE); 2773 } 2774 dev_kfree_skb(skb); 2775 } 2776 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i)); 2777 } 2778 } 2779 2780 static void bnxt_free_one_rx_ring_skbs(struct bnxt *bp, int ring_nr) 2781 { 2782 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr]; 2783 struct pci_dev *pdev = bp->pdev; 2784 struct bnxt_tpa_idx_map *map; 2785 int i, max_idx, max_agg_idx; 2786 2787 max_idx = bp->rx_nr_pages * RX_DESC_CNT; 2788 max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT; 2789 if (!rxr->rx_tpa) 2790 goto skip_rx_tpa_free; 2791 2792 for (i = 0; i < bp->max_tpa; i++) { 2793 struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[i]; 2794 u8 *data = tpa_info->data; 2795 2796 if (!data) 2797 continue; 2798 2799 dma_unmap_single_attrs(&pdev->dev, tpa_info->mapping, 2800 bp->rx_buf_use_size, bp->rx_dir, 2801 DMA_ATTR_WEAK_ORDERING); 2802 2803 tpa_info->data = NULL; 2804 2805 skb_free_frag(data); 2806 } 2807 2808 skip_rx_tpa_free: 2809 if (!rxr->rx_buf_ring) 2810 goto skip_rx_buf_free; 2811 2812 for (i = 0; i < max_idx; i++) { 2813 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[i]; 2814 dma_addr_t mapping = rx_buf->mapping; 2815 void *data = rx_buf->data; 2816 2817 if (!data) 2818 continue; 2819 2820 rx_buf->data = NULL; 2821 if (BNXT_RX_PAGE_MODE(bp)) { 2822 mapping -= bp->rx_dma_offset; 2823 dma_unmap_page_attrs(&pdev->dev, mapping, PAGE_SIZE, 2824 bp->rx_dir, 2825 DMA_ATTR_WEAK_ORDERING); 2826 page_pool_recycle_direct(rxr->page_pool, data); 2827 } else { 2828 dma_unmap_single_attrs(&pdev->dev, mapping, 2829 bp->rx_buf_use_size, bp->rx_dir, 2830 DMA_ATTR_WEAK_ORDERING); 2831 skb_free_frag(data); 2832 } 2833 } 2834 2835 skip_rx_buf_free: 2836 if (!rxr->rx_agg_ring) 2837 goto skip_rx_agg_free; 2838 2839 for (i = 0; i < max_agg_idx; i++) { 2840 struct bnxt_sw_rx_agg_bd *rx_agg_buf = &rxr->rx_agg_ring[i]; 2841 struct page *page = rx_agg_buf->page; 2842 2843 if (!page) 2844 continue; 2845 2846 dma_unmap_page_attrs(&pdev->dev, rx_agg_buf->mapping, 2847 BNXT_RX_PAGE_SIZE, DMA_FROM_DEVICE, 2848 DMA_ATTR_WEAK_ORDERING); 2849 2850 rx_agg_buf->page = NULL; 2851 __clear_bit(i, rxr->rx_agg_bmap); 2852 2853 __free_page(page); 2854 } 2855 2856 skip_rx_agg_free: 2857 if (rxr->rx_page) { 2858 __free_page(rxr->rx_page); 2859 rxr->rx_page = NULL; 2860 } 2861 map = rxr->rx_tpa_idx_map; 2862 if (map) 2863 memset(map->agg_idx_bmap, 0, sizeof(map->agg_idx_bmap)); 2864 } 2865 2866 static void bnxt_free_rx_skbs(struct bnxt *bp) 2867 { 2868 int i; 2869 2870 if (!bp->rx_ring) 2871 return; 2872 2873 for (i = 0; i < bp->rx_nr_rings; i++) 2874 bnxt_free_one_rx_ring_skbs(bp, i); 2875 } 2876 2877 static void bnxt_free_skbs(struct bnxt *bp) 2878 { 2879 bnxt_free_tx_skbs(bp); 2880 bnxt_free_rx_skbs(bp); 2881 } 2882 2883 static void bnxt_init_ctx_mem(struct bnxt_mem_init *mem_init, void *p, int len) 2884 { 2885 u8 init_val = mem_init->init_val; 2886 u16 offset = mem_init->offset; 2887 u8 *p2 = p; 2888 int i; 2889 2890 if (!init_val) 2891 return; 2892 if (offset == BNXT_MEM_INVALID_OFFSET) { 2893 memset(p, init_val, len); 2894 return; 2895 } 2896 for (i = 0; i < len; i += mem_init->size) 2897 *(p2 + i + offset) = init_val; 2898 } 2899 2900 static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem) 2901 { 2902 struct pci_dev *pdev = bp->pdev; 2903 int i; 2904 2905 if (!rmem->pg_arr) 2906 goto skip_pages; 2907 2908 for (i = 0; i < rmem->nr_pages; i++) { 2909 if (!rmem->pg_arr[i]) 2910 continue; 2911 2912 dma_free_coherent(&pdev->dev, rmem->page_size, 2913 rmem->pg_arr[i], rmem->dma_arr[i]); 2914 2915 rmem->pg_arr[i] = NULL; 2916 } 2917 skip_pages: 2918 if (rmem->pg_tbl) { 2919 size_t pg_tbl_size = rmem->nr_pages * 8; 2920 2921 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG) 2922 pg_tbl_size = rmem->page_size; 2923 dma_free_coherent(&pdev->dev, pg_tbl_size, 2924 rmem->pg_tbl, rmem->pg_tbl_map); 2925 rmem->pg_tbl = NULL; 2926 } 2927 if (rmem->vmem_size && *rmem->vmem) { 2928 vfree(*rmem->vmem); 2929 *rmem->vmem = NULL; 2930 } 2931 } 2932 2933 static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem) 2934 { 2935 struct pci_dev *pdev = bp->pdev; 2936 u64 valid_bit = 0; 2937 int i; 2938 2939 if (rmem->flags & (BNXT_RMEM_VALID_PTE_FLAG | BNXT_RMEM_RING_PTE_FLAG)) 2940 valid_bit = PTU_PTE_VALID; 2941 if ((rmem->nr_pages > 1 || rmem->depth > 0) && !rmem->pg_tbl) { 2942 size_t pg_tbl_size = rmem->nr_pages * 8; 2943 2944 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG) 2945 pg_tbl_size = rmem->page_size; 2946 rmem->pg_tbl = dma_alloc_coherent(&pdev->dev, pg_tbl_size, 2947 &rmem->pg_tbl_map, 2948 GFP_KERNEL); 2949 if (!rmem->pg_tbl) 2950 return -ENOMEM; 2951 } 2952 2953 for (i = 0; i < rmem->nr_pages; i++) { 2954 u64 extra_bits = valid_bit; 2955 2956 rmem->pg_arr[i] = dma_alloc_coherent(&pdev->dev, 2957 rmem->page_size, 2958 &rmem->dma_arr[i], 2959 GFP_KERNEL); 2960 if (!rmem->pg_arr[i]) 2961 return -ENOMEM; 2962 2963 if (rmem->mem_init) 2964 bnxt_init_ctx_mem(rmem->mem_init, rmem->pg_arr[i], 2965 rmem->page_size); 2966 if (rmem->nr_pages > 1 || rmem->depth > 0) { 2967 if (i == rmem->nr_pages - 2 && 2968 (rmem->flags & BNXT_RMEM_RING_PTE_FLAG)) 2969 extra_bits |= PTU_PTE_NEXT_TO_LAST; 2970 else if (i == rmem->nr_pages - 1 && 2971 (rmem->flags & BNXT_RMEM_RING_PTE_FLAG)) 2972 extra_bits |= PTU_PTE_LAST; 2973 rmem->pg_tbl[i] = 2974 cpu_to_le64(rmem->dma_arr[i] | extra_bits); 2975 } 2976 } 2977 2978 if (rmem->vmem_size) { 2979 *rmem->vmem = vzalloc(rmem->vmem_size); 2980 if (!(*rmem->vmem)) 2981 return -ENOMEM; 2982 } 2983 return 0; 2984 } 2985 2986 static void bnxt_free_tpa_info(struct bnxt *bp) 2987 { 2988 int i; 2989 2990 for (i = 0; i < bp->rx_nr_rings; i++) { 2991 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 2992 2993 kfree(rxr->rx_tpa_idx_map); 2994 rxr->rx_tpa_idx_map = NULL; 2995 if (rxr->rx_tpa) { 2996 kfree(rxr->rx_tpa[0].agg_arr); 2997 rxr->rx_tpa[0].agg_arr = NULL; 2998 } 2999 kfree(rxr->rx_tpa); 3000 rxr->rx_tpa = NULL; 3001 } 3002 } 3003 3004 static int bnxt_alloc_tpa_info(struct bnxt *bp) 3005 { 3006 int i, j, total_aggs = 0; 3007 3008 bp->max_tpa = MAX_TPA; 3009 if (bp->flags & BNXT_FLAG_CHIP_P5) { 3010 if (!bp->max_tpa_v2) 3011 return 0; 3012 bp->max_tpa = max_t(u16, bp->max_tpa_v2, MAX_TPA_P5); 3013 total_aggs = bp->max_tpa * MAX_SKB_FRAGS; 3014 } 3015 3016 for (i = 0; i < bp->rx_nr_rings; i++) { 3017 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 3018 struct rx_agg_cmp *agg; 3019 3020 rxr->rx_tpa = kcalloc(bp->max_tpa, sizeof(struct bnxt_tpa_info), 3021 GFP_KERNEL); 3022 if (!rxr->rx_tpa) 3023 return -ENOMEM; 3024 3025 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 3026 continue; 3027 agg = kcalloc(total_aggs, sizeof(*agg), GFP_KERNEL); 3028 rxr->rx_tpa[0].agg_arr = agg; 3029 if (!agg) 3030 return -ENOMEM; 3031 for (j = 1; j < bp->max_tpa; j++) 3032 rxr->rx_tpa[j].agg_arr = agg + j * MAX_SKB_FRAGS; 3033 rxr->rx_tpa_idx_map = kzalloc(sizeof(*rxr->rx_tpa_idx_map), 3034 GFP_KERNEL); 3035 if (!rxr->rx_tpa_idx_map) 3036 return -ENOMEM; 3037 } 3038 return 0; 3039 } 3040 3041 static void bnxt_free_rx_rings(struct bnxt *bp) 3042 { 3043 int i; 3044 3045 if (!bp->rx_ring) 3046 return; 3047 3048 bnxt_free_tpa_info(bp); 3049 for (i = 0; i < bp->rx_nr_rings; i++) { 3050 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 3051 struct bnxt_ring_struct *ring; 3052 3053 if (rxr->xdp_prog) 3054 bpf_prog_put(rxr->xdp_prog); 3055 3056 if (xdp_rxq_info_is_reg(&rxr->xdp_rxq)) 3057 xdp_rxq_info_unreg(&rxr->xdp_rxq); 3058 3059 page_pool_destroy(rxr->page_pool); 3060 rxr->page_pool = NULL; 3061 3062 kfree(rxr->rx_agg_bmap); 3063 rxr->rx_agg_bmap = NULL; 3064 3065 ring = &rxr->rx_ring_struct; 3066 bnxt_free_ring(bp, &ring->ring_mem); 3067 3068 ring = &rxr->rx_agg_ring_struct; 3069 bnxt_free_ring(bp, &ring->ring_mem); 3070 } 3071 } 3072 3073 static int bnxt_alloc_rx_page_pool(struct bnxt *bp, 3074 struct bnxt_rx_ring_info *rxr) 3075 { 3076 struct page_pool_params pp = { 0 }; 3077 3078 pp.pool_size = bp->rx_ring_size; 3079 pp.nid = dev_to_node(&bp->pdev->dev); 3080 pp.dev = &bp->pdev->dev; 3081 pp.dma_dir = DMA_BIDIRECTIONAL; 3082 3083 rxr->page_pool = page_pool_create(&pp); 3084 if (IS_ERR(rxr->page_pool)) { 3085 int err = PTR_ERR(rxr->page_pool); 3086 3087 rxr->page_pool = NULL; 3088 return err; 3089 } 3090 return 0; 3091 } 3092 3093 static int bnxt_alloc_rx_rings(struct bnxt *bp) 3094 { 3095 int i, rc = 0, agg_rings = 0; 3096 3097 if (!bp->rx_ring) 3098 return -ENOMEM; 3099 3100 if (bp->flags & BNXT_FLAG_AGG_RINGS) 3101 agg_rings = 1; 3102 3103 for (i = 0; i < bp->rx_nr_rings; i++) { 3104 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 3105 struct bnxt_ring_struct *ring; 3106 3107 ring = &rxr->rx_ring_struct; 3108 3109 rc = bnxt_alloc_rx_page_pool(bp, rxr); 3110 if (rc) 3111 return rc; 3112 3113 rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i, 0); 3114 if (rc < 0) 3115 return rc; 3116 3117 rc = xdp_rxq_info_reg_mem_model(&rxr->xdp_rxq, 3118 MEM_TYPE_PAGE_POOL, 3119 rxr->page_pool); 3120 if (rc) { 3121 xdp_rxq_info_unreg(&rxr->xdp_rxq); 3122 return rc; 3123 } 3124 3125 rc = bnxt_alloc_ring(bp, &ring->ring_mem); 3126 if (rc) 3127 return rc; 3128 3129 ring->grp_idx = i; 3130 if (agg_rings) { 3131 u16 mem_size; 3132 3133 ring = &rxr->rx_agg_ring_struct; 3134 rc = bnxt_alloc_ring(bp, &ring->ring_mem); 3135 if (rc) 3136 return rc; 3137 3138 ring->grp_idx = i; 3139 rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1; 3140 mem_size = rxr->rx_agg_bmap_size / 8; 3141 rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL); 3142 if (!rxr->rx_agg_bmap) 3143 return -ENOMEM; 3144 } 3145 } 3146 if (bp->flags & BNXT_FLAG_TPA) 3147 rc = bnxt_alloc_tpa_info(bp); 3148 return rc; 3149 } 3150 3151 static void bnxt_free_tx_rings(struct bnxt *bp) 3152 { 3153 int i; 3154 struct pci_dev *pdev = bp->pdev; 3155 3156 if (!bp->tx_ring) 3157 return; 3158 3159 for (i = 0; i < bp->tx_nr_rings; i++) { 3160 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 3161 struct bnxt_ring_struct *ring; 3162 3163 if (txr->tx_push) { 3164 dma_free_coherent(&pdev->dev, bp->tx_push_size, 3165 txr->tx_push, txr->tx_push_mapping); 3166 txr->tx_push = NULL; 3167 } 3168 3169 ring = &txr->tx_ring_struct; 3170 3171 bnxt_free_ring(bp, &ring->ring_mem); 3172 } 3173 } 3174 3175 static int bnxt_alloc_tx_rings(struct bnxt *bp) 3176 { 3177 int i, j, rc; 3178 struct pci_dev *pdev = bp->pdev; 3179 3180 bp->tx_push_size = 0; 3181 if (bp->tx_push_thresh) { 3182 int push_size; 3183 3184 push_size = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) + 3185 bp->tx_push_thresh); 3186 3187 if (push_size > 256) { 3188 push_size = 0; 3189 bp->tx_push_thresh = 0; 3190 } 3191 3192 bp->tx_push_size = push_size; 3193 } 3194 3195 for (i = 0, j = 0; i < bp->tx_nr_rings; i++) { 3196 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 3197 struct bnxt_ring_struct *ring; 3198 u8 qidx; 3199 3200 ring = &txr->tx_ring_struct; 3201 3202 rc = bnxt_alloc_ring(bp, &ring->ring_mem); 3203 if (rc) 3204 return rc; 3205 3206 ring->grp_idx = txr->bnapi->index; 3207 if (bp->tx_push_size) { 3208 dma_addr_t mapping; 3209 3210 /* One pre-allocated DMA buffer to backup 3211 * TX push operation 3212 */ 3213 txr->tx_push = dma_alloc_coherent(&pdev->dev, 3214 bp->tx_push_size, 3215 &txr->tx_push_mapping, 3216 GFP_KERNEL); 3217 3218 if (!txr->tx_push) 3219 return -ENOMEM; 3220 3221 mapping = txr->tx_push_mapping + 3222 sizeof(struct tx_push_bd); 3223 txr->data_mapping = cpu_to_le64(mapping); 3224 } 3225 qidx = bp->tc_to_qidx[j]; 3226 ring->queue_id = bp->q_info[qidx].queue_id; 3227 if (i < bp->tx_nr_rings_xdp) 3228 continue; 3229 if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1)) 3230 j++; 3231 } 3232 return 0; 3233 } 3234 3235 static void bnxt_free_cp_arrays(struct bnxt_cp_ring_info *cpr) 3236 { 3237 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 3238 3239 kfree(cpr->cp_desc_ring); 3240 cpr->cp_desc_ring = NULL; 3241 ring->ring_mem.pg_arr = NULL; 3242 kfree(cpr->cp_desc_mapping); 3243 cpr->cp_desc_mapping = NULL; 3244 ring->ring_mem.dma_arr = NULL; 3245 } 3246 3247 static int bnxt_alloc_cp_arrays(struct bnxt_cp_ring_info *cpr, int n) 3248 { 3249 cpr->cp_desc_ring = kcalloc(n, sizeof(*cpr->cp_desc_ring), GFP_KERNEL); 3250 if (!cpr->cp_desc_ring) 3251 return -ENOMEM; 3252 cpr->cp_desc_mapping = kcalloc(n, sizeof(*cpr->cp_desc_mapping), 3253 GFP_KERNEL); 3254 if (!cpr->cp_desc_mapping) 3255 return -ENOMEM; 3256 return 0; 3257 } 3258 3259 static void bnxt_free_all_cp_arrays(struct bnxt *bp) 3260 { 3261 int i; 3262 3263 if (!bp->bnapi) 3264 return; 3265 for (i = 0; i < bp->cp_nr_rings; i++) { 3266 struct bnxt_napi *bnapi = bp->bnapi[i]; 3267 3268 if (!bnapi) 3269 continue; 3270 bnxt_free_cp_arrays(&bnapi->cp_ring); 3271 } 3272 } 3273 3274 static int bnxt_alloc_all_cp_arrays(struct bnxt *bp) 3275 { 3276 int i, n = bp->cp_nr_pages; 3277 3278 for (i = 0; i < bp->cp_nr_rings; i++) { 3279 struct bnxt_napi *bnapi = bp->bnapi[i]; 3280 int rc; 3281 3282 if (!bnapi) 3283 continue; 3284 rc = bnxt_alloc_cp_arrays(&bnapi->cp_ring, n); 3285 if (rc) 3286 return rc; 3287 } 3288 return 0; 3289 } 3290 3291 static void bnxt_free_cp_rings(struct bnxt *bp) 3292 { 3293 int i; 3294 3295 if (!bp->bnapi) 3296 return; 3297 3298 for (i = 0; i < bp->cp_nr_rings; i++) { 3299 struct bnxt_napi *bnapi = bp->bnapi[i]; 3300 struct bnxt_cp_ring_info *cpr; 3301 struct bnxt_ring_struct *ring; 3302 int j; 3303 3304 if (!bnapi) 3305 continue; 3306 3307 cpr = &bnapi->cp_ring; 3308 ring = &cpr->cp_ring_struct; 3309 3310 bnxt_free_ring(bp, &ring->ring_mem); 3311 3312 for (j = 0; j < 2; j++) { 3313 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j]; 3314 3315 if (cpr2) { 3316 ring = &cpr2->cp_ring_struct; 3317 bnxt_free_ring(bp, &ring->ring_mem); 3318 bnxt_free_cp_arrays(cpr2); 3319 kfree(cpr2); 3320 cpr->cp_ring_arr[j] = NULL; 3321 } 3322 } 3323 } 3324 } 3325 3326 static struct bnxt_cp_ring_info *bnxt_alloc_cp_sub_ring(struct bnxt *bp) 3327 { 3328 struct bnxt_ring_mem_info *rmem; 3329 struct bnxt_ring_struct *ring; 3330 struct bnxt_cp_ring_info *cpr; 3331 int rc; 3332 3333 cpr = kzalloc(sizeof(*cpr), GFP_KERNEL); 3334 if (!cpr) 3335 return NULL; 3336 3337 rc = bnxt_alloc_cp_arrays(cpr, bp->cp_nr_pages); 3338 if (rc) { 3339 bnxt_free_cp_arrays(cpr); 3340 kfree(cpr); 3341 return NULL; 3342 } 3343 ring = &cpr->cp_ring_struct; 3344 rmem = &ring->ring_mem; 3345 rmem->nr_pages = bp->cp_nr_pages; 3346 rmem->page_size = HW_CMPD_RING_SIZE; 3347 rmem->pg_arr = (void **)cpr->cp_desc_ring; 3348 rmem->dma_arr = cpr->cp_desc_mapping; 3349 rmem->flags = BNXT_RMEM_RING_PTE_FLAG; 3350 rc = bnxt_alloc_ring(bp, rmem); 3351 if (rc) { 3352 bnxt_free_ring(bp, rmem); 3353 bnxt_free_cp_arrays(cpr); 3354 kfree(cpr); 3355 cpr = NULL; 3356 } 3357 return cpr; 3358 } 3359 3360 static int bnxt_alloc_cp_rings(struct bnxt *bp) 3361 { 3362 bool sh = !!(bp->flags & BNXT_FLAG_SHARED_RINGS); 3363 int i, rc, ulp_base_vec, ulp_msix; 3364 3365 ulp_msix = bnxt_get_ulp_msix_num(bp); 3366 ulp_base_vec = bnxt_get_ulp_msix_base(bp); 3367 for (i = 0; i < bp->cp_nr_rings; i++) { 3368 struct bnxt_napi *bnapi = bp->bnapi[i]; 3369 struct bnxt_cp_ring_info *cpr; 3370 struct bnxt_ring_struct *ring; 3371 3372 if (!bnapi) 3373 continue; 3374 3375 cpr = &bnapi->cp_ring; 3376 cpr->bnapi = bnapi; 3377 ring = &cpr->cp_ring_struct; 3378 3379 rc = bnxt_alloc_ring(bp, &ring->ring_mem); 3380 if (rc) 3381 return rc; 3382 3383 if (ulp_msix && i >= ulp_base_vec) 3384 ring->map_idx = i + ulp_msix; 3385 else 3386 ring->map_idx = i; 3387 3388 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 3389 continue; 3390 3391 if (i < bp->rx_nr_rings) { 3392 struct bnxt_cp_ring_info *cpr2 = 3393 bnxt_alloc_cp_sub_ring(bp); 3394 3395 cpr->cp_ring_arr[BNXT_RX_HDL] = cpr2; 3396 if (!cpr2) 3397 return -ENOMEM; 3398 cpr2->bnapi = bnapi; 3399 } 3400 if ((sh && i < bp->tx_nr_rings) || 3401 (!sh && i >= bp->rx_nr_rings)) { 3402 struct bnxt_cp_ring_info *cpr2 = 3403 bnxt_alloc_cp_sub_ring(bp); 3404 3405 cpr->cp_ring_arr[BNXT_TX_HDL] = cpr2; 3406 if (!cpr2) 3407 return -ENOMEM; 3408 cpr2->bnapi = bnapi; 3409 } 3410 } 3411 return 0; 3412 } 3413 3414 static void bnxt_init_ring_struct(struct bnxt *bp) 3415 { 3416 int i; 3417 3418 for (i = 0; i < bp->cp_nr_rings; i++) { 3419 struct bnxt_napi *bnapi = bp->bnapi[i]; 3420 struct bnxt_ring_mem_info *rmem; 3421 struct bnxt_cp_ring_info *cpr; 3422 struct bnxt_rx_ring_info *rxr; 3423 struct bnxt_tx_ring_info *txr; 3424 struct bnxt_ring_struct *ring; 3425 3426 if (!bnapi) 3427 continue; 3428 3429 cpr = &bnapi->cp_ring; 3430 ring = &cpr->cp_ring_struct; 3431 rmem = &ring->ring_mem; 3432 rmem->nr_pages = bp->cp_nr_pages; 3433 rmem->page_size = HW_CMPD_RING_SIZE; 3434 rmem->pg_arr = (void **)cpr->cp_desc_ring; 3435 rmem->dma_arr = cpr->cp_desc_mapping; 3436 rmem->vmem_size = 0; 3437 3438 rxr = bnapi->rx_ring; 3439 if (!rxr) 3440 goto skip_rx; 3441 3442 ring = &rxr->rx_ring_struct; 3443 rmem = &ring->ring_mem; 3444 rmem->nr_pages = bp->rx_nr_pages; 3445 rmem->page_size = HW_RXBD_RING_SIZE; 3446 rmem->pg_arr = (void **)rxr->rx_desc_ring; 3447 rmem->dma_arr = rxr->rx_desc_mapping; 3448 rmem->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages; 3449 rmem->vmem = (void **)&rxr->rx_buf_ring; 3450 3451 ring = &rxr->rx_agg_ring_struct; 3452 rmem = &ring->ring_mem; 3453 rmem->nr_pages = bp->rx_agg_nr_pages; 3454 rmem->page_size = HW_RXBD_RING_SIZE; 3455 rmem->pg_arr = (void **)rxr->rx_agg_desc_ring; 3456 rmem->dma_arr = rxr->rx_agg_desc_mapping; 3457 rmem->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages; 3458 rmem->vmem = (void **)&rxr->rx_agg_ring; 3459 3460 skip_rx: 3461 txr = bnapi->tx_ring; 3462 if (!txr) 3463 continue; 3464 3465 ring = &txr->tx_ring_struct; 3466 rmem = &ring->ring_mem; 3467 rmem->nr_pages = bp->tx_nr_pages; 3468 rmem->page_size = HW_RXBD_RING_SIZE; 3469 rmem->pg_arr = (void **)txr->tx_desc_ring; 3470 rmem->dma_arr = txr->tx_desc_mapping; 3471 rmem->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages; 3472 rmem->vmem = (void **)&txr->tx_buf_ring; 3473 } 3474 } 3475 3476 static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type) 3477 { 3478 int i; 3479 u32 prod; 3480 struct rx_bd **rx_buf_ring; 3481 3482 rx_buf_ring = (struct rx_bd **)ring->ring_mem.pg_arr; 3483 for (i = 0, prod = 0; i < ring->ring_mem.nr_pages; i++) { 3484 int j; 3485 struct rx_bd *rxbd; 3486 3487 rxbd = rx_buf_ring[i]; 3488 if (!rxbd) 3489 continue; 3490 3491 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) { 3492 rxbd->rx_bd_len_flags_type = cpu_to_le32(type); 3493 rxbd->rx_bd_opaque = prod; 3494 } 3495 } 3496 } 3497 3498 static int bnxt_alloc_one_rx_ring(struct bnxt *bp, int ring_nr) 3499 { 3500 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr]; 3501 struct net_device *dev = bp->dev; 3502 u32 prod; 3503 int i; 3504 3505 prod = rxr->rx_prod; 3506 for (i = 0; i < bp->rx_ring_size; i++) { 3507 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL)) { 3508 netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n", 3509 ring_nr, i, bp->rx_ring_size); 3510 break; 3511 } 3512 prod = NEXT_RX(prod); 3513 } 3514 rxr->rx_prod = prod; 3515 3516 if (!(bp->flags & BNXT_FLAG_AGG_RINGS)) 3517 return 0; 3518 3519 prod = rxr->rx_agg_prod; 3520 for (i = 0; i < bp->rx_agg_ring_size; i++) { 3521 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL)) { 3522 netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n", 3523 ring_nr, i, bp->rx_ring_size); 3524 break; 3525 } 3526 prod = NEXT_RX_AGG(prod); 3527 } 3528 rxr->rx_agg_prod = prod; 3529 3530 if (rxr->rx_tpa) { 3531 dma_addr_t mapping; 3532 u8 *data; 3533 3534 for (i = 0; i < bp->max_tpa; i++) { 3535 data = __bnxt_alloc_rx_frag(bp, &mapping, GFP_KERNEL); 3536 if (!data) 3537 return -ENOMEM; 3538 3539 rxr->rx_tpa[i].data = data; 3540 rxr->rx_tpa[i].data_ptr = data + bp->rx_offset; 3541 rxr->rx_tpa[i].mapping = mapping; 3542 } 3543 } 3544 return 0; 3545 } 3546 3547 static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr) 3548 { 3549 struct bnxt_rx_ring_info *rxr; 3550 struct bnxt_ring_struct *ring; 3551 u32 type; 3552 3553 type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) | 3554 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP; 3555 3556 if (NET_IP_ALIGN == 2) 3557 type |= RX_BD_FLAGS_SOP; 3558 3559 rxr = &bp->rx_ring[ring_nr]; 3560 ring = &rxr->rx_ring_struct; 3561 bnxt_init_rxbd_pages(ring, type); 3562 3563 if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) { 3564 bpf_prog_add(bp->xdp_prog, 1); 3565 rxr->xdp_prog = bp->xdp_prog; 3566 } 3567 ring->fw_ring_id = INVALID_HW_RING_ID; 3568 3569 ring = &rxr->rx_agg_ring_struct; 3570 ring->fw_ring_id = INVALID_HW_RING_ID; 3571 3572 if ((bp->flags & BNXT_FLAG_AGG_RINGS)) { 3573 type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) | 3574 RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP; 3575 3576 bnxt_init_rxbd_pages(ring, type); 3577 } 3578 3579 return bnxt_alloc_one_rx_ring(bp, ring_nr); 3580 } 3581 3582 static void bnxt_init_cp_rings(struct bnxt *bp) 3583 { 3584 int i, j; 3585 3586 for (i = 0; i < bp->cp_nr_rings; i++) { 3587 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring; 3588 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 3589 3590 ring->fw_ring_id = INVALID_HW_RING_ID; 3591 cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks; 3592 cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs; 3593 for (j = 0; j < 2; j++) { 3594 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j]; 3595 3596 if (!cpr2) 3597 continue; 3598 3599 ring = &cpr2->cp_ring_struct; 3600 ring->fw_ring_id = INVALID_HW_RING_ID; 3601 cpr2->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks; 3602 cpr2->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs; 3603 } 3604 } 3605 } 3606 3607 static int bnxt_init_rx_rings(struct bnxt *bp) 3608 { 3609 int i, rc = 0; 3610 3611 if (BNXT_RX_PAGE_MODE(bp)) { 3612 bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM; 3613 bp->rx_dma_offset = XDP_PACKET_HEADROOM; 3614 } else { 3615 bp->rx_offset = BNXT_RX_OFFSET; 3616 bp->rx_dma_offset = BNXT_RX_DMA_OFFSET; 3617 } 3618 3619 for (i = 0; i < bp->rx_nr_rings; i++) { 3620 rc = bnxt_init_one_rx_ring(bp, i); 3621 if (rc) 3622 break; 3623 } 3624 3625 return rc; 3626 } 3627 3628 static int bnxt_init_tx_rings(struct bnxt *bp) 3629 { 3630 u16 i; 3631 3632 bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2, 3633 BNXT_MIN_TX_DESC_CNT); 3634 3635 for (i = 0; i < bp->tx_nr_rings; i++) { 3636 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 3637 struct bnxt_ring_struct *ring = &txr->tx_ring_struct; 3638 3639 ring->fw_ring_id = INVALID_HW_RING_ID; 3640 } 3641 3642 return 0; 3643 } 3644 3645 static void bnxt_free_ring_grps(struct bnxt *bp) 3646 { 3647 kfree(bp->grp_info); 3648 bp->grp_info = NULL; 3649 } 3650 3651 static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init) 3652 { 3653 int i; 3654 3655 if (irq_re_init) { 3656 bp->grp_info = kcalloc(bp->cp_nr_rings, 3657 sizeof(struct bnxt_ring_grp_info), 3658 GFP_KERNEL); 3659 if (!bp->grp_info) 3660 return -ENOMEM; 3661 } 3662 for (i = 0; i < bp->cp_nr_rings; i++) { 3663 if (irq_re_init) 3664 bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID; 3665 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID; 3666 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID; 3667 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID; 3668 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID; 3669 } 3670 return 0; 3671 } 3672 3673 static void bnxt_free_vnics(struct bnxt *bp) 3674 { 3675 kfree(bp->vnic_info); 3676 bp->vnic_info = NULL; 3677 bp->nr_vnics = 0; 3678 } 3679 3680 static int bnxt_alloc_vnics(struct bnxt *bp) 3681 { 3682 int num_vnics = 1; 3683 3684 #ifdef CONFIG_RFS_ACCEL 3685 if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS) 3686 num_vnics += bp->rx_nr_rings; 3687 #endif 3688 3689 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 3690 num_vnics++; 3691 3692 bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info), 3693 GFP_KERNEL); 3694 if (!bp->vnic_info) 3695 return -ENOMEM; 3696 3697 bp->nr_vnics = num_vnics; 3698 return 0; 3699 } 3700 3701 static void bnxt_init_vnics(struct bnxt *bp) 3702 { 3703 int i; 3704 3705 for (i = 0; i < bp->nr_vnics; i++) { 3706 struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; 3707 int j; 3708 3709 vnic->fw_vnic_id = INVALID_HW_RING_ID; 3710 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) 3711 vnic->fw_rss_cos_lb_ctx[j] = INVALID_HW_RING_ID; 3712 3713 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID; 3714 3715 if (bp->vnic_info[i].rss_hash_key) { 3716 if (i == 0) 3717 prandom_bytes(vnic->rss_hash_key, 3718 HW_HASH_KEY_SIZE); 3719 else 3720 memcpy(vnic->rss_hash_key, 3721 bp->vnic_info[0].rss_hash_key, 3722 HW_HASH_KEY_SIZE); 3723 } 3724 } 3725 } 3726 3727 static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg) 3728 { 3729 int pages; 3730 3731 pages = ring_size / desc_per_pg; 3732 3733 if (!pages) 3734 return 1; 3735 3736 pages++; 3737 3738 while (pages & (pages - 1)) 3739 pages++; 3740 3741 return pages; 3742 } 3743 3744 void bnxt_set_tpa_flags(struct bnxt *bp) 3745 { 3746 bp->flags &= ~BNXT_FLAG_TPA; 3747 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS) 3748 return; 3749 if (bp->dev->features & NETIF_F_LRO) 3750 bp->flags |= BNXT_FLAG_LRO; 3751 else if (bp->dev->features & NETIF_F_GRO_HW) 3752 bp->flags |= BNXT_FLAG_GRO; 3753 } 3754 3755 /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must 3756 * be set on entry. 3757 */ 3758 void bnxt_set_ring_params(struct bnxt *bp) 3759 { 3760 u32 ring_size, rx_size, rx_space, max_rx_cmpl; 3761 u32 agg_factor = 0, agg_ring_size = 0; 3762 3763 /* 8 for CRC and VLAN */ 3764 rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8); 3765 3766 rx_space = rx_size + NET_SKB_PAD + 3767 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 3768 3769 bp->rx_copy_thresh = BNXT_RX_COPY_THRESH; 3770 ring_size = bp->rx_ring_size; 3771 bp->rx_agg_ring_size = 0; 3772 bp->rx_agg_nr_pages = 0; 3773 3774 if (bp->flags & BNXT_FLAG_TPA) 3775 agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE); 3776 3777 bp->flags &= ~BNXT_FLAG_JUMBO; 3778 if (rx_space > PAGE_SIZE && !(bp->flags & BNXT_FLAG_NO_AGG_RINGS)) { 3779 u32 jumbo_factor; 3780 3781 bp->flags |= BNXT_FLAG_JUMBO; 3782 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT; 3783 if (jumbo_factor > agg_factor) 3784 agg_factor = jumbo_factor; 3785 } 3786 if (agg_factor) { 3787 if (ring_size > BNXT_MAX_RX_DESC_CNT_JUM_ENA) { 3788 ring_size = BNXT_MAX_RX_DESC_CNT_JUM_ENA; 3789 netdev_warn(bp->dev, "RX ring size reduced from %d to %d because the jumbo ring is now enabled\n", 3790 bp->rx_ring_size, ring_size); 3791 bp->rx_ring_size = ring_size; 3792 } 3793 agg_ring_size = ring_size * agg_factor; 3794 3795 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size, 3796 RX_DESC_CNT); 3797 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) { 3798 u32 tmp = agg_ring_size; 3799 3800 bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES; 3801 agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1; 3802 netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n", 3803 tmp, agg_ring_size); 3804 } 3805 bp->rx_agg_ring_size = agg_ring_size; 3806 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1; 3807 rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN); 3808 rx_space = rx_size + NET_SKB_PAD + 3809 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 3810 } 3811 3812 bp->rx_buf_use_size = rx_size; 3813 bp->rx_buf_size = rx_space; 3814 3815 bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT); 3816 bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1; 3817 3818 ring_size = bp->tx_ring_size; 3819 bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT); 3820 bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1; 3821 3822 max_rx_cmpl = bp->rx_ring_size; 3823 /* MAX TPA needs to be added because TPA_START completions are 3824 * immediately recycled, so the TPA completions are not bound by 3825 * the RX ring size. 3826 */ 3827 if (bp->flags & BNXT_FLAG_TPA) 3828 max_rx_cmpl += bp->max_tpa; 3829 /* RX and TPA completions are 32-byte, all others are 16-byte */ 3830 ring_size = max_rx_cmpl * 2 + agg_ring_size + bp->tx_ring_size; 3831 bp->cp_ring_size = ring_size; 3832 3833 bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT); 3834 if (bp->cp_nr_pages > MAX_CP_PAGES) { 3835 bp->cp_nr_pages = MAX_CP_PAGES; 3836 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1; 3837 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n", 3838 ring_size, bp->cp_ring_size); 3839 } 3840 bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT; 3841 bp->cp_ring_mask = bp->cp_bit - 1; 3842 } 3843 3844 /* Changing allocation mode of RX rings. 3845 * TODO: Update when extending xdp_rxq_info to support allocation modes. 3846 */ 3847 int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode) 3848 { 3849 if (page_mode) { 3850 if (bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU) 3851 return -EOPNOTSUPP; 3852 bp->dev->max_mtu = 3853 min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU); 3854 bp->flags &= ~BNXT_FLAG_AGG_RINGS; 3855 bp->flags |= BNXT_FLAG_NO_AGG_RINGS | BNXT_FLAG_RX_PAGE_MODE; 3856 bp->rx_dir = DMA_BIDIRECTIONAL; 3857 bp->rx_skb_func = bnxt_rx_page_skb; 3858 /* Disable LRO or GRO_HW */ 3859 netdev_update_features(bp->dev); 3860 } else { 3861 bp->dev->max_mtu = bp->max_mtu; 3862 bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE; 3863 bp->rx_dir = DMA_FROM_DEVICE; 3864 bp->rx_skb_func = bnxt_rx_skb; 3865 } 3866 return 0; 3867 } 3868 3869 static void bnxt_free_vnic_attributes(struct bnxt *bp) 3870 { 3871 int i; 3872 struct bnxt_vnic_info *vnic; 3873 struct pci_dev *pdev = bp->pdev; 3874 3875 if (!bp->vnic_info) 3876 return; 3877 3878 for (i = 0; i < bp->nr_vnics; i++) { 3879 vnic = &bp->vnic_info[i]; 3880 3881 kfree(vnic->fw_grp_ids); 3882 vnic->fw_grp_ids = NULL; 3883 3884 kfree(vnic->uc_list); 3885 vnic->uc_list = NULL; 3886 3887 if (vnic->mc_list) { 3888 dma_free_coherent(&pdev->dev, vnic->mc_list_size, 3889 vnic->mc_list, vnic->mc_list_mapping); 3890 vnic->mc_list = NULL; 3891 } 3892 3893 if (vnic->rss_table) { 3894 dma_free_coherent(&pdev->dev, vnic->rss_table_size, 3895 vnic->rss_table, 3896 vnic->rss_table_dma_addr); 3897 vnic->rss_table = NULL; 3898 } 3899 3900 vnic->rss_hash_key = NULL; 3901 vnic->flags = 0; 3902 } 3903 } 3904 3905 static int bnxt_alloc_vnic_attributes(struct bnxt *bp) 3906 { 3907 int i, rc = 0, size; 3908 struct bnxt_vnic_info *vnic; 3909 struct pci_dev *pdev = bp->pdev; 3910 int max_rings; 3911 3912 for (i = 0; i < bp->nr_vnics; i++) { 3913 vnic = &bp->vnic_info[i]; 3914 3915 if (vnic->flags & BNXT_VNIC_UCAST_FLAG) { 3916 int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN; 3917 3918 if (mem_size > 0) { 3919 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL); 3920 if (!vnic->uc_list) { 3921 rc = -ENOMEM; 3922 goto out; 3923 } 3924 } 3925 } 3926 3927 if (vnic->flags & BNXT_VNIC_MCAST_FLAG) { 3928 vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN; 3929 vnic->mc_list = 3930 dma_alloc_coherent(&pdev->dev, 3931 vnic->mc_list_size, 3932 &vnic->mc_list_mapping, 3933 GFP_KERNEL); 3934 if (!vnic->mc_list) { 3935 rc = -ENOMEM; 3936 goto out; 3937 } 3938 } 3939 3940 if (bp->flags & BNXT_FLAG_CHIP_P5) 3941 goto vnic_skip_grps; 3942 3943 if (vnic->flags & BNXT_VNIC_RSS_FLAG) 3944 max_rings = bp->rx_nr_rings; 3945 else 3946 max_rings = 1; 3947 3948 vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL); 3949 if (!vnic->fw_grp_ids) { 3950 rc = -ENOMEM; 3951 goto out; 3952 } 3953 vnic_skip_grps: 3954 if ((bp->flags & BNXT_FLAG_NEW_RSS_CAP) && 3955 !(vnic->flags & BNXT_VNIC_RSS_FLAG)) 3956 continue; 3957 3958 /* Allocate rss table and hash key */ 3959 size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16)); 3960 if (bp->flags & BNXT_FLAG_CHIP_P5) 3961 size = L1_CACHE_ALIGN(BNXT_MAX_RSS_TABLE_SIZE_P5); 3962 3963 vnic->rss_table_size = size + HW_HASH_KEY_SIZE; 3964 vnic->rss_table = dma_alloc_coherent(&pdev->dev, 3965 vnic->rss_table_size, 3966 &vnic->rss_table_dma_addr, 3967 GFP_KERNEL); 3968 if (!vnic->rss_table) { 3969 rc = -ENOMEM; 3970 goto out; 3971 } 3972 3973 vnic->rss_hash_key = ((void *)vnic->rss_table) + size; 3974 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size; 3975 } 3976 return 0; 3977 3978 out: 3979 return rc; 3980 } 3981 3982 static void bnxt_free_hwrm_resources(struct bnxt *bp) 3983 { 3984 struct bnxt_hwrm_wait_token *token; 3985 3986 dma_pool_destroy(bp->hwrm_dma_pool); 3987 bp->hwrm_dma_pool = NULL; 3988 3989 rcu_read_lock(); 3990 hlist_for_each_entry_rcu(token, &bp->hwrm_pending_list, node) 3991 WRITE_ONCE(token->state, BNXT_HWRM_CANCELLED); 3992 rcu_read_unlock(); 3993 } 3994 3995 static int bnxt_alloc_hwrm_resources(struct bnxt *bp) 3996 { 3997 bp->hwrm_dma_pool = dma_pool_create("bnxt_hwrm", &bp->pdev->dev, 3998 BNXT_HWRM_DMA_SIZE, 3999 BNXT_HWRM_DMA_ALIGN, 0); 4000 if (!bp->hwrm_dma_pool) 4001 return -ENOMEM; 4002 4003 INIT_HLIST_HEAD(&bp->hwrm_pending_list); 4004 4005 return 0; 4006 } 4007 4008 static void bnxt_free_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats) 4009 { 4010 kfree(stats->hw_masks); 4011 stats->hw_masks = NULL; 4012 kfree(stats->sw_stats); 4013 stats->sw_stats = NULL; 4014 if (stats->hw_stats) { 4015 dma_free_coherent(&bp->pdev->dev, stats->len, stats->hw_stats, 4016 stats->hw_stats_map); 4017 stats->hw_stats = NULL; 4018 } 4019 } 4020 4021 static int bnxt_alloc_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats, 4022 bool alloc_masks) 4023 { 4024 stats->hw_stats = dma_alloc_coherent(&bp->pdev->dev, stats->len, 4025 &stats->hw_stats_map, GFP_KERNEL); 4026 if (!stats->hw_stats) 4027 return -ENOMEM; 4028 4029 stats->sw_stats = kzalloc(stats->len, GFP_KERNEL); 4030 if (!stats->sw_stats) 4031 goto stats_mem_err; 4032 4033 if (alloc_masks) { 4034 stats->hw_masks = kzalloc(stats->len, GFP_KERNEL); 4035 if (!stats->hw_masks) 4036 goto stats_mem_err; 4037 } 4038 return 0; 4039 4040 stats_mem_err: 4041 bnxt_free_stats_mem(bp, stats); 4042 return -ENOMEM; 4043 } 4044 4045 static void bnxt_fill_masks(u64 *mask_arr, u64 mask, int count) 4046 { 4047 int i; 4048 4049 for (i = 0; i < count; i++) 4050 mask_arr[i] = mask; 4051 } 4052 4053 static void bnxt_copy_hw_masks(u64 *mask_arr, __le64 *hw_mask_arr, int count) 4054 { 4055 int i; 4056 4057 for (i = 0; i < count; i++) 4058 mask_arr[i] = le64_to_cpu(hw_mask_arr[i]); 4059 } 4060 4061 static int bnxt_hwrm_func_qstat_ext(struct bnxt *bp, 4062 struct bnxt_stats_mem *stats) 4063 { 4064 struct hwrm_func_qstats_ext_output *resp; 4065 struct hwrm_func_qstats_ext_input *req; 4066 __le64 *hw_masks; 4067 int rc; 4068 4069 if (!(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED) || 4070 !(bp->flags & BNXT_FLAG_CHIP_P5)) 4071 return -EOPNOTSUPP; 4072 4073 rc = hwrm_req_init(bp, req, HWRM_FUNC_QSTATS_EXT); 4074 if (rc) 4075 return rc; 4076 4077 req->fid = cpu_to_le16(0xffff); 4078 req->flags = FUNC_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK; 4079 4080 resp = hwrm_req_hold(bp, req); 4081 rc = hwrm_req_send(bp, req); 4082 if (!rc) { 4083 hw_masks = &resp->rx_ucast_pkts; 4084 bnxt_copy_hw_masks(stats->hw_masks, hw_masks, stats->len / 8); 4085 } 4086 hwrm_req_drop(bp, req); 4087 return rc; 4088 } 4089 4090 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags); 4091 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags); 4092 4093 static void bnxt_init_stats(struct bnxt *bp) 4094 { 4095 struct bnxt_napi *bnapi = bp->bnapi[0]; 4096 struct bnxt_cp_ring_info *cpr; 4097 struct bnxt_stats_mem *stats; 4098 __le64 *rx_stats, *tx_stats; 4099 int rc, rx_count, tx_count; 4100 u64 *rx_masks, *tx_masks; 4101 u64 mask; 4102 u8 flags; 4103 4104 cpr = &bnapi->cp_ring; 4105 stats = &cpr->stats; 4106 rc = bnxt_hwrm_func_qstat_ext(bp, stats); 4107 if (rc) { 4108 if (bp->flags & BNXT_FLAG_CHIP_P5) 4109 mask = (1ULL << 48) - 1; 4110 else 4111 mask = -1ULL; 4112 bnxt_fill_masks(stats->hw_masks, mask, stats->len / 8); 4113 } 4114 if (bp->flags & BNXT_FLAG_PORT_STATS) { 4115 stats = &bp->port_stats; 4116 rx_stats = stats->hw_stats; 4117 rx_masks = stats->hw_masks; 4118 rx_count = sizeof(struct rx_port_stats) / 8; 4119 tx_stats = rx_stats + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 4120 tx_masks = rx_masks + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 4121 tx_count = sizeof(struct tx_port_stats) / 8; 4122 4123 flags = PORT_QSTATS_REQ_FLAGS_COUNTER_MASK; 4124 rc = bnxt_hwrm_port_qstats(bp, flags); 4125 if (rc) { 4126 mask = (1ULL << 40) - 1; 4127 4128 bnxt_fill_masks(rx_masks, mask, rx_count); 4129 bnxt_fill_masks(tx_masks, mask, tx_count); 4130 } else { 4131 bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count); 4132 bnxt_copy_hw_masks(tx_masks, tx_stats, tx_count); 4133 bnxt_hwrm_port_qstats(bp, 0); 4134 } 4135 } 4136 if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) { 4137 stats = &bp->rx_port_stats_ext; 4138 rx_stats = stats->hw_stats; 4139 rx_masks = stats->hw_masks; 4140 rx_count = sizeof(struct rx_port_stats_ext) / 8; 4141 stats = &bp->tx_port_stats_ext; 4142 tx_stats = stats->hw_stats; 4143 tx_masks = stats->hw_masks; 4144 tx_count = sizeof(struct tx_port_stats_ext) / 8; 4145 4146 flags = PORT_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK; 4147 rc = bnxt_hwrm_port_qstats_ext(bp, flags); 4148 if (rc) { 4149 mask = (1ULL << 40) - 1; 4150 4151 bnxt_fill_masks(rx_masks, mask, rx_count); 4152 if (tx_stats) 4153 bnxt_fill_masks(tx_masks, mask, tx_count); 4154 } else { 4155 bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count); 4156 if (tx_stats) 4157 bnxt_copy_hw_masks(tx_masks, tx_stats, 4158 tx_count); 4159 bnxt_hwrm_port_qstats_ext(bp, 0); 4160 } 4161 } 4162 } 4163 4164 static void bnxt_free_port_stats(struct bnxt *bp) 4165 { 4166 bp->flags &= ~BNXT_FLAG_PORT_STATS; 4167 bp->flags &= ~BNXT_FLAG_PORT_STATS_EXT; 4168 4169 bnxt_free_stats_mem(bp, &bp->port_stats); 4170 bnxt_free_stats_mem(bp, &bp->rx_port_stats_ext); 4171 bnxt_free_stats_mem(bp, &bp->tx_port_stats_ext); 4172 } 4173 4174 static void bnxt_free_ring_stats(struct bnxt *bp) 4175 { 4176 int i; 4177 4178 if (!bp->bnapi) 4179 return; 4180 4181 for (i = 0; i < bp->cp_nr_rings; i++) { 4182 struct bnxt_napi *bnapi = bp->bnapi[i]; 4183 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 4184 4185 bnxt_free_stats_mem(bp, &cpr->stats); 4186 } 4187 } 4188 4189 static int bnxt_alloc_stats(struct bnxt *bp) 4190 { 4191 u32 size, i; 4192 int rc; 4193 4194 size = bp->hw_ring_stats_size; 4195 4196 for (i = 0; i < bp->cp_nr_rings; i++) { 4197 struct bnxt_napi *bnapi = bp->bnapi[i]; 4198 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 4199 4200 cpr->stats.len = size; 4201 rc = bnxt_alloc_stats_mem(bp, &cpr->stats, !i); 4202 if (rc) 4203 return rc; 4204 4205 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID; 4206 } 4207 4208 if (BNXT_VF(bp) || bp->chip_num == CHIP_NUM_58700) 4209 return 0; 4210 4211 if (bp->port_stats.hw_stats) 4212 goto alloc_ext_stats; 4213 4214 bp->port_stats.len = BNXT_PORT_STATS_SIZE; 4215 rc = bnxt_alloc_stats_mem(bp, &bp->port_stats, true); 4216 if (rc) 4217 return rc; 4218 4219 bp->flags |= BNXT_FLAG_PORT_STATS; 4220 4221 alloc_ext_stats: 4222 /* Display extended statistics only if FW supports it */ 4223 if (bp->hwrm_spec_code < 0x10804 || bp->hwrm_spec_code == 0x10900) 4224 if (!(bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED)) 4225 return 0; 4226 4227 if (bp->rx_port_stats_ext.hw_stats) 4228 goto alloc_tx_ext_stats; 4229 4230 bp->rx_port_stats_ext.len = sizeof(struct rx_port_stats_ext); 4231 rc = bnxt_alloc_stats_mem(bp, &bp->rx_port_stats_ext, true); 4232 /* Extended stats are optional */ 4233 if (rc) 4234 return 0; 4235 4236 alloc_tx_ext_stats: 4237 if (bp->tx_port_stats_ext.hw_stats) 4238 return 0; 4239 4240 if (bp->hwrm_spec_code >= 0x10902 || 4241 (bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED)) { 4242 bp->tx_port_stats_ext.len = sizeof(struct tx_port_stats_ext); 4243 rc = bnxt_alloc_stats_mem(bp, &bp->tx_port_stats_ext, true); 4244 /* Extended stats are optional */ 4245 if (rc) 4246 return 0; 4247 } 4248 bp->flags |= BNXT_FLAG_PORT_STATS_EXT; 4249 return 0; 4250 } 4251 4252 static void bnxt_clear_ring_indices(struct bnxt *bp) 4253 { 4254 int i; 4255 4256 if (!bp->bnapi) 4257 return; 4258 4259 for (i = 0; i < bp->cp_nr_rings; i++) { 4260 struct bnxt_napi *bnapi = bp->bnapi[i]; 4261 struct bnxt_cp_ring_info *cpr; 4262 struct bnxt_rx_ring_info *rxr; 4263 struct bnxt_tx_ring_info *txr; 4264 4265 if (!bnapi) 4266 continue; 4267 4268 cpr = &bnapi->cp_ring; 4269 cpr->cp_raw_cons = 0; 4270 4271 txr = bnapi->tx_ring; 4272 if (txr) { 4273 txr->tx_prod = 0; 4274 txr->tx_cons = 0; 4275 } 4276 4277 rxr = bnapi->rx_ring; 4278 if (rxr) { 4279 rxr->rx_prod = 0; 4280 rxr->rx_agg_prod = 0; 4281 rxr->rx_sw_agg_prod = 0; 4282 rxr->rx_next_cons = 0; 4283 } 4284 } 4285 } 4286 4287 static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool irq_reinit) 4288 { 4289 #ifdef CONFIG_RFS_ACCEL 4290 int i; 4291 4292 /* Under rtnl_lock and all our NAPIs have been disabled. It's 4293 * safe to delete the hash table. 4294 */ 4295 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) { 4296 struct hlist_head *head; 4297 struct hlist_node *tmp; 4298 struct bnxt_ntuple_filter *fltr; 4299 4300 head = &bp->ntp_fltr_hash_tbl[i]; 4301 hlist_for_each_entry_safe(fltr, tmp, head, hash) { 4302 hlist_del(&fltr->hash); 4303 kfree(fltr); 4304 } 4305 } 4306 if (irq_reinit) { 4307 kfree(bp->ntp_fltr_bmap); 4308 bp->ntp_fltr_bmap = NULL; 4309 } 4310 bp->ntp_fltr_count = 0; 4311 #endif 4312 } 4313 4314 static int bnxt_alloc_ntp_fltrs(struct bnxt *bp) 4315 { 4316 #ifdef CONFIG_RFS_ACCEL 4317 int i, rc = 0; 4318 4319 if (!(bp->flags & BNXT_FLAG_RFS)) 4320 return 0; 4321 4322 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) 4323 INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]); 4324 4325 bp->ntp_fltr_count = 0; 4326 bp->ntp_fltr_bmap = kcalloc(BITS_TO_LONGS(BNXT_NTP_FLTR_MAX_FLTR), 4327 sizeof(long), 4328 GFP_KERNEL); 4329 4330 if (!bp->ntp_fltr_bmap) 4331 rc = -ENOMEM; 4332 4333 return rc; 4334 #else 4335 return 0; 4336 #endif 4337 } 4338 4339 static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init) 4340 { 4341 bnxt_free_vnic_attributes(bp); 4342 bnxt_free_tx_rings(bp); 4343 bnxt_free_rx_rings(bp); 4344 bnxt_free_cp_rings(bp); 4345 bnxt_free_all_cp_arrays(bp); 4346 bnxt_free_ntp_fltrs(bp, irq_re_init); 4347 if (irq_re_init) { 4348 bnxt_free_ring_stats(bp); 4349 if (!(bp->phy_flags & BNXT_PHY_FL_PORT_STATS_NO_RESET) || 4350 test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 4351 bnxt_free_port_stats(bp); 4352 bnxt_free_ring_grps(bp); 4353 bnxt_free_vnics(bp); 4354 kfree(bp->tx_ring_map); 4355 bp->tx_ring_map = NULL; 4356 kfree(bp->tx_ring); 4357 bp->tx_ring = NULL; 4358 kfree(bp->rx_ring); 4359 bp->rx_ring = NULL; 4360 kfree(bp->bnapi); 4361 bp->bnapi = NULL; 4362 } else { 4363 bnxt_clear_ring_indices(bp); 4364 } 4365 } 4366 4367 static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init) 4368 { 4369 int i, j, rc, size, arr_size; 4370 void *bnapi; 4371 4372 if (irq_re_init) { 4373 /* Allocate bnapi mem pointer array and mem block for 4374 * all queues 4375 */ 4376 arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) * 4377 bp->cp_nr_rings); 4378 size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi)); 4379 bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL); 4380 if (!bnapi) 4381 return -ENOMEM; 4382 4383 bp->bnapi = bnapi; 4384 bnapi += arr_size; 4385 for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) { 4386 bp->bnapi[i] = bnapi; 4387 bp->bnapi[i]->index = i; 4388 bp->bnapi[i]->bp = bp; 4389 if (bp->flags & BNXT_FLAG_CHIP_P5) { 4390 struct bnxt_cp_ring_info *cpr = 4391 &bp->bnapi[i]->cp_ring; 4392 4393 cpr->cp_ring_struct.ring_mem.flags = 4394 BNXT_RMEM_RING_PTE_FLAG; 4395 } 4396 } 4397 4398 bp->rx_ring = kcalloc(bp->rx_nr_rings, 4399 sizeof(struct bnxt_rx_ring_info), 4400 GFP_KERNEL); 4401 if (!bp->rx_ring) 4402 return -ENOMEM; 4403 4404 for (i = 0; i < bp->rx_nr_rings; i++) { 4405 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 4406 4407 if (bp->flags & BNXT_FLAG_CHIP_P5) { 4408 rxr->rx_ring_struct.ring_mem.flags = 4409 BNXT_RMEM_RING_PTE_FLAG; 4410 rxr->rx_agg_ring_struct.ring_mem.flags = 4411 BNXT_RMEM_RING_PTE_FLAG; 4412 } 4413 rxr->bnapi = bp->bnapi[i]; 4414 bp->bnapi[i]->rx_ring = &bp->rx_ring[i]; 4415 } 4416 4417 bp->tx_ring = kcalloc(bp->tx_nr_rings, 4418 sizeof(struct bnxt_tx_ring_info), 4419 GFP_KERNEL); 4420 if (!bp->tx_ring) 4421 return -ENOMEM; 4422 4423 bp->tx_ring_map = kcalloc(bp->tx_nr_rings, sizeof(u16), 4424 GFP_KERNEL); 4425 4426 if (!bp->tx_ring_map) 4427 return -ENOMEM; 4428 4429 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 4430 j = 0; 4431 else 4432 j = bp->rx_nr_rings; 4433 4434 for (i = 0; i < bp->tx_nr_rings; i++, j++) { 4435 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 4436 4437 if (bp->flags & BNXT_FLAG_CHIP_P5) 4438 txr->tx_ring_struct.ring_mem.flags = 4439 BNXT_RMEM_RING_PTE_FLAG; 4440 txr->bnapi = bp->bnapi[j]; 4441 bp->bnapi[j]->tx_ring = txr; 4442 bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i; 4443 if (i >= bp->tx_nr_rings_xdp) { 4444 txr->txq_index = i - bp->tx_nr_rings_xdp; 4445 bp->bnapi[j]->tx_int = bnxt_tx_int; 4446 } else { 4447 bp->bnapi[j]->flags |= BNXT_NAPI_FLAG_XDP; 4448 bp->bnapi[j]->tx_int = bnxt_tx_int_xdp; 4449 } 4450 } 4451 4452 rc = bnxt_alloc_stats(bp); 4453 if (rc) 4454 goto alloc_mem_err; 4455 bnxt_init_stats(bp); 4456 4457 rc = bnxt_alloc_ntp_fltrs(bp); 4458 if (rc) 4459 goto alloc_mem_err; 4460 4461 rc = bnxt_alloc_vnics(bp); 4462 if (rc) 4463 goto alloc_mem_err; 4464 } 4465 4466 rc = bnxt_alloc_all_cp_arrays(bp); 4467 if (rc) 4468 goto alloc_mem_err; 4469 4470 bnxt_init_ring_struct(bp); 4471 4472 rc = bnxt_alloc_rx_rings(bp); 4473 if (rc) 4474 goto alloc_mem_err; 4475 4476 rc = bnxt_alloc_tx_rings(bp); 4477 if (rc) 4478 goto alloc_mem_err; 4479 4480 rc = bnxt_alloc_cp_rings(bp); 4481 if (rc) 4482 goto alloc_mem_err; 4483 4484 bp->vnic_info[0].flags |= BNXT_VNIC_RSS_FLAG | BNXT_VNIC_MCAST_FLAG | 4485 BNXT_VNIC_UCAST_FLAG; 4486 rc = bnxt_alloc_vnic_attributes(bp); 4487 if (rc) 4488 goto alloc_mem_err; 4489 return 0; 4490 4491 alloc_mem_err: 4492 bnxt_free_mem(bp, true); 4493 return rc; 4494 } 4495 4496 static void bnxt_disable_int(struct bnxt *bp) 4497 { 4498 int i; 4499 4500 if (!bp->bnapi) 4501 return; 4502 4503 for (i = 0; i < bp->cp_nr_rings; i++) { 4504 struct bnxt_napi *bnapi = bp->bnapi[i]; 4505 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 4506 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 4507 4508 if (ring->fw_ring_id != INVALID_HW_RING_ID) 4509 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons); 4510 } 4511 } 4512 4513 static int bnxt_cp_num_to_irq_num(struct bnxt *bp, int n) 4514 { 4515 struct bnxt_napi *bnapi = bp->bnapi[n]; 4516 struct bnxt_cp_ring_info *cpr; 4517 4518 cpr = &bnapi->cp_ring; 4519 return cpr->cp_ring_struct.map_idx; 4520 } 4521 4522 static void bnxt_disable_int_sync(struct bnxt *bp) 4523 { 4524 int i; 4525 4526 if (!bp->irq_tbl) 4527 return; 4528 4529 atomic_inc(&bp->intr_sem); 4530 4531 bnxt_disable_int(bp); 4532 for (i = 0; i < bp->cp_nr_rings; i++) { 4533 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 4534 4535 synchronize_irq(bp->irq_tbl[map_idx].vector); 4536 } 4537 } 4538 4539 static void bnxt_enable_int(struct bnxt *bp) 4540 { 4541 int i; 4542 4543 atomic_set(&bp->intr_sem, 0); 4544 for (i = 0; i < bp->cp_nr_rings; i++) { 4545 struct bnxt_napi *bnapi = bp->bnapi[i]; 4546 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 4547 4548 bnxt_db_nq_arm(bp, &cpr->cp_db, cpr->cp_raw_cons); 4549 } 4550 } 4551 4552 int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp, unsigned long *bmap, int bmap_size, 4553 bool async_only) 4554 { 4555 DECLARE_BITMAP(async_events_bmap, 256); 4556 u32 *events = (u32 *)async_events_bmap; 4557 struct hwrm_func_drv_rgtr_output *resp; 4558 struct hwrm_func_drv_rgtr_input *req; 4559 u32 flags; 4560 int rc, i; 4561 4562 rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_RGTR); 4563 if (rc) 4564 return rc; 4565 4566 req->enables = cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE | 4567 FUNC_DRV_RGTR_REQ_ENABLES_VER | 4568 FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD); 4569 4570 req->os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX); 4571 flags = FUNC_DRV_RGTR_REQ_FLAGS_16BIT_VER_MODE; 4572 if (bp->fw_cap & BNXT_FW_CAP_HOT_RESET) 4573 flags |= FUNC_DRV_RGTR_REQ_FLAGS_HOT_RESET_SUPPORT; 4574 if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) 4575 flags |= FUNC_DRV_RGTR_REQ_FLAGS_ERROR_RECOVERY_SUPPORT | 4576 FUNC_DRV_RGTR_REQ_FLAGS_MASTER_SUPPORT; 4577 req->flags = cpu_to_le32(flags); 4578 req->ver_maj_8b = DRV_VER_MAJ; 4579 req->ver_min_8b = DRV_VER_MIN; 4580 req->ver_upd_8b = DRV_VER_UPD; 4581 req->ver_maj = cpu_to_le16(DRV_VER_MAJ); 4582 req->ver_min = cpu_to_le16(DRV_VER_MIN); 4583 req->ver_upd = cpu_to_le16(DRV_VER_UPD); 4584 4585 if (BNXT_PF(bp)) { 4586 u32 data[8]; 4587 int i; 4588 4589 memset(data, 0, sizeof(data)); 4590 for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) { 4591 u16 cmd = bnxt_vf_req_snif[i]; 4592 unsigned int bit, idx; 4593 4594 idx = cmd / 32; 4595 bit = cmd % 32; 4596 data[idx] |= 1 << bit; 4597 } 4598 4599 for (i = 0; i < 8; i++) 4600 req->vf_req_fwd[i] = cpu_to_le32(data[i]); 4601 4602 req->enables |= 4603 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD); 4604 } 4605 4606 if (bp->fw_cap & BNXT_FW_CAP_OVS_64BIT_HANDLE) 4607 req->flags |= cpu_to_le32( 4608 FUNC_DRV_RGTR_REQ_FLAGS_FLOW_HANDLE_64BIT_MODE); 4609 4610 memset(async_events_bmap, 0, sizeof(async_events_bmap)); 4611 for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++) { 4612 u16 event_id = bnxt_async_events_arr[i]; 4613 4614 if (event_id == ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY && 4615 !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) 4616 continue; 4617 __set_bit(bnxt_async_events_arr[i], async_events_bmap); 4618 } 4619 if (bmap && bmap_size) { 4620 for (i = 0; i < bmap_size; i++) { 4621 if (test_bit(i, bmap)) 4622 __set_bit(i, async_events_bmap); 4623 } 4624 } 4625 for (i = 0; i < 8; i++) 4626 req->async_event_fwd[i] |= cpu_to_le32(events[i]); 4627 4628 if (async_only) 4629 req->enables = 4630 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD); 4631 4632 resp = hwrm_req_hold(bp, req); 4633 rc = hwrm_req_send(bp, req); 4634 if (!rc) { 4635 set_bit(BNXT_STATE_DRV_REGISTERED, &bp->state); 4636 if (resp->flags & 4637 cpu_to_le32(FUNC_DRV_RGTR_RESP_FLAGS_IF_CHANGE_SUPPORTED)) 4638 bp->fw_cap |= BNXT_FW_CAP_IF_CHANGE; 4639 } 4640 hwrm_req_drop(bp, req); 4641 return rc; 4642 } 4643 4644 int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp) 4645 { 4646 struct hwrm_func_drv_unrgtr_input *req; 4647 int rc; 4648 4649 if (!test_and_clear_bit(BNXT_STATE_DRV_REGISTERED, &bp->state)) 4650 return 0; 4651 4652 rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_UNRGTR); 4653 if (rc) 4654 return rc; 4655 return hwrm_req_send(bp, req); 4656 } 4657 4658 static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type) 4659 { 4660 struct hwrm_tunnel_dst_port_free_input *req; 4661 int rc; 4662 4663 if (tunnel_type == TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN && 4664 bp->vxlan_fw_dst_port_id == INVALID_HW_RING_ID) 4665 return 0; 4666 if (tunnel_type == TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE && 4667 bp->nge_fw_dst_port_id == INVALID_HW_RING_ID) 4668 return 0; 4669 4670 rc = hwrm_req_init(bp, req, HWRM_TUNNEL_DST_PORT_FREE); 4671 if (rc) 4672 return rc; 4673 4674 req->tunnel_type = tunnel_type; 4675 4676 switch (tunnel_type) { 4677 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN: 4678 req->tunnel_dst_port_id = cpu_to_le16(bp->vxlan_fw_dst_port_id); 4679 bp->vxlan_port = 0; 4680 bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID; 4681 break; 4682 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE: 4683 req->tunnel_dst_port_id = cpu_to_le16(bp->nge_fw_dst_port_id); 4684 bp->nge_port = 0; 4685 bp->nge_fw_dst_port_id = INVALID_HW_RING_ID; 4686 break; 4687 default: 4688 break; 4689 } 4690 4691 rc = hwrm_req_send(bp, req); 4692 if (rc) 4693 netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n", 4694 rc); 4695 return rc; 4696 } 4697 4698 static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port, 4699 u8 tunnel_type) 4700 { 4701 struct hwrm_tunnel_dst_port_alloc_output *resp; 4702 struct hwrm_tunnel_dst_port_alloc_input *req; 4703 int rc; 4704 4705 rc = hwrm_req_init(bp, req, HWRM_TUNNEL_DST_PORT_ALLOC); 4706 if (rc) 4707 return rc; 4708 4709 req->tunnel_type = tunnel_type; 4710 req->tunnel_dst_port_val = port; 4711 4712 resp = hwrm_req_hold(bp, req); 4713 rc = hwrm_req_send(bp, req); 4714 if (rc) { 4715 netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n", 4716 rc); 4717 goto err_out; 4718 } 4719 4720 switch (tunnel_type) { 4721 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN: 4722 bp->vxlan_port = port; 4723 bp->vxlan_fw_dst_port_id = 4724 le16_to_cpu(resp->tunnel_dst_port_id); 4725 break; 4726 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE: 4727 bp->nge_port = port; 4728 bp->nge_fw_dst_port_id = le16_to_cpu(resp->tunnel_dst_port_id); 4729 break; 4730 default: 4731 break; 4732 } 4733 4734 err_out: 4735 hwrm_req_drop(bp, req); 4736 return rc; 4737 } 4738 4739 static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id) 4740 { 4741 struct hwrm_cfa_l2_set_rx_mask_input *req; 4742 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 4743 int rc; 4744 4745 rc = hwrm_req_init(bp, req, HWRM_CFA_L2_SET_RX_MASK); 4746 if (rc) 4747 return rc; 4748 4749 req->vnic_id = cpu_to_le32(vnic->fw_vnic_id); 4750 if (vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_MCAST) { 4751 req->num_mc_entries = cpu_to_le32(vnic->mc_list_count); 4752 req->mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping); 4753 } 4754 req->mask = cpu_to_le32(vnic->rx_mask); 4755 return hwrm_req_send_silent(bp, req); 4756 } 4757 4758 #ifdef CONFIG_RFS_ACCEL 4759 static int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp, 4760 struct bnxt_ntuple_filter *fltr) 4761 { 4762 struct hwrm_cfa_ntuple_filter_free_input *req; 4763 int rc; 4764 4765 rc = hwrm_req_init(bp, req, HWRM_CFA_NTUPLE_FILTER_FREE); 4766 if (rc) 4767 return rc; 4768 4769 req->ntuple_filter_id = fltr->filter_id; 4770 return hwrm_req_send(bp, req); 4771 } 4772 4773 #define BNXT_NTP_FLTR_FLAGS \ 4774 (CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID | \ 4775 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE | \ 4776 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR | \ 4777 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE | \ 4778 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR | \ 4779 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK | \ 4780 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR | \ 4781 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK | \ 4782 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL | \ 4783 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT | \ 4784 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK | \ 4785 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT | \ 4786 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK | \ 4787 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID) 4788 4789 #define BNXT_NTP_TUNNEL_FLTR_FLAG \ 4790 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE 4791 4792 static int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp, 4793 struct bnxt_ntuple_filter *fltr) 4794 { 4795 struct hwrm_cfa_ntuple_filter_alloc_output *resp; 4796 struct hwrm_cfa_ntuple_filter_alloc_input *req; 4797 struct flow_keys *keys = &fltr->fkeys; 4798 struct bnxt_vnic_info *vnic; 4799 u32 flags = 0; 4800 int rc; 4801 4802 rc = hwrm_req_init(bp, req, HWRM_CFA_NTUPLE_FILTER_ALLOC); 4803 if (rc) 4804 return rc; 4805 4806 req->l2_filter_id = bp->vnic_info[0].fw_l2_filter_id[fltr->l2_fltr_idx]; 4807 4808 if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2) { 4809 flags = CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DEST_RFS_RING_IDX; 4810 req->dst_id = cpu_to_le16(fltr->rxq); 4811 } else { 4812 vnic = &bp->vnic_info[fltr->rxq + 1]; 4813 req->dst_id = cpu_to_le16(vnic->fw_vnic_id); 4814 } 4815 req->flags = cpu_to_le32(flags); 4816 req->enables = cpu_to_le32(BNXT_NTP_FLTR_FLAGS); 4817 4818 req->ethertype = htons(ETH_P_IP); 4819 memcpy(req->src_macaddr, fltr->src_mac_addr, ETH_ALEN); 4820 req->ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4; 4821 req->ip_protocol = keys->basic.ip_proto; 4822 4823 if (keys->basic.n_proto == htons(ETH_P_IPV6)) { 4824 int i; 4825 4826 req->ethertype = htons(ETH_P_IPV6); 4827 req->ip_addr_type = 4828 CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6; 4829 *(struct in6_addr *)&req->src_ipaddr[0] = 4830 keys->addrs.v6addrs.src; 4831 *(struct in6_addr *)&req->dst_ipaddr[0] = 4832 keys->addrs.v6addrs.dst; 4833 for (i = 0; i < 4; i++) { 4834 req->src_ipaddr_mask[i] = cpu_to_be32(0xffffffff); 4835 req->dst_ipaddr_mask[i] = cpu_to_be32(0xffffffff); 4836 } 4837 } else { 4838 req->src_ipaddr[0] = keys->addrs.v4addrs.src; 4839 req->src_ipaddr_mask[0] = cpu_to_be32(0xffffffff); 4840 req->dst_ipaddr[0] = keys->addrs.v4addrs.dst; 4841 req->dst_ipaddr_mask[0] = cpu_to_be32(0xffffffff); 4842 } 4843 if (keys->control.flags & FLOW_DIS_ENCAPSULATION) { 4844 req->enables |= cpu_to_le32(BNXT_NTP_TUNNEL_FLTR_FLAG); 4845 req->tunnel_type = 4846 CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL; 4847 } 4848 4849 req->src_port = keys->ports.src; 4850 req->src_port_mask = cpu_to_be16(0xffff); 4851 req->dst_port = keys->ports.dst; 4852 req->dst_port_mask = cpu_to_be16(0xffff); 4853 4854 resp = hwrm_req_hold(bp, req); 4855 rc = hwrm_req_send(bp, req); 4856 if (!rc) 4857 fltr->filter_id = resp->ntuple_filter_id; 4858 hwrm_req_drop(bp, req); 4859 return rc; 4860 } 4861 #endif 4862 4863 static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx, 4864 const u8 *mac_addr) 4865 { 4866 struct hwrm_cfa_l2_filter_alloc_output *resp; 4867 struct hwrm_cfa_l2_filter_alloc_input *req; 4868 int rc; 4869 4870 rc = hwrm_req_init(bp, req, HWRM_CFA_L2_FILTER_ALLOC); 4871 if (rc) 4872 return rc; 4873 4874 req->flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX); 4875 if (!BNXT_CHIP_TYPE_NITRO_A0(bp)) 4876 req->flags |= 4877 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST); 4878 req->dst_id = cpu_to_le16(bp->vnic_info[vnic_id].fw_vnic_id); 4879 req->enables = 4880 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR | 4881 CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID | 4882 CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK); 4883 memcpy(req->l2_addr, mac_addr, ETH_ALEN); 4884 req->l2_addr_mask[0] = 0xff; 4885 req->l2_addr_mask[1] = 0xff; 4886 req->l2_addr_mask[2] = 0xff; 4887 req->l2_addr_mask[3] = 0xff; 4888 req->l2_addr_mask[4] = 0xff; 4889 req->l2_addr_mask[5] = 0xff; 4890 4891 resp = hwrm_req_hold(bp, req); 4892 rc = hwrm_req_send(bp, req); 4893 if (!rc) 4894 bp->vnic_info[vnic_id].fw_l2_filter_id[idx] = 4895 resp->l2_filter_id; 4896 hwrm_req_drop(bp, req); 4897 return rc; 4898 } 4899 4900 static int bnxt_hwrm_clear_vnic_filter(struct bnxt *bp) 4901 { 4902 struct hwrm_cfa_l2_filter_free_input *req; 4903 u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */ 4904 int rc; 4905 4906 /* Any associated ntuple filters will also be cleared by firmware. */ 4907 rc = hwrm_req_init(bp, req, HWRM_CFA_L2_FILTER_FREE); 4908 if (rc) 4909 return rc; 4910 hwrm_req_hold(bp, req); 4911 for (i = 0; i < num_of_vnics; i++) { 4912 struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; 4913 4914 for (j = 0; j < vnic->uc_filter_count; j++) { 4915 req->l2_filter_id = vnic->fw_l2_filter_id[j]; 4916 4917 rc = hwrm_req_send(bp, req); 4918 } 4919 vnic->uc_filter_count = 0; 4920 } 4921 hwrm_req_drop(bp, req); 4922 return rc; 4923 } 4924 4925 static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags) 4926 { 4927 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 4928 u16 max_aggs = VNIC_TPA_CFG_REQ_MAX_AGGS_MAX; 4929 struct hwrm_vnic_tpa_cfg_input *req; 4930 int rc; 4931 4932 if (vnic->fw_vnic_id == INVALID_HW_RING_ID) 4933 return 0; 4934 4935 rc = hwrm_req_init(bp, req, HWRM_VNIC_TPA_CFG); 4936 if (rc) 4937 return rc; 4938 4939 if (tpa_flags) { 4940 u16 mss = bp->dev->mtu - 40; 4941 u32 nsegs, n, segs = 0, flags; 4942 4943 flags = VNIC_TPA_CFG_REQ_FLAGS_TPA | 4944 VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA | 4945 VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE | 4946 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN | 4947 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ; 4948 if (tpa_flags & BNXT_FLAG_GRO) 4949 flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO; 4950 4951 req->flags = cpu_to_le32(flags); 4952 4953 req->enables = 4954 cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS | 4955 VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS | 4956 VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN); 4957 4958 /* Number of segs are log2 units, and first packet is not 4959 * included as part of this units. 4960 */ 4961 if (mss <= BNXT_RX_PAGE_SIZE) { 4962 n = BNXT_RX_PAGE_SIZE / mss; 4963 nsegs = (MAX_SKB_FRAGS - 1) * n; 4964 } else { 4965 n = mss / BNXT_RX_PAGE_SIZE; 4966 if (mss & (BNXT_RX_PAGE_SIZE - 1)) 4967 n++; 4968 nsegs = (MAX_SKB_FRAGS - n) / n; 4969 } 4970 4971 if (bp->flags & BNXT_FLAG_CHIP_P5) { 4972 segs = MAX_TPA_SEGS_P5; 4973 max_aggs = bp->max_tpa; 4974 } else { 4975 segs = ilog2(nsegs); 4976 } 4977 req->max_agg_segs = cpu_to_le16(segs); 4978 req->max_aggs = cpu_to_le16(max_aggs); 4979 4980 req->min_agg_len = cpu_to_le32(512); 4981 } 4982 req->vnic_id = cpu_to_le16(vnic->fw_vnic_id); 4983 4984 return hwrm_req_send(bp, req); 4985 } 4986 4987 static u16 bnxt_cp_ring_from_grp(struct bnxt *bp, struct bnxt_ring_struct *ring) 4988 { 4989 struct bnxt_ring_grp_info *grp_info; 4990 4991 grp_info = &bp->grp_info[ring->grp_idx]; 4992 return grp_info->cp_fw_ring_id; 4993 } 4994 4995 static u16 bnxt_cp_ring_for_rx(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) 4996 { 4997 if (bp->flags & BNXT_FLAG_CHIP_P5) { 4998 struct bnxt_napi *bnapi = rxr->bnapi; 4999 struct bnxt_cp_ring_info *cpr; 5000 5001 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_RX_HDL]; 5002 return cpr->cp_ring_struct.fw_ring_id; 5003 } else { 5004 return bnxt_cp_ring_from_grp(bp, &rxr->rx_ring_struct); 5005 } 5006 } 5007 5008 static u16 bnxt_cp_ring_for_tx(struct bnxt *bp, struct bnxt_tx_ring_info *txr) 5009 { 5010 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5011 struct bnxt_napi *bnapi = txr->bnapi; 5012 struct bnxt_cp_ring_info *cpr; 5013 5014 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_TX_HDL]; 5015 return cpr->cp_ring_struct.fw_ring_id; 5016 } else { 5017 return bnxt_cp_ring_from_grp(bp, &txr->tx_ring_struct); 5018 } 5019 } 5020 5021 static int bnxt_alloc_rss_indir_tbl(struct bnxt *bp) 5022 { 5023 int entries; 5024 5025 if (bp->flags & BNXT_FLAG_CHIP_P5) 5026 entries = BNXT_MAX_RSS_TABLE_ENTRIES_P5; 5027 else 5028 entries = HW_HASH_INDEX_SIZE; 5029 5030 bp->rss_indir_tbl_entries = entries; 5031 bp->rss_indir_tbl = kmalloc_array(entries, sizeof(*bp->rss_indir_tbl), 5032 GFP_KERNEL); 5033 if (!bp->rss_indir_tbl) 5034 return -ENOMEM; 5035 return 0; 5036 } 5037 5038 static void bnxt_set_dflt_rss_indir_tbl(struct bnxt *bp) 5039 { 5040 u16 max_rings, max_entries, pad, i; 5041 5042 if (!bp->rx_nr_rings) 5043 return; 5044 5045 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 5046 max_rings = bp->rx_nr_rings - 1; 5047 else 5048 max_rings = bp->rx_nr_rings; 5049 5050 max_entries = bnxt_get_rxfh_indir_size(bp->dev); 5051 5052 for (i = 0; i < max_entries; i++) 5053 bp->rss_indir_tbl[i] = ethtool_rxfh_indir_default(i, max_rings); 5054 5055 pad = bp->rss_indir_tbl_entries - max_entries; 5056 if (pad) 5057 memset(&bp->rss_indir_tbl[i], 0, pad * sizeof(u16)); 5058 } 5059 5060 static u16 bnxt_get_max_rss_ring(struct bnxt *bp) 5061 { 5062 u16 i, tbl_size, max_ring = 0; 5063 5064 if (!bp->rss_indir_tbl) 5065 return 0; 5066 5067 tbl_size = bnxt_get_rxfh_indir_size(bp->dev); 5068 for (i = 0; i < tbl_size; i++) 5069 max_ring = max(max_ring, bp->rss_indir_tbl[i]); 5070 return max_ring; 5071 } 5072 5073 int bnxt_get_nr_rss_ctxs(struct bnxt *bp, int rx_rings) 5074 { 5075 if (bp->flags & BNXT_FLAG_CHIP_P5) 5076 return DIV_ROUND_UP(rx_rings, BNXT_RSS_TABLE_ENTRIES_P5); 5077 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 5078 return 2; 5079 return 1; 5080 } 5081 5082 static void __bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic) 5083 { 5084 bool no_rss = !(vnic->flags & BNXT_VNIC_RSS_FLAG); 5085 u16 i, j; 5086 5087 /* Fill the RSS indirection table with ring group ids */ 5088 for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++) { 5089 if (!no_rss) 5090 j = bp->rss_indir_tbl[i]; 5091 vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]); 5092 } 5093 } 5094 5095 static void __bnxt_fill_hw_rss_tbl_p5(struct bnxt *bp, 5096 struct bnxt_vnic_info *vnic) 5097 { 5098 __le16 *ring_tbl = vnic->rss_table; 5099 struct bnxt_rx_ring_info *rxr; 5100 u16 tbl_size, i; 5101 5102 tbl_size = bnxt_get_rxfh_indir_size(bp->dev); 5103 5104 for (i = 0; i < tbl_size; i++) { 5105 u16 ring_id, j; 5106 5107 j = bp->rss_indir_tbl[i]; 5108 rxr = &bp->rx_ring[j]; 5109 5110 ring_id = rxr->rx_ring_struct.fw_ring_id; 5111 *ring_tbl++ = cpu_to_le16(ring_id); 5112 ring_id = bnxt_cp_ring_for_rx(bp, rxr); 5113 *ring_tbl++ = cpu_to_le16(ring_id); 5114 } 5115 } 5116 5117 static void bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic) 5118 { 5119 if (bp->flags & BNXT_FLAG_CHIP_P5) 5120 __bnxt_fill_hw_rss_tbl_p5(bp, vnic); 5121 else 5122 __bnxt_fill_hw_rss_tbl(bp, vnic); 5123 } 5124 5125 static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss) 5126 { 5127 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5128 struct hwrm_vnic_rss_cfg_input *req; 5129 int rc; 5130 5131 if ((bp->flags & BNXT_FLAG_CHIP_P5) || 5132 vnic->fw_rss_cos_lb_ctx[0] == INVALID_HW_RING_ID) 5133 return 0; 5134 5135 rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_CFG); 5136 if (rc) 5137 return rc; 5138 5139 if (set_rss) { 5140 bnxt_fill_hw_rss_tbl(bp, vnic); 5141 req->hash_type = cpu_to_le32(bp->rss_hash_cfg); 5142 req->hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT; 5143 req->ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr); 5144 req->hash_key_tbl_addr = 5145 cpu_to_le64(vnic->rss_hash_key_dma_addr); 5146 } 5147 req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]); 5148 return hwrm_req_send(bp, req); 5149 } 5150 5151 static int bnxt_hwrm_vnic_set_rss_p5(struct bnxt *bp, u16 vnic_id, bool set_rss) 5152 { 5153 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5154 struct hwrm_vnic_rss_cfg_input *req; 5155 dma_addr_t ring_tbl_map; 5156 u32 i, nr_ctxs; 5157 int rc; 5158 5159 rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_CFG); 5160 if (rc) 5161 return rc; 5162 5163 req->vnic_id = cpu_to_le16(vnic->fw_vnic_id); 5164 if (!set_rss) 5165 return hwrm_req_send(bp, req); 5166 5167 bnxt_fill_hw_rss_tbl(bp, vnic); 5168 req->hash_type = cpu_to_le32(bp->rss_hash_cfg); 5169 req->hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT; 5170 req->hash_key_tbl_addr = cpu_to_le64(vnic->rss_hash_key_dma_addr); 5171 ring_tbl_map = vnic->rss_table_dma_addr; 5172 nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings); 5173 5174 hwrm_req_hold(bp, req); 5175 for (i = 0; i < nr_ctxs; ring_tbl_map += BNXT_RSS_TABLE_SIZE_P5, i++) { 5176 req->ring_grp_tbl_addr = cpu_to_le64(ring_tbl_map); 5177 req->ring_table_pair_index = i; 5178 req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[i]); 5179 rc = hwrm_req_send(bp, req); 5180 if (rc) 5181 goto exit; 5182 } 5183 5184 exit: 5185 hwrm_req_drop(bp, req); 5186 return rc; 5187 } 5188 5189 static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id) 5190 { 5191 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5192 struct hwrm_vnic_plcmodes_cfg_input *req; 5193 int rc; 5194 5195 rc = hwrm_req_init(bp, req, HWRM_VNIC_PLCMODES_CFG); 5196 if (rc) 5197 return rc; 5198 5199 req->flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT | 5200 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 | 5201 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6); 5202 req->enables = 5203 cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID | 5204 VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID); 5205 /* thresholds not implemented in firmware yet */ 5206 req->jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh); 5207 req->hds_threshold = cpu_to_le16(bp->rx_copy_thresh); 5208 req->vnic_id = cpu_to_le32(vnic->fw_vnic_id); 5209 return hwrm_req_send(bp, req); 5210 } 5211 5212 static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id, 5213 u16 ctx_idx) 5214 { 5215 struct hwrm_vnic_rss_cos_lb_ctx_free_input *req; 5216 5217 if (hwrm_req_init(bp, req, HWRM_VNIC_RSS_COS_LB_CTX_FREE)) 5218 return; 5219 5220 req->rss_cos_lb_ctx_id = 5221 cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx]); 5222 5223 hwrm_req_send(bp, req); 5224 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID; 5225 } 5226 5227 static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp) 5228 { 5229 int i, j; 5230 5231 for (i = 0; i < bp->nr_vnics; i++) { 5232 struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; 5233 5234 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) { 5235 if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID) 5236 bnxt_hwrm_vnic_ctx_free_one(bp, i, j); 5237 } 5238 } 5239 bp->rsscos_nr_ctxs = 0; 5240 } 5241 5242 static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id, u16 ctx_idx) 5243 { 5244 struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp; 5245 struct hwrm_vnic_rss_cos_lb_ctx_alloc_input *req; 5246 int rc; 5247 5248 rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC); 5249 if (rc) 5250 return rc; 5251 5252 resp = hwrm_req_hold(bp, req); 5253 rc = hwrm_req_send(bp, req); 5254 if (!rc) 5255 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = 5256 le16_to_cpu(resp->rss_cos_lb_ctx_id); 5257 hwrm_req_drop(bp, req); 5258 5259 return rc; 5260 } 5261 5262 static u32 bnxt_get_roce_vnic_mode(struct bnxt *bp) 5263 { 5264 if (bp->flags & BNXT_FLAG_ROCE_MIRROR_CAP) 5265 return VNIC_CFG_REQ_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE; 5266 return VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE; 5267 } 5268 5269 int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id) 5270 { 5271 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5272 struct hwrm_vnic_cfg_input *req; 5273 unsigned int ring = 0, grp_idx; 5274 u16 def_vlan = 0; 5275 int rc; 5276 5277 rc = hwrm_req_init(bp, req, HWRM_VNIC_CFG); 5278 if (rc) 5279 return rc; 5280 5281 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5282 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0]; 5283 5284 req->default_rx_ring_id = 5285 cpu_to_le16(rxr->rx_ring_struct.fw_ring_id); 5286 req->default_cmpl_ring_id = 5287 cpu_to_le16(bnxt_cp_ring_for_rx(bp, rxr)); 5288 req->enables = 5289 cpu_to_le32(VNIC_CFG_REQ_ENABLES_DEFAULT_RX_RING_ID | 5290 VNIC_CFG_REQ_ENABLES_DEFAULT_CMPL_RING_ID); 5291 goto vnic_mru; 5292 } 5293 req->enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP); 5294 /* Only RSS support for now TBD: COS & LB */ 5295 if (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID) { 5296 req->rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]); 5297 req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE | 5298 VNIC_CFG_REQ_ENABLES_MRU); 5299 } else if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) { 5300 req->rss_rule = 5301 cpu_to_le16(bp->vnic_info[0].fw_rss_cos_lb_ctx[0]); 5302 req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE | 5303 VNIC_CFG_REQ_ENABLES_MRU); 5304 req->flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE); 5305 } else { 5306 req->rss_rule = cpu_to_le16(0xffff); 5307 } 5308 5309 if (BNXT_CHIP_TYPE_NITRO_A0(bp) && 5310 (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID)) { 5311 req->cos_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[1]); 5312 req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_COS_RULE); 5313 } else { 5314 req->cos_rule = cpu_to_le16(0xffff); 5315 } 5316 5317 if (vnic->flags & BNXT_VNIC_RSS_FLAG) 5318 ring = 0; 5319 else if (vnic->flags & BNXT_VNIC_RFS_FLAG) 5320 ring = vnic_id - 1; 5321 else if ((vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp)) 5322 ring = bp->rx_nr_rings - 1; 5323 5324 grp_idx = bp->rx_ring[ring].bnapi->index; 5325 req->dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id); 5326 req->lb_rule = cpu_to_le16(0xffff); 5327 vnic_mru: 5328 req->mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + VLAN_HLEN); 5329 5330 req->vnic_id = cpu_to_le16(vnic->fw_vnic_id); 5331 #ifdef CONFIG_BNXT_SRIOV 5332 if (BNXT_VF(bp)) 5333 def_vlan = bp->vf.vlan; 5334 #endif 5335 if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan) 5336 req->flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE); 5337 if (!vnic_id && bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP)) 5338 req->flags |= cpu_to_le32(bnxt_get_roce_vnic_mode(bp)); 5339 5340 return hwrm_req_send(bp, req); 5341 } 5342 5343 static void bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id) 5344 { 5345 if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) { 5346 struct hwrm_vnic_free_input *req; 5347 5348 if (hwrm_req_init(bp, req, HWRM_VNIC_FREE)) 5349 return; 5350 5351 req->vnic_id = 5352 cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id); 5353 5354 hwrm_req_send(bp, req); 5355 bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID; 5356 } 5357 } 5358 5359 static void bnxt_hwrm_vnic_free(struct bnxt *bp) 5360 { 5361 u16 i; 5362 5363 for (i = 0; i < bp->nr_vnics; i++) 5364 bnxt_hwrm_vnic_free_one(bp, i); 5365 } 5366 5367 static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id, 5368 unsigned int start_rx_ring_idx, 5369 unsigned int nr_rings) 5370 { 5371 unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings; 5372 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5373 struct hwrm_vnic_alloc_output *resp; 5374 struct hwrm_vnic_alloc_input *req; 5375 int rc; 5376 5377 rc = hwrm_req_init(bp, req, HWRM_VNIC_ALLOC); 5378 if (rc) 5379 return rc; 5380 5381 if (bp->flags & BNXT_FLAG_CHIP_P5) 5382 goto vnic_no_ring_grps; 5383 5384 /* map ring groups to this vnic */ 5385 for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) { 5386 grp_idx = bp->rx_ring[i].bnapi->index; 5387 if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) { 5388 netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n", 5389 j, nr_rings); 5390 break; 5391 } 5392 vnic->fw_grp_ids[j] = bp->grp_info[grp_idx].fw_grp_id; 5393 } 5394 5395 vnic_no_ring_grps: 5396 for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++) 5397 vnic->fw_rss_cos_lb_ctx[i] = INVALID_HW_RING_ID; 5398 if (vnic_id == 0) 5399 req->flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT); 5400 5401 resp = hwrm_req_hold(bp, req); 5402 rc = hwrm_req_send(bp, req); 5403 if (!rc) 5404 vnic->fw_vnic_id = le32_to_cpu(resp->vnic_id); 5405 hwrm_req_drop(bp, req); 5406 return rc; 5407 } 5408 5409 static int bnxt_hwrm_vnic_qcaps(struct bnxt *bp) 5410 { 5411 struct hwrm_vnic_qcaps_output *resp; 5412 struct hwrm_vnic_qcaps_input *req; 5413 int rc; 5414 5415 bp->hw_ring_stats_size = sizeof(struct ctx_hw_stats); 5416 bp->flags &= ~(BNXT_FLAG_NEW_RSS_CAP | BNXT_FLAG_ROCE_MIRROR_CAP); 5417 if (bp->hwrm_spec_code < 0x10600) 5418 return 0; 5419 5420 rc = hwrm_req_init(bp, req, HWRM_VNIC_QCAPS); 5421 if (rc) 5422 return rc; 5423 5424 resp = hwrm_req_hold(bp, req); 5425 rc = hwrm_req_send(bp, req); 5426 if (!rc) { 5427 u32 flags = le32_to_cpu(resp->flags); 5428 5429 if (!(bp->flags & BNXT_FLAG_CHIP_P5) && 5430 (flags & VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP)) 5431 bp->flags |= BNXT_FLAG_NEW_RSS_CAP; 5432 if (flags & 5433 VNIC_QCAPS_RESP_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_CAP) 5434 bp->flags |= BNXT_FLAG_ROCE_MIRROR_CAP; 5435 5436 /* Older P5 fw before EXT_HW_STATS support did not set 5437 * VLAN_STRIP_CAP properly. 5438 */ 5439 if ((flags & VNIC_QCAPS_RESP_FLAGS_VLAN_STRIP_CAP) || 5440 (BNXT_CHIP_P5_THOR(bp) && 5441 !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))) 5442 bp->fw_cap |= BNXT_FW_CAP_VLAN_RX_STRIP; 5443 bp->max_tpa_v2 = le16_to_cpu(resp->max_aggs_supported); 5444 if (bp->max_tpa_v2) { 5445 if (BNXT_CHIP_P5_THOR(bp)) 5446 bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P5; 5447 else 5448 bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P5_SR2; 5449 } 5450 } 5451 hwrm_req_drop(bp, req); 5452 return rc; 5453 } 5454 5455 static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp) 5456 { 5457 struct hwrm_ring_grp_alloc_output *resp; 5458 struct hwrm_ring_grp_alloc_input *req; 5459 int rc; 5460 u16 i; 5461 5462 if (bp->flags & BNXT_FLAG_CHIP_P5) 5463 return 0; 5464 5465 rc = hwrm_req_init(bp, req, HWRM_RING_GRP_ALLOC); 5466 if (rc) 5467 return rc; 5468 5469 resp = hwrm_req_hold(bp, req); 5470 for (i = 0; i < bp->rx_nr_rings; i++) { 5471 unsigned int grp_idx = bp->rx_ring[i].bnapi->index; 5472 5473 req->cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id); 5474 req->rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id); 5475 req->ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id); 5476 req->sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx); 5477 5478 rc = hwrm_req_send(bp, req); 5479 5480 if (rc) 5481 break; 5482 5483 bp->grp_info[grp_idx].fw_grp_id = 5484 le32_to_cpu(resp->ring_group_id); 5485 } 5486 hwrm_req_drop(bp, req); 5487 return rc; 5488 } 5489 5490 static void bnxt_hwrm_ring_grp_free(struct bnxt *bp) 5491 { 5492 struct hwrm_ring_grp_free_input *req; 5493 u16 i; 5494 5495 if (!bp->grp_info || (bp->flags & BNXT_FLAG_CHIP_P5)) 5496 return; 5497 5498 if (hwrm_req_init(bp, req, HWRM_RING_GRP_FREE)) 5499 return; 5500 5501 hwrm_req_hold(bp, req); 5502 for (i = 0; i < bp->cp_nr_rings; i++) { 5503 if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID) 5504 continue; 5505 req->ring_group_id = 5506 cpu_to_le32(bp->grp_info[i].fw_grp_id); 5507 5508 hwrm_req_send(bp, req); 5509 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID; 5510 } 5511 hwrm_req_drop(bp, req); 5512 } 5513 5514 static int hwrm_ring_alloc_send_msg(struct bnxt *bp, 5515 struct bnxt_ring_struct *ring, 5516 u32 ring_type, u32 map_index) 5517 { 5518 struct hwrm_ring_alloc_output *resp; 5519 struct hwrm_ring_alloc_input *req; 5520 struct bnxt_ring_mem_info *rmem = &ring->ring_mem; 5521 struct bnxt_ring_grp_info *grp_info; 5522 int rc, err = 0; 5523 u16 ring_id; 5524 5525 rc = hwrm_req_init(bp, req, HWRM_RING_ALLOC); 5526 if (rc) 5527 goto exit; 5528 5529 req->enables = 0; 5530 if (rmem->nr_pages > 1) { 5531 req->page_tbl_addr = cpu_to_le64(rmem->pg_tbl_map); 5532 /* Page size is in log2 units */ 5533 req->page_size = BNXT_PAGE_SHIFT; 5534 req->page_tbl_depth = 1; 5535 } else { 5536 req->page_tbl_addr = cpu_to_le64(rmem->dma_arr[0]); 5537 } 5538 req->fbo = 0; 5539 /* Association of ring index with doorbell index and MSIX number */ 5540 req->logical_id = cpu_to_le16(map_index); 5541 5542 switch (ring_type) { 5543 case HWRM_RING_ALLOC_TX: { 5544 struct bnxt_tx_ring_info *txr; 5545 5546 txr = container_of(ring, struct bnxt_tx_ring_info, 5547 tx_ring_struct); 5548 req->ring_type = RING_ALLOC_REQ_RING_TYPE_TX; 5549 /* Association of transmit ring with completion ring */ 5550 grp_info = &bp->grp_info[ring->grp_idx]; 5551 req->cmpl_ring_id = cpu_to_le16(bnxt_cp_ring_for_tx(bp, txr)); 5552 req->length = cpu_to_le32(bp->tx_ring_mask + 1); 5553 req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx); 5554 req->queue_id = cpu_to_le16(ring->queue_id); 5555 break; 5556 } 5557 case HWRM_RING_ALLOC_RX: 5558 req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX; 5559 req->length = cpu_to_le32(bp->rx_ring_mask + 1); 5560 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5561 u16 flags = 0; 5562 5563 /* Association of rx ring with stats context */ 5564 grp_info = &bp->grp_info[ring->grp_idx]; 5565 req->rx_buf_size = cpu_to_le16(bp->rx_buf_use_size); 5566 req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx); 5567 req->enables |= cpu_to_le32( 5568 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID); 5569 if (NET_IP_ALIGN == 2) 5570 flags = RING_ALLOC_REQ_FLAGS_RX_SOP_PAD; 5571 req->flags = cpu_to_le16(flags); 5572 } 5573 break; 5574 case HWRM_RING_ALLOC_AGG: 5575 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5576 req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX_AGG; 5577 /* Association of agg ring with rx ring */ 5578 grp_info = &bp->grp_info[ring->grp_idx]; 5579 req->rx_ring_id = cpu_to_le16(grp_info->rx_fw_ring_id); 5580 req->rx_buf_size = cpu_to_le16(BNXT_RX_PAGE_SIZE); 5581 req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx); 5582 req->enables |= cpu_to_le32( 5583 RING_ALLOC_REQ_ENABLES_RX_RING_ID_VALID | 5584 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID); 5585 } else { 5586 req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX; 5587 } 5588 req->length = cpu_to_le32(bp->rx_agg_ring_mask + 1); 5589 break; 5590 case HWRM_RING_ALLOC_CMPL: 5591 req->ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL; 5592 req->length = cpu_to_le32(bp->cp_ring_mask + 1); 5593 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5594 /* Association of cp ring with nq */ 5595 grp_info = &bp->grp_info[map_index]; 5596 req->nq_ring_id = cpu_to_le16(grp_info->cp_fw_ring_id); 5597 req->cq_handle = cpu_to_le64(ring->handle); 5598 req->enables |= cpu_to_le32( 5599 RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID); 5600 } else if (bp->flags & BNXT_FLAG_USING_MSIX) { 5601 req->int_mode = RING_ALLOC_REQ_INT_MODE_MSIX; 5602 } 5603 break; 5604 case HWRM_RING_ALLOC_NQ: 5605 req->ring_type = RING_ALLOC_REQ_RING_TYPE_NQ; 5606 req->length = cpu_to_le32(bp->cp_ring_mask + 1); 5607 if (bp->flags & BNXT_FLAG_USING_MSIX) 5608 req->int_mode = RING_ALLOC_REQ_INT_MODE_MSIX; 5609 break; 5610 default: 5611 netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n", 5612 ring_type); 5613 return -1; 5614 } 5615 5616 resp = hwrm_req_hold(bp, req); 5617 rc = hwrm_req_send(bp, req); 5618 err = le16_to_cpu(resp->error_code); 5619 ring_id = le16_to_cpu(resp->ring_id); 5620 hwrm_req_drop(bp, req); 5621 5622 exit: 5623 if (rc || err) { 5624 netdev_err(bp->dev, "hwrm_ring_alloc type %d failed. rc:%x err:%x\n", 5625 ring_type, rc, err); 5626 return -EIO; 5627 } 5628 ring->fw_ring_id = ring_id; 5629 return rc; 5630 } 5631 5632 static int bnxt_hwrm_set_async_event_cr(struct bnxt *bp, int idx) 5633 { 5634 int rc; 5635 5636 if (BNXT_PF(bp)) { 5637 struct hwrm_func_cfg_input *req; 5638 5639 rc = hwrm_req_init(bp, req, HWRM_FUNC_CFG); 5640 if (rc) 5641 return rc; 5642 5643 req->fid = cpu_to_le16(0xffff); 5644 req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR); 5645 req->async_event_cr = cpu_to_le16(idx); 5646 return hwrm_req_send(bp, req); 5647 } else { 5648 struct hwrm_func_vf_cfg_input *req; 5649 5650 rc = hwrm_req_init(bp, req, HWRM_FUNC_VF_CFG); 5651 if (rc) 5652 return rc; 5653 5654 req->enables = 5655 cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR); 5656 req->async_event_cr = cpu_to_le16(idx); 5657 return hwrm_req_send(bp, req); 5658 } 5659 } 5660 5661 static void bnxt_set_db(struct bnxt *bp, struct bnxt_db_info *db, u32 ring_type, 5662 u32 map_idx, u32 xid) 5663 { 5664 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5665 if (BNXT_PF(bp)) 5666 db->doorbell = bp->bar1 + DB_PF_OFFSET_P5; 5667 else 5668 db->doorbell = bp->bar1 + DB_VF_OFFSET_P5; 5669 switch (ring_type) { 5670 case HWRM_RING_ALLOC_TX: 5671 db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SQ; 5672 break; 5673 case HWRM_RING_ALLOC_RX: 5674 case HWRM_RING_ALLOC_AGG: 5675 db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SRQ; 5676 break; 5677 case HWRM_RING_ALLOC_CMPL: 5678 db->db_key64 = DBR_PATH_L2; 5679 break; 5680 case HWRM_RING_ALLOC_NQ: 5681 db->db_key64 = DBR_PATH_L2; 5682 break; 5683 } 5684 db->db_key64 |= (u64)xid << DBR_XID_SFT; 5685 } else { 5686 db->doorbell = bp->bar1 + map_idx * 0x80; 5687 switch (ring_type) { 5688 case HWRM_RING_ALLOC_TX: 5689 db->db_key32 = DB_KEY_TX; 5690 break; 5691 case HWRM_RING_ALLOC_RX: 5692 case HWRM_RING_ALLOC_AGG: 5693 db->db_key32 = DB_KEY_RX; 5694 break; 5695 case HWRM_RING_ALLOC_CMPL: 5696 db->db_key32 = DB_KEY_CP; 5697 break; 5698 } 5699 } 5700 } 5701 5702 static int bnxt_hwrm_ring_alloc(struct bnxt *bp) 5703 { 5704 bool agg_rings = !!(bp->flags & BNXT_FLAG_AGG_RINGS); 5705 int i, rc = 0; 5706 u32 type; 5707 5708 if (bp->flags & BNXT_FLAG_CHIP_P5) 5709 type = HWRM_RING_ALLOC_NQ; 5710 else 5711 type = HWRM_RING_ALLOC_CMPL; 5712 for (i = 0; i < bp->cp_nr_rings; i++) { 5713 struct bnxt_napi *bnapi = bp->bnapi[i]; 5714 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 5715 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 5716 u32 map_idx = ring->map_idx; 5717 unsigned int vector; 5718 5719 vector = bp->irq_tbl[map_idx].vector; 5720 disable_irq_nosync(vector); 5721 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 5722 if (rc) { 5723 enable_irq(vector); 5724 goto err_out; 5725 } 5726 bnxt_set_db(bp, &cpr->cp_db, type, map_idx, ring->fw_ring_id); 5727 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons); 5728 enable_irq(vector); 5729 bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id; 5730 5731 if (!i) { 5732 rc = bnxt_hwrm_set_async_event_cr(bp, ring->fw_ring_id); 5733 if (rc) 5734 netdev_warn(bp->dev, "Failed to set async event completion ring.\n"); 5735 } 5736 } 5737 5738 type = HWRM_RING_ALLOC_TX; 5739 for (i = 0; i < bp->tx_nr_rings; i++) { 5740 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 5741 struct bnxt_ring_struct *ring; 5742 u32 map_idx; 5743 5744 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5745 struct bnxt_napi *bnapi = txr->bnapi; 5746 struct bnxt_cp_ring_info *cpr, *cpr2; 5747 u32 type2 = HWRM_RING_ALLOC_CMPL; 5748 5749 cpr = &bnapi->cp_ring; 5750 cpr2 = cpr->cp_ring_arr[BNXT_TX_HDL]; 5751 ring = &cpr2->cp_ring_struct; 5752 ring->handle = BNXT_TX_HDL; 5753 map_idx = bnapi->index; 5754 rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx); 5755 if (rc) 5756 goto err_out; 5757 bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx, 5758 ring->fw_ring_id); 5759 bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons); 5760 } 5761 ring = &txr->tx_ring_struct; 5762 map_idx = i; 5763 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 5764 if (rc) 5765 goto err_out; 5766 bnxt_set_db(bp, &txr->tx_db, type, map_idx, ring->fw_ring_id); 5767 } 5768 5769 type = HWRM_RING_ALLOC_RX; 5770 for (i = 0; i < bp->rx_nr_rings; i++) { 5771 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 5772 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct; 5773 struct bnxt_napi *bnapi = rxr->bnapi; 5774 u32 map_idx = bnapi->index; 5775 5776 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 5777 if (rc) 5778 goto err_out; 5779 bnxt_set_db(bp, &rxr->rx_db, type, map_idx, ring->fw_ring_id); 5780 /* If we have agg rings, post agg buffers first. */ 5781 if (!agg_rings) 5782 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 5783 bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id; 5784 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5785 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 5786 u32 type2 = HWRM_RING_ALLOC_CMPL; 5787 struct bnxt_cp_ring_info *cpr2; 5788 5789 cpr2 = cpr->cp_ring_arr[BNXT_RX_HDL]; 5790 ring = &cpr2->cp_ring_struct; 5791 ring->handle = BNXT_RX_HDL; 5792 rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx); 5793 if (rc) 5794 goto err_out; 5795 bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx, 5796 ring->fw_ring_id); 5797 bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons); 5798 } 5799 } 5800 5801 if (agg_rings) { 5802 type = HWRM_RING_ALLOC_AGG; 5803 for (i = 0; i < bp->rx_nr_rings; i++) { 5804 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 5805 struct bnxt_ring_struct *ring = 5806 &rxr->rx_agg_ring_struct; 5807 u32 grp_idx = ring->grp_idx; 5808 u32 map_idx = grp_idx + bp->rx_nr_rings; 5809 5810 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 5811 if (rc) 5812 goto err_out; 5813 5814 bnxt_set_db(bp, &rxr->rx_agg_db, type, map_idx, 5815 ring->fw_ring_id); 5816 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); 5817 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 5818 bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id; 5819 } 5820 } 5821 err_out: 5822 return rc; 5823 } 5824 5825 static int hwrm_ring_free_send_msg(struct bnxt *bp, 5826 struct bnxt_ring_struct *ring, 5827 u32 ring_type, int cmpl_ring_id) 5828 { 5829 struct hwrm_ring_free_output *resp; 5830 struct hwrm_ring_free_input *req; 5831 u16 error_code = 0; 5832 int rc; 5833 5834 if (BNXT_NO_FW_ACCESS(bp)) 5835 return 0; 5836 5837 rc = hwrm_req_init(bp, req, HWRM_RING_FREE); 5838 if (rc) 5839 goto exit; 5840 5841 req->cmpl_ring = cpu_to_le16(cmpl_ring_id); 5842 req->ring_type = ring_type; 5843 req->ring_id = cpu_to_le16(ring->fw_ring_id); 5844 5845 resp = hwrm_req_hold(bp, req); 5846 rc = hwrm_req_send(bp, req); 5847 error_code = le16_to_cpu(resp->error_code); 5848 hwrm_req_drop(bp, req); 5849 exit: 5850 if (rc || error_code) { 5851 netdev_err(bp->dev, "hwrm_ring_free type %d failed. rc:%x err:%x\n", 5852 ring_type, rc, error_code); 5853 return -EIO; 5854 } 5855 return 0; 5856 } 5857 5858 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path) 5859 { 5860 u32 type; 5861 int i; 5862 5863 if (!bp->bnapi) 5864 return; 5865 5866 for (i = 0; i < bp->tx_nr_rings; i++) { 5867 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 5868 struct bnxt_ring_struct *ring = &txr->tx_ring_struct; 5869 5870 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 5871 u32 cmpl_ring_id = bnxt_cp_ring_for_tx(bp, txr); 5872 5873 hwrm_ring_free_send_msg(bp, ring, 5874 RING_FREE_REQ_RING_TYPE_TX, 5875 close_path ? cmpl_ring_id : 5876 INVALID_HW_RING_ID); 5877 ring->fw_ring_id = INVALID_HW_RING_ID; 5878 } 5879 } 5880 5881 for (i = 0; i < bp->rx_nr_rings; i++) { 5882 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 5883 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct; 5884 u32 grp_idx = rxr->bnapi->index; 5885 5886 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 5887 u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr); 5888 5889 hwrm_ring_free_send_msg(bp, ring, 5890 RING_FREE_REQ_RING_TYPE_RX, 5891 close_path ? cmpl_ring_id : 5892 INVALID_HW_RING_ID); 5893 ring->fw_ring_id = INVALID_HW_RING_ID; 5894 bp->grp_info[grp_idx].rx_fw_ring_id = 5895 INVALID_HW_RING_ID; 5896 } 5897 } 5898 5899 if (bp->flags & BNXT_FLAG_CHIP_P5) 5900 type = RING_FREE_REQ_RING_TYPE_RX_AGG; 5901 else 5902 type = RING_FREE_REQ_RING_TYPE_RX; 5903 for (i = 0; i < bp->rx_nr_rings; i++) { 5904 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 5905 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct; 5906 u32 grp_idx = rxr->bnapi->index; 5907 5908 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 5909 u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr); 5910 5911 hwrm_ring_free_send_msg(bp, ring, type, 5912 close_path ? cmpl_ring_id : 5913 INVALID_HW_RING_ID); 5914 ring->fw_ring_id = INVALID_HW_RING_ID; 5915 bp->grp_info[grp_idx].agg_fw_ring_id = 5916 INVALID_HW_RING_ID; 5917 } 5918 } 5919 5920 /* The completion rings are about to be freed. After that the 5921 * IRQ doorbell will not work anymore. So we need to disable 5922 * IRQ here. 5923 */ 5924 bnxt_disable_int_sync(bp); 5925 5926 if (bp->flags & BNXT_FLAG_CHIP_P5) 5927 type = RING_FREE_REQ_RING_TYPE_NQ; 5928 else 5929 type = RING_FREE_REQ_RING_TYPE_L2_CMPL; 5930 for (i = 0; i < bp->cp_nr_rings; i++) { 5931 struct bnxt_napi *bnapi = bp->bnapi[i]; 5932 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 5933 struct bnxt_ring_struct *ring; 5934 int j; 5935 5936 for (j = 0; j < 2; j++) { 5937 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j]; 5938 5939 if (cpr2) { 5940 ring = &cpr2->cp_ring_struct; 5941 if (ring->fw_ring_id == INVALID_HW_RING_ID) 5942 continue; 5943 hwrm_ring_free_send_msg(bp, ring, 5944 RING_FREE_REQ_RING_TYPE_L2_CMPL, 5945 INVALID_HW_RING_ID); 5946 ring->fw_ring_id = INVALID_HW_RING_ID; 5947 } 5948 } 5949 ring = &cpr->cp_ring_struct; 5950 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 5951 hwrm_ring_free_send_msg(bp, ring, type, 5952 INVALID_HW_RING_ID); 5953 ring->fw_ring_id = INVALID_HW_RING_ID; 5954 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID; 5955 } 5956 } 5957 } 5958 5959 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max, 5960 bool shared); 5961 5962 static int bnxt_hwrm_get_rings(struct bnxt *bp) 5963 { 5964 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 5965 struct hwrm_func_qcfg_output *resp; 5966 struct hwrm_func_qcfg_input *req; 5967 int rc; 5968 5969 if (bp->hwrm_spec_code < 0x10601) 5970 return 0; 5971 5972 rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG); 5973 if (rc) 5974 return rc; 5975 5976 req->fid = cpu_to_le16(0xffff); 5977 resp = hwrm_req_hold(bp, req); 5978 rc = hwrm_req_send(bp, req); 5979 if (rc) { 5980 hwrm_req_drop(bp, req); 5981 return rc; 5982 } 5983 5984 hw_resc->resv_tx_rings = le16_to_cpu(resp->alloc_tx_rings); 5985 if (BNXT_NEW_RM(bp)) { 5986 u16 cp, stats; 5987 5988 hw_resc->resv_rx_rings = le16_to_cpu(resp->alloc_rx_rings); 5989 hw_resc->resv_hw_ring_grps = 5990 le32_to_cpu(resp->alloc_hw_ring_grps); 5991 hw_resc->resv_vnics = le16_to_cpu(resp->alloc_vnics); 5992 cp = le16_to_cpu(resp->alloc_cmpl_rings); 5993 stats = le16_to_cpu(resp->alloc_stat_ctx); 5994 hw_resc->resv_irqs = cp; 5995 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5996 int rx = hw_resc->resv_rx_rings; 5997 int tx = hw_resc->resv_tx_rings; 5998 5999 if (bp->flags & BNXT_FLAG_AGG_RINGS) 6000 rx >>= 1; 6001 if (cp < (rx + tx)) { 6002 bnxt_trim_rings(bp, &rx, &tx, cp, false); 6003 if (bp->flags & BNXT_FLAG_AGG_RINGS) 6004 rx <<= 1; 6005 hw_resc->resv_rx_rings = rx; 6006 hw_resc->resv_tx_rings = tx; 6007 } 6008 hw_resc->resv_irqs = le16_to_cpu(resp->alloc_msix); 6009 hw_resc->resv_hw_ring_grps = rx; 6010 } 6011 hw_resc->resv_cp_rings = cp; 6012 hw_resc->resv_stat_ctxs = stats; 6013 } 6014 hwrm_req_drop(bp, req); 6015 return 0; 6016 } 6017 6018 int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings) 6019 { 6020 struct hwrm_func_qcfg_output *resp; 6021 struct hwrm_func_qcfg_input *req; 6022 int rc; 6023 6024 if (bp->hwrm_spec_code < 0x10601) 6025 return 0; 6026 6027 rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG); 6028 if (rc) 6029 return rc; 6030 6031 req->fid = cpu_to_le16(fid); 6032 resp = hwrm_req_hold(bp, req); 6033 rc = hwrm_req_send(bp, req); 6034 if (!rc) 6035 *tx_rings = le16_to_cpu(resp->alloc_tx_rings); 6036 6037 hwrm_req_drop(bp, req); 6038 return rc; 6039 } 6040 6041 static bool bnxt_rfs_supported(struct bnxt *bp); 6042 6043 static struct hwrm_func_cfg_input * 6044 __bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6045 int ring_grps, int cp_rings, int stats, int vnics) 6046 { 6047 struct hwrm_func_cfg_input *req; 6048 u32 enables = 0; 6049 6050 if (hwrm_req_init(bp, req, HWRM_FUNC_CFG)) 6051 return NULL; 6052 6053 req->fid = cpu_to_le16(0xffff); 6054 enables |= tx_rings ? FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS : 0; 6055 req->num_tx_rings = cpu_to_le16(tx_rings); 6056 if (BNXT_NEW_RM(bp)) { 6057 enables |= rx_rings ? FUNC_CFG_REQ_ENABLES_NUM_RX_RINGS : 0; 6058 enables |= stats ? FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0; 6059 if (bp->flags & BNXT_FLAG_CHIP_P5) { 6060 enables |= cp_rings ? FUNC_CFG_REQ_ENABLES_NUM_MSIX : 0; 6061 enables |= tx_rings + ring_grps ? 6062 FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0; 6063 enables |= rx_rings ? 6064 FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0; 6065 } else { 6066 enables |= cp_rings ? 6067 FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0; 6068 enables |= ring_grps ? 6069 FUNC_CFG_REQ_ENABLES_NUM_HW_RING_GRPS | 6070 FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0; 6071 } 6072 enables |= vnics ? FUNC_CFG_REQ_ENABLES_NUM_VNICS : 0; 6073 6074 req->num_rx_rings = cpu_to_le16(rx_rings); 6075 if (bp->flags & BNXT_FLAG_CHIP_P5) { 6076 req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps); 6077 req->num_msix = cpu_to_le16(cp_rings); 6078 req->num_rsscos_ctxs = 6079 cpu_to_le16(DIV_ROUND_UP(ring_grps, 64)); 6080 } else { 6081 req->num_cmpl_rings = cpu_to_le16(cp_rings); 6082 req->num_hw_ring_grps = cpu_to_le16(ring_grps); 6083 req->num_rsscos_ctxs = cpu_to_le16(1); 6084 if (!(bp->flags & BNXT_FLAG_NEW_RSS_CAP) && 6085 bnxt_rfs_supported(bp)) 6086 req->num_rsscos_ctxs = 6087 cpu_to_le16(ring_grps + 1); 6088 } 6089 req->num_stat_ctxs = cpu_to_le16(stats); 6090 req->num_vnics = cpu_to_le16(vnics); 6091 } 6092 req->enables = cpu_to_le32(enables); 6093 return req; 6094 } 6095 6096 static struct hwrm_func_vf_cfg_input * 6097 __bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6098 int ring_grps, int cp_rings, int stats, int vnics) 6099 { 6100 struct hwrm_func_vf_cfg_input *req; 6101 u32 enables = 0; 6102 6103 if (hwrm_req_init(bp, req, HWRM_FUNC_VF_CFG)) 6104 return NULL; 6105 6106 enables |= tx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_TX_RINGS : 0; 6107 enables |= rx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_RX_RINGS | 6108 FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0; 6109 enables |= stats ? FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0; 6110 if (bp->flags & BNXT_FLAG_CHIP_P5) { 6111 enables |= tx_rings + ring_grps ? 6112 FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0; 6113 } else { 6114 enables |= cp_rings ? 6115 FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0; 6116 enables |= ring_grps ? 6117 FUNC_VF_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0; 6118 } 6119 enables |= vnics ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0; 6120 enables |= FUNC_VF_CFG_REQ_ENABLES_NUM_L2_CTXS; 6121 6122 req->num_l2_ctxs = cpu_to_le16(BNXT_VF_MAX_L2_CTX); 6123 req->num_tx_rings = cpu_to_le16(tx_rings); 6124 req->num_rx_rings = cpu_to_le16(rx_rings); 6125 if (bp->flags & BNXT_FLAG_CHIP_P5) { 6126 req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps); 6127 req->num_rsscos_ctxs = cpu_to_le16(DIV_ROUND_UP(ring_grps, 64)); 6128 } else { 6129 req->num_cmpl_rings = cpu_to_le16(cp_rings); 6130 req->num_hw_ring_grps = cpu_to_le16(ring_grps); 6131 req->num_rsscos_ctxs = cpu_to_le16(BNXT_VF_MAX_RSS_CTX); 6132 } 6133 req->num_stat_ctxs = cpu_to_le16(stats); 6134 req->num_vnics = cpu_to_le16(vnics); 6135 6136 req->enables = cpu_to_le32(enables); 6137 return req; 6138 } 6139 6140 static int 6141 bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6142 int ring_grps, int cp_rings, int stats, int vnics) 6143 { 6144 struct hwrm_func_cfg_input *req; 6145 int rc; 6146 6147 req = __bnxt_hwrm_reserve_pf_rings(bp, tx_rings, rx_rings, ring_grps, 6148 cp_rings, stats, vnics); 6149 if (!req) 6150 return -ENOMEM; 6151 6152 if (!req->enables) { 6153 hwrm_req_drop(bp, req); 6154 return 0; 6155 } 6156 6157 rc = hwrm_req_send(bp, req); 6158 if (rc) 6159 return rc; 6160 6161 if (bp->hwrm_spec_code < 0x10601) 6162 bp->hw_resc.resv_tx_rings = tx_rings; 6163 6164 return bnxt_hwrm_get_rings(bp); 6165 } 6166 6167 static int 6168 bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6169 int ring_grps, int cp_rings, int stats, int vnics) 6170 { 6171 struct hwrm_func_vf_cfg_input *req; 6172 int rc; 6173 6174 if (!BNXT_NEW_RM(bp)) { 6175 bp->hw_resc.resv_tx_rings = tx_rings; 6176 return 0; 6177 } 6178 6179 req = __bnxt_hwrm_reserve_vf_rings(bp, tx_rings, rx_rings, ring_grps, 6180 cp_rings, stats, vnics); 6181 if (!req) 6182 return -ENOMEM; 6183 6184 rc = hwrm_req_send(bp, req); 6185 if (rc) 6186 return rc; 6187 6188 return bnxt_hwrm_get_rings(bp); 6189 } 6190 6191 static int bnxt_hwrm_reserve_rings(struct bnxt *bp, int tx, int rx, int grp, 6192 int cp, int stat, int vnic) 6193 { 6194 if (BNXT_PF(bp)) 6195 return bnxt_hwrm_reserve_pf_rings(bp, tx, rx, grp, cp, stat, 6196 vnic); 6197 else 6198 return bnxt_hwrm_reserve_vf_rings(bp, tx, rx, grp, cp, stat, 6199 vnic); 6200 } 6201 6202 int bnxt_nq_rings_in_use(struct bnxt *bp) 6203 { 6204 int cp = bp->cp_nr_rings; 6205 int ulp_msix, ulp_base; 6206 6207 ulp_msix = bnxt_get_ulp_msix_num(bp); 6208 if (ulp_msix) { 6209 ulp_base = bnxt_get_ulp_msix_base(bp); 6210 cp += ulp_msix; 6211 if ((ulp_base + ulp_msix) > cp) 6212 cp = ulp_base + ulp_msix; 6213 } 6214 return cp; 6215 } 6216 6217 static int bnxt_cp_rings_in_use(struct bnxt *bp) 6218 { 6219 int cp; 6220 6221 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 6222 return bnxt_nq_rings_in_use(bp); 6223 6224 cp = bp->tx_nr_rings + bp->rx_nr_rings; 6225 return cp; 6226 } 6227 6228 static int bnxt_get_func_stat_ctxs(struct bnxt *bp) 6229 { 6230 int ulp_stat = bnxt_get_ulp_stat_ctxs(bp); 6231 int cp = bp->cp_nr_rings; 6232 6233 if (!ulp_stat) 6234 return cp; 6235 6236 if (bnxt_nq_rings_in_use(bp) > cp + bnxt_get_ulp_msix_num(bp)) 6237 return bnxt_get_ulp_msix_base(bp) + ulp_stat; 6238 6239 return cp + ulp_stat; 6240 } 6241 6242 /* Check if a default RSS map needs to be setup. This function is only 6243 * used on older firmware that does not require reserving RX rings. 6244 */ 6245 static void bnxt_check_rss_tbl_no_rmgr(struct bnxt *bp) 6246 { 6247 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 6248 6249 /* The RSS map is valid for RX rings set to resv_rx_rings */ 6250 if (hw_resc->resv_rx_rings != bp->rx_nr_rings) { 6251 hw_resc->resv_rx_rings = bp->rx_nr_rings; 6252 if (!netif_is_rxfh_configured(bp->dev)) 6253 bnxt_set_dflt_rss_indir_tbl(bp); 6254 } 6255 } 6256 6257 static bool bnxt_need_reserve_rings(struct bnxt *bp) 6258 { 6259 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 6260 int cp = bnxt_cp_rings_in_use(bp); 6261 int nq = bnxt_nq_rings_in_use(bp); 6262 int rx = bp->rx_nr_rings, stat; 6263 int vnic = 1, grp = rx; 6264 6265 if (hw_resc->resv_tx_rings != bp->tx_nr_rings && 6266 bp->hwrm_spec_code >= 0x10601) 6267 return true; 6268 6269 /* Old firmware does not need RX ring reservations but we still 6270 * need to setup a default RSS map when needed. With new firmware 6271 * we go through RX ring reservations first and then set up the 6272 * RSS map for the successfully reserved RX rings when needed. 6273 */ 6274 if (!BNXT_NEW_RM(bp)) { 6275 bnxt_check_rss_tbl_no_rmgr(bp); 6276 return false; 6277 } 6278 if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5)) 6279 vnic = rx + 1; 6280 if (bp->flags & BNXT_FLAG_AGG_RINGS) 6281 rx <<= 1; 6282 stat = bnxt_get_func_stat_ctxs(bp); 6283 if (hw_resc->resv_rx_rings != rx || hw_resc->resv_cp_rings != cp || 6284 hw_resc->resv_vnics != vnic || hw_resc->resv_stat_ctxs != stat || 6285 (hw_resc->resv_hw_ring_grps != grp && 6286 !(bp->flags & BNXT_FLAG_CHIP_P5))) 6287 return true; 6288 if ((bp->flags & BNXT_FLAG_CHIP_P5) && BNXT_PF(bp) && 6289 hw_resc->resv_irqs != nq) 6290 return true; 6291 return false; 6292 } 6293 6294 static int __bnxt_reserve_rings(struct bnxt *bp) 6295 { 6296 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 6297 int cp = bnxt_nq_rings_in_use(bp); 6298 int tx = bp->tx_nr_rings; 6299 int rx = bp->rx_nr_rings; 6300 int grp, rx_rings, rc; 6301 int vnic = 1, stat; 6302 bool sh = false; 6303 6304 if (!bnxt_need_reserve_rings(bp)) 6305 return 0; 6306 6307 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 6308 sh = true; 6309 if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5)) 6310 vnic = rx + 1; 6311 if (bp->flags & BNXT_FLAG_AGG_RINGS) 6312 rx <<= 1; 6313 grp = bp->rx_nr_rings; 6314 stat = bnxt_get_func_stat_ctxs(bp); 6315 6316 rc = bnxt_hwrm_reserve_rings(bp, tx, rx, grp, cp, stat, vnic); 6317 if (rc) 6318 return rc; 6319 6320 tx = hw_resc->resv_tx_rings; 6321 if (BNXT_NEW_RM(bp)) { 6322 rx = hw_resc->resv_rx_rings; 6323 cp = hw_resc->resv_irqs; 6324 grp = hw_resc->resv_hw_ring_grps; 6325 vnic = hw_resc->resv_vnics; 6326 stat = hw_resc->resv_stat_ctxs; 6327 } 6328 6329 rx_rings = rx; 6330 if (bp->flags & BNXT_FLAG_AGG_RINGS) { 6331 if (rx >= 2) { 6332 rx_rings = rx >> 1; 6333 } else { 6334 if (netif_running(bp->dev)) 6335 return -ENOMEM; 6336 6337 bp->flags &= ~BNXT_FLAG_AGG_RINGS; 6338 bp->flags |= BNXT_FLAG_NO_AGG_RINGS; 6339 bp->dev->hw_features &= ~NETIF_F_LRO; 6340 bp->dev->features &= ~NETIF_F_LRO; 6341 bnxt_set_ring_params(bp); 6342 } 6343 } 6344 rx_rings = min_t(int, rx_rings, grp); 6345 cp = min_t(int, cp, bp->cp_nr_rings); 6346 if (stat > bnxt_get_ulp_stat_ctxs(bp)) 6347 stat -= bnxt_get_ulp_stat_ctxs(bp); 6348 cp = min_t(int, cp, stat); 6349 rc = bnxt_trim_rings(bp, &rx_rings, &tx, cp, sh); 6350 if (bp->flags & BNXT_FLAG_AGG_RINGS) 6351 rx = rx_rings << 1; 6352 cp = sh ? max_t(int, tx, rx_rings) : tx + rx_rings; 6353 bp->tx_nr_rings = tx; 6354 6355 /* If we cannot reserve all the RX rings, reset the RSS map only 6356 * if absolutely necessary 6357 */ 6358 if (rx_rings != bp->rx_nr_rings) { 6359 netdev_warn(bp->dev, "Able to reserve only %d out of %d requested RX rings\n", 6360 rx_rings, bp->rx_nr_rings); 6361 if (netif_is_rxfh_configured(bp->dev) && 6362 (bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings) != 6363 bnxt_get_nr_rss_ctxs(bp, rx_rings) || 6364 bnxt_get_max_rss_ring(bp) >= rx_rings)) { 6365 netdev_warn(bp->dev, "RSS table entries reverting to default\n"); 6366 bp->dev->priv_flags &= ~IFF_RXFH_CONFIGURED; 6367 } 6368 } 6369 bp->rx_nr_rings = rx_rings; 6370 bp->cp_nr_rings = cp; 6371 6372 if (!tx || !rx || !cp || !grp || !vnic || !stat) 6373 return -ENOMEM; 6374 6375 if (!netif_is_rxfh_configured(bp->dev)) 6376 bnxt_set_dflt_rss_indir_tbl(bp); 6377 6378 return rc; 6379 } 6380 6381 static int bnxt_hwrm_check_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6382 int ring_grps, int cp_rings, int stats, 6383 int vnics) 6384 { 6385 struct hwrm_func_vf_cfg_input *req; 6386 u32 flags; 6387 6388 if (!BNXT_NEW_RM(bp)) 6389 return 0; 6390 6391 req = __bnxt_hwrm_reserve_vf_rings(bp, tx_rings, rx_rings, ring_grps, 6392 cp_rings, stats, vnics); 6393 flags = FUNC_VF_CFG_REQ_FLAGS_TX_ASSETS_TEST | 6394 FUNC_VF_CFG_REQ_FLAGS_RX_ASSETS_TEST | 6395 FUNC_VF_CFG_REQ_FLAGS_CMPL_ASSETS_TEST | 6396 FUNC_VF_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST | 6397 FUNC_VF_CFG_REQ_FLAGS_VNIC_ASSETS_TEST | 6398 FUNC_VF_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST; 6399 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 6400 flags |= FUNC_VF_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST; 6401 6402 req->flags = cpu_to_le32(flags); 6403 return hwrm_req_send_silent(bp, req); 6404 } 6405 6406 static int bnxt_hwrm_check_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6407 int ring_grps, int cp_rings, int stats, 6408 int vnics) 6409 { 6410 struct hwrm_func_cfg_input *req; 6411 u32 flags; 6412 6413 req = __bnxt_hwrm_reserve_pf_rings(bp, tx_rings, rx_rings, ring_grps, 6414 cp_rings, stats, vnics); 6415 flags = FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST; 6416 if (BNXT_NEW_RM(bp)) { 6417 flags |= FUNC_CFG_REQ_FLAGS_RX_ASSETS_TEST | 6418 FUNC_CFG_REQ_FLAGS_CMPL_ASSETS_TEST | 6419 FUNC_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST | 6420 FUNC_CFG_REQ_FLAGS_VNIC_ASSETS_TEST; 6421 if (bp->flags & BNXT_FLAG_CHIP_P5) 6422 flags |= FUNC_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST | 6423 FUNC_CFG_REQ_FLAGS_NQ_ASSETS_TEST; 6424 else 6425 flags |= FUNC_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST; 6426 } 6427 6428 req->flags = cpu_to_le32(flags); 6429 return hwrm_req_send_silent(bp, req); 6430 } 6431 6432 static int bnxt_hwrm_check_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6433 int ring_grps, int cp_rings, int stats, 6434 int vnics) 6435 { 6436 if (bp->hwrm_spec_code < 0x10801) 6437 return 0; 6438 6439 if (BNXT_PF(bp)) 6440 return bnxt_hwrm_check_pf_rings(bp, tx_rings, rx_rings, 6441 ring_grps, cp_rings, stats, 6442 vnics); 6443 6444 return bnxt_hwrm_check_vf_rings(bp, tx_rings, rx_rings, ring_grps, 6445 cp_rings, stats, vnics); 6446 } 6447 6448 static void bnxt_hwrm_coal_params_qcaps(struct bnxt *bp) 6449 { 6450 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 6451 struct hwrm_ring_aggint_qcaps_output *resp; 6452 struct hwrm_ring_aggint_qcaps_input *req; 6453 int rc; 6454 6455 coal_cap->cmpl_params = BNXT_LEGACY_COAL_CMPL_PARAMS; 6456 coal_cap->num_cmpl_dma_aggr_max = 63; 6457 coal_cap->num_cmpl_dma_aggr_during_int_max = 63; 6458 coal_cap->cmpl_aggr_dma_tmr_max = 65535; 6459 coal_cap->cmpl_aggr_dma_tmr_during_int_max = 65535; 6460 coal_cap->int_lat_tmr_min_max = 65535; 6461 coal_cap->int_lat_tmr_max_max = 65535; 6462 coal_cap->num_cmpl_aggr_int_max = 65535; 6463 coal_cap->timer_units = 80; 6464 6465 if (bp->hwrm_spec_code < 0x10902) 6466 return; 6467 6468 if (hwrm_req_init(bp, req, HWRM_RING_AGGINT_QCAPS)) 6469 return; 6470 6471 resp = hwrm_req_hold(bp, req); 6472 rc = hwrm_req_send_silent(bp, req); 6473 if (!rc) { 6474 coal_cap->cmpl_params = le32_to_cpu(resp->cmpl_params); 6475 coal_cap->nq_params = le32_to_cpu(resp->nq_params); 6476 coal_cap->num_cmpl_dma_aggr_max = 6477 le16_to_cpu(resp->num_cmpl_dma_aggr_max); 6478 coal_cap->num_cmpl_dma_aggr_during_int_max = 6479 le16_to_cpu(resp->num_cmpl_dma_aggr_during_int_max); 6480 coal_cap->cmpl_aggr_dma_tmr_max = 6481 le16_to_cpu(resp->cmpl_aggr_dma_tmr_max); 6482 coal_cap->cmpl_aggr_dma_tmr_during_int_max = 6483 le16_to_cpu(resp->cmpl_aggr_dma_tmr_during_int_max); 6484 coal_cap->int_lat_tmr_min_max = 6485 le16_to_cpu(resp->int_lat_tmr_min_max); 6486 coal_cap->int_lat_tmr_max_max = 6487 le16_to_cpu(resp->int_lat_tmr_max_max); 6488 coal_cap->num_cmpl_aggr_int_max = 6489 le16_to_cpu(resp->num_cmpl_aggr_int_max); 6490 coal_cap->timer_units = le16_to_cpu(resp->timer_units); 6491 } 6492 hwrm_req_drop(bp, req); 6493 } 6494 6495 static u16 bnxt_usec_to_coal_tmr(struct bnxt *bp, u16 usec) 6496 { 6497 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 6498 6499 return usec * 1000 / coal_cap->timer_units; 6500 } 6501 6502 static void bnxt_hwrm_set_coal_params(struct bnxt *bp, 6503 struct bnxt_coal *hw_coal, 6504 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req) 6505 { 6506 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 6507 u16 val, tmr, max, flags = hw_coal->flags; 6508 u32 cmpl_params = coal_cap->cmpl_params; 6509 6510 max = hw_coal->bufs_per_record * 128; 6511 if (hw_coal->budget) 6512 max = hw_coal->bufs_per_record * hw_coal->budget; 6513 max = min_t(u16, max, coal_cap->num_cmpl_aggr_int_max); 6514 6515 val = clamp_t(u16, hw_coal->coal_bufs, 1, max); 6516 req->num_cmpl_aggr_int = cpu_to_le16(val); 6517 6518 val = min_t(u16, val, coal_cap->num_cmpl_dma_aggr_max); 6519 req->num_cmpl_dma_aggr = cpu_to_le16(val); 6520 6521 val = clamp_t(u16, hw_coal->coal_bufs_irq, 1, 6522 coal_cap->num_cmpl_dma_aggr_during_int_max); 6523 req->num_cmpl_dma_aggr_during_int = cpu_to_le16(val); 6524 6525 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks); 6526 tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_max_max); 6527 req->int_lat_tmr_max = cpu_to_le16(tmr); 6528 6529 /* min timer set to 1/2 of interrupt timer */ 6530 if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_INT_LAT_TMR_MIN) { 6531 val = tmr / 2; 6532 val = clamp_t(u16, val, 1, coal_cap->int_lat_tmr_min_max); 6533 req->int_lat_tmr_min = cpu_to_le16(val); 6534 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE); 6535 } 6536 6537 /* buf timer set to 1/4 of interrupt timer */ 6538 val = clamp_t(u16, tmr / 4, 1, coal_cap->cmpl_aggr_dma_tmr_max); 6539 req->cmpl_aggr_dma_tmr = cpu_to_le16(val); 6540 6541 if (cmpl_params & 6542 RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_NUM_CMPL_DMA_AGGR_DURING_INT) { 6543 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks_irq); 6544 val = clamp_t(u16, tmr, 1, 6545 coal_cap->cmpl_aggr_dma_tmr_during_int_max); 6546 req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(val); 6547 req->enables |= 6548 cpu_to_le16(BNXT_COAL_CMPL_AGGR_TMR_DURING_INT_ENABLE); 6549 } 6550 6551 if ((cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_RING_IDLE) && 6552 hw_coal->idle_thresh && hw_coal->coal_ticks < hw_coal->idle_thresh) 6553 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE; 6554 req->flags = cpu_to_le16(flags); 6555 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_ENABLES); 6556 } 6557 6558 static int __bnxt_hwrm_set_coal_nq(struct bnxt *bp, struct bnxt_napi *bnapi, 6559 struct bnxt_coal *hw_coal) 6560 { 6561 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req; 6562 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 6563 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 6564 u32 nq_params = coal_cap->nq_params; 6565 u16 tmr; 6566 int rc; 6567 6568 if (!(nq_params & RING_AGGINT_QCAPS_RESP_NQ_PARAMS_INT_LAT_TMR_MIN)) 6569 return 0; 6570 6571 rc = hwrm_req_init(bp, req, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS); 6572 if (rc) 6573 return rc; 6574 6575 req->ring_id = cpu_to_le16(cpr->cp_ring_struct.fw_ring_id); 6576 req->flags = 6577 cpu_to_le16(RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_IS_NQ); 6578 6579 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks) / 2; 6580 tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_min_max); 6581 req->int_lat_tmr_min = cpu_to_le16(tmr); 6582 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE); 6583 return hwrm_req_send(bp, req); 6584 } 6585 6586 int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi) 6587 { 6588 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req_rx; 6589 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 6590 struct bnxt_coal coal; 6591 int rc; 6592 6593 /* Tick values in micro seconds. 6594 * 1 coal_buf x bufs_per_record = 1 completion record. 6595 */ 6596 memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal)); 6597 6598 coal.coal_ticks = cpr->rx_ring_coal.coal_ticks; 6599 coal.coal_bufs = cpr->rx_ring_coal.coal_bufs; 6600 6601 if (!bnapi->rx_ring) 6602 return -ENODEV; 6603 6604 rc = hwrm_req_init(bp, req_rx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS); 6605 if (rc) 6606 return rc; 6607 6608 bnxt_hwrm_set_coal_params(bp, &coal, req_rx); 6609 6610 req_rx->ring_id = cpu_to_le16(bnxt_cp_ring_for_rx(bp, bnapi->rx_ring)); 6611 6612 return hwrm_req_send(bp, req_rx); 6613 } 6614 6615 int bnxt_hwrm_set_coal(struct bnxt *bp) 6616 { 6617 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req_rx, *req_tx, 6618 *req; 6619 int i, rc; 6620 6621 rc = hwrm_req_init(bp, req_rx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS); 6622 if (rc) 6623 return rc; 6624 6625 rc = hwrm_req_init(bp, req_tx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS); 6626 if (rc) { 6627 hwrm_req_drop(bp, req_rx); 6628 return rc; 6629 } 6630 6631 bnxt_hwrm_set_coal_params(bp, &bp->rx_coal, req_rx); 6632 bnxt_hwrm_set_coal_params(bp, &bp->tx_coal, req_tx); 6633 6634 hwrm_req_hold(bp, req_rx); 6635 hwrm_req_hold(bp, req_tx); 6636 for (i = 0; i < bp->cp_nr_rings; i++) { 6637 struct bnxt_napi *bnapi = bp->bnapi[i]; 6638 struct bnxt_coal *hw_coal; 6639 u16 ring_id; 6640 6641 req = req_rx; 6642 if (!bnapi->rx_ring) { 6643 ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring); 6644 req = req_tx; 6645 } else { 6646 ring_id = bnxt_cp_ring_for_rx(bp, bnapi->rx_ring); 6647 } 6648 req->ring_id = cpu_to_le16(ring_id); 6649 6650 rc = hwrm_req_send(bp, req); 6651 if (rc) 6652 break; 6653 6654 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 6655 continue; 6656 6657 if (bnapi->rx_ring && bnapi->tx_ring) { 6658 req = req_tx; 6659 ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring); 6660 req->ring_id = cpu_to_le16(ring_id); 6661 rc = hwrm_req_send(bp, req); 6662 if (rc) 6663 break; 6664 } 6665 if (bnapi->rx_ring) 6666 hw_coal = &bp->rx_coal; 6667 else 6668 hw_coal = &bp->tx_coal; 6669 __bnxt_hwrm_set_coal_nq(bp, bnapi, hw_coal); 6670 } 6671 hwrm_req_drop(bp, req_rx); 6672 hwrm_req_drop(bp, req_tx); 6673 return rc; 6674 } 6675 6676 static void bnxt_hwrm_stat_ctx_free(struct bnxt *bp) 6677 { 6678 struct hwrm_stat_ctx_clr_stats_input *req0 = NULL; 6679 struct hwrm_stat_ctx_free_input *req; 6680 int i; 6681 6682 if (!bp->bnapi) 6683 return; 6684 6685 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 6686 return; 6687 6688 if (hwrm_req_init(bp, req, HWRM_STAT_CTX_FREE)) 6689 return; 6690 if (BNXT_FW_MAJ(bp) <= 20) { 6691 if (hwrm_req_init(bp, req0, HWRM_STAT_CTX_CLR_STATS)) { 6692 hwrm_req_drop(bp, req); 6693 return; 6694 } 6695 hwrm_req_hold(bp, req0); 6696 } 6697 hwrm_req_hold(bp, req); 6698 for (i = 0; i < bp->cp_nr_rings; i++) { 6699 struct bnxt_napi *bnapi = bp->bnapi[i]; 6700 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 6701 6702 if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) { 6703 req->stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id); 6704 if (req0) { 6705 req0->stat_ctx_id = req->stat_ctx_id; 6706 hwrm_req_send(bp, req0); 6707 } 6708 hwrm_req_send(bp, req); 6709 6710 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID; 6711 } 6712 } 6713 hwrm_req_drop(bp, req); 6714 if (req0) 6715 hwrm_req_drop(bp, req0); 6716 } 6717 6718 static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp) 6719 { 6720 struct hwrm_stat_ctx_alloc_output *resp; 6721 struct hwrm_stat_ctx_alloc_input *req; 6722 int rc, i; 6723 6724 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 6725 return 0; 6726 6727 rc = hwrm_req_init(bp, req, HWRM_STAT_CTX_ALLOC); 6728 if (rc) 6729 return rc; 6730 6731 req->stats_dma_length = cpu_to_le16(bp->hw_ring_stats_size); 6732 req->update_period_ms = cpu_to_le32(bp->stats_coal_ticks / 1000); 6733 6734 resp = hwrm_req_hold(bp, req); 6735 for (i = 0; i < bp->cp_nr_rings; i++) { 6736 struct bnxt_napi *bnapi = bp->bnapi[i]; 6737 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 6738 6739 req->stats_dma_addr = cpu_to_le64(cpr->stats.hw_stats_map); 6740 6741 rc = hwrm_req_send(bp, req); 6742 if (rc) 6743 break; 6744 6745 cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id); 6746 6747 bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id; 6748 } 6749 hwrm_req_drop(bp, req); 6750 return rc; 6751 } 6752 6753 static int bnxt_hwrm_func_qcfg(struct bnxt *bp) 6754 { 6755 struct hwrm_func_qcfg_output *resp; 6756 struct hwrm_func_qcfg_input *req; 6757 u32 min_db_offset = 0; 6758 u16 flags; 6759 int rc; 6760 6761 rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG); 6762 if (rc) 6763 return rc; 6764 6765 req->fid = cpu_to_le16(0xffff); 6766 resp = hwrm_req_hold(bp, req); 6767 rc = hwrm_req_send(bp, req); 6768 if (rc) 6769 goto func_qcfg_exit; 6770 6771 #ifdef CONFIG_BNXT_SRIOV 6772 if (BNXT_VF(bp)) { 6773 struct bnxt_vf_info *vf = &bp->vf; 6774 6775 vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK; 6776 } else { 6777 bp->pf.registered_vfs = le16_to_cpu(resp->registered_vfs); 6778 } 6779 #endif 6780 flags = le16_to_cpu(resp->flags); 6781 if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED | 6782 FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED)) { 6783 bp->fw_cap |= BNXT_FW_CAP_LLDP_AGENT; 6784 if (flags & FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED) 6785 bp->fw_cap |= BNXT_FW_CAP_DCBX_AGENT; 6786 } 6787 if (BNXT_PF(bp) && (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST)) 6788 bp->flags |= BNXT_FLAG_MULTI_HOST; 6789 if (flags & FUNC_QCFG_RESP_FLAGS_RING_MONITOR_ENABLED) 6790 bp->fw_cap |= BNXT_FW_CAP_RING_MONITOR; 6791 6792 switch (resp->port_partition_type) { 6793 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0: 6794 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5: 6795 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0: 6796 bp->port_partition_type = resp->port_partition_type; 6797 break; 6798 } 6799 if (bp->hwrm_spec_code < 0x10707 || 6800 resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEB) 6801 bp->br_mode = BRIDGE_MODE_VEB; 6802 else if (resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEPA) 6803 bp->br_mode = BRIDGE_MODE_VEPA; 6804 else 6805 bp->br_mode = BRIDGE_MODE_UNDEF; 6806 6807 bp->max_mtu = le16_to_cpu(resp->max_mtu_configured); 6808 if (!bp->max_mtu) 6809 bp->max_mtu = BNXT_MAX_MTU; 6810 6811 if (bp->db_size) 6812 goto func_qcfg_exit; 6813 6814 if (bp->flags & BNXT_FLAG_CHIP_P5) { 6815 if (BNXT_PF(bp)) 6816 min_db_offset = DB_PF_OFFSET_P5; 6817 else 6818 min_db_offset = DB_VF_OFFSET_P5; 6819 } 6820 bp->db_size = PAGE_ALIGN(le16_to_cpu(resp->l2_doorbell_bar_size_kb) * 6821 1024); 6822 if (!bp->db_size || bp->db_size > pci_resource_len(bp->pdev, 2) || 6823 bp->db_size <= min_db_offset) 6824 bp->db_size = pci_resource_len(bp->pdev, 2); 6825 6826 func_qcfg_exit: 6827 hwrm_req_drop(bp, req); 6828 return rc; 6829 } 6830 6831 static void bnxt_init_ctx_initializer(struct bnxt_ctx_mem_info *ctx, 6832 struct hwrm_func_backing_store_qcaps_output *resp) 6833 { 6834 struct bnxt_mem_init *mem_init; 6835 u16 init_mask; 6836 u8 init_val; 6837 u8 *offset; 6838 int i; 6839 6840 init_val = resp->ctx_kind_initializer; 6841 init_mask = le16_to_cpu(resp->ctx_init_mask); 6842 offset = &resp->qp_init_offset; 6843 mem_init = &ctx->mem_init[BNXT_CTX_MEM_INIT_QP]; 6844 for (i = 0; i < BNXT_CTX_MEM_INIT_MAX; i++, mem_init++, offset++) { 6845 mem_init->init_val = init_val; 6846 mem_init->offset = BNXT_MEM_INVALID_OFFSET; 6847 if (!init_mask) 6848 continue; 6849 if (i == BNXT_CTX_MEM_INIT_STAT) 6850 offset = &resp->stat_init_offset; 6851 if (init_mask & (1 << i)) 6852 mem_init->offset = *offset * 4; 6853 else 6854 mem_init->init_val = 0; 6855 } 6856 ctx->mem_init[BNXT_CTX_MEM_INIT_QP].size = ctx->qp_entry_size; 6857 ctx->mem_init[BNXT_CTX_MEM_INIT_SRQ].size = ctx->srq_entry_size; 6858 ctx->mem_init[BNXT_CTX_MEM_INIT_CQ].size = ctx->cq_entry_size; 6859 ctx->mem_init[BNXT_CTX_MEM_INIT_VNIC].size = ctx->vnic_entry_size; 6860 ctx->mem_init[BNXT_CTX_MEM_INIT_STAT].size = ctx->stat_entry_size; 6861 ctx->mem_init[BNXT_CTX_MEM_INIT_MRAV].size = ctx->mrav_entry_size; 6862 } 6863 6864 static int bnxt_hwrm_func_backing_store_qcaps(struct bnxt *bp) 6865 { 6866 struct hwrm_func_backing_store_qcaps_output *resp; 6867 struct hwrm_func_backing_store_qcaps_input *req; 6868 int rc; 6869 6870 if (bp->hwrm_spec_code < 0x10902 || BNXT_VF(bp) || bp->ctx) 6871 return 0; 6872 6873 rc = hwrm_req_init(bp, req, HWRM_FUNC_BACKING_STORE_QCAPS); 6874 if (rc) 6875 return rc; 6876 6877 resp = hwrm_req_hold(bp, req); 6878 rc = hwrm_req_send_silent(bp, req); 6879 if (!rc) { 6880 struct bnxt_ctx_pg_info *ctx_pg; 6881 struct bnxt_ctx_mem_info *ctx; 6882 int i, tqm_rings; 6883 6884 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 6885 if (!ctx) { 6886 rc = -ENOMEM; 6887 goto ctx_err; 6888 } 6889 ctx->qp_max_entries = le32_to_cpu(resp->qp_max_entries); 6890 ctx->qp_min_qp1_entries = le16_to_cpu(resp->qp_min_qp1_entries); 6891 ctx->qp_max_l2_entries = le16_to_cpu(resp->qp_max_l2_entries); 6892 ctx->qp_entry_size = le16_to_cpu(resp->qp_entry_size); 6893 ctx->srq_max_l2_entries = le16_to_cpu(resp->srq_max_l2_entries); 6894 ctx->srq_max_entries = le32_to_cpu(resp->srq_max_entries); 6895 ctx->srq_entry_size = le16_to_cpu(resp->srq_entry_size); 6896 ctx->cq_max_l2_entries = le16_to_cpu(resp->cq_max_l2_entries); 6897 ctx->cq_max_entries = le32_to_cpu(resp->cq_max_entries); 6898 ctx->cq_entry_size = le16_to_cpu(resp->cq_entry_size); 6899 ctx->vnic_max_vnic_entries = 6900 le16_to_cpu(resp->vnic_max_vnic_entries); 6901 ctx->vnic_max_ring_table_entries = 6902 le16_to_cpu(resp->vnic_max_ring_table_entries); 6903 ctx->vnic_entry_size = le16_to_cpu(resp->vnic_entry_size); 6904 ctx->stat_max_entries = le32_to_cpu(resp->stat_max_entries); 6905 ctx->stat_entry_size = le16_to_cpu(resp->stat_entry_size); 6906 ctx->tqm_entry_size = le16_to_cpu(resp->tqm_entry_size); 6907 ctx->tqm_min_entries_per_ring = 6908 le32_to_cpu(resp->tqm_min_entries_per_ring); 6909 ctx->tqm_max_entries_per_ring = 6910 le32_to_cpu(resp->tqm_max_entries_per_ring); 6911 ctx->tqm_entries_multiple = resp->tqm_entries_multiple; 6912 if (!ctx->tqm_entries_multiple) 6913 ctx->tqm_entries_multiple = 1; 6914 ctx->mrav_max_entries = le32_to_cpu(resp->mrav_max_entries); 6915 ctx->mrav_entry_size = le16_to_cpu(resp->mrav_entry_size); 6916 ctx->mrav_num_entries_units = 6917 le16_to_cpu(resp->mrav_num_entries_units); 6918 ctx->tim_entry_size = le16_to_cpu(resp->tim_entry_size); 6919 ctx->tim_max_entries = le32_to_cpu(resp->tim_max_entries); 6920 6921 bnxt_init_ctx_initializer(ctx, resp); 6922 6923 ctx->tqm_fp_rings_count = resp->tqm_fp_rings_count; 6924 if (!ctx->tqm_fp_rings_count) 6925 ctx->tqm_fp_rings_count = bp->max_q; 6926 else if (ctx->tqm_fp_rings_count > BNXT_MAX_TQM_FP_RINGS) 6927 ctx->tqm_fp_rings_count = BNXT_MAX_TQM_FP_RINGS; 6928 6929 tqm_rings = ctx->tqm_fp_rings_count + BNXT_MAX_TQM_SP_RINGS; 6930 ctx_pg = kcalloc(tqm_rings, sizeof(*ctx_pg), GFP_KERNEL); 6931 if (!ctx_pg) { 6932 kfree(ctx); 6933 rc = -ENOMEM; 6934 goto ctx_err; 6935 } 6936 for (i = 0; i < tqm_rings; i++, ctx_pg++) 6937 ctx->tqm_mem[i] = ctx_pg; 6938 bp->ctx = ctx; 6939 } else { 6940 rc = 0; 6941 } 6942 ctx_err: 6943 hwrm_req_drop(bp, req); 6944 return rc; 6945 } 6946 6947 static void bnxt_hwrm_set_pg_attr(struct bnxt_ring_mem_info *rmem, u8 *pg_attr, 6948 __le64 *pg_dir) 6949 { 6950 if (!rmem->nr_pages) 6951 return; 6952 6953 BNXT_SET_CTX_PAGE_ATTR(*pg_attr); 6954 if (rmem->depth >= 1) { 6955 if (rmem->depth == 2) 6956 *pg_attr |= 2; 6957 else 6958 *pg_attr |= 1; 6959 *pg_dir = cpu_to_le64(rmem->pg_tbl_map); 6960 } else { 6961 *pg_dir = cpu_to_le64(rmem->dma_arr[0]); 6962 } 6963 } 6964 6965 #define FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES \ 6966 (FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP | \ 6967 FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ | \ 6968 FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ | \ 6969 FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC | \ 6970 FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) 6971 6972 static int bnxt_hwrm_func_backing_store_cfg(struct bnxt *bp, u32 enables) 6973 { 6974 struct hwrm_func_backing_store_cfg_input *req; 6975 struct bnxt_ctx_mem_info *ctx = bp->ctx; 6976 struct bnxt_ctx_pg_info *ctx_pg; 6977 void **__req = (void **)&req; 6978 u32 req_len = sizeof(*req); 6979 __le32 *num_entries; 6980 __le64 *pg_dir; 6981 u32 flags = 0; 6982 u8 *pg_attr; 6983 u32 ena; 6984 int rc; 6985 int i; 6986 6987 if (!ctx) 6988 return 0; 6989 6990 if (req_len > bp->hwrm_max_ext_req_len) 6991 req_len = BNXT_BACKING_STORE_CFG_LEGACY_LEN; 6992 rc = __hwrm_req_init(bp, __req, HWRM_FUNC_BACKING_STORE_CFG, req_len); 6993 if (rc) 6994 return rc; 6995 6996 req->enables = cpu_to_le32(enables); 6997 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP) { 6998 ctx_pg = &ctx->qp_mem; 6999 req->qp_num_entries = cpu_to_le32(ctx_pg->entries); 7000 req->qp_num_qp1_entries = cpu_to_le16(ctx->qp_min_qp1_entries); 7001 req->qp_num_l2_entries = cpu_to_le16(ctx->qp_max_l2_entries); 7002 req->qp_entry_size = cpu_to_le16(ctx->qp_entry_size); 7003 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 7004 &req->qpc_pg_size_qpc_lvl, 7005 &req->qpc_page_dir); 7006 } 7007 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ) { 7008 ctx_pg = &ctx->srq_mem; 7009 req->srq_num_entries = cpu_to_le32(ctx_pg->entries); 7010 req->srq_num_l2_entries = cpu_to_le16(ctx->srq_max_l2_entries); 7011 req->srq_entry_size = cpu_to_le16(ctx->srq_entry_size); 7012 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 7013 &req->srq_pg_size_srq_lvl, 7014 &req->srq_page_dir); 7015 } 7016 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ) { 7017 ctx_pg = &ctx->cq_mem; 7018 req->cq_num_entries = cpu_to_le32(ctx_pg->entries); 7019 req->cq_num_l2_entries = cpu_to_le16(ctx->cq_max_l2_entries); 7020 req->cq_entry_size = cpu_to_le16(ctx->cq_entry_size); 7021 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 7022 &req->cq_pg_size_cq_lvl, 7023 &req->cq_page_dir); 7024 } 7025 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC) { 7026 ctx_pg = &ctx->vnic_mem; 7027 req->vnic_num_vnic_entries = 7028 cpu_to_le16(ctx->vnic_max_vnic_entries); 7029 req->vnic_num_ring_table_entries = 7030 cpu_to_le16(ctx->vnic_max_ring_table_entries); 7031 req->vnic_entry_size = cpu_to_le16(ctx->vnic_entry_size); 7032 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 7033 &req->vnic_pg_size_vnic_lvl, 7034 &req->vnic_page_dir); 7035 } 7036 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) { 7037 ctx_pg = &ctx->stat_mem; 7038 req->stat_num_entries = cpu_to_le32(ctx->stat_max_entries); 7039 req->stat_entry_size = cpu_to_le16(ctx->stat_entry_size); 7040 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 7041 &req->stat_pg_size_stat_lvl, 7042 &req->stat_page_dir); 7043 } 7044 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV) { 7045 ctx_pg = &ctx->mrav_mem; 7046 req->mrav_num_entries = cpu_to_le32(ctx_pg->entries); 7047 if (ctx->mrav_num_entries_units) 7048 flags |= 7049 FUNC_BACKING_STORE_CFG_REQ_FLAGS_MRAV_RESERVATION_SPLIT; 7050 req->mrav_entry_size = cpu_to_le16(ctx->mrav_entry_size); 7051 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 7052 &req->mrav_pg_size_mrav_lvl, 7053 &req->mrav_page_dir); 7054 } 7055 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM) { 7056 ctx_pg = &ctx->tim_mem; 7057 req->tim_num_entries = cpu_to_le32(ctx_pg->entries); 7058 req->tim_entry_size = cpu_to_le16(ctx->tim_entry_size); 7059 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 7060 &req->tim_pg_size_tim_lvl, 7061 &req->tim_page_dir); 7062 } 7063 for (i = 0, num_entries = &req->tqm_sp_num_entries, 7064 pg_attr = &req->tqm_sp_pg_size_tqm_sp_lvl, 7065 pg_dir = &req->tqm_sp_page_dir, 7066 ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP; 7067 i < BNXT_MAX_TQM_RINGS; 7068 i++, num_entries++, pg_attr++, pg_dir++, ena <<= 1) { 7069 if (!(enables & ena)) 7070 continue; 7071 7072 req->tqm_entry_size = cpu_to_le16(ctx->tqm_entry_size); 7073 ctx_pg = ctx->tqm_mem[i]; 7074 *num_entries = cpu_to_le32(ctx_pg->entries); 7075 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, pg_attr, pg_dir); 7076 } 7077 req->flags = cpu_to_le32(flags); 7078 return hwrm_req_send(bp, req); 7079 } 7080 7081 static int bnxt_alloc_ctx_mem_blk(struct bnxt *bp, 7082 struct bnxt_ctx_pg_info *ctx_pg) 7083 { 7084 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem; 7085 7086 rmem->page_size = BNXT_PAGE_SIZE; 7087 rmem->pg_arr = ctx_pg->ctx_pg_arr; 7088 rmem->dma_arr = ctx_pg->ctx_dma_arr; 7089 rmem->flags = BNXT_RMEM_VALID_PTE_FLAG; 7090 if (rmem->depth >= 1) 7091 rmem->flags |= BNXT_RMEM_USE_FULL_PAGE_FLAG; 7092 return bnxt_alloc_ring(bp, rmem); 7093 } 7094 7095 static int bnxt_alloc_ctx_pg_tbls(struct bnxt *bp, 7096 struct bnxt_ctx_pg_info *ctx_pg, u32 mem_size, 7097 u8 depth, struct bnxt_mem_init *mem_init) 7098 { 7099 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem; 7100 int rc; 7101 7102 if (!mem_size) 7103 return -EINVAL; 7104 7105 ctx_pg->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE); 7106 if (ctx_pg->nr_pages > MAX_CTX_TOTAL_PAGES) { 7107 ctx_pg->nr_pages = 0; 7108 return -EINVAL; 7109 } 7110 if (ctx_pg->nr_pages > MAX_CTX_PAGES || depth > 1) { 7111 int nr_tbls, i; 7112 7113 rmem->depth = 2; 7114 ctx_pg->ctx_pg_tbl = kcalloc(MAX_CTX_PAGES, sizeof(ctx_pg), 7115 GFP_KERNEL); 7116 if (!ctx_pg->ctx_pg_tbl) 7117 return -ENOMEM; 7118 nr_tbls = DIV_ROUND_UP(ctx_pg->nr_pages, MAX_CTX_PAGES); 7119 rmem->nr_pages = nr_tbls; 7120 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg); 7121 if (rc) 7122 return rc; 7123 for (i = 0; i < nr_tbls; i++) { 7124 struct bnxt_ctx_pg_info *pg_tbl; 7125 7126 pg_tbl = kzalloc(sizeof(*pg_tbl), GFP_KERNEL); 7127 if (!pg_tbl) 7128 return -ENOMEM; 7129 ctx_pg->ctx_pg_tbl[i] = pg_tbl; 7130 rmem = &pg_tbl->ring_mem; 7131 rmem->pg_tbl = ctx_pg->ctx_pg_arr[i]; 7132 rmem->pg_tbl_map = ctx_pg->ctx_dma_arr[i]; 7133 rmem->depth = 1; 7134 rmem->nr_pages = MAX_CTX_PAGES; 7135 rmem->mem_init = mem_init; 7136 if (i == (nr_tbls - 1)) { 7137 int rem = ctx_pg->nr_pages % MAX_CTX_PAGES; 7138 7139 if (rem) 7140 rmem->nr_pages = rem; 7141 } 7142 rc = bnxt_alloc_ctx_mem_blk(bp, pg_tbl); 7143 if (rc) 7144 break; 7145 } 7146 } else { 7147 rmem->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE); 7148 if (rmem->nr_pages > 1 || depth) 7149 rmem->depth = 1; 7150 rmem->mem_init = mem_init; 7151 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg); 7152 } 7153 return rc; 7154 } 7155 7156 static void bnxt_free_ctx_pg_tbls(struct bnxt *bp, 7157 struct bnxt_ctx_pg_info *ctx_pg) 7158 { 7159 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem; 7160 7161 if (rmem->depth > 1 || ctx_pg->nr_pages > MAX_CTX_PAGES || 7162 ctx_pg->ctx_pg_tbl) { 7163 int i, nr_tbls = rmem->nr_pages; 7164 7165 for (i = 0; i < nr_tbls; i++) { 7166 struct bnxt_ctx_pg_info *pg_tbl; 7167 struct bnxt_ring_mem_info *rmem2; 7168 7169 pg_tbl = ctx_pg->ctx_pg_tbl[i]; 7170 if (!pg_tbl) 7171 continue; 7172 rmem2 = &pg_tbl->ring_mem; 7173 bnxt_free_ring(bp, rmem2); 7174 ctx_pg->ctx_pg_arr[i] = NULL; 7175 kfree(pg_tbl); 7176 ctx_pg->ctx_pg_tbl[i] = NULL; 7177 } 7178 kfree(ctx_pg->ctx_pg_tbl); 7179 ctx_pg->ctx_pg_tbl = NULL; 7180 } 7181 bnxt_free_ring(bp, rmem); 7182 ctx_pg->nr_pages = 0; 7183 } 7184 7185 void bnxt_free_ctx_mem(struct bnxt *bp) 7186 { 7187 struct bnxt_ctx_mem_info *ctx = bp->ctx; 7188 int i; 7189 7190 if (!ctx) 7191 return; 7192 7193 if (ctx->tqm_mem[0]) { 7194 for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++) 7195 bnxt_free_ctx_pg_tbls(bp, ctx->tqm_mem[i]); 7196 kfree(ctx->tqm_mem[0]); 7197 ctx->tqm_mem[0] = NULL; 7198 } 7199 7200 bnxt_free_ctx_pg_tbls(bp, &ctx->tim_mem); 7201 bnxt_free_ctx_pg_tbls(bp, &ctx->mrav_mem); 7202 bnxt_free_ctx_pg_tbls(bp, &ctx->stat_mem); 7203 bnxt_free_ctx_pg_tbls(bp, &ctx->vnic_mem); 7204 bnxt_free_ctx_pg_tbls(bp, &ctx->cq_mem); 7205 bnxt_free_ctx_pg_tbls(bp, &ctx->srq_mem); 7206 bnxt_free_ctx_pg_tbls(bp, &ctx->qp_mem); 7207 ctx->flags &= ~BNXT_CTX_FLAG_INITED; 7208 } 7209 7210 static int bnxt_alloc_ctx_mem(struct bnxt *bp) 7211 { 7212 struct bnxt_ctx_pg_info *ctx_pg; 7213 struct bnxt_ctx_mem_info *ctx; 7214 struct bnxt_mem_init *init; 7215 u32 mem_size, ena, entries; 7216 u32 entries_sp, min; 7217 u32 num_mr, num_ah; 7218 u32 extra_srqs = 0; 7219 u32 extra_qps = 0; 7220 u8 pg_lvl = 1; 7221 int i, rc; 7222 7223 rc = bnxt_hwrm_func_backing_store_qcaps(bp); 7224 if (rc) { 7225 netdev_err(bp->dev, "Failed querying context mem capability, rc = %d.\n", 7226 rc); 7227 return rc; 7228 } 7229 ctx = bp->ctx; 7230 if (!ctx || (ctx->flags & BNXT_CTX_FLAG_INITED)) 7231 return 0; 7232 7233 if ((bp->flags & BNXT_FLAG_ROCE_CAP) && !is_kdump_kernel()) { 7234 pg_lvl = 2; 7235 extra_qps = 65536; 7236 extra_srqs = 8192; 7237 } 7238 7239 ctx_pg = &ctx->qp_mem; 7240 ctx_pg->entries = ctx->qp_min_qp1_entries + ctx->qp_max_l2_entries + 7241 extra_qps; 7242 if (ctx->qp_entry_size) { 7243 mem_size = ctx->qp_entry_size * ctx_pg->entries; 7244 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_QP]; 7245 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, init); 7246 if (rc) 7247 return rc; 7248 } 7249 7250 ctx_pg = &ctx->srq_mem; 7251 ctx_pg->entries = ctx->srq_max_l2_entries + extra_srqs; 7252 if (ctx->srq_entry_size) { 7253 mem_size = ctx->srq_entry_size * ctx_pg->entries; 7254 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_SRQ]; 7255 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, init); 7256 if (rc) 7257 return rc; 7258 } 7259 7260 ctx_pg = &ctx->cq_mem; 7261 ctx_pg->entries = ctx->cq_max_l2_entries + extra_qps * 2; 7262 if (ctx->cq_entry_size) { 7263 mem_size = ctx->cq_entry_size * ctx_pg->entries; 7264 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_CQ]; 7265 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, init); 7266 if (rc) 7267 return rc; 7268 } 7269 7270 ctx_pg = &ctx->vnic_mem; 7271 ctx_pg->entries = ctx->vnic_max_vnic_entries + 7272 ctx->vnic_max_ring_table_entries; 7273 if (ctx->vnic_entry_size) { 7274 mem_size = ctx->vnic_entry_size * ctx_pg->entries; 7275 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_VNIC]; 7276 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, init); 7277 if (rc) 7278 return rc; 7279 } 7280 7281 ctx_pg = &ctx->stat_mem; 7282 ctx_pg->entries = ctx->stat_max_entries; 7283 if (ctx->stat_entry_size) { 7284 mem_size = ctx->stat_entry_size * ctx_pg->entries; 7285 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_STAT]; 7286 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, init); 7287 if (rc) 7288 return rc; 7289 } 7290 7291 ena = 0; 7292 if (!(bp->flags & BNXT_FLAG_ROCE_CAP)) 7293 goto skip_rdma; 7294 7295 ctx_pg = &ctx->mrav_mem; 7296 /* 128K extra is needed to accommodate static AH context 7297 * allocation by f/w. 7298 */ 7299 num_mr = 1024 * 256; 7300 num_ah = 1024 * 128; 7301 ctx_pg->entries = num_mr + num_ah; 7302 if (ctx->mrav_entry_size) { 7303 mem_size = ctx->mrav_entry_size * ctx_pg->entries; 7304 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_MRAV]; 7305 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 2, init); 7306 if (rc) 7307 return rc; 7308 } 7309 ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV; 7310 if (ctx->mrav_num_entries_units) 7311 ctx_pg->entries = 7312 ((num_mr / ctx->mrav_num_entries_units) << 16) | 7313 (num_ah / ctx->mrav_num_entries_units); 7314 7315 ctx_pg = &ctx->tim_mem; 7316 ctx_pg->entries = ctx->qp_mem.entries; 7317 if (ctx->tim_entry_size) { 7318 mem_size = ctx->tim_entry_size * ctx_pg->entries; 7319 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, NULL); 7320 if (rc) 7321 return rc; 7322 } 7323 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM; 7324 7325 skip_rdma: 7326 min = ctx->tqm_min_entries_per_ring; 7327 entries_sp = ctx->vnic_max_vnic_entries + ctx->qp_max_l2_entries + 7328 2 * (extra_qps + ctx->qp_min_qp1_entries) + min; 7329 entries_sp = roundup(entries_sp, ctx->tqm_entries_multiple); 7330 entries = ctx->qp_max_l2_entries + 2 * (extra_qps + ctx->qp_min_qp1_entries); 7331 entries = roundup(entries, ctx->tqm_entries_multiple); 7332 entries = clamp_t(u32, entries, min, ctx->tqm_max_entries_per_ring); 7333 for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++) { 7334 ctx_pg = ctx->tqm_mem[i]; 7335 ctx_pg->entries = i ? entries : entries_sp; 7336 if (ctx->tqm_entry_size) { 7337 mem_size = ctx->tqm_entry_size * ctx_pg->entries; 7338 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, 7339 NULL); 7340 if (rc) 7341 return rc; 7342 } 7343 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP << i; 7344 } 7345 ena |= FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES; 7346 rc = bnxt_hwrm_func_backing_store_cfg(bp, ena); 7347 if (rc) { 7348 netdev_err(bp->dev, "Failed configuring context mem, rc = %d.\n", 7349 rc); 7350 return rc; 7351 } 7352 ctx->flags |= BNXT_CTX_FLAG_INITED; 7353 return 0; 7354 } 7355 7356 int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp, bool all) 7357 { 7358 struct hwrm_func_resource_qcaps_output *resp; 7359 struct hwrm_func_resource_qcaps_input *req; 7360 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 7361 int rc; 7362 7363 rc = hwrm_req_init(bp, req, HWRM_FUNC_RESOURCE_QCAPS); 7364 if (rc) 7365 return rc; 7366 7367 req->fid = cpu_to_le16(0xffff); 7368 resp = hwrm_req_hold(bp, req); 7369 rc = hwrm_req_send_silent(bp, req); 7370 if (rc) 7371 goto hwrm_func_resc_qcaps_exit; 7372 7373 hw_resc->max_tx_sch_inputs = le16_to_cpu(resp->max_tx_scheduler_inputs); 7374 if (!all) 7375 goto hwrm_func_resc_qcaps_exit; 7376 7377 hw_resc->min_rsscos_ctxs = le16_to_cpu(resp->min_rsscos_ctx); 7378 hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx); 7379 hw_resc->min_cp_rings = le16_to_cpu(resp->min_cmpl_rings); 7380 hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings); 7381 hw_resc->min_tx_rings = le16_to_cpu(resp->min_tx_rings); 7382 hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings); 7383 hw_resc->min_rx_rings = le16_to_cpu(resp->min_rx_rings); 7384 hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings); 7385 hw_resc->min_hw_ring_grps = le16_to_cpu(resp->min_hw_ring_grps); 7386 hw_resc->max_hw_ring_grps = le16_to_cpu(resp->max_hw_ring_grps); 7387 hw_resc->min_l2_ctxs = le16_to_cpu(resp->min_l2_ctxs); 7388 hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs); 7389 hw_resc->min_vnics = le16_to_cpu(resp->min_vnics); 7390 hw_resc->max_vnics = le16_to_cpu(resp->max_vnics); 7391 hw_resc->min_stat_ctxs = le16_to_cpu(resp->min_stat_ctx); 7392 hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx); 7393 7394 if (bp->flags & BNXT_FLAG_CHIP_P5) { 7395 u16 max_msix = le16_to_cpu(resp->max_msix); 7396 7397 hw_resc->max_nqs = max_msix; 7398 hw_resc->max_hw_ring_grps = hw_resc->max_rx_rings; 7399 } 7400 7401 if (BNXT_PF(bp)) { 7402 struct bnxt_pf_info *pf = &bp->pf; 7403 7404 pf->vf_resv_strategy = 7405 le16_to_cpu(resp->vf_reservation_strategy); 7406 if (pf->vf_resv_strategy > BNXT_VF_RESV_STRATEGY_MINIMAL_STATIC) 7407 pf->vf_resv_strategy = BNXT_VF_RESV_STRATEGY_MAXIMAL; 7408 } 7409 hwrm_func_resc_qcaps_exit: 7410 hwrm_req_drop(bp, req); 7411 return rc; 7412 } 7413 7414 static int __bnxt_hwrm_ptp_qcfg(struct bnxt *bp) 7415 { 7416 struct hwrm_port_mac_ptp_qcfg_output *resp; 7417 struct hwrm_port_mac_ptp_qcfg_input *req; 7418 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; 7419 u8 flags; 7420 int rc; 7421 7422 if (bp->hwrm_spec_code < 0x10801) { 7423 rc = -ENODEV; 7424 goto no_ptp; 7425 } 7426 7427 rc = hwrm_req_init(bp, req, HWRM_PORT_MAC_PTP_QCFG); 7428 if (rc) 7429 goto no_ptp; 7430 7431 req->port_id = cpu_to_le16(bp->pf.port_id); 7432 resp = hwrm_req_hold(bp, req); 7433 rc = hwrm_req_send(bp, req); 7434 if (rc) 7435 goto exit; 7436 7437 flags = resp->flags; 7438 if (!(flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_HWRM_ACCESS)) { 7439 rc = -ENODEV; 7440 goto exit; 7441 } 7442 if (!ptp) { 7443 ptp = kzalloc(sizeof(*ptp), GFP_KERNEL); 7444 if (!ptp) { 7445 rc = -ENOMEM; 7446 goto exit; 7447 } 7448 ptp->bp = bp; 7449 bp->ptp_cfg = ptp; 7450 } 7451 if (flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_PARTIAL_DIRECT_ACCESS_REF_CLOCK) { 7452 ptp->refclk_regs[0] = le32_to_cpu(resp->ts_ref_clock_reg_lower); 7453 ptp->refclk_regs[1] = le32_to_cpu(resp->ts_ref_clock_reg_upper); 7454 } else if (bp->flags & BNXT_FLAG_CHIP_P5) { 7455 ptp->refclk_regs[0] = BNXT_TS_REG_TIMESYNC_TS0_LOWER; 7456 ptp->refclk_regs[1] = BNXT_TS_REG_TIMESYNC_TS0_UPPER; 7457 } else { 7458 rc = -ENODEV; 7459 goto exit; 7460 } 7461 rc = bnxt_ptp_init(bp); 7462 if (rc) 7463 netdev_warn(bp->dev, "PTP initialization failed.\n"); 7464 exit: 7465 hwrm_req_drop(bp, req); 7466 if (!rc) 7467 return 0; 7468 7469 no_ptp: 7470 bnxt_ptp_clear(bp); 7471 kfree(ptp); 7472 bp->ptp_cfg = NULL; 7473 return rc; 7474 } 7475 7476 static int __bnxt_hwrm_func_qcaps(struct bnxt *bp) 7477 { 7478 struct hwrm_func_qcaps_output *resp; 7479 struct hwrm_func_qcaps_input *req; 7480 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 7481 u32 flags, flags_ext; 7482 int rc; 7483 7484 rc = hwrm_req_init(bp, req, HWRM_FUNC_QCAPS); 7485 if (rc) 7486 return rc; 7487 7488 req->fid = cpu_to_le16(0xffff); 7489 resp = hwrm_req_hold(bp, req); 7490 rc = hwrm_req_send(bp, req); 7491 if (rc) 7492 goto hwrm_func_qcaps_exit; 7493 7494 flags = le32_to_cpu(resp->flags); 7495 if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED) 7496 bp->flags |= BNXT_FLAG_ROCEV1_CAP; 7497 if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED) 7498 bp->flags |= BNXT_FLAG_ROCEV2_CAP; 7499 if (flags & FUNC_QCAPS_RESP_FLAGS_PCIE_STATS_SUPPORTED) 7500 bp->fw_cap |= BNXT_FW_CAP_PCIE_STATS_SUPPORTED; 7501 if (flags & FUNC_QCAPS_RESP_FLAGS_HOT_RESET_CAPABLE) 7502 bp->fw_cap |= BNXT_FW_CAP_HOT_RESET; 7503 if (flags & FUNC_QCAPS_RESP_FLAGS_EXT_STATS_SUPPORTED) 7504 bp->fw_cap |= BNXT_FW_CAP_EXT_STATS_SUPPORTED; 7505 if (flags & FUNC_QCAPS_RESP_FLAGS_ERROR_RECOVERY_CAPABLE) 7506 bp->fw_cap |= BNXT_FW_CAP_ERROR_RECOVERY; 7507 if (flags & FUNC_QCAPS_RESP_FLAGS_ERR_RECOVER_RELOAD) 7508 bp->fw_cap |= BNXT_FW_CAP_ERR_RECOVER_RELOAD; 7509 if (!(flags & FUNC_QCAPS_RESP_FLAGS_VLAN_ACCELERATION_TX_DISABLED)) 7510 bp->fw_cap |= BNXT_FW_CAP_VLAN_TX_INSERT; 7511 if (flags & FUNC_QCAPS_RESP_FLAGS_DBG_QCAPS_CMD_SUPPORTED) 7512 bp->fw_cap |= BNXT_FW_CAP_DBG_QCAPS; 7513 7514 flags_ext = le32_to_cpu(resp->flags_ext); 7515 if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_EXT_HW_STATS_SUPPORTED) 7516 bp->fw_cap |= BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED; 7517 if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_PTP_PPS_SUPPORTED)) 7518 bp->fw_cap |= BNXT_FW_CAP_PTP_PPS; 7519 if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_HOT_RESET_IF_SUPPORT)) 7520 bp->fw_cap |= BNXT_FW_CAP_HOT_RESET_IF; 7521 if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_FW_LIVEPATCH_SUPPORTED)) 7522 bp->fw_cap |= BNXT_FW_CAP_LIVEPATCH; 7523 7524 bp->tx_push_thresh = 0; 7525 if ((flags & FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED) && 7526 BNXT_FW_MAJ(bp) > 217) 7527 bp->tx_push_thresh = BNXT_TX_PUSH_THRESH; 7528 7529 hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx); 7530 hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings); 7531 hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings); 7532 hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings); 7533 hw_resc->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps); 7534 if (!hw_resc->max_hw_ring_grps) 7535 hw_resc->max_hw_ring_grps = hw_resc->max_tx_rings; 7536 hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs); 7537 hw_resc->max_vnics = le16_to_cpu(resp->max_vnics); 7538 hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx); 7539 7540 if (BNXT_PF(bp)) { 7541 struct bnxt_pf_info *pf = &bp->pf; 7542 7543 pf->fw_fid = le16_to_cpu(resp->fid); 7544 pf->port_id = le16_to_cpu(resp->port_id); 7545 memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN); 7546 pf->first_vf_id = le16_to_cpu(resp->first_vf_id); 7547 pf->max_vfs = le16_to_cpu(resp->max_vfs); 7548 pf->max_encap_records = le32_to_cpu(resp->max_encap_records); 7549 pf->max_decap_records = le32_to_cpu(resp->max_decap_records); 7550 pf->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows); 7551 pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows); 7552 pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows); 7553 pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows); 7554 bp->flags &= ~BNXT_FLAG_WOL_CAP; 7555 if (flags & FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED) 7556 bp->flags |= BNXT_FLAG_WOL_CAP; 7557 if (flags & FUNC_QCAPS_RESP_FLAGS_PTP_SUPPORTED) { 7558 __bnxt_hwrm_ptp_qcfg(bp); 7559 } else { 7560 bnxt_ptp_clear(bp); 7561 kfree(bp->ptp_cfg); 7562 bp->ptp_cfg = NULL; 7563 } 7564 } else { 7565 #ifdef CONFIG_BNXT_SRIOV 7566 struct bnxt_vf_info *vf = &bp->vf; 7567 7568 vf->fw_fid = le16_to_cpu(resp->fid); 7569 memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN); 7570 #endif 7571 } 7572 7573 hwrm_func_qcaps_exit: 7574 hwrm_req_drop(bp, req); 7575 return rc; 7576 } 7577 7578 static void bnxt_hwrm_dbg_qcaps(struct bnxt *bp) 7579 { 7580 struct hwrm_dbg_qcaps_output *resp; 7581 struct hwrm_dbg_qcaps_input *req; 7582 int rc; 7583 7584 bp->fw_dbg_cap = 0; 7585 if (!(bp->fw_cap & BNXT_FW_CAP_DBG_QCAPS)) 7586 return; 7587 7588 rc = hwrm_req_init(bp, req, HWRM_DBG_QCAPS); 7589 if (rc) 7590 return; 7591 7592 req->fid = cpu_to_le16(0xffff); 7593 resp = hwrm_req_hold(bp, req); 7594 rc = hwrm_req_send(bp, req); 7595 if (rc) 7596 goto hwrm_dbg_qcaps_exit; 7597 7598 bp->fw_dbg_cap = le32_to_cpu(resp->flags); 7599 7600 hwrm_dbg_qcaps_exit: 7601 hwrm_req_drop(bp, req); 7602 } 7603 7604 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp); 7605 7606 static int bnxt_hwrm_func_qcaps(struct bnxt *bp) 7607 { 7608 int rc; 7609 7610 rc = __bnxt_hwrm_func_qcaps(bp); 7611 if (rc) 7612 return rc; 7613 7614 bnxt_hwrm_dbg_qcaps(bp); 7615 7616 rc = bnxt_hwrm_queue_qportcfg(bp); 7617 if (rc) { 7618 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %d\n", rc); 7619 return rc; 7620 } 7621 if (bp->hwrm_spec_code >= 0x10803) { 7622 rc = bnxt_alloc_ctx_mem(bp); 7623 if (rc) 7624 return rc; 7625 rc = bnxt_hwrm_func_resc_qcaps(bp, true); 7626 if (!rc) 7627 bp->fw_cap |= BNXT_FW_CAP_NEW_RM; 7628 } 7629 return 0; 7630 } 7631 7632 static int bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(struct bnxt *bp) 7633 { 7634 struct hwrm_cfa_adv_flow_mgnt_qcaps_output *resp; 7635 struct hwrm_cfa_adv_flow_mgnt_qcaps_input *req; 7636 u32 flags; 7637 int rc; 7638 7639 if (!(bp->fw_cap & BNXT_FW_CAP_CFA_ADV_FLOW)) 7640 return 0; 7641 7642 rc = hwrm_req_init(bp, req, HWRM_CFA_ADV_FLOW_MGNT_QCAPS); 7643 if (rc) 7644 return rc; 7645 7646 resp = hwrm_req_hold(bp, req); 7647 rc = hwrm_req_send(bp, req); 7648 if (rc) 7649 goto hwrm_cfa_adv_qcaps_exit; 7650 7651 flags = le32_to_cpu(resp->flags); 7652 if (flags & 7653 CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_V2_SUPPORTED) 7654 bp->fw_cap |= BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2; 7655 7656 hwrm_cfa_adv_qcaps_exit: 7657 hwrm_req_drop(bp, req); 7658 return rc; 7659 } 7660 7661 static int __bnxt_alloc_fw_health(struct bnxt *bp) 7662 { 7663 if (bp->fw_health) 7664 return 0; 7665 7666 bp->fw_health = kzalloc(sizeof(*bp->fw_health), GFP_KERNEL); 7667 if (!bp->fw_health) 7668 return -ENOMEM; 7669 7670 mutex_init(&bp->fw_health->lock); 7671 return 0; 7672 } 7673 7674 static int bnxt_alloc_fw_health(struct bnxt *bp) 7675 { 7676 int rc; 7677 7678 if (!(bp->fw_cap & BNXT_FW_CAP_HOT_RESET) && 7679 !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) 7680 return 0; 7681 7682 rc = __bnxt_alloc_fw_health(bp); 7683 if (rc) { 7684 bp->fw_cap &= ~BNXT_FW_CAP_HOT_RESET; 7685 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY; 7686 return rc; 7687 } 7688 7689 return 0; 7690 } 7691 7692 static void __bnxt_map_fw_health_reg(struct bnxt *bp, u32 reg) 7693 { 7694 writel(reg & BNXT_GRC_BASE_MASK, bp->bar0 + 7695 BNXT_GRCPF_REG_WINDOW_BASE_OUT + 7696 BNXT_FW_HEALTH_WIN_MAP_OFF); 7697 } 7698 7699 static void bnxt_inv_fw_health_reg(struct bnxt *bp) 7700 { 7701 struct bnxt_fw_health *fw_health = bp->fw_health; 7702 u32 reg_type; 7703 7704 if (!fw_health) 7705 return; 7706 7707 reg_type = BNXT_FW_HEALTH_REG_TYPE(fw_health->regs[BNXT_FW_HEALTH_REG]); 7708 if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC) 7709 fw_health->status_reliable = false; 7710 7711 reg_type = BNXT_FW_HEALTH_REG_TYPE(fw_health->regs[BNXT_FW_RESET_CNT_REG]); 7712 if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC) 7713 fw_health->resets_reliable = false; 7714 } 7715 7716 static void bnxt_try_map_fw_health_reg(struct bnxt *bp) 7717 { 7718 void __iomem *hs; 7719 u32 status_loc; 7720 u32 reg_type; 7721 u32 sig; 7722 7723 if (bp->fw_health) 7724 bp->fw_health->status_reliable = false; 7725 7726 __bnxt_map_fw_health_reg(bp, HCOMM_STATUS_STRUCT_LOC); 7727 hs = bp->bar0 + BNXT_FW_HEALTH_WIN_OFF(HCOMM_STATUS_STRUCT_LOC); 7728 7729 sig = readl(hs + offsetof(struct hcomm_status, sig_ver)); 7730 if ((sig & HCOMM_STATUS_SIGNATURE_MASK) != HCOMM_STATUS_SIGNATURE_VAL) { 7731 if (!bp->chip_num) { 7732 __bnxt_map_fw_health_reg(bp, BNXT_GRC_REG_BASE); 7733 bp->chip_num = readl(bp->bar0 + 7734 BNXT_FW_HEALTH_WIN_BASE + 7735 BNXT_GRC_REG_CHIP_NUM); 7736 } 7737 if (!BNXT_CHIP_P5(bp)) 7738 return; 7739 7740 status_loc = BNXT_GRC_REG_STATUS_P5 | 7741 BNXT_FW_HEALTH_REG_TYPE_BAR0; 7742 } else { 7743 status_loc = readl(hs + offsetof(struct hcomm_status, 7744 fw_status_loc)); 7745 } 7746 7747 if (__bnxt_alloc_fw_health(bp)) { 7748 netdev_warn(bp->dev, "no memory for firmware status checks\n"); 7749 return; 7750 } 7751 7752 bp->fw_health->regs[BNXT_FW_HEALTH_REG] = status_loc; 7753 reg_type = BNXT_FW_HEALTH_REG_TYPE(status_loc); 7754 if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC) { 7755 __bnxt_map_fw_health_reg(bp, status_loc); 7756 bp->fw_health->mapped_regs[BNXT_FW_HEALTH_REG] = 7757 BNXT_FW_HEALTH_WIN_OFF(status_loc); 7758 } 7759 7760 bp->fw_health->status_reliable = true; 7761 } 7762 7763 static int bnxt_map_fw_health_regs(struct bnxt *bp) 7764 { 7765 struct bnxt_fw_health *fw_health = bp->fw_health; 7766 u32 reg_base = 0xffffffff; 7767 int i; 7768 7769 bp->fw_health->status_reliable = false; 7770 bp->fw_health->resets_reliable = false; 7771 /* Only pre-map the monitoring GRC registers using window 3 */ 7772 for (i = 0; i < 4; i++) { 7773 u32 reg = fw_health->regs[i]; 7774 7775 if (BNXT_FW_HEALTH_REG_TYPE(reg) != BNXT_FW_HEALTH_REG_TYPE_GRC) 7776 continue; 7777 if (reg_base == 0xffffffff) 7778 reg_base = reg & BNXT_GRC_BASE_MASK; 7779 if ((reg & BNXT_GRC_BASE_MASK) != reg_base) 7780 return -ERANGE; 7781 fw_health->mapped_regs[i] = BNXT_FW_HEALTH_WIN_OFF(reg); 7782 } 7783 bp->fw_health->status_reliable = true; 7784 bp->fw_health->resets_reliable = true; 7785 if (reg_base == 0xffffffff) 7786 return 0; 7787 7788 __bnxt_map_fw_health_reg(bp, reg_base); 7789 return 0; 7790 } 7791 7792 static void bnxt_remap_fw_health_regs(struct bnxt *bp) 7793 { 7794 if (!bp->fw_health) 7795 return; 7796 7797 if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) { 7798 bp->fw_health->status_reliable = true; 7799 bp->fw_health->resets_reliable = true; 7800 } else { 7801 bnxt_try_map_fw_health_reg(bp); 7802 } 7803 } 7804 7805 static int bnxt_hwrm_error_recovery_qcfg(struct bnxt *bp) 7806 { 7807 struct bnxt_fw_health *fw_health = bp->fw_health; 7808 struct hwrm_error_recovery_qcfg_output *resp; 7809 struct hwrm_error_recovery_qcfg_input *req; 7810 int rc, i; 7811 7812 if (!(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) 7813 return 0; 7814 7815 rc = hwrm_req_init(bp, req, HWRM_ERROR_RECOVERY_QCFG); 7816 if (rc) 7817 return rc; 7818 7819 resp = hwrm_req_hold(bp, req); 7820 rc = hwrm_req_send(bp, req); 7821 if (rc) 7822 goto err_recovery_out; 7823 fw_health->flags = le32_to_cpu(resp->flags); 7824 if ((fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) && 7825 !(bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL)) { 7826 rc = -EINVAL; 7827 goto err_recovery_out; 7828 } 7829 fw_health->polling_dsecs = le32_to_cpu(resp->driver_polling_freq); 7830 fw_health->master_func_wait_dsecs = 7831 le32_to_cpu(resp->master_func_wait_period); 7832 fw_health->normal_func_wait_dsecs = 7833 le32_to_cpu(resp->normal_func_wait_period); 7834 fw_health->post_reset_wait_dsecs = 7835 le32_to_cpu(resp->master_func_wait_period_after_reset); 7836 fw_health->post_reset_max_wait_dsecs = 7837 le32_to_cpu(resp->max_bailout_time_after_reset); 7838 fw_health->regs[BNXT_FW_HEALTH_REG] = 7839 le32_to_cpu(resp->fw_health_status_reg); 7840 fw_health->regs[BNXT_FW_HEARTBEAT_REG] = 7841 le32_to_cpu(resp->fw_heartbeat_reg); 7842 fw_health->regs[BNXT_FW_RESET_CNT_REG] = 7843 le32_to_cpu(resp->fw_reset_cnt_reg); 7844 fw_health->regs[BNXT_FW_RESET_INPROG_REG] = 7845 le32_to_cpu(resp->reset_inprogress_reg); 7846 fw_health->fw_reset_inprog_reg_mask = 7847 le32_to_cpu(resp->reset_inprogress_reg_mask); 7848 fw_health->fw_reset_seq_cnt = resp->reg_array_cnt; 7849 if (fw_health->fw_reset_seq_cnt >= 16) { 7850 rc = -EINVAL; 7851 goto err_recovery_out; 7852 } 7853 for (i = 0; i < fw_health->fw_reset_seq_cnt; i++) { 7854 fw_health->fw_reset_seq_regs[i] = 7855 le32_to_cpu(resp->reset_reg[i]); 7856 fw_health->fw_reset_seq_vals[i] = 7857 le32_to_cpu(resp->reset_reg_val[i]); 7858 fw_health->fw_reset_seq_delay_msec[i] = 7859 resp->delay_after_reset[i]; 7860 } 7861 err_recovery_out: 7862 hwrm_req_drop(bp, req); 7863 if (!rc) 7864 rc = bnxt_map_fw_health_regs(bp); 7865 if (rc) 7866 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY; 7867 return rc; 7868 } 7869 7870 static int bnxt_hwrm_func_reset(struct bnxt *bp) 7871 { 7872 struct hwrm_func_reset_input *req; 7873 int rc; 7874 7875 rc = hwrm_req_init(bp, req, HWRM_FUNC_RESET); 7876 if (rc) 7877 return rc; 7878 7879 req->enables = 0; 7880 hwrm_req_timeout(bp, req, HWRM_RESET_TIMEOUT); 7881 return hwrm_req_send(bp, req); 7882 } 7883 7884 static void bnxt_nvm_cfg_ver_get(struct bnxt *bp) 7885 { 7886 struct hwrm_nvm_get_dev_info_output nvm_info; 7887 7888 if (!bnxt_hwrm_nvm_get_dev_info(bp, &nvm_info)) 7889 snprintf(bp->nvm_cfg_ver, FW_VER_STR_LEN, "%d.%d.%d", 7890 nvm_info.nvm_cfg_ver_maj, nvm_info.nvm_cfg_ver_min, 7891 nvm_info.nvm_cfg_ver_upd); 7892 } 7893 7894 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp) 7895 { 7896 struct hwrm_queue_qportcfg_output *resp; 7897 struct hwrm_queue_qportcfg_input *req; 7898 u8 i, j, *qptr; 7899 bool no_rdma; 7900 int rc = 0; 7901 7902 rc = hwrm_req_init(bp, req, HWRM_QUEUE_QPORTCFG); 7903 if (rc) 7904 return rc; 7905 7906 resp = hwrm_req_hold(bp, req); 7907 rc = hwrm_req_send(bp, req); 7908 if (rc) 7909 goto qportcfg_exit; 7910 7911 if (!resp->max_configurable_queues) { 7912 rc = -EINVAL; 7913 goto qportcfg_exit; 7914 } 7915 bp->max_tc = resp->max_configurable_queues; 7916 bp->max_lltc = resp->max_configurable_lossless_queues; 7917 if (bp->max_tc > BNXT_MAX_QUEUE) 7918 bp->max_tc = BNXT_MAX_QUEUE; 7919 7920 no_rdma = !(bp->flags & BNXT_FLAG_ROCE_CAP); 7921 qptr = &resp->queue_id0; 7922 for (i = 0, j = 0; i < bp->max_tc; i++) { 7923 bp->q_info[j].queue_id = *qptr; 7924 bp->q_ids[i] = *qptr++; 7925 bp->q_info[j].queue_profile = *qptr++; 7926 bp->tc_to_qidx[j] = j; 7927 if (!BNXT_CNPQ(bp->q_info[j].queue_profile) || 7928 (no_rdma && BNXT_PF(bp))) 7929 j++; 7930 } 7931 bp->max_q = bp->max_tc; 7932 bp->max_tc = max_t(u8, j, 1); 7933 7934 if (resp->queue_cfg_info & QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG) 7935 bp->max_tc = 1; 7936 7937 if (bp->max_lltc > bp->max_tc) 7938 bp->max_lltc = bp->max_tc; 7939 7940 qportcfg_exit: 7941 hwrm_req_drop(bp, req); 7942 return rc; 7943 } 7944 7945 static int bnxt_hwrm_poll(struct bnxt *bp) 7946 { 7947 struct hwrm_ver_get_input *req; 7948 int rc; 7949 7950 rc = hwrm_req_init(bp, req, HWRM_VER_GET); 7951 if (rc) 7952 return rc; 7953 7954 req->hwrm_intf_maj = HWRM_VERSION_MAJOR; 7955 req->hwrm_intf_min = HWRM_VERSION_MINOR; 7956 req->hwrm_intf_upd = HWRM_VERSION_UPDATE; 7957 7958 hwrm_req_flags(bp, req, BNXT_HWRM_CTX_SILENT | BNXT_HWRM_FULL_WAIT); 7959 rc = hwrm_req_send(bp, req); 7960 return rc; 7961 } 7962 7963 static int bnxt_hwrm_ver_get(struct bnxt *bp) 7964 { 7965 struct hwrm_ver_get_output *resp; 7966 struct hwrm_ver_get_input *req; 7967 u16 fw_maj, fw_min, fw_bld, fw_rsv; 7968 u32 dev_caps_cfg, hwrm_ver; 7969 int rc, len; 7970 7971 rc = hwrm_req_init(bp, req, HWRM_VER_GET); 7972 if (rc) 7973 return rc; 7974 7975 hwrm_req_flags(bp, req, BNXT_HWRM_FULL_WAIT); 7976 bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN; 7977 req->hwrm_intf_maj = HWRM_VERSION_MAJOR; 7978 req->hwrm_intf_min = HWRM_VERSION_MINOR; 7979 req->hwrm_intf_upd = HWRM_VERSION_UPDATE; 7980 7981 resp = hwrm_req_hold(bp, req); 7982 rc = hwrm_req_send(bp, req); 7983 if (rc) 7984 goto hwrm_ver_get_exit; 7985 7986 memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output)); 7987 7988 bp->hwrm_spec_code = resp->hwrm_intf_maj_8b << 16 | 7989 resp->hwrm_intf_min_8b << 8 | 7990 resp->hwrm_intf_upd_8b; 7991 if (resp->hwrm_intf_maj_8b < 1) { 7992 netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n", 7993 resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b, 7994 resp->hwrm_intf_upd_8b); 7995 netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n"); 7996 } 7997 7998 hwrm_ver = HWRM_VERSION_MAJOR << 16 | HWRM_VERSION_MINOR << 8 | 7999 HWRM_VERSION_UPDATE; 8000 8001 if (bp->hwrm_spec_code > hwrm_ver) 8002 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d", 8003 HWRM_VERSION_MAJOR, HWRM_VERSION_MINOR, 8004 HWRM_VERSION_UPDATE); 8005 else 8006 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d", 8007 resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b, 8008 resp->hwrm_intf_upd_8b); 8009 8010 fw_maj = le16_to_cpu(resp->hwrm_fw_major); 8011 if (bp->hwrm_spec_code > 0x10803 && fw_maj) { 8012 fw_min = le16_to_cpu(resp->hwrm_fw_minor); 8013 fw_bld = le16_to_cpu(resp->hwrm_fw_build); 8014 fw_rsv = le16_to_cpu(resp->hwrm_fw_patch); 8015 len = FW_VER_STR_LEN; 8016 } else { 8017 fw_maj = resp->hwrm_fw_maj_8b; 8018 fw_min = resp->hwrm_fw_min_8b; 8019 fw_bld = resp->hwrm_fw_bld_8b; 8020 fw_rsv = resp->hwrm_fw_rsvd_8b; 8021 len = BC_HWRM_STR_LEN; 8022 } 8023 bp->fw_ver_code = BNXT_FW_VER_CODE(fw_maj, fw_min, fw_bld, fw_rsv); 8024 snprintf(bp->fw_ver_str, len, "%d.%d.%d.%d", fw_maj, fw_min, fw_bld, 8025 fw_rsv); 8026 8027 if (strlen(resp->active_pkg_name)) { 8028 int fw_ver_len = strlen(bp->fw_ver_str); 8029 8030 snprintf(bp->fw_ver_str + fw_ver_len, 8031 FW_VER_STR_LEN - fw_ver_len - 1, "/pkg %s", 8032 resp->active_pkg_name); 8033 bp->fw_cap |= BNXT_FW_CAP_PKG_VER; 8034 } 8035 8036 bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout); 8037 if (!bp->hwrm_cmd_timeout) 8038 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT; 8039 bp->hwrm_cmd_max_timeout = le16_to_cpu(resp->max_req_timeout) * 1000; 8040 if (!bp->hwrm_cmd_max_timeout) 8041 bp->hwrm_cmd_max_timeout = HWRM_CMD_MAX_TIMEOUT; 8042 else if (bp->hwrm_cmd_max_timeout > HWRM_CMD_MAX_TIMEOUT) 8043 netdev_warn(bp->dev, "Device requests max timeout of %d seconds, may trigger hung task watchdog\n", 8044 bp->hwrm_cmd_max_timeout / 1000); 8045 8046 if (resp->hwrm_intf_maj_8b >= 1) { 8047 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len); 8048 bp->hwrm_max_ext_req_len = le16_to_cpu(resp->max_ext_req_len); 8049 } 8050 if (bp->hwrm_max_ext_req_len < HWRM_MAX_REQ_LEN) 8051 bp->hwrm_max_ext_req_len = HWRM_MAX_REQ_LEN; 8052 8053 bp->chip_num = le16_to_cpu(resp->chip_num); 8054 bp->chip_rev = resp->chip_rev; 8055 if (bp->chip_num == CHIP_NUM_58700 && !resp->chip_rev && 8056 !resp->chip_metal) 8057 bp->flags |= BNXT_FLAG_CHIP_NITRO_A0; 8058 8059 dev_caps_cfg = le32_to_cpu(resp->dev_caps_cfg); 8060 if ((dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) && 8061 (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED)) 8062 bp->fw_cap |= BNXT_FW_CAP_SHORT_CMD; 8063 8064 if (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_KONG_MB_CHNL_SUPPORTED) 8065 bp->fw_cap |= BNXT_FW_CAP_KONG_MB_CHNL; 8066 8067 if (dev_caps_cfg & 8068 VER_GET_RESP_DEV_CAPS_CFG_FLOW_HANDLE_64BIT_SUPPORTED) 8069 bp->fw_cap |= BNXT_FW_CAP_OVS_64BIT_HANDLE; 8070 8071 if (dev_caps_cfg & 8072 VER_GET_RESP_DEV_CAPS_CFG_TRUSTED_VF_SUPPORTED) 8073 bp->fw_cap |= BNXT_FW_CAP_TRUSTED_VF; 8074 8075 if (dev_caps_cfg & 8076 VER_GET_RESP_DEV_CAPS_CFG_CFA_ADV_FLOW_MGNT_SUPPORTED) 8077 bp->fw_cap |= BNXT_FW_CAP_CFA_ADV_FLOW; 8078 8079 hwrm_ver_get_exit: 8080 hwrm_req_drop(bp, req); 8081 return rc; 8082 } 8083 8084 int bnxt_hwrm_fw_set_time(struct bnxt *bp) 8085 { 8086 struct hwrm_fw_set_time_input *req; 8087 struct tm tm; 8088 time64_t now = ktime_get_real_seconds(); 8089 int rc; 8090 8091 if ((BNXT_VF(bp) && bp->hwrm_spec_code < 0x10901) || 8092 bp->hwrm_spec_code < 0x10400) 8093 return -EOPNOTSUPP; 8094 8095 time64_to_tm(now, 0, &tm); 8096 rc = hwrm_req_init(bp, req, HWRM_FW_SET_TIME); 8097 if (rc) 8098 return rc; 8099 8100 req->year = cpu_to_le16(1900 + tm.tm_year); 8101 req->month = 1 + tm.tm_mon; 8102 req->day = tm.tm_mday; 8103 req->hour = tm.tm_hour; 8104 req->minute = tm.tm_min; 8105 req->second = tm.tm_sec; 8106 return hwrm_req_send(bp, req); 8107 } 8108 8109 static void bnxt_add_one_ctr(u64 hw, u64 *sw, u64 mask) 8110 { 8111 u64 sw_tmp; 8112 8113 hw &= mask; 8114 sw_tmp = (*sw & ~mask) | hw; 8115 if (hw < (*sw & mask)) 8116 sw_tmp += mask + 1; 8117 WRITE_ONCE(*sw, sw_tmp); 8118 } 8119 8120 static void __bnxt_accumulate_stats(__le64 *hw_stats, u64 *sw_stats, u64 *masks, 8121 int count, bool ignore_zero) 8122 { 8123 int i; 8124 8125 for (i = 0; i < count; i++) { 8126 u64 hw = le64_to_cpu(READ_ONCE(hw_stats[i])); 8127 8128 if (ignore_zero && !hw) 8129 continue; 8130 8131 if (masks[i] == -1ULL) 8132 sw_stats[i] = hw; 8133 else 8134 bnxt_add_one_ctr(hw, &sw_stats[i], masks[i]); 8135 } 8136 } 8137 8138 static void bnxt_accumulate_stats(struct bnxt_stats_mem *stats) 8139 { 8140 if (!stats->hw_stats) 8141 return; 8142 8143 __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats, 8144 stats->hw_masks, stats->len / 8, false); 8145 } 8146 8147 static void bnxt_accumulate_all_stats(struct bnxt *bp) 8148 { 8149 struct bnxt_stats_mem *ring0_stats; 8150 bool ignore_zero = false; 8151 int i; 8152 8153 /* Chip bug. Counter intermittently becomes 0. */ 8154 if (bp->flags & BNXT_FLAG_CHIP_P5) 8155 ignore_zero = true; 8156 8157 for (i = 0; i < bp->cp_nr_rings; i++) { 8158 struct bnxt_napi *bnapi = bp->bnapi[i]; 8159 struct bnxt_cp_ring_info *cpr; 8160 struct bnxt_stats_mem *stats; 8161 8162 cpr = &bnapi->cp_ring; 8163 stats = &cpr->stats; 8164 if (!i) 8165 ring0_stats = stats; 8166 __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats, 8167 ring0_stats->hw_masks, 8168 ring0_stats->len / 8, ignore_zero); 8169 } 8170 if (bp->flags & BNXT_FLAG_PORT_STATS) { 8171 struct bnxt_stats_mem *stats = &bp->port_stats; 8172 __le64 *hw_stats = stats->hw_stats; 8173 u64 *sw_stats = stats->sw_stats; 8174 u64 *masks = stats->hw_masks; 8175 int cnt; 8176 8177 cnt = sizeof(struct rx_port_stats) / 8; 8178 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false); 8179 8180 hw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 8181 sw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 8182 masks += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 8183 cnt = sizeof(struct tx_port_stats) / 8; 8184 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false); 8185 } 8186 if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) { 8187 bnxt_accumulate_stats(&bp->rx_port_stats_ext); 8188 bnxt_accumulate_stats(&bp->tx_port_stats_ext); 8189 } 8190 } 8191 8192 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags) 8193 { 8194 struct hwrm_port_qstats_input *req; 8195 struct bnxt_pf_info *pf = &bp->pf; 8196 int rc; 8197 8198 if (!(bp->flags & BNXT_FLAG_PORT_STATS)) 8199 return 0; 8200 8201 if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED)) 8202 return -EOPNOTSUPP; 8203 8204 rc = hwrm_req_init(bp, req, HWRM_PORT_QSTATS); 8205 if (rc) 8206 return rc; 8207 8208 req->flags = flags; 8209 req->port_id = cpu_to_le16(pf->port_id); 8210 req->tx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map + 8211 BNXT_TX_PORT_STATS_BYTE_OFFSET); 8212 req->rx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map); 8213 return hwrm_req_send(bp, req); 8214 } 8215 8216 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags) 8217 { 8218 struct hwrm_queue_pri2cos_qcfg_output *resp_qc; 8219 struct hwrm_queue_pri2cos_qcfg_input *req_qc; 8220 struct hwrm_port_qstats_ext_output *resp_qs; 8221 struct hwrm_port_qstats_ext_input *req_qs; 8222 struct bnxt_pf_info *pf = &bp->pf; 8223 u32 tx_stat_size; 8224 int rc; 8225 8226 if (!(bp->flags & BNXT_FLAG_PORT_STATS_EXT)) 8227 return 0; 8228 8229 if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED)) 8230 return -EOPNOTSUPP; 8231 8232 rc = hwrm_req_init(bp, req_qs, HWRM_PORT_QSTATS_EXT); 8233 if (rc) 8234 return rc; 8235 8236 req_qs->flags = flags; 8237 req_qs->port_id = cpu_to_le16(pf->port_id); 8238 req_qs->rx_stat_size = cpu_to_le16(sizeof(struct rx_port_stats_ext)); 8239 req_qs->rx_stat_host_addr = cpu_to_le64(bp->rx_port_stats_ext.hw_stats_map); 8240 tx_stat_size = bp->tx_port_stats_ext.hw_stats ? 8241 sizeof(struct tx_port_stats_ext) : 0; 8242 req_qs->tx_stat_size = cpu_to_le16(tx_stat_size); 8243 req_qs->tx_stat_host_addr = cpu_to_le64(bp->tx_port_stats_ext.hw_stats_map); 8244 resp_qs = hwrm_req_hold(bp, req_qs); 8245 rc = hwrm_req_send(bp, req_qs); 8246 if (!rc) { 8247 bp->fw_rx_stats_ext_size = 8248 le16_to_cpu(resp_qs->rx_stat_size) / 8; 8249 if (BNXT_FW_MAJ(bp) < 220 && 8250 bp->fw_rx_stats_ext_size > BNXT_RX_STATS_EXT_NUM_LEGACY) 8251 bp->fw_rx_stats_ext_size = BNXT_RX_STATS_EXT_NUM_LEGACY; 8252 8253 bp->fw_tx_stats_ext_size = tx_stat_size ? 8254 le16_to_cpu(resp_qs->tx_stat_size) / 8 : 0; 8255 } else { 8256 bp->fw_rx_stats_ext_size = 0; 8257 bp->fw_tx_stats_ext_size = 0; 8258 } 8259 hwrm_req_drop(bp, req_qs); 8260 8261 if (flags) 8262 return rc; 8263 8264 if (bp->fw_tx_stats_ext_size <= 8265 offsetof(struct tx_port_stats_ext, pfc_pri0_tx_duration_us) / 8) { 8266 bp->pri2cos_valid = 0; 8267 return rc; 8268 } 8269 8270 rc = hwrm_req_init(bp, req_qc, HWRM_QUEUE_PRI2COS_QCFG); 8271 if (rc) 8272 return rc; 8273 8274 req_qc->flags = cpu_to_le32(QUEUE_PRI2COS_QCFG_REQ_FLAGS_IVLAN); 8275 8276 resp_qc = hwrm_req_hold(bp, req_qc); 8277 rc = hwrm_req_send(bp, req_qc); 8278 if (!rc) { 8279 u8 *pri2cos; 8280 int i, j; 8281 8282 pri2cos = &resp_qc->pri0_cos_queue_id; 8283 for (i = 0; i < 8; i++) { 8284 u8 queue_id = pri2cos[i]; 8285 u8 queue_idx; 8286 8287 /* Per port queue IDs start from 0, 10, 20, etc */ 8288 queue_idx = queue_id % 10; 8289 if (queue_idx > BNXT_MAX_QUEUE) { 8290 bp->pri2cos_valid = false; 8291 hwrm_req_drop(bp, req_qc); 8292 return rc; 8293 } 8294 for (j = 0; j < bp->max_q; j++) { 8295 if (bp->q_ids[j] == queue_id) 8296 bp->pri2cos_idx[i] = queue_idx; 8297 } 8298 } 8299 bp->pri2cos_valid = true; 8300 } 8301 hwrm_req_drop(bp, req_qc); 8302 8303 return rc; 8304 } 8305 8306 static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp) 8307 { 8308 bnxt_hwrm_tunnel_dst_port_free(bp, 8309 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN); 8310 bnxt_hwrm_tunnel_dst_port_free(bp, 8311 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE); 8312 } 8313 8314 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa) 8315 { 8316 int rc, i; 8317 u32 tpa_flags = 0; 8318 8319 if (set_tpa) 8320 tpa_flags = bp->flags & BNXT_FLAG_TPA; 8321 else if (BNXT_NO_FW_ACCESS(bp)) 8322 return 0; 8323 for (i = 0; i < bp->nr_vnics; i++) { 8324 rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags); 8325 if (rc) { 8326 netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n", 8327 i, rc); 8328 return rc; 8329 } 8330 } 8331 return 0; 8332 } 8333 8334 static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp) 8335 { 8336 int i; 8337 8338 for (i = 0; i < bp->nr_vnics; i++) 8339 bnxt_hwrm_vnic_set_rss(bp, i, false); 8340 } 8341 8342 static void bnxt_clear_vnic(struct bnxt *bp) 8343 { 8344 if (!bp->vnic_info) 8345 return; 8346 8347 bnxt_hwrm_clear_vnic_filter(bp); 8348 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) { 8349 /* clear all RSS setting before free vnic ctx */ 8350 bnxt_hwrm_clear_vnic_rss(bp); 8351 bnxt_hwrm_vnic_ctx_free(bp); 8352 } 8353 /* before free the vnic, undo the vnic tpa settings */ 8354 if (bp->flags & BNXT_FLAG_TPA) 8355 bnxt_set_tpa(bp, false); 8356 bnxt_hwrm_vnic_free(bp); 8357 if (bp->flags & BNXT_FLAG_CHIP_P5) 8358 bnxt_hwrm_vnic_ctx_free(bp); 8359 } 8360 8361 static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path, 8362 bool irq_re_init) 8363 { 8364 bnxt_clear_vnic(bp); 8365 bnxt_hwrm_ring_free(bp, close_path); 8366 bnxt_hwrm_ring_grp_free(bp); 8367 if (irq_re_init) { 8368 bnxt_hwrm_stat_ctx_free(bp); 8369 bnxt_hwrm_free_tunnel_ports(bp); 8370 } 8371 } 8372 8373 static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode) 8374 { 8375 struct hwrm_func_cfg_input *req; 8376 u8 evb_mode; 8377 int rc; 8378 8379 if (br_mode == BRIDGE_MODE_VEB) 8380 evb_mode = FUNC_CFG_REQ_EVB_MODE_VEB; 8381 else if (br_mode == BRIDGE_MODE_VEPA) 8382 evb_mode = FUNC_CFG_REQ_EVB_MODE_VEPA; 8383 else 8384 return -EINVAL; 8385 8386 rc = hwrm_req_init(bp, req, HWRM_FUNC_CFG); 8387 if (rc) 8388 return rc; 8389 8390 req->fid = cpu_to_le16(0xffff); 8391 req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_EVB_MODE); 8392 req->evb_mode = evb_mode; 8393 return hwrm_req_send(bp, req); 8394 } 8395 8396 static int bnxt_hwrm_set_cache_line_size(struct bnxt *bp, int size) 8397 { 8398 struct hwrm_func_cfg_input *req; 8399 int rc; 8400 8401 if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10803) 8402 return 0; 8403 8404 rc = hwrm_req_init(bp, req, HWRM_FUNC_CFG); 8405 if (rc) 8406 return rc; 8407 8408 req->fid = cpu_to_le16(0xffff); 8409 req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_CACHE_LINESIZE); 8410 req->options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_64; 8411 if (size == 128) 8412 req->options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_128; 8413 8414 return hwrm_req_send(bp, req); 8415 } 8416 8417 static int __bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id) 8418 { 8419 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 8420 int rc; 8421 8422 if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) 8423 goto skip_rss_ctx; 8424 8425 /* allocate context for vnic */ 8426 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 0); 8427 if (rc) { 8428 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n", 8429 vnic_id, rc); 8430 goto vnic_setup_err; 8431 } 8432 bp->rsscos_nr_ctxs++; 8433 8434 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 8435 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 1); 8436 if (rc) { 8437 netdev_err(bp->dev, "hwrm vnic %d cos ctx alloc failure rc: %x\n", 8438 vnic_id, rc); 8439 goto vnic_setup_err; 8440 } 8441 bp->rsscos_nr_ctxs++; 8442 } 8443 8444 skip_rss_ctx: 8445 /* configure default vnic, ring grp */ 8446 rc = bnxt_hwrm_vnic_cfg(bp, vnic_id); 8447 if (rc) { 8448 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n", 8449 vnic_id, rc); 8450 goto vnic_setup_err; 8451 } 8452 8453 /* Enable RSS hashing on vnic */ 8454 rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true); 8455 if (rc) { 8456 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n", 8457 vnic_id, rc); 8458 goto vnic_setup_err; 8459 } 8460 8461 if (bp->flags & BNXT_FLAG_AGG_RINGS) { 8462 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id); 8463 if (rc) { 8464 netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n", 8465 vnic_id, rc); 8466 } 8467 } 8468 8469 vnic_setup_err: 8470 return rc; 8471 } 8472 8473 static int __bnxt_setup_vnic_p5(struct bnxt *bp, u16 vnic_id) 8474 { 8475 int rc, i, nr_ctxs; 8476 8477 nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings); 8478 for (i = 0; i < nr_ctxs; i++) { 8479 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, i); 8480 if (rc) { 8481 netdev_err(bp->dev, "hwrm vnic %d ctx %d alloc failure rc: %x\n", 8482 vnic_id, i, rc); 8483 break; 8484 } 8485 bp->rsscos_nr_ctxs++; 8486 } 8487 if (i < nr_ctxs) 8488 return -ENOMEM; 8489 8490 rc = bnxt_hwrm_vnic_set_rss_p5(bp, vnic_id, true); 8491 if (rc) { 8492 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %d\n", 8493 vnic_id, rc); 8494 return rc; 8495 } 8496 rc = bnxt_hwrm_vnic_cfg(bp, vnic_id); 8497 if (rc) { 8498 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n", 8499 vnic_id, rc); 8500 return rc; 8501 } 8502 if (bp->flags & BNXT_FLAG_AGG_RINGS) { 8503 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id); 8504 if (rc) { 8505 netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n", 8506 vnic_id, rc); 8507 } 8508 } 8509 return rc; 8510 } 8511 8512 static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id) 8513 { 8514 if (bp->flags & BNXT_FLAG_CHIP_P5) 8515 return __bnxt_setup_vnic_p5(bp, vnic_id); 8516 else 8517 return __bnxt_setup_vnic(bp, vnic_id); 8518 } 8519 8520 static int bnxt_alloc_rfs_vnics(struct bnxt *bp) 8521 { 8522 #ifdef CONFIG_RFS_ACCEL 8523 int i, rc = 0; 8524 8525 if (bp->flags & BNXT_FLAG_CHIP_P5) 8526 return 0; 8527 8528 for (i = 0; i < bp->rx_nr_rings; i++) { 8529 struct bnxt_vnic_info *vnic; 8530 u16 vnic_id = i + 1; 8531 u16 ring_id = i; 8532 8533 if (vnic_id >= bp->nr_vnics) 8534 break; 8535 8536 vnic = &bp->vnic_info[vnic_id]; 8537 vnic->flags |= BNXT_VNIC_RFS_FLAG; 8538 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP) 8539 vnic->flags |= BNXT_VNIC_RFS_NEW_RSS_FLAG; 8540 rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, ring_id, 1); 8541 if (rc) { 8542 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n", 8543 vnic_id, rc); 8544 break; 8545 } 8546 rc = bnxt_setup_vnic(bp, vnic_id); 8547 if (rc) 8548 break; 8549 } 8550 return rc; 8551 #else 8552 return 0; 8553 #endif 8554 } 8555 8556 /* Allow PF, trusted VFs and VFs with default VLAN to be in promiscuous mode */ 8557 static bool bnxt_promisc_ok(struct bnxt *bp) 8558 { 8559 #ifdef CONFIG_BNXT_SRIOV 8560 if (BNXT_VF(bp) && !bp->vf.vlan && !bnxt_is_trusted_vf(bp, &bp->vf)) 8561 return false; 8562 #endif 8563 return true; 8564 } 8565 8566 static int bnxt_setup_nitroa0_vnic(struct bnxt *bp) 8567 { 8568 unsigned int rc = 0; 8569 8570 rc = bnxt_hwrm_vnic_alloc(bp, 1, bp->rx_nr_rings - 1, 1); 8571 if (rc) { 8572 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n", 8573 rc); 8574 return rc; 8575 } 8576 8577 rc = bnxt_hwrm_vnic_cfg(bp, 1); 8578 if (rc) { 8579 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n", 8580 rc); 8581 return rc; 8582 } 8583 return rc; 8584 } 8585 8586 static int bnxt_cfg_rx_mode(struct bnxt *); 8587 static bool bnxt_mc_list_updated(struct bnxt *, u32 *); 8588 8589 static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init) 8590 { 8591 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 8592 int rc = 0; 8593 unsigned int rx_nr_rings = bp->rx_nr_rings; 8594 8595 if (irq_re_init) { 8596 rc = bnxt_hwrm_stat_ctx_alloc(bp); 8597 if (rc) { 8598 netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n", 8599 rc); 8600 goto err_out; 8601 } 8602 } 8603 8604 rc = bnxt_hwrm_ring_alloc(bp); 8605 if (rc) { 8606 netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc); 8607 goto err_out; 8608 } 8609 8610 rc = bnxt_hwrm_ring_grp_alloc(bp); 8611 if (rc) { 8612 netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc); 8613 goto err_out; 8614 } 8615 8616 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 8617 rx_nr_rings--; 8618 8619 /* default vnic 0 */ 8620 rc = bnxt_hwrm_vnic_alloc(bp, 0, 0, rx_nr_rings); 8621 if (rc) { 8622 netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc); 8623 goto err_out; 8624 } 8625 8626 rc = bnxt_setup_vnic(bp, 0); 8627 if (rc) 8628 goto err_out; 8629 8630 if (bp->flags & BNXT_FLAG_RFS) { 8631 rc = bnxt_alloc_rfs_vnics(bp); 8632 if (rc) 8633 goto err_out; 8634 } 8635 8636 if (bp->flags & BNXT_FLAG_TPA) { 8637 rc = bnxt_set_tpa(bp, true); 8638 if (rc) 8639 goto err_out; 8640 } 8641 8642 if (BNXT_VF(bp)) 8643 bnxt_update_vf_mac(bp); 8644 8645 /* Filter for default vnic 0 */ 8646 rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr); 8647 if (rc) { 8648 if (BNXT_VF(bp) && rc == -ENODEV) 8649 netdev_err(bp->dev, "Cannot configure L2 filter while PF is unavailable\n"); 8650 else 8651 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc); 8652 goto err_out; 8653 } 8654 vnic->uc_filter_count = 1; 8655 8656 vnic->rx_mask = 0; 8657 if (test_bit(BNXT_STATE_HALF_OPEN, &bp->state)) 8658 goto skip_rx_mask; 8659 8660 if (bp->dev->flags & IFF_BROADCAST) 8661 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST; 8662 8663 if (bp->dev->flags & IFF_PROMISC) 8664 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 8665 8666 if (bp->dev->flags & IFF_ALLMULTI) { 8667 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 8668 vnic->mc_list_count = 0; 8669 } else if (bp->dev->flags & IFF_MULTICAST) { 8670 u32 mask = 0; 8671 8672 bnxt_mc_list_updated(bp, &mask); 8673 vnic->rx_mask |= mask; 8674 } 8675 8676 rc = bnxt_cfg_rx_mode(bp); 8677 if (rc) 8678 goto err_out; 8679 8680 skip_rx_mask: 8681 rc = bnxt_hwrm_set_coal(bp); 8682 if (rc) 8683 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n", 8684 rc); 8685 8686 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 8687 rc = bnxt_setup_nitroa0_vnic(bp); 8688 if (rc) 8689 netdev_err(bp->dev, "Special vnic setup failure for NS2 A0 rc: %x\n", 8690 rc); 8691 } 8692 8693 if (BNXT_VF(bp)) { 8694 bnxt_hwrm_func_qcfg(bp); 8695 netdev_update_features(bp->dev); 8696 } 8697 8698 return 0; 8699 8700 err_out: 8701 bnxt_hwrm_resource_free(bp, 0, true); 8702 8703 return rc; 8704 } 8705 8706 static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init) 8707 { 8708 bnxt_hwrm_resource_free(bp, 1, irq_re_init); 8709 return 0; 8710 } 8711 8712 static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init) 8713 { 8714 bnxt_init_cp_rings(bp); 8715 bnxt_init_rx_rings(bp); 8716 bnxt_init_tx_rings(bp); 8717 bnxt_init_ring_grps(bp, irq_re_init); 8718 bnxt_init_vnics(bp); 8719 8720 return bnxt_init_chip(bp, irq_re_init); 8721 } 8722 8723 static int bnxt_set_real_num_queues(struct bnxt *bp) 8724 { 8725 int rc; 8726 struct net_device *dev = bp->dev; 8727 8728 rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings - 8729 bp->tx_nr_rings_xdp); 8730 if (rc) 8731 return rc; 8732 8733 rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings); 8734 if (rc) 8735 return rc; 8736 8737 #ifdef CONFIG_RFS_ACCEL 8738 if (bp->flags & BNXT_FLAG_RFS) 8739 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings); 8740 #endif 8741 8742 return rc; 8743 } 8744 8745 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max, 8746 bool shared) 8747 { 8748 int _rx = *rx, _tx = *tx; 8749 8750 if (shared) { 8751 *rx = min_t(int, _rx, max); 8752 *tx = min_t(int, _tx, max); 8753 } else { 8754 if (max < 2) 8755 return -ENOMEM; 8756 8757 while (_rx + _tx > max) { 8758 if (_rx > _tx && _rx > 1) 8759 _rx--; 8760 else if (_tx > 1) 8761 _tx--; 8762 } 8763 *rx = _rx; 8764 *tx = _tx; 8765 } 8766 return 0; 8767 } 8768 8769 static void bnxt_setup_msix(struct bnxt *bp) 8770 { 8771 const int len = sizeof(bp->irq_tbl[0].name); 8772 struct net_device *dev = bp->dev; 8773 int tcs, i; 8774 8775 tcs = netdev_get_num_tc(dev); 8776 if (tcs) { 8777 int i, off, count; 8778 8779 for (i = 0; i < tcs; i++) { 8780 count = bp->tx_nr_rings_per_tc; 8781 off = i * count; 8782 netdev_set_tc_queue(dev, i, count, off); 8783 } 8784 } 8785 8786 for (i = 0; i < bp->cp_nr_rings; i++) { 8787 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 8788 char *attr; 8789 8790 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 8791 attr = "TxRx"; 8792 else if (i < bp->rx_nr_rings) 8793 attr = "rx"; 8794 else 8795 attr = "tx"; 8796 8797 snprintf(bp->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name, 8798 attr, i); 8799 bp->irq_tbl[map_idx].handler = bnxt_msix; 8800 } 8801 } 8802 8803 static void bnxt_setup_inta(struct bnxt *bp) 8804 { 8805 const int len = sizeof(bp->irq_tbl[0].name); 8806 8807 if (netdev_get_num_tc(bp->dev)) 8808 netdev_reset_tc(bp->dev); 8809 8810 snprintf(bp->irq_tbl[0].name, len, "%s-%s-%d", bp->dev->name, "TxRx", 8811 0); 8812 bp->irq_tbl[0].handler = bnxt_inta; 8813 } 8814 8815 static int bnxt_init_int_mode(struct bnxt *bp); 8816 8817 static int bnxt_setup_int_mode(struct bnxt *bp) 8818 { 8819 int rc; 8820 8821 if (!bp->irq_tbl) { 8822 rc = bnxt_init_int_mode(bp); 8823 if (rc || !bp->irq_tbl) 8824 return rc ?: -ENODEV; 8825 } 8826 8827 if (bp->flags & BNXT_FLAG_USING_MSIX) 8828 bnxt_setup_msix(bp); 8829 else 8830 bnxt_setup_inta(bp); 8831 8832 rc = bnxt_set_real_num_queues(bp); 8833 return rc; 8834 } 8835 8836 #ifdef CONFIG_RFS_ACCEL 8837 static unsigned int bnxt_get_max_func_rss_ctxs(struct bnxt *bp) 8838 { 8839 return bp->hw_resc.max_rsscos_ctxs; 8840 } 8841 8842 static unsigned int bnxt_get_max_func_vnics(struct bnxt *bp) 8843 { 8844 return bp->hw_resc.max_vnics; 8845 } 8846 #endif 8847 8848 unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp) 8849 { 8850 return bp->hw_resc.max_stat_ctxs; 8851 } 8852 8853 unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp) 8854 { 8855 return bp->hw_resc.max_cp_rings; 8856 } 8857 8858 static unsigned int bnxt_get_max_func_cp_rings_for_en(struct bnxt *bp) 8859 { 8860 unsigned int cp = bp->hw_resc.max_cp_rings; 8861 8862 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 8863 cp -= bnxt_get_ulp_msix_num(bp); 8864 8865 return cp; 8866 } 8867 8868 static unsigned int bnxt_get_max_func_irqs(struct bnxt *bp) 8869 { 8870 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 8871 8872 if (bp->flags & BNXT_FLAG_CHIP_P5) 8873 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_nqs); 8874 8875 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_cp_rings); 8876 } 8877 8878 static void bnxt_set_max_func_irqs(struct bnxt *bp, unsigned int max_irqs) 8879 { 8880 bp->hw_resc.max_irqs = max_irqs; 8881 } 8882 8883 unsigned int bnxt_get_avail_cp_rings_for_en(struct bnxt *bp) 8884 { 8885 unsigned int cp; 8886 8887 cp = bnxt_get_max_func_cp_rings_for_en(bp); 8888 if (bp->flags & BNXT_FLAG_CHIP_P5) 8889 return cp - bp->rx_nr_rings - bp->tx_nr_rings; 8890 else 8891 return cp - bp->cp_nr_rings; 8892 } 8893 8894 unsigned int bnxt_get_avail_stat_ctxs_for_en(struct bnxt *bp) 8895 { 8896 return bnxt_get_max_func_stat_ctxs(bp) - bnxt_get_func_stat_ctxs(bp); 8897 } 8898 8899 int bnxt_get_avail_msix(struct bnxt *bp, int num) 8900 { 8901 int max_cp = bnxt_get_max_func_cp_rings(bp); 8902 int max_irq = bnxt_get_max_func_irqs(bp); 8903 int total_req = bp->cp_nr_rings + num; 8904 int max_idx, avail_msix; 8905 8906 max_idx = bp->total_irqs; 8907 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 8908 max_idx = min_t(int, bp->total_irqs, max_cp); 8909 avail_msix = max_idx - bp->cp_nr_rings; 8910 if (!BNXT_NEW_RM(bp) || avail_msix >= num) 8911 return avail_msix; 8912 8913 if (max_irq < total_req) { 8914 num = max_irq - bp->cp_nr_rings; 8915 if (num <= 0) 8916 return 0; 8917 } 8918 return num; 8919 } 8920 8921 static int bnxt_get_num_msix(struct bnxt *bp) 8922 { 8923 if (!BNXT_NEW_RM(bp)) 8924 return bnxt_get_max_func_irqs(bp); 8925 8926 return bnxt_nq_rings_in_use(bp); 8927 } 8928 8929 static int bnxt_init_msix(struct bnxt *bp) 8930 { 8931 int i, total_vecs, max, rc = 0, min = 1, ulp_msix; 8932 struct msix_entry *msix_ent; 8933 8934 total_vecs = bnxt_get_num_msix(bp); 8935 max = bnxt_get_max_func_irqs(bp); 8936 if (total_vecs > max) 8937 total_vecs = max; 8938 8939 if (!total_vecs) 8940 return 0; 8941 8942 msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL); 8943 if (!msix_ent) 8944 return -ENOMEM; 8945 8946 for (i = 0; i < total_vecs; i++) { 8947 msix_ent[i].entry = i; 8948 msix_ent[i].vector = 0; 8949 } 8950 8951 if (!(bp->flags & BNXT_FLAG_SHARED_RINGS)) 8952 min = 2; 8953 8954 total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs); 8955 ulp_msix = bnxt_get_ulp_msix_num(bp); 8956 if (total_vecs < 0 || total_vecs < ulp_msix) { 8957 rc = -ENODEV; 8958 goto msix_setup_exit; 8959 } 8960 8961 bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL); 8962 if (bp->irq_tbl) { 8963 for (i = 0; i < total_vecs; i++) 8964 bp->irq_tbl[i].vector = msix_ent[i].vector; 8965 8966 bp->total_irqs = total_vecs; 8967 /* Trim rings based upon num of vectors allocated */ 8968 rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings, 8969 total_vecs - ulp_msix, min == 1); 8970 if (rc) 8971 goto msix_setup_exit; 8972 8973 bp->cp_nr_rings = (min == 1) ? 8974 max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) : 8975 bp->tx_nr_rings + bp->rx_nr_rings; 8976 8977 } else { 8978 rc = -ENOMEM; 8979 goto msix_setup_exit; 8980 } 8981 bp->flags |= BNXT_FLAG_USING_MSIX; 8982 kfree(msix_ent); 8983 return 0; 8984 8985 msix_setup_exit: 8986 netdev_err(bp->dev, "bnxt_init_msix err: %x\n", rc); 8987 kfree(bp->irq_tbl); 8988 bp->irq_tbl = NULL; 8989 pci_disable_msix(bp->pdev); 8990 kfree(msix_ent); 8991 return rc; 8992 } 8993 8994 static int bnxt_init_inta(struct bnxt *bp) 8995 { 8996 bp->irq_tbl = kzalloc(sizeof(struct bnxt_irq), GFP_KERNEL); 8997 if (!bp->irq_tbl) 8998 return -ENOMEM; 8999 9000 bp->total_irqs = 1; 9001 bp->rx_nr_rings = 1; 9002 bp->tx_nr_rings = 1; 9003 bp->cp_nr_rings = 1; 9004 bp->flags |= BNXT_FLAG_SHARED_RINGS; 9005 bp->irq_tbl[0].vector = bp->pdev->irq; 9006 return 0; 9007 } 9008 9009 static int bnxt_init_int_mode(struct bnxt *bp) 9010 { 9011 int rc = -ENODEV; 9012 9013 if (bp->flags & BNXT_FLAG_MSIX_CAP) 9014 rc = bnxt_init_msix(bp); 9015 9016 if (!(bp->flags & BNXT_FLAG_USING_MSIX) && BNXT_PF(bp)) { 9017 /* fallback to INTA */ 9018 rc = bnxt_init_inta(bp); 9019 } 9020 return rc; 9021 } 9022 9023 static void bnxt_clear_int_mode(struct bnxt *bp) 9024 { 9025 if (bp->flags & BNXT_FLAG_USING_MSIX) 9026 pci_disable_msix(bp->pdev); 9027 9028 kfree(bp->irq_tbl); 9029 bp->irq_tbl = NULL; 9030 bp->flags &= ~BNXT_FLAG_USING_MSIX; 9031 } 9032 9033 int bnxt_reserve_rings(struct bnxt *bp, bool irq_re_init) 9034 { 9035 int tcs = netdev_get_num_tc(bp->dev); 9036 bool irq_cleared = false; 9037 int rc; 9038 9039 if (!bnxt_need_reserve_rings(bp)) 9040 return 0; 9041 9042 if (irq_re_init && BNXT_NEW_RM(bp) && 9043 bnxt_get_num_msix(bp) != bp->total_irqs) { 9044 bnxt_ulp_irq_stop(bp); 9045 bnxt_clear_int_mode(bp); 9046 irq_cleared = true; 9047 } 9048 rc = __bnxt_reserve_rings(bp); 9049 if (irq_cleared) { 9050 if (!rc) 9051 rc = bnxt_init_int_mode(bp); 9052 bnxt_ulp_irq_restart(bp, rc); 9053 } 9054 if (rc) { 9055 netdev_err(bp->dev, "ring reservation/IRQ init failure rc: %d\n", rc); 9056 return rc; 9057 } 9058 if (tcs && (bp->tx_nr_rings_per_tc * tcs != bp->tx_nr_rings)) { 9059 netdev_err(bp->dev, "tx ring reservation failure\n"); 9060 netdev_reset_tc(bp->dev); 9061 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 9062 return -ENOMEM; 9063 } 9064 return 0; 9065 } 9066 9067 static void bnxt_free_irq(struct bnxt *bp) 9068 { 9069 struct bnxt_irq *irq; 9070 int i; 9071 9072 #ifdef CONFIG_RFS_ACCEL 9073 free_irq_cpu_rmap(bp->dev->rx_cpu_rmap); 9074 bp->dev->rx_cpu_rmap = NULL; 9075 #endif 9076 if (!bp->irq_tbl || !bp->bnapi) 9077 return; 9078 9079 for (i = 0; i < bp->cp_nr_rings; i++) { 9080 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 9081 9082 irq = &bp->irq_tbl[map_idx]; 9083 if (irq->requested) { 9084 if (irq->have_cpumask) { 9085 irq_set_affinity_hint(irq->vector, NULL); 9086 free_cpumask_var(irq->cpu_mask); 9087 irq->have_cpumask = 0; 9088 } 9089 free_irq(irq->vector, bp->bnapi[i]); 9090 } 9091 9092 irq->requested = 0; 9093 } 9094 } 9095 9096 static int bnxt_request_irq(struct bnxt *bp) 9097 { 9098 int i, j, rc = 0; 9099 unsigned long flags = 0; 9100 #ifdef CONFIG_RFS_ACCEL 9101 struct cpu_rmap *rmap; 9102 #endif 9103 9104 rc = bnxt_setup_int_mode(bp); 9105 if (rc) { 9106 netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n", 9107 rc); 9108 return rc; 9109 } 9110 #ifdef CONFIG_RFS_ACCEL 9111 rmap = bp->dev->rx_cpu_rmap; 9112 #endif 9113 if (!(bp->flags & BNXT_FLAG_USING_MSIX)) 9114 flags = IRQF_SHARED; 9115 9116 for (i = 0, j = 0; i < bp->cp_nr_rings; i++) { 9117 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 9118 struct bnxt_irq *irq = &bp->irq_tbl[map_idx]; 9119 9120 #ifdef CONFIG_RFS_ACCEL 9121 if (rmap && bp->bnapi[i]->rx_ring) { 9122 rc = irq_cpu_rmap_add(rmap, irq->vector); 9123 if (rc) 9124 netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n", 9125 j); 9126 j++; 9127 } 9128 #endif 9129 rc = request_irq(irq->vector, irq->handler, flags, irq->name, 9130 bp->bnapi[i]); 9131 if (rc) 9132 break; 9133 9134 irq->requested = 1; 9135 9136 if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) { 9137 int numa_node = dev_to_node(&bp->pdev->dev); 9138 9139 irq->have_cpumask = 1; 9140 cpumask_set_cpu(cpumask_local_spread(i, numa_node), 9141 irq->cpu_mask); 9142 rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask); 9143 if (rc) { 9144 netdev_warn(bp->dev, 9145 "Set affinity failed, IRQ = %d\n", 9146 irq->vector); 9147 break; 9148 } 9149 } 9150 } 9151 return rc; 9152 } 9153 9154 static void bnxt_del_napi(struct bnxt *bp) 9155 { 9156 int i; 9157 9158 if (!bp->bnapi) 9159 return; 9160 9161 for (i = 0; i < bp->cp_nr_rings; i++) { 9162 struct bnxt_napi *bnapi = bp->bnapi[i]; 9163 9164 __netif_napi_del(&bnapi->napi); 9165 } 9166 /* We called __netif_napi_del(), we need 9167 * to respect an RCU grace period before freeing napi structures. 9168 */ 9169 synchronize_net(); 9170 } 9171 9172 static void bnxt_init_napi(struct bnxt *bp) 9173 { 9174 int i; 9175 unsigned int cp_nr_rings = bp->cp_nr_rings; 9176 struct bnxt_napi *bnapi; 9177 9178 if (bp->flags & BNXT_FLAG_USING_MSIX) { 9179 int (*poll_fn)(struct napi_struct *, int) = bnxt_poll; 9180 9181 if (bp->flags & BNXT_FLAG_CHIP_P5) 9182 poll_fn = bnxt_poll_p5; 9183 else if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 9184 cp_nr_rings--; 9185 for (i = 0; i < cp_nr_rings; i++) { 9186 bnapi = bp->bnapi[i]; 9187 netif_napi_add(bp->dev, &bnapi->napi, poll_fn, 64); 9188 } 9189 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 9190 bnapi = bp->bnapi[cp_nr_rings]; 9191 netif_napi_add(bp->dev, &bnapi->napi, 9192 bnxt_poll_nitroa0, 64); 9193 } 9194 } else { 9195 bnapi = bp->bnapi[0]; 9196 netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll, 64); 9197 } 9198 } 9199 9200 static void bnxt_disable_napi(struct bnxt *bp) 9201 { 9202 int i; 9203 9204 if (!bp->bnapi || 9205 test_and_set_bit(BNXT_STATE_NAPI_DISABLED, &bp->state)) 9206 return; 9207 9208 for (i = 0; i < bp->cp_nr_rings; i++) { 9209 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring; 9210 9211 napi_disable(&bp->bnapi[i]->napi); 9212 if (bp->bnapi[i]->rx_ring) 9213 cancel_work_sync(&cpr->dim.work); 9214 } 9215 } 9216 9217 static void bnxt_enable_napi(struct bnxt *bp) 9218 { 9219 int i; 9220 9221 clear_bit(BNXT_STATE_NAPI_DISABLED, &bp->state); 9222 for (i = 0; i < bp->cp_nr_rings; i++) { 9223 struct bnxt_napi *bnapi = bp->bnapi[i]; 9224 struct bnxt_cp_ring_info *cpr; 9225 9226 cpr = &bnapi->cp_ring; 9227 if (bnapi->in_reset) 9228 cpr->sw_stats.rx.rx_resets++; 9229 bnapi->in_reset = false; 9230 9231 if (bnapi->rx_ring) { 9232 INIT_WORK(&cpr->dim.work, bnxt_dim_work); 9233 cpr->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; 9234 } 9235 napi_enable(&bnapi->napi); 9236 } 9237 } 9238 9239 void bnxt_tx_disable(struct bnxt *bp) 9240 { 9241 int i; 9242 struct bnxt_tx_ring_info *txr; 9243 9244 if (bp->tx_ring) { 9245 for (i = 0; i < bp->tx_nr_rings; i++) { 9246 txr = &bp->tx_ring[i]; 9247 WRITE_ONCE(txr->dev_state, BNXT_DEV_STATE_CLOSING); 9248 } 9249 } 9250 /* Make sure napi polls see @dev_state change */ 9251 synchronize_net(); 9252 /* Drop carrier first to prevent TX timeout */ 9253 netif_carrier_off(bp->dev); 9254 /* Stop all TX queues */ 9255 netif_tx_disable(bp->dev); 9256 } 9257 9258 void bnxt_tx_enable(struct bnxt *bp) 9259 { 9260 int i; 9261 struct bnxt_tx_ring_info *txr; 9262 9263 for (i = 0; i < bp->tx_nr_rings; i++) { 9264 txr = &bp->tx_ring[i]; 9265 WRITE_ONCE(txr->dev_state, 0); 9266 } 9267 /* Make sure napi polls see @dev_state change */ 9268 synchronize_net(); 9269 netif_tx_wake_all_queues(bp->dev); 9270 if (bp->link_info.link_up) 9271 netif_carrier_on(bp->dev); 9272 } 9273 9274 static char *bnxt_report_fec(struct bnxt_link_info *link_info) 9275 { 9276 u8 active_fec = link_info->active_fec_sig_mode & 9277 PORT_PHY_QCFG_RESP_ACTIVE_FEC_MASK; 9278 9279 switch (active_fec) { 9280 default: 9281 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_NONE_ACTIVE: 9282 return "None"; 9283 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE74_ACTIVE: 9284 return "Clause 74 BaseR"; 9285 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE91_ACTIVE: 9286 return "Clause 91 RS(528,514)"; 9287 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_1XN_ACTIVE: 9288 return "Clause 91 RS544_1XN"; 9289 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_IEEE_ACTIVE: 9290 return "Clause 91 RS(544,514)"; 9291 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_1XN_ACTIVE: 9292 return "Clause 91 RS272_1XN"; 9293 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_IEEE_ACTIVE: 9294 return "Clause 91 RS(272,257)"; 9295 } 9296 } 9297 9298 void bnxt_report_link(struct bnxt *bp) 9299 { 9300 if (bp->link_info.link_up) { 9301 const char *signal = ""; 9302 const char *flow_ctrl; 9303 const char *duplex; 9304 u32 speed; 9305 u16 fec; 9306 9307 netif_carrier_on(bp->dev); 9308 speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed); 9309 if (speed == SPEED_UNKNOWN) { 9310 netdev_info(bp->dev, "NIC Link is Up, speed unknown\n"); 9311 return; 9312 } 9313 if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL) 9314 duplex = "full"; 9315 else 9316 duplex = "half"; 9317 if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH) 9318 flow_ctrl = "ON - receive & transmit"; 9319 else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX) 9320 flow_ctrl = "ON - transmit"; 9321 else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX) 9322 flow_ctrl = "ON - receive"; 9323 else 9324 flow_ctrl = "none"; 9325 if (bp->link_info.phy_qcfg_resp.option_flags & 9326 PORT_PHY_QCFG_RESP_OPTION_FLAGS_SIGNAL_MODE_KNOWN) { 9327 u8 sig_mode = bp->link_info.active_fec_sig_mode & 9328 PORT_PHY_QCFG_RESP_SIGNAL_MODE_MASK; 9329 switch (sig_mode) { 9330 case PORT_PHY_QCFG_RESP_SIGNAL_MODE_NRZ: 9331 signal = "(NRZ) "; 9332 break; 9333 case PORT_PHY_QCFG_RESP_SIGNAL_MODE_PAM4: 9334 signal = "(PAM4) "; 9335 break; 9336 default: 9337 break; 9338 } 9339 } 9340 netdev_info(bp->dev, "NIC Link is Up, %u Mbps %s%s duplex, Flow control: %s\n", 9341 speed, signal, duplex, flow_ctrl); 9342 if (bp->phy_flags & BNXT_PHY_FL_EEE_CAP) 9343 netdev_info(bp->dev, "EEE is %s\n", 9344 bp->eee.eee_active ? "active" : 9345 "not active"); 9346 fec = bp->link_info.fec_cfg; 9347 if (!(fec & PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED)) 9348 netdev_info(bp->dev, "FEC autoneg %s encoding: %s\n", 9349 (fec & BNXT_FEC_AUTONEG) ? "on" : "off", 9350 bnxt_report_fec(&bp->link_info)); 9351 } else { 9352 netif_carrier_off(bp->dev); 9353 netdev_err(bp->dev, "NIC Link is Down\n"); 9354 } 9355 } 9356 9357 static bool bnxt_phy_qcaps_no_speed(struct hwrm_port_phy_qcaps_output *resp) 9358 { 9359 if (!resp->supported_speeds_auto_mode && 9360 !resp->supported_speeds_force_mode && 9361 !resp->supported_pam4_speeds_auto_mode && 9362 !resp->supported_pam4_speeds_force_mode) 9363 return true; 9364 return false; 9365 } 9366 9367 static int bnxt_hwrm_phy_qcaps(struct bnxt *bp) 9368 { 9369 struct bnxt_link_info *link_info = &bp->link_info; 9370 struct hwrm_port_phy_qcaps_output *resp; 9371 struct hwrm_port_phy_qcaps_input *req; 9372 int rc = 0; 9373 9374 if (bp->hwrm_spec_code < 0x10201) 9375 return 0; 9376 9377 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_QCAPS); 9378 if (rc) 9379 return rc; 9380 9381 resp = hwrm_req_hold(bp, req); 9382 rc = hwrm_req_send(bp, req); 9383 if (rc) 9384 goto hwrm_phy_qcaps_exit; 9385 9386 bp->phy_flags = resp->flags; 9387 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) { 9388 struct ethtool_eee *eee = &bp->eee; 9389 u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode); 9390 9391 eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0); 9392 bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) & 9393 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK; 9394 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) & 9395 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK; 9396 } 9397 9398 if (bp->hwrm_spec_code >= 0x10a01) { 9399 if (bnxt_phy_qcaps_no_speed(resp)) { 9400 link_info->phy_state = BNXT_PHY_STATE_DISABLED; 9401 netdev_warn(bp->dev, "Ethernet link disabled\n"); 9402 } else if (link_info->phy_state == BNXT_PHY_STATE_DISABLED) { 9403 link_info->phy_state = BNXT_PHY_STATE_ENABLED; 9404 netdev_info(bp->dev, "Ethernet link enabled\n"); 9405 /* Phy re-enabled, reprobe the speeds */ 9406 link_info->support_auto_speeds = 0; 9407 link_info->support_pam4_auto_speeds = 0; 9408 } 9409 } 9410 if (resp->supported_speeds_auto_mode) 9411 link_info->support_auto_speeds = 9412 le16_to_cpu(resp->supported_speeds_auto_mode); 9413 if (resp->supported_pam4_speeds_auto_mode) 9414 link_info->support_pam4_auto_speeds = 9415 le16_to_cpu(resp->supported_pam4_speeds_auto_mode); 9416 9417 bp->port_count = resp->port_cnt; 9418 9419 hwrm_phy_qcaps_exit: 9420 hwrm_req_drop(bp, req); 9421 return rc; 9422 } 9423 9424 static bool bnxt_support_dropped(u16 advertising, u16 supported) 9425 { 9426 u16 diff = advertising ^ supported; 9427 9428 return ((supported | diff) != supported); 9429 } 9430 9431 int bnxt_update_link(struct bnxt *bp, bool chng_link_state) 9432 { 9433 struct bnxt_link_info *link_info = &bp->link_info; 9434 struct hwrm_port_phy_qcfg_output *resp; 9435 struct hwrm_port_phy_qcfg_input *req; 9436 u8 link_up = link_info->link_up; 9437 bool support_changed = false; 9438 int rc; 9439 9440 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_QCFG); 9441 if (rc) 9442 return rc; 9443 9444 resp = hwrm_req_hold(bp, req); 9445 rc = hwrm_req_send(bp, req); 9446 if (rc) { 9447 hwrm_req_drop(bp, req); 9448 if (BNXT_VF(bp) && rc == -ENODEV) { 9449 netdev_warn(bp->dev, "Cannot obtain link state while PF unavailable.\n"); 9450 rc = 0; 9451 } 9452 return rc; 9453 } 9454 9455 memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp)); 9456 link_info->phy_link_status = resp->link; 9457 link_info->duplex = resp->duplex_cfg; 9458 if (bp->hwrm_spec_code >= 0x10800) 9459 link_info->duplex = resp->duplex_state; 9460 link_info->pause = resp->pause; 9461 link_info->auto_mode = resp->auto_mode; 9462 link_info->auto_pause_setting = resp->auto_pause; 9463 link_info->lp_pause = resp->link_partner_adv_pause; 9464 link_info->force_pause_setting = resp->force_pause; 9465 link_info->duplex_setting = resp->duplex_cfg; 9466 if (link_info->phy_link_status == BNXT_LINK_LINK) 9467 link_info->link_speed = le16_to_cpu(resp->link_speed); 9468 else 9469 link_info->link_speed = 0; 9470 link_info->force_link_speed = le16_to_cpu(resp->force_link_speed); 9471 link_info->force_pam4_link_speed = 9472 le16_to_cpu(resp->force_pam4_link_speed); 9473 link_info->support_speeds = le16_to_cpu(resp->support_speeds); 9474 link_info->support_pam4_speeds = le16_to_cpu(resp->support_pam4_speeds); 9475 link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask); 9476 link_info->auto_pam4_link_speeds = 9477 le16_to_cpu(resp->auto_pam4_link_speed_mask); 9478 link_info->lp_auto_link_speeds = 9479 le16_to_cpu(resp->link_partner_adv_speeds); 9480 link_info->lp_auto_pam4_link_speeds = 9481 resp->link_partner_pam4_adv_speeds; 9482 link_info->preemphasis = le32_to_cpu(resp->preemphasis); 9483 link_info->phy_ver[0] = resp->phy_maj; 9484 link_info->phy_ver[1] = resp->phy_min; 9485 link_info->phy_ver[2] = resp->phy_bld; 9486 link_info->media_type = resp->media_type; 9487 link_info->phy_type = resp->phy_type; 9488 link_info->transceiver = resp->xcvr_pkg_type; 9489 link_info->phy_addr = resp->eee_config_phy_addr & 9490 PORT_PHY_QCFG_RESP_PHY_ADDR_MASK; 9491 link_info->module_status = resp->module_status; 9492 9493 if (bp->phy_flags & BNXT_PHY_FL_EEE_CAP) { 9494 struct ethtool_eee *eee = &bp->eee; 9495 u16 fw_speeds; 9496 9497 eee->eee_active = 0; 9498 if (resp->eee_config_phy_addr & 9499 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) { 9500 eee->eee_active = 1; 9501 fw_speeds = le16_to_cpu( 9502 resp->link_partner_adv_eee_link_speed_mask); 9503 eee->lp_advertised = 9504 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0); 9505 } 9506 9507 /* Pull initial EEE config */ 9508 if (!chng_link_state) { 9509 if (resp->eee_config_phy_addr & 9510 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED) 9511 eee->eee_enabled = 1; 9512 9513 fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask); 9514 eee->advertised = 9515 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0); 9516 9517 if (resp->eee_config_phy_addr & 9518 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) { 9519 __le32 tmr; 9520 9521 eee->tx_lpi_enabled = 1; 9522 tmr = resp->xcvr_identifier_type_tx_lpi_timer; 9523 eee->tx_lpi_timer = le32_to_cpu(tmr) & 9524 PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK; 9525 } 9526 } 9527 } 9528 9529 link_info->fec_cfg = PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED; 9530 if (bp->hwrm_spec_code >= 0x10504) { 9531 link_info->fec_cfg = le16_to_cpu(resp->fec_cfg); 9532 link_info->active_fec_sig_mode = resp->active_fec_signal_mode; 9533 } 9534 /* TODO: need to add more logic to report VF link */ 9535 if (chng_link_state) { 9536 if (link_info->phy_link_status == BNXT_LINK_LINK) 9537 link_info->link_up = 1; 9538 else 9539 link_info->link_up = 0; 9540 if (link_up != link_info->link_up) 9541 bnxt_report_link(bp); 9542 } else { 9543 /* alwasy link down if not require to update link state */ 9544 link_info->link_up = 0; 9545 } 9546 hwrm_req_drop(bp, req); 9547 9548 if (!BNXT_PHY_CFG_ABLE(bp)) 9549 return 0; 9550 9551 /* Check if any advertised speeds are no longer supported. The caller 9552 * holds the link_lock mutex, so we can modify link_info settings. 9553 */ 9554 if (bnxt_support_dropped(link_info->advertising, 9555 link_info->support_auto_speeds)) { 9556 link_info->advertising = link_info->support_auto_speeds; 9557 support_changed = true; 9558 } 9559 if (bnxt_support_dropped(link_info->advertising_pam4, 9560 link_info->support_pam4_auto_speeds)) { 9561 link_info->advertising_pam4 = link_info->support_pam4_auto_speeds; 9562 support_changed = true; 9563 } 9564 if (support_changed && (link_info->autoneg & BNXT_AUTONEG_SPEED)) 9565 bnxt_hwrm_set_link_setting(bp, true, false); 9566 return 0; 9567 } 9568 9569 static void bnxt_get_port_module_status(struct bnxt *bp) 9570 { 9571 struct bnxt_link_info *link_info = &bp->link_info; 9572 struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp; 9573 u8 module_status; 9574 9575 if (bnxt_update_link(bp, true)) 9576 return; 9577 9578 module_status = link_info->module_status; 9579 switch (module_status) { 9580 case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX: 9581 case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN: 9582 case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG: 9583 netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n", 9584 bp->pf.port_id); 9585 if (bp->hwrm_spec_code >= 0x10201) { 9586 netdev_warn(bp->dev, "Module part number %s\n", 9587 resp->phy_vendor_partnumber); 9588 } 9589 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX) 9590 netdev_warn(bp->dev, "TX is disabled\n"); 9591 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN) 9592 netdev_warn(bp->dev, "SFP+ module is shutdown\n"); 9593 } 9594 } 9595 9596 static void 9597 bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req) 9598 { 9599 if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) { 9600 if (bp->hwrm_spec_code >= 0x10201) 9601 req->auto_pause = 9602 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE; 9603 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX) 9604 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX; 9605 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX) 9606 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX; 9607 req->enables |= 9608 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE); 9609 } else { 9610 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX) 9611 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX; 9612 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX) 9613 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX; 9614 req->enables |= 9615 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE); 9616 if (bp->hwrm_spec_code >= 0x10201) { 9617 req->auto_pause = req->force_pause; 9618 req->enables |= cpu_to_le32( 9619 PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE); 9620 } 9621 } 9622 } 9623 9624 static void bnxt_hwrm_set_link_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req) 9625 { 9626 if (bp->link_info.autoneg & BNXT_AUTONEG_SPEED) { 9627 req->auto_mode |= PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK; 9628 if (bp->link_info.advertising) { 9629 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK); 9630 req->auto_link_speed_mask = cpu_to_le16(bp->link_info.advertising); 9631 } 9632 if (bp->link_info.advertising_pam4) { 9633 req->enables |= 9634 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAM4_LINK_SPEED_MASK); 9635 req->auto_link_pam4_speed_mask = 9636 cpu_to_le16(bp->link_info.advertising_pam4); 9637 } 9638 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE); 9639 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG); 9640 } else { 9641 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE); 9642 if (bp->link_info.req_signal_mode == BNXT_SIG_MODE_PAM4) { 9643 req->force_pam4_link_speed = cpu_to_le16(bp->link_info.req_link_speed); 9644 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAM4_LINK_SPEED); 9645 } else { 9646 req->force_link_speed = cpu_to_le16(bp->link_info.req_link_speed); 9647 } 9648 } 9649 9650 /* tell chimp that the setting takes effect immediately */ 9651 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY); 9652 } 9653 9654 int bnxt_hwrm_set_pause(struct bnxt *bp) 9655 { 9656 struct hwrm_port_phy_cfg_input *req; 9657 int rc; 9658 9659 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG); 9660 if (rc) 9661 return rc; 9662 9663 bnxt_hwrm_set_pause_common(bp, req); 9664 9665 if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) || 9666 bp->link_info.force_link_chng) 9667 bnxt_hwrm_set_link_common(bp, req); 9668 9669 rc = hwrm_req_send(bp, req); 9670 if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) { 9671 /* since changing of pause setting doesn't trigger any link 9672 * change event, the driver needs to update the current pause 9673 * result upon successfully return of the phy_cfg command 9674 */ 9675 bp->link_info.pause = 9676 bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl; 9677 bp->link_info.auto_pause_setting = 0; 9678 if (!bp->link_info.force_link_chng) 9679 bnxt_report_link(bp); 9680 } 9681 bp->link_info.force_link_chng = false; 9682 return rc; 9683 } 9684 9685 static void bnxt_hwrm_set_eee(struct bnxt *bp, 9686 struct hwrm_port_phy_cfg_input *req) 9687 { 9688 struct ethtool_eee *eee = &bp->eee; 9689 9690 if (eee->eee_enabled) { 9691 u16 eee_speeds; 9692 u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE; 9693 9694 if (eee->tx_lpi_enabled) 9695 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE; 9696 else 9697 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE; 9698 9699 req->flags |= cpu_to_le32(flags); 9700 eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised); 9701 req->eee_link_speed_mask = cpu_to_le16(eee_speeds); 9702 req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer); 9703 } else { 9704 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE); 9705 } 9706 } 9707 9708 int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee) 9709 { 9710 struct hwrm_port_phy_cfg_input *req; 9711 int rc; 9712 9713 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG); 9714 if (rc) 9715 return rc; 9716 9717 if (set_pause) 9718 bnxt_hwrm_set_pause_common(bp, req); 9719 9720 bnxt_hwrm_set_link_common(bp, req); 9721 9722 if (set_eee) 9723 bnxt_hwrm_set_eee(bp, req); 9724 return hwrm_req_send(bp, req); 9725 } 9726 9727 static int bnxt_hwrm_shutdown_link(struct bnxt *bp) 9728 { 9729 struct hwrm_port_phy_cfg_input *req; 9730 int rc; 9731 9732 if (!BNXT_SINGLE_PF(bp)) 9733 return 0; 9734 9735 if (pci_num_vf(bp->pdev) && 9736 !(bp->phy_flags & BNXT_PHY_FL_FW_MANAGED_LKDN)) 9737 return 0; 9738 9739 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG); 9740 if (rc) 9741 return rc; 9742 9743 req->flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN); 9744 return hwrm_req_send(bp, req); 9745 } 9746 9747 static int bnxt_fw_reset_via_optee(struct bnxt *bp) 9748 { 9749 #ifdef CONFIG_TEE_BNXT_FW 9750 int rc = tee_bnxt_fw_load(); 9751 9752 if (rc) 9753 netdev_err(bp->dev, "Failed FW reset via OP-TEE, rc=%d\n", rc); 9754 9755 return rc; 9756 #else 9757 netdev_err(bp->dev, "OP-TEE not supported\n"); 9758 return -ENODEV; 9759 #endif 9760 } 9761 9762 static int bnxt_try_recover_fw(struct bnxt *bp) 9763 { 9764 if (bp->fw_health && bp->fw_health->status_reliable) { 9765 int retry = 0, rc; 9766 u32 sts; 9767 9768 do { 9769 sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG); 9770 rc = bnxt_hwrm_poll(bp); 9771 if (!BNXT_FW_IS_BOOTING(sts) && 9772 !BNXT_FW_IS_RECOVERING(sts)) 9773 break; 9774 retry++; 9775 } while (rc == -EBUSY && retry < BNXT_FW_RETRY); 9776 9777 if (!BNXT_FW_IS_HEALTHY(sts)) { 9778 netdev_err(bp->dev, 9779 "Firmware not responding, status: 0x%x\n", 9780 sts); 9781 rc = -ENODEV; 9782 } 9783 if (sts & FW_STATUS_REG_CRASHED_NO_MASTER) { 9784 netdev_warn(bp->dev, "Firmware recover via OP-TEE requested\n"); 9785 return bnxt_fw_reset_via_optee(bp); 9786 } 9787 return rc; 9788 } 9789 9790 return -ENODEV; 9791 } 9792 9793 int bnxt_cancel_reservations(struct bnxt *bp, bool fw_reset) 9794 { 9795 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 9796 int rc; 9797 9798 if (!BNXT_NEW_RM(bp)) 9799 return 0; /* no resource reservations required */ 9800 9801 rc = bnxt_hwrm_func_resc_qcaps(bp, true); 9802 if (rc) 9803 netdev_err(bp->dev, "resc_qcaps failed\n"); 9804 9805 hw_resc->resv_cp_rings = 0; 9806 hw_resc->resv_stat_ctxs = 0; 9807 hw_resc->resv_irqs = 0; 9808 hw_resc->resv_tx_rings = 0; 9809 hw_resc->resv_rx_rings = 0; 9810 hw_resc->resv_hw_ring_grps = 0; 9811 hw_resc->resv_vnics = 0; 9812 if (!fw_reset) { 9813 bp->tx_nr_rings = 0; 9814 bp->rx_nr_rings = 0; 9815 } 9816 9817 return rc; 9818 } 9819 9820 static int bnxt_hwrm_if_change(struct bnxt *bp, bool up) 9821 { 9822 struct hwrm_func_drv_if_change_output *resp; 9823 struct hwrm_func_drv_if_change_input *req; 9824 bool fw_reset = !bp->irq_tbl; 9825 bool resc_reinit = false; 9826 int rc, retry = 0; 9827 u32 flags = 0; 9828 9829 if (!(bp->fw_cap & BNXT_FW_CAP_IF_CHANGE)) 9830 return 0; 9831 9832 rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_IF_CHANGE); 9833 if (rc) 9834 return rc; 9835 9836 if (up) 9837 req->flags = cpu_to_le32(FUNC_DRV_IF_CHANGE_REQ_FLAGS_UP); 9838 resp = hwrm_req_hold(bp, req); 9839 9840 hwrm_req_flags(bp, req, BNXT_HWRM_FULL_WAIT); 9841 while (retry < BNXT_FW_IF_RETRY) { 9842 rc = hwrm_req_send(bp, req); 9843 if (rc != -EAGAIN) 9844 break; 9845 9846 msleep(50); 9847 retry++; 9848 } 9849 9850 if (rc == -EAGAIN) { 9851 hwrm_req_drop(bp, req); 9852 return rc; 9853 } else if (!rc) { 9854 flags = le32_to_cpu(resp->flags); 9855 } else if (up) { 9856 rc = bnxt_try_recover_fw(bp); 9857 fw_reset = true; 9858 } 9859 hwrm_req_drop(bp, req); 9860 if (rc) 9861 return rc; 9862 9863 if (!up) { 9864 bnxt_inv_fw_health_reg(bp); 9865 return 0; 9866 } 9867 9868 if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_RESC_CHANGE) 9869 resc_reinit = true; 9870 if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_HOT_FW_RESET_DONE) 9871 fw_reset = true; 9872 else 9873 bnxt_remap_fw_health_regs(bp); 9874 9875 if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state) && !fw_reset) { 9876 netdev_err(bp->dev, "RESET_DONE not set during FW reset.\n"); 9877 set_bit(BNXT_STATE_ABORT_ERR, &bp->state); 9878 return -ENODEV; 9879 } 9880 if (resc_reinit || fw_reset) { 9881 if (fw_reset) { 9882 set_bit(BNXT_STATE_FW_RESET_DET, &bp->state); 9883 if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 9884 bnxt_ulp_stop(bp); 9885 bnxt_free_ctx_mem(bp); 9886 kfree(bp->ctx); 9887 bp->ctx = NULL; 9888 bnxt_dcb_free(bp); 9889 rc = bnxt_fw_init_one(bp); 9890 if (rc) { 9891 clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state); 9892 set_bit(BNXT_STATE_ABORT_ERR, &bp->state); 9893 return rc; 9894 } 9895 bnxt_clear_int_mode(bp); 9896 rc = bnxt_init_int_mode(bp); 9897 if (rc) { 9898 clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state); 9899 netdev_err(bp->dev, "init int mode failed\n"); 9900 return rc; 9901 } 9902 } 9903 rc = bnxt_cancel_reservations(bp, fw_reset); 9904 } 9905 return rc; 9906 } 9907 9908 static int bnxt_hwrm_port_led_qcaps(struct bnxt *bp) 9909 { 9910 struct hwrm_port_led_qcaps_output *resp; 9911 struct hwrm_port_led_qcaps_input *req; 9912 struct bnxt_pf_info *pf = &bp->pf; 9913 int rc; 9914 9915 bp->num_leds = 0; 9916 if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10601) 9917 return 0; 9918 9919 rc = hwrm_req_init(bp, req, HWRM_PORT_LED_QCAPS); 9920 if (rc) 9921 return rc; 9922 9923 req->port_id = cpu_to_le16(pf->port_id); 9924 resp = hwrm_req_hold(bp, req); 9925 rc = hwrm_req_send(bp, req); 9926 if (rc) { 9927 hwrm_req_drop(bp, req); 9928 return rc; 9929 } 9930 if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) { 9931 int i; 9932 9933 bp->num_leds = resp->num_leds; 9934 memcpy(bp->leds, &resp->led0_id, sizeof(bp->leds[0]) * 9935 bp->num_leds); 9936 for (i = 0; i < bp->num_leds; i++) { 9937 struct bnxt_led_info *led = &bp->leds[i]; 9938 __le16 caps = led->led_state_caps; 9939 9940 if (!led->led_group_id || 9941 !BNXT_LED_ALT_BLINK_CAP(caps)) { 9942 bp->num_leds = 0; 9943 break; 9944 } 9945 } 9946 } 9947 hwrm_req_drop(bp, req); 9948 return 0; 9949 } 9950 9951 int bnxt_hwrm_alloc_wol_fltr(struct bnxt *bp) 9952 { 9953 struct hwrm_wol_filter_alloc_output *resp; 9954 struct hwrm_wol_filter_alloc_input *req; 9955 int rc; 9956 9957 rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_ALLOC); 9958 if (rc) 9959 return rc; 9960 9961 req->port_id = cpu_to_le16(bp->pf.port_id); 9962 req->wol_type = WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT; 9963 req->enables = cpu_to_le32(WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS); 9964 memcpy(req->mac_address, bp->dev->dev_addr, ETH_ALEN); 9965 9966 resp = hwrm_req_hold(bp, req); 9967 rc = hwrm_req_send(bp, req); 9968 if (!rc) 9969 bp->wol_filter_id = resp->wol_filter_id; 9970 hwrm_req_drop(bp, req); 9971 return rc; 9972 } 9973 9974 int bnxt_hwrm_free_wol_fltr(struct bnxt *bp) 9975 { 9976 struct hwrm_wol_filter_free_input *req; 9977 int rc; 9978 9979 rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_FREE); 9980 if (rc) 9981 return rc; 9982 9983 req->port_id = cpu_to_le16(bp->pf.port_id); 9984 req->enables = cpu_to_le32(WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID); 9985 req->wol_filter_id = bp->wol_filter_id; 9986 9987 return hwrm_req_send(bp, req); 9988 } 9989 9990 static u16 bnxt_hwrm_get_wol_fltrs(struct bnxt *bp, u16 handle) 9991 { 9992 struct hwrm_wol_filter_qcfg_output *resp; 9993 struct hwrm_wol_filter_qcfg_input *req; 9994 u16 next_handle = 0; 9995 int rc; 9996 9997 rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_QCFG); 9998 if (rc) 9999 return rc; 10000 10001 req->port_id = cpu_to_le16(bp->pf.port_id); 10002 req->handle = cpu_to_le16(handle); 10003 resp = hwrm_req_hold(bp, req); 10004 rc = hwrm_req_send(bp, req); 10005 if (!rc) { 10006 next_handle = le16_to_cpu(resp->next_handle); 10007 if (next_handle != 0) { 10008 if (resp->wol_type == 10009 WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT) { 10010 bp->wol = 1; 10011 bp->wol_filter_id = resp->wol_filter_id; 10012 } 10013 } 10014 } 10015 hwrm_req_drop(bp, req); 10016 return next_handle; 10017 } 10018 10019 static void bnxt_get_wol_settings(struct bnxt *bp) 10020 { 10021 u16 handle = 0; 10022 10023 bp->wol = 0; 10024 if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_WOL_CAP)) 10025 return; 10026 10027 do { 10028 handle = bnxt_hwrm_get_wol_fltrs(bp, handle); 10029 } while (handle && handle != 0xffff); 10030 } 10031 10032 #ifdef CONFIG_BNXT_HWMON 10033 static ssize_t bnxt_show_temp(struct device *dev, 10034 struct device_attribute *devattr, char *buf) 10035 { 10036 struct hwrm_temp_monitor_query_output *resp; 10037 struct hwrm_temp_monitor_query_input *req; 10038 struct bnxt *bp = dev_get_drvdata(dev); 10039 u32 len = 0; 10040 int rc; 10041 10042 rc = hwrm_req_init(bp, req, HWRM_TEMP_MONITOR_QUERY); 10043 if (rc) 10044 return rc; 10045 resp = hwrm_req_hold(bp, req); 10046 rc = hwrm_req_send(bp, req); 10047 if (!rc) 10048 len = sprintf(buf, "%u\n", resp->temp * 1000); /* display millidegree */ 10049 hwrm_req_drop(bp, req); 10050 if (rc) 10051 return rc; 10052 return len; 10053 } 10054 static SENSOR_DEVICE_ATTR(temp1_input, 0444, bnxt_show_temp, NULL, 0); 10055 10056 static struct attribute *bnxt_attrs[] = { 10057 &sensor_dev_attr_temp1_input.dev_attr.attr, 10058 NULL 10059 }; 10060 ATTRIBUTE_GROUPS(bnxt); 10061 10062 static void bnxt_hwmon_close(struct bnxt *bp) 10063 { 10064 if (bp->hwmon_dev) { 10065 hwmon_device_unregister(bp->hwmon_dev); 10066 bp->hwmon_dev = NULL; 10067 } 10068 } 10069 10070 static void bnxt_hwmon_open(struct bnxt *bp) 10071 { 10072 struct hwrm_temp_monitor_query_input *req; 10073 struct pci_dev *pdev = bp->pdev; 10074 int rc; 10075 10076 rc = hwrm_req_init(bp, req, HWRM_TEMP_MONITOR_QUERY); 10077 if (!rc) 10078 rc = hwrm_req_send_silent(bp, req); 10079 if (rc == -EACCES || rc == -EOPNOTSUPP) { 10080 bnxt_hwmon_close(bp); 10081 return; 10082 } 10083 10084 if (bp->hwmon_dev) 10085 return; 10086 10087 bp->hwmon_dev = hwmon_device_register_with_groups(&pdev->dev, 10088 DRV_MODULE_NAME, bp, 10089 bnxt_groups); 10090 if (IS_ERR(bp->hwmon_dev)) { 10091 bp->hwmon_dev = NULL; 10092 dev_warn(&pdev->dev, "Cannot register hwmon device\n"); 10093 } 10094 } 10095 #else 10096 static void bnxt_hwmon_close(struct bnxt *bp) 10097 { 10098 } 10099 10100 static void bnxt_hwmon_open(struct bnxt *bp) 10101 { 10102 } 10103 #endif 10104 10105 static bool bnxt_eee_config_ok(struct bnxt *bp) 10106 { 10107 struct ethtool_eee *eee = &bp->eee; 10108 struct bnxt_link_info *link_info = &bp->link_info; 10109 10110 if (!(bp->phy_flags & BNXT_PHY_FL_EEE_CAP)) 10111 return true; 10112 10113 if (eee->eee_enabled) { 10114 u32 advertising = 10115 _bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0); 10116 10117 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) { 10118 eee->eee_enabled = 0; 10119 return false; 10120 } 10121 if (eee->advertised & ~advertising) { 10122 eee->advertised = advertising & eee->supported; 10123 return false; 10124 } 10125 } 10126 return true; 10127 } 10128 10129 static int bnxt_update_phy_setting(struct bnxt *bp) 10130 { 10131 int rc; 10132 bool update_link = false; 10133 bool update_pause = false; 10134 bool update_eee = false; 10135 struct bnxt_link_info *link_info = &bp->link_info; 10136 10137 rc = bnxt_update_link(bp, true); 10138 if (rc) { 10139 netdev_err(bp->dev, "failed to update link (rc: %x)\n", 10140 rc); 10141 return rc; 10142 } 10143 if (!BNXT_SINGLE_PF(bp)) 10144 return 0; 10145 10146 if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) && 10147 (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) != 10148 link_info->req_flow_ctrl) 10149 update_pause = true; 10150 if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) && 10151 link_info->force_pause_setting != link_info->req_flow_ctrl) 10152 update_pause = true; 10153 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) { 10154 if (BNXT_AUTO_MODE(link_info->auto_mode)) 10155 update_link = true; 10156 if (link_info->req_signal_mode == BNXT_SIG_MODE_NRZ && 10157 link_info->req_link_speed != link_info->force_link_speed) 10158 update_link = true; 10159 else if (link_info->req_signal_mode == BNXT_SIG_MODE_PAM4 && 10160 link_info->req_link_speed != link_info->force_pam4_link_speed) 10161 update_link = true; 10162 if (link_info->req_duplex != link_info->duplex_setting) 10163 update_link = true; 10164 } else { 10165 if (link_info->auto_mode == BNXT_LINK_AUTO_NONE) 10166 update_link = true; 10167 if (link_info->advertising != link_info->auto_link_speeds || 10168 link_info->advertising_pam4 != link_info->auto_pam4_link_speeds) 10169 update_link = true; 10170 } 10171 10172 /* The last close may have shutdown the link, so need to call 10173 * PHY_CFG to bring it back up. 10174 */ 10175 if (!bp->link_info.link_up) 10176 update_link = true; 10177 10178 if (!bnxt_eee_config_ok(bp)) 10179 update_eee = true; 10180 10181 if (update_link) 10182 rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee); 10183 else if (update_pause) 10184 rc = bnxt_hwrm_set_pause(bp); 10185 if (rc) { 10186 netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n", 10187 rc); 10188 return rc; 10189 } 10190 10191 return rc; 10192 } 10193 10194 /* Common routine to pre-map certain register block to different GRC window. 10195 * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows 10196 * in PF and 3 windows in VF that can be customized to map in different 10197 * register blocks. 10198 */ 10199 static void bnxt_preset_reg_win(struct bnxt *bp) 10200 { 10201 if (BNXT_PF(bp)) { 10202 /* CAG registers map to GRC window #4 */ 10203 writel(BNXT_CAG_REG_BASE, 10204 bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12); 10205 } 10206 } 10207 10208 static int bnxt_init_dflt_ring_mode(struct bnxt *bp); 10209 10210 static int bnxt_reinit_after_abort(struct bnxt *bp) 10211 { 10212 int rc; 10213 10214 if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 10215 return -EBUSY; 10216 10217 if (bp->dev->reg_state == NETREG_UNREGISTERED) 10218 return -ENODEV; 10219 10220 rc = bnxt_fw_init_one(bp); 10221 if (!rc) { 10222 bnxt_clear_int_mode(bp); 10223 rc = bnxt_init_int_mode(bp); 10224 if (!rc) { 10225 clear_bit(BNXT_STATE_ABORT_ERR, &bp->state); 10226 set_bit(BNXT_STATE_FW_RESET_DET, &bp->state); 10227 } 10228 } 10229 return rc; 10230 } 10231 10232 static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) 10233 { 10234 int rc = 0; 10235 10236 bnxt_preset_reg_win(bp); 10237 netif_carrier_off(bp->dev); 10238 if (irq_re_init) { 10239 /* Reserve rings now if none were reserved at driver probe. */ 10240 rc = bnxt_init_dflt_ring_mode(bp); 10241 if (rc) { 10242 netdev_err(bp->dev, "Failed to reserve default rings at open\n"); 10243 return rc; 10244 } 10245 } 10246 rc = bnxt_reserve_rings(bp, irq_re_init); 10247 if (rc) 10248 return rc; 10249 if ((bp->flags & BNXT_FLAG_RFS) && 10250 !(bp->flags & BNXT_FLAG_USING_MSIX)) { 10251 /* disable RFS if falling back to INTA */ 10252 bp->dev->hw_features &= ~NETIF_F_NTUPLE; 10253 bp->flags &= ~BNXT_FLAG_RFS; 10254 } 10255 10256 rc = bnxt_alloc_mem(bp, irq_re_init); 10257 if (rc) { 10258 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc); 10259 goto open_err_free_mem; 10260 } 10261 10262 if (irq_re_init) { 10263 bnxt_init_napi(bp); 10264 rc = bnxt_request_irq(bp); 10265 if (rc) { 10266 netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc); 10267 goto open_err_irq; 10268 } 10269 } 10270 10271 rc = bnxt_init_nic(bp, irq_re_init); 10272 if (rc) { 10273 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc); 10274 goto open_err_irq; 10275 } 10276 10277 bnxt_enable_napi(bp); 10278 bnxt_debug_dev_init(bp); 10279 10280 if (link_re_init) { 10281 mutex_lock(&bp->link_lock); 10282 rc = bnxt_update_phy_setting(bp); 10283 mutex_unlock(&bp->link_lock); 10284 if (rc) { 10285 netdev_warn(bp->dev, "failed to update phy settings\n"); 10286 if (BNXT_SINGLE_PF(bp)) { 10287 bp->link_info.phy_retry = true; 10288 bp->link_info.phy_retry_expires = 10289 jiffies + 5 * HZ; 10290 } 10291 } 10292 } 10293 10294 if (irq_re_init) 10295 udp_tunnel_nic_reset_ntf(bp->dev); 10296 10297 set_bit(BNXT_STATE_OPEN, &bp->state); 10298 bnxt_enable_int(bp); 10299 /* Enable TX queues */ 10300 bnxt_tx_enable(bp); 10301 mod_timer(&bp->timer, jiffies + bp->current_interval); 10302 /* Poll link status and check for SFP+ module status */ 10303 mutex_lock(&bp->link_lock); 10304 bnxt_get_port_module_status(bp); 10305 mutex_unlock(&bp->link_lock); 10306 10307 /* VF-reps may need to be re-opened after the PF is re-opened */ 10308 if (BNXT_PF(bp)) 10309 bnxt_vf_reps_open(bp); 10310 return 0; 10311 10312 open_err_irq: 10313 bnxt_del_napi(bp); 10314 10315 open_err_free_mem: 10316 bnxt_free_skbs(bp); 10317 bnxt_free_irq(bp); 10318 bnxt_free_mem(bp, true); 10319 return rc; 10320 } 10321 10322 /* rtnl_lock held */ 10323 int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) 10324 { 10325 int rc = 0; 10326 10327 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) 10328 rc = -EIO; 10329 if (!rc) 10330 rc = __bnxt_open_nic(bp, irq_re_init, link_re_init); 10331 if (rc) { 10332 netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc); 10333 dev_close(bp->dev); 10334 } 10335 return rc; 10336 } 10337 10338 /* rtnl_lock held, open the NIC half way by allocating all resources, but 10339 * NAPI, IRQ, and TX are not enabled. This is mainly used for offline 10340 * self tests. 10341 */ 10342 int bnxt_half_open_nic(struct bnxt *bp) 10343 { 10344 int rc = 0; 10345 10346 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) { 10347 netdev_err(bp->dev, "A previous firmware reset has not completed, aborting half open\n"); 10348 rc = -ENODEV; 10349 goto half_open_err; 10350 } 10351 10352 rc = bnxt_alloc_mem(bp, true); 10353 if (rc) { 10354 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc); 10355 goto half_open_err; 10356 } 10357 set_bit(BNXT_STATE_HALF_OPEN, &bp->state); 10358 rc = bnxt_init_nic(bp, true); 10359 if (rc) { 10360 clear_bit(BNXT_STATE_HALF_OPEN, &bp->state); 10361 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc); 10362 goto half_open_err; 10363 } 10364 return 0; 10365 10366 half_open_err: 10367 bnxt_free_skbs(bp); 10368 bnxt_free_mem(bp, true); 10369 dev_close(bp->dev); 10370 return rc; 10371 } 10372 10373 /* rtnl_lock held, this call can only be made after a previous successful 10374 * call to bnxt_half_open_nic(). 10375 */ 10376 void bnxt_half_close_nic(struct bnxt *bp) 10377 { 10378 bnxt_hwrm_resource_free(bp, false, true); 10379 bnxt_free_skbs(bp); 10380 bnxt_free_mem(bp, true); 10381 clear_bit(BNXT_STATE_HALF_OPEN, &bp->state); 10382 } 10383 10384 void bnxt_reenable_sriov(struct bnxt *bp) 10385 { 10386 if (BNXT_PF(bp)) { 10387 struct bnxt_pf_info *pf = &bp->pf; 10388 int n = pf->active_vfs; 10389 10390 if (n) 10391 bnxt_cfg_hw_sriov(bp, &n, true); 10392 } 10393 } 10394 10395 static int bnxt_open(struct net_device *dev) 10396 { 10397 struct bnxt *bp = netdev_priv(dev); 10398 int rc; 10399 10400 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) { 10401 rc = bnxt_reinit_after_abort(bp); 10402 if (rc) { 10403 if (rc == -EBUSY) 10404 netdev_err(bp->dev, "A previous firmware reset has not completed, aborting\n"); 10405 else 10406 netdev_err(bp->dev, "Failed to reinitialize after aborted firmware reset\n"); 10407 return -ENODEV; 10408 } 10409 } 10410 10411 rc = bnxt_hwrm_if_change(bp, true); 10412 if (rc) 10413 return rc; 10414 10415 rc = __bnxt_open_nic(bp, true, true); 10416 if (rc) { 10417 bnxt_hwrm_if_change(bp, false); 10418 } else { 10419 if (test_and_clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state)) { 10420 if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) { 10421 bnxt_ulp_start(bp, 0); 10422 bnxt_reenable_sriov(bp); 10423 } 10424 } 10425 bnxt_hwmon_open(bp); 10426 } 10427 10428 return rc; 10429 } 10430 10431 static bool bnxt_drv_busy(struct bnxt *bp) 10432 { 10433 return (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state) || 10434 test_bit(BNXT_STATE_READ_STATS, &bp->state)); 10435 } 10436 10437 static void bnxt_get_ring_stats(struct bnxt *bp, 10438 struct rtnl_link_stats64 *stats); 10439 10440 static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init, 10441 bool link_re_init) 10442 { 10443 /* Close the VF-reps before closing PF */ 10444 if (BNXT_PF(bp)) 10445 bnxt_vf_reps_close(bp); 10446 10447 /* Change device state to avoid TX queue wake up's */ 10448 bnxt_tx_disable(bp); 10449 10450 clear_bit(BNXT_STATE_OPEN, &bp->state); 10451 smp_mb__after_atomic(); 10452 while (bnxt_drv_busy(bp)) 10453 msleep(20); 10454 10455 /* Flush rings and and disable interrupts */ 10456 bnxt_shutdown_nic(bp, irq_re_init); 10457 10458 /* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */ 10459 10460 bnxt_debug_dev_exit(bp); 10461 bnxt_disable_napi(bp); 10462 del_timer_sync(&bp->timer); 10463 bnxt_free_skbs(bp); 10464 10465 /* Save ring stats before shutdown */ 10466 if (bp->bnapi && irq_re_init) 10467 bnxt_get_ring_stats(bp, &bp->net_stats_prev); 10468 if (irq_re_init) { 10469 bnxt_free_irq(bp); 10470 bnxt_del_napi(bp); 10471 } 10472 bnxt_free_mem(bp, irq_re_init); 10473 } 10474 10475 int bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) 10476 { 10477 int rc = 0; 10478 10479 if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) { 10480 /* If we get here, it means firmware reset is in progress 10481 * while we are trying to close. We can safely proceed with 10482 * the close because we are holding rtnl_lock(). Some firmware 10483 * messages may fail as we proceed to close. We set the 10484 * ABORT_ERR flag here so that the FW reset thread will later 10485 * abort when it gets the rtnl_lock() and sees the flag. 10486 */ 10487 netdev_warn(bp->dev, "FW reset in progress during close, FW reset will be aborted\n"); 10488 set_bit(BNXT_STATE_ABORT_ERR, &bp->state); 10489 } 10490 10491 #ifdef CONFIG_BNXT_SRIOV 10492 if (bp->sriov_cfg) { 10493 rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait, 10494 !bp->sriov_cfg, 10495 BNXT_SRIOV_CFG_WAIT_TMO); 10496 if (rc) 10497 netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete!\n"); 10498 } 10499 #endif 10500 __bnxt_close_nic(bp, irq_re_init, link_re_init); 10501 return rc; 10502 } 10503 10504 static int bnxt_close(struct net_device *dev) 10505 { 10506 struct bnxt *bp = netdev_priv(dev); 10507 10508 bnxt_hwmon_close(bp); 10509 bnxt_close_nic(bp, true, true); 10510 bnxt_hwrm_shutdown_link(bp); 10511 bnxt_hwrm_if_change(bp, false); 10512 return 0; 10513 } 10514 10515 static int bnxt_hwrm_port_phy_read(struct bnxt *bp, u16 phy_addr, u16 reg, 10516 u16 *val) 10517 { 10518 struct hwrm_port_phy_mdio_read_output *resp; 10519 struct hwrm_port_phy_mdio_read_input *req; 10520 int rc; 10521 10522 if (bp->hwrm_spec_code < 0x10a00) 10523 return -EOPNOTSUPP; 10524 10525 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_MDIO_READ); 10526 if (rc) 10527 return rc; 10528 10529 req->port_id = cpu_to_le16(bp->pf.port_id); 10530 req->phy_addr = phy_addr; 10531 req->reg_addr = cpu_to_le16(reg & 0x1f); 10532 if (mdio_phy_id_is_c45(phy_addr)) { 10533 req->cl45_mdio = 1; 10534 req->phy_addr = mdio_phy_id_prtad(phy_addr); 10535 req->dev_addr = mdio_phy_id_devad(phy_addr); 10536 req->reg_addr = cpu_to_le16(reg); 10537 } 10538 10539 resp = hwrm_req_hold(bp, req); 10540 rc = hwrm_req_send(bp, req); 10541 if (!rc) 10542 *val = le16_to_cpu(resp->reg_data); 10543 hwrm_req_drop(bp, req); 10544 return rc; 10545 } 10546 10547 static int bnxt_hwrm_port_phy_write(struct bnxt *bp, u16 phy_addr, u16 reg, 10548 u16 val) 10549 { 10550 struct hwrm_port_phy_mdio_write_input *req; 10551 int rc; 10552 10553 if (bp->hwrm_spec_code < 0x10a00) 10554 return -EOPNOTSUPP; 10555 10556 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_MDIO_WRITE); 10557 if (rc) 10558 return rc; 10559 10560 req->port_id = cpu_to_le16(bp->pf.port_id); 10561 req->phy_addr = phy_addr; 10562 req->reg_addr = cpu_to_le16(reg & 0x1f); 10563 if (mdio_phy_id_is_c45(phy_addr)) { 10564 req->cl45_mdio = 1; 10565 req->phy_addr = mdio_phy_id_prtad(phy_addr); 10566 req->dev_addr = mdio_phy_id_devad(phy_addr); 10567 req->reg_addr = cpu_to_le16(reg); 10568 } 10569 req->reg_data = cpu_to_le16(val); 10570 10571 return hwrm_req_send(bp, req); 10572 } 10573 10574 /* rtnl_lock held */ 10575 static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) 10576 { 10577 struct mii_ioctl_data *mdio = if_mii(ifr); 10578 struct bnxt *bp = netdev_priv(dev); 10579 int rc; 10580 10581 switch (cmd) { 10582 case SIOCGMIIPHY: 10583 mdio->phy_id = bp->link_info.phy_addr; 10584 10585 fallthrough; 10586 case SIOCGMIIREG: { 10587 u16 mii_regval = 0; 10588 10589 if (!netif_running(dev)) 10590 return -EAGAIN; 10591 10592 rc = bnxt_hwrm_port_phy_read(bp, mdio->phy_id, mdio->reg_num, 10593 &mii_regval); 10594 mdio->val_out = mii_regval; 10595 return rc; 10596 } 10597 10598 case SIOCSMIIREG: 10599 if (!netif_running(dev)) 10600 return -EAGAIN; 10601 10602 return bnxt_hwrm_port_phy_write(bp, mdio->phy_id, mdio->reg_num, 10603 mdio->val_in); 10604 10605 case SIOCSHWTSTAMP: 10606 return bnxt_hwtstamp_set(dev, ifr); 10607 10608 case SIOCGHWTSTAMP: 10609 return bnxt_hwtstamp_get(dev, ifr); 10610 10611 default: 10612 /* do nothing */ 10613 break; 10614 } 10615 return -EOPNOTSUPP; 10616 } 10617 10618 static void bnxt_get_ring_stats(struct bnxt *bp, 10619 struct rtnl_link_stats64 *stats) 10620 { 10621 int i; 10622 10623 for (i = 0; i < bp->cp_nr_rings; i++) { 10624 struct bnxt_napi *bnapi = bp->bnapi[i]; 10625 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 10626 u64 *sw = cpr->stats.sw_stats; 10627 10628 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_ucast_pkts); 10629 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts); 10630 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_bcast_pkts); 10631 10632 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_ucast_pkts); 10633 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_mcast_pkts); 10634 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_bcast_pkts); 10635 10636 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_ucast_bytes); 10637 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_mcast_bytes); 10638 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_bcast_bytes); 10639 10640 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_ucast_bytes); 10641 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_mcast_bytes); 10642 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_bcast_bytes); 10643 10644 stats->rx_missed_errors += 10645 BNXT_GET_RING_STATS64(sw, rx_discard_pkts); 10646 10647 stats->multicast += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts); 10648 10649 stats->tx_dropped += BNXT_GET_RING_STATS64(sw, tx_error_pkts); 10650 10651 stats->rx_dropped += 10652 cpr->sw_stats.rx.rx_netpoll_discards + 10653 cpr->sw_stats.rx.rx_oom_discards; 10654 } 10655 } 10656 10657 static void bnxt_add_prev_stats(struct bnxt *bp, 10658 struct rtnl_link_stats64 *stats) 10659 { 10660 struct rtnl_link_stats64 *prev_stats = &bp->net_stats_prev; 10661 10662 stats->rx_packets += prev_stats->rx_packets; 10663 stats->tx_packets += prev_stats->tx_packets; 10664 stats->rx_bytes += prev_stats->rx_bytes; 10665 stats->tx_bytes += prev_stats->tx_bytes; 10666 stats->rx_missed_errors += prev_stats->rx_missed_errors; 10667 stats->multicast += prev_stats->multicast; 10668 stats->rx_dropped += prev_stats->rx_dropped; 10669 stats->tx_dropped += prev_stats->tx_dropped; 10670 } 10671 10672 static void 10673 bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) 10674 { 10675 struct bnxt *bp = netdev_priv(dev); 10676 10677 set_bit(BNXT_STATE_READ_STATS, &bp->state); 10678 /* Make sure bnxt_close_nic() sees that we are reading stats before 10679 * we check the BNXT_STATE_OPEN flag. 10680 */ 10681 smp_mb__after_atomic(); 10682 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 10683 clear_bit(BNXT_STATE_READ_STATS, &bp->state); 10684 *stats = bp->net_stats_prev; 10685 return; 10686 } 10687 10688 bnxt_get_ring_stats(bp, stats); 10689 bnxt_add_prev_stats(bp, stats); 10690 10691 if (bp->flags & BNXT_FLAG_PORT_STATS) { 10692 u64 *rx = bp->port_stats.sw_stats; 10693 u64 *tx = bp->port_stats.sw_stats + 10694 BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 10695 10696 stats->rx_crc_errors = 10697 BNXT_GET_RX_PORT_STATS64(rx, rx_fcs_err_frames); 10698 stats->rx_frame_errors = 10699 BNXT_GET_RX_PORT_STATS64(rx, rx_align_err_frames); 10700 stats->rx_length_errors = 10701 BNXT_GET_RX_PORT_STATS64(rx, rx_undrsz_frames) + 10702 BNXT_GET_RX_PORT_STATS64(rx, rx_ovrsz_frames) + 10703 BNXT_GET_RX_PORT_STATS64(rx, rx_runt_frames); 10704 stats->rx_errors = 10705 BNXT_GET_RX_PORT_STATS64(rx, rx_false_carrier_frames) + 10706 BNXT_GET_RX_PORT_STATS64(rx, rx_jbr_frames); 10707 stats->collisions = 10708 BNXT_GET_TX_PORT_STATS64(tx, tx_total_collisions); 10709 stats->tx_fifo_errors = 10710 BNXT_GET_TX_PORT_STATS64(tx, tx_fifo_underruns); 10711 stats->tx_errors = BNXT_GET_TX_PORT_STATS64(tx, tx_err); 10712 } 10713 clear_bit(BNXT_STATE_READ_STATS, &bp->state); 10714 } 10715 10716 static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask) 10717 { 10718 struct net_device *dev = bp->dev; 10719 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 10720 struct netdev_hw_addr *ha; 10721 u8 *haddr; 10722 int mc_count = 0; 10723 bool update = false; 10724 int off = 0; 10725 10726 netdev_for_each_mc_addr(ha, dev) { 10727 if (mc_count >= BNXT_MAX_MC_ADDRS) { 10728 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 10729 vnic->mc_list_count = 0; 10730 return false; 10731 } 10732 haddr = ha->addr; 10733 if (!ether_addr_equal(haddr, vnic->mc_list + off)) { 10734 memcpy(vnic->mc_list + off, haddr, ETH_ALEN); 10735 update = true; 10736 } 10737 off += ETH_ALEN; 10738 mc_count++; 10739 } 10740 if (mc_count) 10741 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST; 10742 10743 if (mc_count != vnic->mc_list_count) { 10744 vnic->mc_list_count = mc_count; 10745 update = true; 10746 } 10747 return update; 10748 } 10749 10750 static bool bnxt_uc_list_updated(struct bnxt *bp) 10751 { 10752 struct net_device *dev = bp->dev; 10753 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 10754 struct netdev_hw_addr *ha; 10755 int off = 0; 10756 10757 if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1)) 10758 return true; 10759 10760 netdev_for_each_uc_addr(ha, dev) { 10761 if (!ether_addr_equal(ha->addr, vnic->uc_list + off)) 10762 return true; 10763 10764 off += ETH_ALEN; 10765 } 10766 return false; 10767 } 10768 10769 static void bnxt_set_rx_mode(struct net_device *dev) 10770 { 10771 struct bnxt *bp = netdev_priv(dev); 10772 struct bnxt_vnic_info *vnic; 10773 bool mc_update = false; 10774 bool uc_update; 10775 u32 mask; 10776 10777 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) 10778 return; 10779 10780 vnic = &bp->vnic_info[0]; 10781 mask = vnic->rx_mask; 10782 mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS | 10783 CFA_L2_SET_RX_MASK_REQ_MASK_MCAST | 10784 CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST | 10785 CFA_L2_SET_RX_MASK_REQ_MASK_BCAST); 10786 10787 if (dev->flags & IFF_PROMISC) 10788 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 10789 10790 uc_update = bnxt_uc_list_updated(bp); 10791 10792 if (dev->flags & IFF_BROADCAST) 10793 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST; 10794 if (dev->flags & IFF_ALLMULTI) { 10795 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 10796 vnic->mc_list_count = 0; 10797 } else if (dev->flags & IFF_MULTICAST) { 10798 mc_update = bnxt_mc_list_updated(bp, &mask); 10799 } 10800 10801 if (mask != vnic->rx_mask || uc_update || mc_update) { 10802 vnic->rx_mask = mask; 10803 10804 set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event); 10805 bnxt_queue_sp_work(bp); 10806 } 10807 } 10808 10809 static int bnxt_cfg_rx_mode(struct bnxt *bp) 10810 { 10811 struct net_device *dev = bp->dev; 10812 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 10813 struct hwrm_cfa_l2_filter_free_input *req; 10814 struct netdev_hw_addr *ha; 10815 int i, off = 0, rc; 10816 bool uc_update; 10817 10818 netif_addr_lock_bh(dev); 10819 uc_update = bnxt_uc_list_updated(bp); 10820 netif_addr_unlock_bh(dev); 10821 10822 if (!uc_update) 10823 goto skip_uc; 10824 10825 rc = hwrm_req_init(bp, req, HWRM_CFA_L2_FILTER_FREE); 10826 if (rc) 10827 return rc; 10828 hwrm_req_hold(bp, req); 10829 for (i = 1; i < vnic->uc_filter_count; i++) { 10830 req->l2_filter_id = vnic->fw_l2_filter_id[i]; 10831 10832 rc = hwrm_req_send(bp, req); 10833 } 10834 hwrm_req_drop(bp, req); 10835 10836 vnic->uc_filter_count = 1; 10837 10838 netif_addr_lock_bh(dev); 10839 if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) { 10840 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 10841 } else { 10842 netdev_for_each_uc_addr(ha, dev) { 10843 memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN); 10844 off += ETH_ALEN; 10845 vnic->uc_filter_count++; 10846 } 10847 } 10848 netif_addr_unlock_bh(dev); 10849 10850 for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) { 10851 rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off); 10852 if (rc) { 10853 if (BNXT_VF(bp) && rc == -ENODEV) { 10854 if (!test_and_set_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state)) 10855 netdev_warn(bp->dev, "Cannot configure L2 filters while PF is unavailable, will retry\n"); 10856 else 10857 netdev_dbg(bp->dev, "PF still unavailable while configuring L2 filters.\n"); 10858 rc = 0; 10859 } else { 10860 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc); 10861 } 10862 vnic->uc_filter_count = i; 10863 return rc; 10864 } 10865 } 10866 if (test_and_clear_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state)) 10867 netdev_notice(bp->dev, "Retry of L2 filter configuration successful.\n"); 10868 10869 skip_uc: 10870 if ((vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS) && 10871 !bnxt_promisc_ok(bp)) 10872 vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 10873 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0); 10874 if (rc && (vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_MCAST)) { 10875 netdev_info(bp->dev, "Failed setting MC filters rc: %d, turning on ALL_MCAST mode\n", 10876 rc); 10877 vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_MCAST; 10878 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 10879 vnic->mc_list_count = 0; 10880 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0); 10881 } 10882 if (rc) 10883 netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %d\n", 10884 rc); 10885 10886 return rc; 10887 } 10888 10889 static bool bnxt_can_reserve_rings(struct bnxt *bp) 10890 { 10891 #ifdef CONFIG_BNXT_SRIOV 10892 if (BNXT_NEW_RM(bp) && BNXT_VF(bp)) { 10893 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 10894 10895 /* No minimum rings were provisioned by the PF. Don't 10896 * reserve rings by default when device is down. 10897 */ 10898 if (hw_resc->min_tx_rings || hw_resc->resv_tx_rings) 10899 return true; 10900 10901 if (!netif_running(bp->dev)) 10902 return false; 10903 } 10904 #endif 10905 return true; 10906 } 10907 10908 /* If the chip and firmware supports RFS */ 10909 static bool bnxt_rfs_supported(struct bnxt *bp) 10910 { 10911 if (bp->flags & BNXT_FLAG_CHIP_P5) { 10912 if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2) 10913 return true; 10914 return false; 10915 } 10916 /* 212 firmware is broken for aRFS */ 10917 if (BNXT_FW_MAJ(bp) == 212) 10918 return false; 10919 if (BNXT_PF(bp) && !BNXT_CHIP_TYPE_NITRO_A0(bp)) 10920 return true; 10921 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP) 10922 return true; 10923 return false; 10924 } 10925 10926 /* If runtime conditions support RFS */ 10927 static bool bnxt_rfs_capable(struct bnxt *bp) 10928 { 10929 #ifdef CONFIG_RFS_ACCEL 10930 int vnics, max_vnics, max_rss_ctxs; 10931 10932 if (bp->flags & BNXT_FLAG_CHIP_P5) 10933 return bnxt_rfs_supported(bp); 10934 if (!(bp->flags & BNXT_FLAG_MSIX_CAP) || !bnxt_can_reserve_rings(bp)) 10935 return false; 10936 10937 vnics = 1 + bp->rx_nr_rings; 10938 max_vnics = bnxt_get_max_func_vnics(bp); 10939 max_rss_ctxs = bnxt_get_max_func_rss_ctxs(bp); 10940 10941 /* RSS contexts not a limiting factor */ 10942 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP) 10943 max_rss_ctxs = max_vnics; 10944 if (vnics > max_vnics || vnics > max_rss_ctxs) { 10945 if (bp->rx_nr_rings > 1) 10946 netdev_warn(bp->dev, 10947 "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n", 10948 min(max_rss_ctxs - 1, max_vnics - 1)); 10949 return false; 10950 } 10951 10952 if (!BNXT_NEW_RM(bp)) 10953 return true; 10954 10955 if (vnics == bp->hw_resc.resv_vnics) 10956 return true; 10957 10958 bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, vnics); 10959 if (vnics <= bp->hw_resc.resv_vnics) 10960 return true; 10961 10962 netdev_warn(bp->dev, "Unable to reserve resources to support NTUPLE filters.\n"); 10963 bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, 1); 10964 return false; 10965 #else 10966 return false; 10967 #endif 10968 } 10969 10970 static netdev_features_t bnxt_fix_features(struct net_device *dev, 10971 netdev_features_t features) 10972 { 10973 struct bnxt *bp = netdev_priv(dev); 10974 netdev_features_t vlan_features; 10975 10976 if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp)) 10977 features &= ~NETIF_F_NTUPLE; 10978 10979 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS) 10980 features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); 10981 10982 if (!(features & NETIF_F_GRO)) 10983 features &= ~NETIF_F_GRO_HW; 10984 10985 if (features & NETIF_F_GRO_HW) 10986 features &= ~NETIF_F_LRO; 10987 10988 /* Both CTAG and STAG VLAN accelaration on the RX side have to be 10989 * turned on or off together. 10990 */ 10991 vlan_features = features & BNXT_HW_FEATURE_VLAN_ALL_RX; 10992 if (vlan_features != BNXT_HW_FEATURE_VLAN_ALL_RX) { 10993 if (dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX) 10994 features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX; 10995 else if (vlan_features) 10996 features |= BNXT_HW_FEATURE_VLAN_ALL_RX; 10997 } 10998 #ifdef CONFIG_BNXT_SRIOV 10999 if (BNXT_VF(bp) && bp->vf.vlan) 11000 features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX; 11001 #endif 11002 return features; 11003 } 11004 11005 static int bnxt_set_features(struct net_device *dev, netdev_features_t features) 11006 { 11007 struct bnxt *bp = netdev_priv(dev); 11008 u32 flags = bp->flags; 11009 u32 changes; 11010 int rc = 0; 11011 bool re_init = false; 11012 bool update_tpa = false; 11013 11014 flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS; 11015 if (features & NETIF_F_GRO_HW) 11016 flags |= BNXT_FLAG_GRO; 11017 else if (features & NETIF_F_LRO) 11018 flags |= BNXT_FLAG_LRO; 11019 11020 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS) 11021 flags &= ~BNXT_FLAG_TPA; 11022 11023 if (features & BNXT_HW_FEATURE_VLAN_ALL_RX) 11024 flags |= BNXT_FLAG_STRIP_VLAN; 11025 11026 if (features & NETIF_F_NTUPLE) 11027 flags |= BNXT_FLAG_RFS; 11028 11029 changes = flags ^ bp->flags; 11030 if (changes & BNXT_FLAG_TPA) { 11031 update_tpa = true; 11032 if ((bp->flags & BNXT_FLAG_TPA) == 0 || 11033 (flags & BNXT_FLAG_TPA) == 0 || 11034 (bp->flags & BNXT_FLAG_CHIP_P5)) 11035 re_init = true; 11036 } 11037 11038 if (changes & ~BNXT_FLAG_TPA) 11039 re_init = true; 11040 11041 if (flags != bp->flags) { 11042 u32 old_flags = bp->flags; 11043 11044 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 11045 bp->flags = flags; 11046 if (update_tpa) 11047 bnxt_set_ring_params(bp); 11048 return rc; 11049 } 11050 11051 if (re_init) { 11052 bnxt_close_nic(bp, false, false); 11053 bp->flags = flags; 11054 if (update_tpa) 11055 bnxt_set_ring_params(bp); 11056 11057 return bnxt_open_nic(bp, false, false); 11058 } 11059 if (update_tpa) { 11060 bp->flags = flags; 11061 rc = bnxt_set_tpa(bp, 11062 (flags & BNXT_FLAG_TPA) ? 11063 true : false); 11064 if (rc) 11065 bp->flags = old_flags; 11066 } 11067 } 11068 return rc; 11069 } 11070 11071 static bool bnxt_exthdr_check(struct bnxt *bp, struct sk_buff *skb, int nw_off, 11072 u8 **nextp) 11073 { 11074 struct ipv6hdr *ip6h = (struct ipv6hdr *)(skb->data + nw_off); 11075 int hdr_count = 0; 11076 u8 *nexthdr; 11077 int start; 11078 11079 /* Check that there are at most 2 IPv6 extension headers, no 11080 * fragment header, and each is <= 64 bytes. 11081 */ 11082 start = nw_off + sizeof(*ip6h); 11083 nexthdr = &ip6h->nexthdr; 11084 while (ipv6_ext_hdr(*nexthdr)) { 11085 struct ipv6_opt_hdr *hp; 11086 int hdrlen; 11087 11088 if (hdr_count >= 3 || *nexthdr == NEXTHDR_NONE || 11089 *nexthdr == NEXTHDR_FRAGMENT) 11090 return false; 11091 hp = __skb_header_pointer(NULL, start, sizeof(*hp), skb->data, 11092 skb_headlen(skb), NULL); 11093 if (!hp) 11094 return false; 11095 if (*nexthdr == NEXTHDR_AUTH) 11096 hdrlen = ipv6_authlen(hp); 11097 else 11098 hdrlen = ipv6_optlen(hp); 11099 11100 if (hdrlen > 64) 11101 return false; 11102 nexthdr = &hp->nexthdr; 11103 start += hdrlen; 11104 hdr_count++; 11105 } 11106 if (nextp) { 11107 /* Caller will check inner protocol */ 11108 if (skb->encapsulation) { 11109 *nextp = nexthdr; 11110 return true; 11111 } 11112 *nextp = NULL; 11113 } 11114 /* Only support TCP/UDP for non-tunneled ipv6 and inner ipv6 */ 11115 return *nexthdr == IPPROTO_TCP || *nexthdr == IPPROTO_UDP; 11116 } 11117 11118 /* For UDP, we can only handle 1 Vxlan port and 1 Geneve port. */ 11119 static bool bnxt_udp_tunl_check(struct bnxt *bp, struct sk_buff *skb) 11120 { 11121 struct udphdr *uh = udp_hdr(skb); 11122 __be16 udp_port = uh->dest; 11123 11124 if (udp_port != bp->vxlan_port && udp_port != bp->nge_port) 11125 return false; 11126 if (skb->inner_protocol_type == ENCAP_TYPE_ETHER) { 11127 struct ethhdr *eh = inner_eth_hdr(skb); 11128 11129 switch (eh->h_proto) { 11130 case htons(ETH_P_IP): 11131 return true; 11132 case htons(ETH_P_IPV6): 11133 return bnxt_exthdr_check(bp, skb, 11134 skb_inner_network_offset(skb), 11135 NULL); 11136 } 11137 } 11138 return false; 11139 } 11140 11141 static bool bnxt_tunl_check(struct bnxt *bp, struct sk_buff *skb, u8 l4_proto) 11142 { 11143 switch (l4_proto) { 11144 case IPPROTO_UDP: 11145 return bnxt_udp_tunl_check(bp, skb); 11146 case IPPROTO_IPIP: 11147 return true; 11148 case IPPROTO_GRE: { 11149 switch (skb->inner_protocol) { 11150 default: 11151 return false; 11152 case htons(ETH_P_IP): 11153 return true; 11154 case htons(ETH_P_IPV6): 11155 fallthrough; 11156 } 11157 } 11158 case IPPROTO_IPV6: 11159 /* Check ext headers of inner ipv6 */ 11160 return bnxt_exthdr_check(bp, skb, skb_inner_network_offset(skb), 11161 NULL); 11162 } 11163 return false; 11164 } 11165 11166 static netdev_features_t bnxt_features_check(struct sk_buff *skb, 11167 struct net_device *dev, 11168 netdev_features_t features) 11169 { 11170 struct bnxt *bp = netdev_priv(dev); 11171 u8 *l4_proto; 11172 11173 features = vlan_features_check(skb, features); 11174 switch (vlan_get_protocol(skb)) { 11175 case htons(ETH_P_IP): 11176 if (!skb->encapsulation) 11177 return features; 11178 l4_proto = &ip_hdr(skb)->protocol; 11179 if (bnxt_tunl_check(bp, skb, *l4_proto)) 11180 return features; 11181 break; 11182 case htons(ETH_P_IPV6): 11183 if (!bnxt_exthdr_check(bp, skb, skb_network_offset(skb), 11184 &l4_proto)) 11185 break; 11186 if (!l4_proto || bnxt_tunl_check(bp, skb, *l4_proto)) 11187 return features; 11188 break; 11189 } 11190 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 11191 } 11192 11193 int bnxt_dbg_hwrm_rd_reg(struct bnxt *bp, u32 reg_off, u16 num_words, 11194 u32 *reg_buf) 11195 { 11196 struct hwrm_dbg_read_direct_output *resp; 11197 struct hwrm_dbg_read_direct_input *req; 11198 __le32 *dbg_reg_buf; 11199 dma_addr_t mapping; 11200 int rc, i; 11201 11202 rc = hwrm_req_init(bp, req, HWRM_DBG_READ_DIRECT); 11203 if (rc) 11204 return rc; 11205 11206 dbg_reg_buf = hwrm_req_dma_slice(bp, req, num_words * 4, 11207 &mapping); 11208 if (!dbg_reg_buf) { 11209 rc = -ENOMEM; 11210 goto dbg_rd_reg_exit; 11211 } 11212 11213 req->host_dest_addr = cpu_to_le64(mapping); 11214 11215 resp = hwrm_req_hold(bp, req); 11216 req->read_addr = cpu_to_le32(reg_off + CHIMP_REG_VIEW_ADDR); 11217 req->read_len32 = cpu_to_le32(num_words); 11218 11219 rc = hwrm_req_send(bp, req); 11220 if (rc || resp->error_code) { 11221 rc = -EIO; 11222 goto dbg_rd_reg_exit; 11223 } 11224 for (i = 0; i < num_words; i++) 11225 reg_buf[i] = le32_to_cpu(dbg_reg_buf[i]); 11226 11227 dbg_rd_reg_exit: 11228 hwrm_req_drop(bp, req); 11229 return rc; 11230 } 11231 11232 static int bnxt_dbg_hwrm_ring_info_get(struct bnxt *bp, u8 ring_type, 11233 u32 ring_id, u32 *prod, u32 *cons) 11234 { 11235 struct hwrm_dbg_ring_info_get_output *resp; 11236 struct hwrm_dbg_ring_info_get_input *req; 11237 int rc; 11238 11239 rc = hwrm_req_init(bp, req, HWRM_DBG_RING_INFO_GET); 11240 if (rc) 11241 return rc; 11242 11243 req->ring_type = ring_type; 11244 req->fw_ring_id = cpu_to_le32(ring_id); 11245 resp = hwrm_req_hold(bp, req); 11246 rc = hwrm_req_send(bp, req); 11247 if (!rc) { 11248 *prod = le32_to_cpu(resp->producer_index); 11249 *cons = le32_to_cpu(resp->consumer_index); 11250 } 11251 hwrm_req_drop(bp, req); 11252 return rc; 11253 } 11254 11255 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi) 11256 { 11257 struct bnxt_tx_ring_info *txr = bnapi->tx_ring; 11258 int i = bnapi->index; 11259 11260 if (!txr) 11261 return; 11262 11263 netdev_info(bnapi->bp->dev, "[%d]: tx{fw_ring: %d prod: %x cons: %x}\n", 11264 i, txr->tx_ring_struct.fw_ring_id, txr->tx_prod, 11265 txr->tx_cons); 11266 } 11267 11268 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi) 11269 { 11270 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 11271 int i = bnapi->index; 11272 11273 if (!rxr) 11274 return; 11275 11276 netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n", 11277 i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod, 11278 rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod, 11279 rxr->rx_sw_agg_prod); 11280 } 11281 11282 static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi) 11283 { 11284 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 11285 int i = bnapi->index; 11286 11287 netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n", 11288 i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons); 11289 } 11290 11291 static void bnxt_dbg_dump_states(struct bnxt *bp) 11292 { 11293 int i; 11294 struct bnxt_napi *bnapi; 11295 11296 for (i = 0; i < bp->cp_nr_rings; i++) { 11297 bnapi = bp->bnapi[i]; 11298 if (netif_msg_drv(bp)) { 11299 bnxt_dump_tx_sw_state(bnapi); 11300 bnxt_dump_rx_sw_state(bnapi); 11301 bnxt_dump_cp_sw_state(bnapi); 11302 } 11303 } 11304 } 11305 11306 static int bnxt_hwrm_rx_ring_reset(struct bnxt *bp, int ring_nr) 11307 { 11308 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr]; 11309 struct hwrm_ring_reset_input *req; 11310 struct bnxt_napi *bnapi = rxr->bnapi; 11311 struct bnxt_cp_ring_info *cpr; 11312 u16 cp_ring_id; 11313 int rc; 11314 11315 rc = hwrm_req_init(bp, req, HWRM_RING_RESET); 11316 if (rc) 11317 return rc; 11318 11319 cpr = &bnapi->cp_ring; 11320 cp_ring_id = cpr->cp_ring_struct.fw_ring_id; 11321 req->cmpl_ring = cpu_to_le16(cp_ring_id); 11322 req->ring_type = RING_RESET_REQ_RING_TYPE_RX_RING_GRP; 11323 req->ring_id = cpu_to_le16(bp->grp_info[bnapi->index].fw_grp_id); 11324 return hwrm_req_send_silent(bp, req); 11325 } 11326 11327 static void bnxt_reset_task(struct bnxt *bp, bool silent) 11328 { 11329 if (!silent) 11330 bnxt_dbg_dump_states(bp); 11331 if (netif_running(bp->dev)) { 11332 int rc; 11333 11334 if (silent) { 11335 bnxt_close_nic(bp, false, false); 11336 bnxt_open_nic(bp, false, false); 11337 } else { 11338 bnxt_ulp_stop(bp); 11339 bnxt_close_nic(bp, true, false); 11340 rc = bnxt_open_nic(bp, true, false); 11341 bnxt_ulp_start(bp, rc); 11342 } 11343 } 11344 } 11345 11346 static void bnxt_tx_timeout(struct net_device *dev, unsigned int txqueue) 11347 { 11348 struct bnxt *bp = netdev_priv(dev); 11349 11350 netdev_err(bp->dev, "TX timeout detected, starting reset task!\n"); 11351 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event); 11352 bnxt_queue_sp_work(bp); 11353 } 11354 11355 static void bnxt_fw_health_check(struct bnxt *bp) 11356 { 11357 struct bnxt_fw_health *fw_health = bp->fw_health; 11358 u32 val; 11359 11360 if (!fw_health->enabled || test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 11361 return; 11362 11363 /* Make sure it is enabled before checking the tmr_counter. */ 11364 smp_rmb(); 11365 if (fw_health->tmr_counter) { 11366 fw_health->tmr_counter--; 11367 return; 11368 } 11369 11370 val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG); 11371 if (val == fw_health->last_fw_heartbeat) { 11372 fw_health->arrests++; 11373 goto fw_reset; 11374 } 11375 11376 fw_health->last_fw_heartbeat = val; 11377 11378 val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG); 11379 if (val != fw_health->last_fw_reset_cnt) { 11380 fw_health->discoveries++; 11381 goto fw_reset; 11382 } 11383 11384 fw_health->tmr_counter = fw_health->tmr_multiplier; 11385 return; 11386 11387 fw_reset: 11388 set_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event); 11389 bnxt_queue_sp_work(bp); 11390 } 11391 11392 static void bnxt_timer(struct timer_list *t) 11393 { 11394 struct bnxt *bp = from_timer(bp, t, timer); 11395 struct net_device *dev = bp->dev; 11396 11397 if (!netif_running(dev) || !test_bit(BNXT_STATE_OPEN, &bp->state)) 11398 return; 11399 11400 if (atomic_read(&bp->intr_sem) != 0) 11401 goto bnxt_restart_timer; 11402 11403 if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) 11404 bnxt_fw_health_check(bp); 11405 11406 if (bp->link_info.link_up && bp->stats_coal_ticks) { 11407 set_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event); 11408 bnxt_queue_sp_work(bp); 11409 } 11410 11411 if (bnxt_tc_flower_enabled(bp)) { 11412 set_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event); 11413 bnxt_queue_sp_work(bp); 11414 } 11415 11416 #ifdef CONFIG_RFS_ACCEL 11417 if ((bp->flags & BNXT_FLAG_RFS) && bp->ntp_fltr_count) { 11418 set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event); 11419 bnxt_queue_sp_work(bp); 11420 } 11421 #endif /*CONFIG_RFS_ACCEL*/ 11422 11423 if (bp->link_info.phy_retry) { 11424 if (time_after(jiffies, bp->link_info.phy_retry_expires)) { 11425 bp->link_info.phy_retry = false; 11426 netdev_warn(bp->dev, "failed to update phy settings after maximum retries.\n"); 11427 } else { 11428 set_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event); 11429 bnxt_queue_sp_work(bp); 11430 } 11431 } 11432 11433 if (test_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state)) { 11434 set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event); 11435 bnxt_queue_sp_work(bp); 11436 } 11437 11438 if ((bp->flags & BNXT_FLAG_CHIP_P5) && !bp->chip_rev && 11439 netif_carrier_ok(dev)) { 11440 set_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event); 11441 bnxt_queue_sp_work(bp); 11442 } 11443 bnxt_restart_timer: 11444 mod_timer(&bp->timer, jiffies + bp->current_interval); 11445 } 11446 11447 static void bnxt_rtnl_lock_sp(struct bnxt *bp) 11448 { 11449 /* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK 11450 * set. If the device is being closed, bnxt_close() may be holding 11451 * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear. So we 11452 * must clear BNXT_STATE_IN_SP_TASK before holding rtnl(). 11453 */ 11454 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 11455 rtnl_lock(); 11456 } 11457 11458 static void bnxt_rtnl_unlock_sp(struct bnxt *bp) 11459 { 11460 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 11461 rtnl_unlock(); 11462 } 11463 11464 /* Only called from bnxt_sp_task() */ 11465 static void bnxt_reset(struct bnxt *bp, bool silent) 11466 { 11467 bnxt_rtnl_lock_sp(bp); 11468 if (test_bit(BNXT_STATE_OPEN, &bp->state)) 11469 bnxt_reset_task(bp, silent); 11470 bnxt_rtnl_unlock_sp(bp); 11471 } 11472 11473 /* Only called from bnxt_sp_task() */ 11474 static void bnxt_rx_ring_reset(struct bnxt *bp) 11475 { 11476 int i; 11477 11478 bnxt_rtnl_lock_sp(bp); 11479 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 11480 bnxt_rtnl_unlock_sp(bp); 11481 return; 11482 } 11483 /* Disable and flush TPA before resetting the RX ring */ 11484 if (bp->flags & BNXT_FLAG_TPA) 11485 bnxt_set_tpa(bp, false); 11486 for (i = 0; i < bp->rx_nr_rings; i++) { 11487 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 11488 struct bnxt_cp_ring_info *cpr; 11489 int rc; 11490 11491 if (!rxr->bnapi->in_reset) 11492 continue; 11493 11494 rc = bnxt_hwrm_rx_ring_reset(bp, i); 11495 if (rc) { 11496 if (rc == -EINVAL || rc == -EOPNOTSUPP) 11497 netdev_info_once(bp->dev, "RX ring reset not supported by firmware, falling back to global reset\n"); 11498 else 11499 netdev_warn(bp->dev, "RX ring reset failed, rc = %d, falling back to global reset\n", 11500 rc); 11501 bnxt_reset_task(bp, true); 11502 break; 11503 } 11504 bnxt_free_one_rx_ring_skbs(bp, i); 11505 rxr->rx_prod = 0; 11506 rxr->rx_agg_prod = 0; 11507 rxr->rx_sw_agg_prod = 0; 11508 rxr->rx_next_cons = 0; 11509 rxr->bnapi->in_reset = false; 11510 bnxt_alloc_one_rx_ring(bp, i); 11511 cpr = &rxr->bnapi->cp_ring; 11512 cpr->sw_stats.rx.rx_resets++; 11513 if (bp->flags & BNXT_FLAG_AGG_RINGS) 11514 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); 11515 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 11516 } 11517 if (bp->flags & BNXT_FLAG_TPA) 11518 bnxt_set_tpa(bp, true); 11519 bnxt_rtnl_unlock_sp(bp); 11520 } 11521 11522 static void bnxt_fw_reset_close(struct bnxt *bp) 11523 { 11524 bnxt_ulp_stop(bp); 11525 /* When firmware is in fatal state, quiesce device and disable 11526 * bus master to prevent any potential bad DMAs before freeing 11527 * kernel memory. 11528 */ 11529 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) { 11530 u16 val = 0; 11531 11532 pci_read_config_word(bp->pdev, PCI_SUBSYSTEM_ID, &val); 11533 if (val == 0xffff) 11534 bp->fw_reset_min_dsecs = 0; 11535 bnxt_tx_disable(bp); 11536 bnxt_disable_napi(bp); 11537 bnxt_disable_int_sync(bp); 11538 bnxt_free_irq(bp); 11539 bnxt_clear_int_mode(bp); 11540 pci_disable_device(bp->pdev); 11541 } 11542 __bnxt_close_nic(bp, true, false); 11543 bnxt_vf_reps_free(bp); 11544 bnxt_clear_int_mode(bp); 11545 bnxt_hwrm_func_drv_unrgtr(bp); 11546 if (pci_is_enabled(bp->pdev)) 11547 pci_disable_device(bp->pdev); 11548 bnxt_free_ctx_mem(bp); 11549 kfree(bp->ctx); 11550 bp->ctx = NULL; 11551 } 11552 11553 static bool is_bnxt_fw_ok(struct bnxt *bp) 11554 { 11555 struct bnxt_fw_health *fw_health = bp->fw_health; 11556 bool no_heartbeat = false, has_reset = false; 11557 u32 val; 11558 11559 val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG); 11560 if (val == fw_health->last_fw_heartbeat) 11561 no_heartbeat = true; 11562 11563 val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG); 11564 if (val != fw_health->last_fw_reset_cnt) 11565 has_reset = true; 11566 11567 if (!no_heartbeat && has_reset) 11568 return true; 11569 11570 return false; 11571 } 11572 11573 /* rtnl_lock is acquired before calling this function */ 11574 static void bnxt_force_fw_reset(struct bnxt *bp) 11575 { 11576 struct bnxt_fw_health *fw_health = bp->fw_health; 11577 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; 11578 u32 wait_dsecs; 11579 11580 if (!test_bit(BNXT_STATE_OPEN, &bp->state) || 11581 test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 11582 return; 11583 11584 if (ptp) { 11585 spin_lock_bh(&ptp->ptp_lock); 11586 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 11587 spin_unlock_bh(&ptp->ptp_lock); 11588 } else { 11589 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 11590 } 11591 bnxt_fw_reset_close(bp); 11592 wait_dsecs = fw_health->master_func_wait_dsecs; 11593 if (fw_health->primary) { 11594 if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) 11595 wait_dsecs = 0; 11596 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW; 11597 } else { 11598 bp->fw_reset_timestamp = jiffies + wait_dsecs * HZ / 10; 11599 wait_dsecs = fw_health->normal_func_wait_dsecs; 11600 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 11601 } 11602 11603 bp->fw_reset_min_dsecs = fw_health->post_reset_wait_dsecs; 11604 bp->fw_reset_max_dsecs = fw_health->post_reset_max_wait_dsecs; 11605 bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10); 11606 } 11607 11608 void bnxt_fw_exception(struct bnxt *bp) 11609 { 11610 netdev_warn(bp->dev, "Detected firmware fatal condition, initiating reset\n"); 11611 set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state); 11612 bnxt_rtnl_lock_sp(bp); 11613 bnxt_force_fw_reset(bp); 11614 bnxt_rtnl_unlock_sp(bp); 11615 } 11616 11617 /* Returns the number of registered VFs, or 1 if VF configuration is pending, or 11618 * < 0 on error. 11619 */ 11620 static int bnxt_get_registered_vfs(struct bnxt *bp) 11621 { 11622 #ifdef CONFIG_BNXT_SRIOV 11623 int rc; 11624 11625 if (!BNXT_PF(bp)) 11626 return 0; 11627 11628 rc = bnxt_hwrm_func_qcfg(bp); 11629 if (rc) { 11630 netdev_err(bp->dev, "func_qcfg cmd failed, rc = %d\n", rc); 11631 return rc; 11632 } 11633 if (bp->pf.registered_vfs) 11634 return bp->pf.registered_vfs; 11635 if (bp->sriov_cfg) 11636 return 1; 11637 #endif 11638 return 0; 11639 } 11640 11641 void bnxt_fw_reset(struct bnxt *bp) 11642 { 11643 bnxt_rtnl_lock_sp(bp); 11644 if (test_bit(BNXT_STATE_OPEN, &bp->state) && 11645 !test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) { 11646 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; 11647 int n = 0, tmo; 11648 11649 if (ptp) { 11650 spin_lock_bh(&ptp->ptp_lock); 11651 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 11652 spin_unlock_bh(&ptp->ptp_lock); 11653 } else { 11654 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 11655 } 11656 if (bp->pf.active_vfs && 11657 !test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) 11658 n = bnxt_get_registered_vfs(bp); 11659 if (n < 0) { 11660 netdev_err(bp->dev, "Firmware reset aborted, rc = %d\n", 11661 n); 11662 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 11663 dev_close(bp->dev); 11664 goto fw_reset_exit; 11665 } else if (n > 0) { 11666 u16 vf_tmo_dsecs = n * 10; 11667 11668 if (bp->fw_reset_max_dsecs < vf_tmo_dsecs) 11669 bp->fw_reset_max_dsecs = vf_tmo_dsecs; 11670 bp->fw_reset_state = 11671 BNXT_FW_RESET_STATE_POLL_VF; 11672 bnxt_queue_fw_reset_work(bp, HZ / 10); 11673 goto fw_reset_exit; 11674 } 11675 bnxt_fw_reset_close(bp); 11676 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) { 11677 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN; 11678 tmo = HZ / 10; 11679 } else { 11680 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 11681 tmo = bp->fw_reset_min_dsecs * HZ / 10; 11682 } 11683 bnxt_queue_fw_reset_work(bp, tmo); 11684 } 11685 fw_reset_exit: 11686 bnxt_rtnl_unlock_sp(bp); 11687 } 11688 11689 static void bnxt_chk_missed_irq(struct bnxt *bp) 11690 { 11691 int i; 11692 11693 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 11694 return; 11695 11696 for (i = 0; i < bp->cp_nr_rings; i++) { 11697 struct bnxt_napi *bnapi = bp->bnapi[i]; 11698 struct bnxt_cp_ring_info *cpr; 11699 u32 fw_ring_id; 11700 int j; 11701 11702 if (!bnapi) 11703 continue; 11704 11705 cpr = &bnapi->cp_ring; 11706 for (j = 0; j < 2; j++) { 11707 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j]; 11708 u32 val[2]; 11709 11710 if (!cpr2 || cpr2->has_more_work || 11711 !bnxt_has_work(bp, cpr2)) 11712 continue; 11713 11714 if (cpr2->cp_raw_cons != cpr2->last_cp_raw_cons) { 11715 cpr2->last_cp_raw_cons = cpr2->cp_raw_cons; 11716 continue; 11717 } 11718 fw_ring_id = cpr2->cp_ring_struct.fw_ring_id; 11719 bnxt_dbg_hwrm_ring_info_get(bp, 11720 DBG_RING_INFO_GET_REQ_RING_TYPE_L2_CMPL, 11721 fw_ring_id, &val[0], &val[1]); 11722 cpr->sw_stats.cmn.missed_irqs++; 11723 } 11724 } 11725 } 11726 11727 static void bnxt_cfg_ntp_filters(struct bnxt *); 11728 11729 static void bnxt_init_ethtool_link_settings(struct bnxt *bp) 11730 { 11731 struct bnxt_link_info *link_info = &bp->link_info; 11732 11733 if (BNXT_AUTO_MODE(link_info->auto_mode)) { 11734 link_info->autoneg = BNXT_AUTONEG_SPEED; 11735 if (bp->hwrm_spec_code >= 0x10201) { 11736 if (link_info->auto_pause_setting & 11737 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE) 11738 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL; 11739 } else { 11740 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL; 11741 } 11742 link_info->advertising = link_info->auto_link_speeds; 11743 link_info->advertising_pam4 = link_info->auto_pam4_link_speeds; 11744 } else { 11745 link_info->req_link_speed = link_info->force_link_speed; 11746 link_info->req_signal_mode = BNXT_SIG_MODE_NRZ; 11747 if (link_info->force_pam4_link_speed) { 11748 link_info->req_link_speed = 11749 link_info->force_pam4_link_speed; 11750 link_info->req_signal_mode = BNXT_SIG_MODE_PAM4; 11751 } 11752 link_info->req_duplex = link_info->duplex_setting; 11753 } 11754 if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) 11755 link_info->req_flow_ctrl = 11756 link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH; 11757 else 11758 link_info->req_flow_ctrl = link_info->force_pause_setting; 11759 } 11760 11761 static void bnxt_fw_echo_reply(struct bnxt *bp) 11762 { 11763 struct bnxt_fw_health *fw_health = bp->fw_health; 11764 struct hwrm_func_echo_response_input *req; 11765 int rc; 11766 11767 rc = hwrm_req_init(bp, req, HWRM_FUNC_ECHO_RESPONSE); 11768 if (rc) 11769 return; 11770 req->event_data1 = cpu_to_le32(fw_health->echo_req_data1); 11771 req->event_data2 = cpu_to_le32(fw_health->echo_req_data2); 11772 hwrm_req_send(bp, req); 11773 } 11774 11775 static void bnxt_sp_task(struct work_struct *work) 11776 { 11777 struct bnxt *bp = container_of(work, struct bnxt, sp_task); 11778 11779 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 11780 smp_mb__after_atomic(); 11781 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 11782 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 11783 return; 11784 } 11785 11786 if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event)) 11787 bnxt_cfg_rx_mode(bp); 11788 11789 if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event)) 11790 bnxt_cfg_ntp_filters(bp); 11791 if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event)) 11792 bnxt_hwrm_exec_fwd_req(bp); 11793 if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event)) { 11794 bnxt_hwrm_port_qstats(bp, 0); 11795 bnxt_hwrm_port_qstats_ext(bp, 0); 11796 bnxt_accumulate_all_stats(bp); 11797 } 11798 11799 if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) { 11800 int rc; 11801 11802 mutex_lock(&bp->link_lock); 11803 if (test_and_clear_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, 11804 &bp->sp_event)) 11805 bnxt_hwrm_phy_qcaps(bp); 11806 11807 rc = bnxt_update_link(bp, true); 11808 if (rc) 11809 netdev_err(bp->dev, "SP task can't update link (rc: %x)\n", 11810 rc); 11811 11812 if (test_and_clear_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT, 11813 &bp->sp_event)) 11814 bnxt_init_ethtool_link_settings(bp); 11815 mutex_unlock(&bp->link_lock); 11816 } 11817 if (test_and_clear_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event)) { 11818 int rc; 11819 11820 mutex_lock(&bp->link_lock); 11821 rc = bnxt_update_phy_setting(bp); 11822 mutex_unlock(&bp->link_lock); 11823 if (rc) { 11824 netdev_warn(bp->dev, "update phy settings retry failed\n"); 11825 } else { 11826 bp->link_info.phy_retry = false; 11827 netdev_info(bp->dev, "update phy settings retry succeeded\n"); 11828 } 11829 } 11830 if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event)) { 11831 mutex_lock(&bp->link_lock); 11832 bnxt_get_port_module_status(bp); 11833 mutex_unlock(&bp->link_lock); 11834 } 11835 11836 if (test_and_clear_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event)) 11837 bnxt_tc_flow_stats_work(bp); 11838 11839 if (test_and_clear_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event)) 11840 bnxt_chk_missed_irq(bp); 11841 11842 if (test_and_clear_bit(BNXT_FW_ECHO_REQUEST_SP_EVENT, &bp->sp_event)) 11843 bnxt_fw_echo_reply(bp); 11844 11845 /* These functions below will clear BNXT_STATE_IN_SP_TASK. They 11846 * must be the last functions to be called before exiting. 11847 */ 11848 if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event)) 11849 bnxt_reset(bp, false); 11850 11851 if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event)) 11852 bnxt_reset(bp, true); 11853 11854 if (test_and_clear_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event)) 11855 bnxt_rx_ring_reset(bp); 11856 11857 if (test_and_clear_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event)) { 11858 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state) || 11859 test_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state)) 11860 bnxt_devlink_health_fw_report(bp); 11861 else 11862 bnxt_fw_reset(bp); 11863 } 11864 11865 if (test_and_clear_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event)) { 11866 if (!is_bnxt_fw_ok(bp)) 11867 bnxt_devlink_health_fw_report(bp); 11868 } 11869 11870 smp_mb__before_atomic(); 11871 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 11872 } 11873 11874 /* Under rtnl_lock */ 11875 int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs, 11876 int tx_xdp) 11877 { 11878 int max_rx, max_tx, tx_sets = 1; 11879 int tx_rings_needed, stats; 11880 int rx_rings = rx; 11881 int cp, vnics, rc; 11882 11883 if (tcs) 11884 tx_sets = tcs; 11885 11886 rc = bnxt_get_max_rings(bp, &max_rx, &max_tx, sh); 11887 if (rc) 11888 return rc; 11889 11890 if (max_rx < rx) 11891 return -ENOMEM; 11892 11893 tx_rings_needed = tx * tx_sets + tx_xdp; 11894 if (max_tx < tx_rings_needed) 11895 return -ENOMEM; 11896 11897 vnics = 1; 11898 if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS) 11899 vnics += rx_rings; 11900 11901 if (bp->flags & BNXT_FLAG_AGG_RINGS) 11902 rx_rings <<= 1; 11903 cp = sh ? max_t(int, tx_rings_needed, rx) : tx_rings_needed + rx; 11904 stats = cp; 11905 if (BNXT_NEW_RM(bp)) { 11906 cp += bnxt_get_ulp_msix_num(bp); 11907 stats += bnxt_get_ulp_stat_ctxs(bp); 11908 } 11909 return bnxt_hwrm_check_rings(bp, tx_rings_needed, rx_rings, rx, cp, 11910 stats, vnics); 11911 } 11912 11913 static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev) 11914 { 11915 if (bp->bar2) { 11916 pci_iounmap(pdev, bp->bar2); 11917 bp->bar2 = NULL; 11918 } 11919 11920 if (bp->bar1) { 11921 pci_iounmap(pdev, bp->bar1); 11922 bp->bar1 = NULL; 11923 } 11924 11925 if (bp->bar0) { 11926 pci_iounmap(pdev, bp->bar0); 11927 bp->bar0 = NULL; 11928 } 11929 } 11930 11931 static void bnxt_cleanup_pci(struct bnxt *bp) 11932 { 11933 bnxt_unmap_bars(bp, bp->pdev); 11934 pci_release_regions(bp->pdev); 11935 if (pci_is_enabled(bp->pdev)) 11936 pci_disable_device(bp->pdev); 11937 } 11938 11939 static void bnxt_init_dflt_coal(struct bnxt *bp) 11940 { 11941 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 11942 struct bnxt_coal *coal; 11943 u16 flags = 0; 11944 11945 if (coal_cap->cmpl_params & 11946 RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_TIMER_RESET) 11947 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET; 11948 11949 /* Tick values in micro seconds. 11950 * 1 coal_buf x bufs_per_record = 1 completion record. 11951 */ 11952 coal = &bp->rx_coal; 11953 coal->coal_ticks = 10; 11954 coal->coal_bufs = 30; 11955 coal->coal_ticks_irq = 1; 11956 coal->coal_bufs_irq = 2; 11957 coal->idle_thresh = 50; 11958 coal->bufs_per_record = 2; 11959 coal->budget = 64; /* NAPI budget */ 11960 coal->flags = flags; 11961 11962 coal = &bp->tx_coal; 11963 coal->coal_ticks = 28; 11964 coal->coal_bufs = 30; 11965 coal->coal_ticks_irq = 2; 11966 coal->coal_bufs_irq = 2; 11967 coal->bufs_per_record = 1; 11968 coal->flags = flags; 11969 11970 bp->stats_coal_ticks = BNXT_DEF_STATS_COAL_TICKS; 11971 } 11972 11973 static int bnxt_fw_init_one_p1(struct bnxt *bp) 11974 { 11975 int rc; 11976 11977 bp->fw_cap = 0; 11978 rc = bnxt_hwrm_ver_get(bp); 11979 bnxt_try_map_fw_health_reg(bp); 11980 if (rc) { 11981 rc = bnxt_try_recover_fw(bp); 11982 if (rc) 11983 return rc; 11984 rc = bnxt_hwrm_ver_get(bp); 11985 if (rc) 11986 return rc; 11987 } 11988 11989 bnxt_nvm_cfg_ver_get(bp); 11990 11991 rc = bnxt_hwrm_func_reset(bp); 11992 if (rc) 11993 return -ENODEV; 11994 11995 bnxt_hwrm_fw_set_time(bp); 11996 return 0; 11997 } 11998 11999 static int bnxt_fw_init_one_p2(struct bnxt *bp) 12000 { 12001 int rc; 12002 12003 /* Get the MAX capabilities for this function */ 12004 rc = bnxt_hwrm_func_qcaps(bp); 12005 if (rc) { 12006 netdev_err(bp->dev, "hwrm query capability failure rc: %x\n", 12007 rc); 12008 return -ENODEV; 12009 } 12010 12011 rc = bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(bp); 12012 if (rc) 12013 netdev_warn(bp->dev, "hwrm query adv flow mgnt failure rc: %d\n", 12014 rc); 12015 12016 if (bnxt_alloc_fw_health(bp)) { 12017 netdev_warn(bp->dev, "no memory for firmware error recovery\n"); 12018 } else { 12019 rc = bnxt_hwrm_error_recovery_qcfg(bp); 12020 if (rc) 12021 netdev_warn(bp->dev, "hwrm query error recovery failure rc: %d\n", 12022 rc); 12023 } 12024 12025 rc = bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false); 12026 if (rc) 12027 return -ENODEV; 12028 12029 bnxt_hwrm_func_qcfg(bp); 12030 bnxt_hwrm_vnic_qcaps(bp); 12031 bnxt_hwrm_port_led_qcaps(bp); 12032 bnxt_ethtool_init(bp); 12033 bnxt_dcb_init(bp); 12034 return 0; 12035 } 12036 12037 static void bnxt_set_dflt_rss_hash_type(struct bnxt *bp) 12038 { 12039 bp->flags &= ~BNXT_FLAG_UDP_RSS_CAP; 12040 bp->rss_hash_cfg = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 | 12041 VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 | 12042 VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 | 12043 VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6; 12044 if (BNXT_CHIP_P4_PLUS(bp) && bp->hwrm_spec_code >= 0x10501) { 12045 bp->flags |= BNXT_FLAG_UDP_RSS_CAP; 12046 bp->rss_hash_cfg |= VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 | 12047 VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6; 12048 } 12049 } 12050 12051 static void bnxt_set_dflt_rfs(struct bnxt *bp) 12052 { 12053 struct net_device *dev = bp->dev; 12054 12055 dev->hw_features &= ~NETIF_F_NTUPLE; 12056 dev->features &= ~NETIF_F_NTUPLE; 12057 bp->flags &= ~BNXT_FLAG_RFS; 12058 if (bnxt_rfs_supported(bp)) { 12059 dev->hw_features |= NETIF_F_NTUPLE; 12060 if (bnxt_rfs_capable(bp)) { 12061 bp->flags |= BNXT_FLAG_RFS; 12062 dev->features |= NETIF_F_NTUPLE; 12063 } 12064 } 12065 } 12066 12067 static void bnxt_fw_init_one_p3(struct bnxt *bp) 12068 { 12069 struct pci_dev *pdev = bp->pdev; 12070 12071 bnxt_set_dflt_rss_hash_type(bp); 12072 bnxt_set_dflt_rfs(bp); 12073 12074 bnxt_get_wol_settings(bp); 12075 if (bp->flags & BNXT_FLAG_WOL_CAP) 12076 device_set_wakeup_enable(&pdev->dev, bp->wol); 12077 else 12078 device_set_wakeup_capable(&pdev->dev, false); 12079 12080 bnxt_hwrm_set_cache_line_size(bp, cache_line_size()); 12081 bnxt_hwrm_coal_params_qcaps(bp); 12082 } 12083 12084 static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt); 12085 12086 int bnxt_fw_init_one(struct bnxt *bp) 12087 { 12088 int rc; 12089 12090 rc = bnxt_fw_init_one_p1(bp); 12091 if (rc) { 12092 netdev_err(bp->dev, "Firmware init phase 1 failed\n"); 12093 return rc; 12094 } 12095 rc = bnxt_fw_init_one_p2(bp); 12096 if (rc) { 12097 netdev_err(bp->dev, "Firmware init phase 2 failed\n"); 12098 return rc; 12099 } 12100 rc = bnxt_probe_phy(bp, false); 12101 if (rc) 12102 return rc; 12103 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, false); 12104 if (rc) 12105 return rc; 12106 12107 /* In case fw capabilities have changed, destroy the unneeded 12108 * reporters and create newly capable ones. 12109 */ 12110 bnxt_dl_fw_reporters_destroy(bp, false); 12111 bnxt_dl_fw_reporters_create(bp); 12112 bnxt_fw_init_one_p3(bp); 12113 return 0; 12114 } 12115 12116 static void bnxt_fw_reset_writel(struct bnxt *bp, int reg_idx) 12117 { 12118 struct bnxt_fw_health *fw_health = bp->fw_health; 12119 u32 reg = fw_health->fw_reset_seq_regs[reg_idx]; 12120 u32 val = fw_health->fw_reset_seq_vals[reg_idx]; 12121 u32 reg_type, reg_off, delay_msecs; 12122 12123 delay_msecs = fw_health->fw_reset_seq_delay_msec[reg_idx]; 12124 reg_type = BNXT_FW_HEALTH_REG_TYPE(reg); 12125 reg_off = BNXT_FW_HEALTH_REG_OFF(reg); 12126 switch (reg_type) { 12127 case BNXT_FW_HEALTH_REG_TYPE_CFG: 12128 pci_write_config_dword(bp->pdev, reg_off, val); 12129 break; 12130 case BNXT_FW_HEALTH_REG_TYPE_GRC: 12131 writel(reg_off & BNXT_GRC_BASE_MASK, 12132 bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4); 12133 reg_off = (reg_off & BNXT_GRC_OFFSET_MASK) + 0x2000; 12134 fallthrough; 12135 case BNXT_FW_HEALTH_REG_TYPE_BAR0: 12136 writel(val, bp->bar0 + reg_off); 12137 break; 12138 case BNXT_FW_HEALTH_REG_TYPE_BAR1: 12139 writel(val, bp->bar1 + reg_off); 12140 break; 12141 } 12142 if (delay_msecs) { 12143 pci_read_config_dword(bp->pdev, 0, &val); 12144 msleep(delay_msecs); 12145 } 12146 } 12147 12148 bool bnxt_hwrm_reset_permitted(struct bnxt *bp) 12149 { 12150 struct hwrm_func_qcfg_output *resp; 12151 struct hwrm_func_qcfg_input *req; 12152 bool result = true; /* firmware will enforce if unknown */ 12153 12154 if (~bp->fw_cap & BNXT_FW_CAP_HOT_RESET_IF) 12155 return result; 12156 12157 if (hwrm_req_init(bp, req, HWRM_FUNC_QCFG)) 12158 return result; 12159 12160 req->fid = cpu_to_le16(0xffff); 12161 resp = hwrm_req_hold(bp, req); 12162 if (!hwrm_req_send(bp, req)) 12163 result = !!(le16_to_cpu(resp->flags) & 12164 FUNC_QCFG_RESP_FLAGS_HOT_RESET_ALLOWED); 12165 hwrm_req_drop(bp, req); 12166 return result; 12167 } 12168 12169 static void bnxt_reset_all(struct bnxt *bp) 12170 { 12171 struct bnxt_fw_health *fw_health = bp->fw_health; 12172 int i, rc; 12173 12174 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) { 12175 bnxt_fw_reset_via_optee(bp); 12176 bp->fw_reset_timestamp = jiffies; 12177 return; 12178 } 12179 12180 if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_HOST) { 12181 for (i = 0; i < fw_health->fw_reset_seq_cnt; i++) 12182 bnxt_fw_reset_writel(bp, i); 12183 } else if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) { 12184 struct hwrm_fw_reset_input *req; 12185 12186 rc = hwrm_req_init(bp, req, HWRM_FW_RESET); 12187 if (!rc) { 12188 req->target_id = cpu_to_le16(HWRM_TARGET_ID_KONG); 12189 req->embedded_proc_type = FW_RESET_REQ_EMBEDDED_PROC_TYPE_CHIP; 12190 req->selfrst_status = FW_RESET_REQ_SELFRST_STATUS_SELFRSTASAP; 12191 req->flags = FW_RESET_REQ_FLAGS_RESET_GRACEFUL; 12192 rc = hwrm_req_send(bp, req); 12193 } 12194 if (rc != -ENODEV) 12195 netdev_warn(bp->dev, "Unable to reset FW rc=%d\n", rc); 12196 } 12197 bp->fw_reset_timestamp = jiffies; 12198 } 12199 12200 static bool bnxt_fw_reset_timeout(struct bnxt *bp) 12201 { 12202 return time_after(jiffies, bp->fw_reset_timestamp + 12203 (bp->fw_reset_max_dsecs * HZ / 10)); 12204 } 12205 12206 static void bnxt_fw_reset_abort(struct bnxt *bp, int rc) 12207 { 12208 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 12209 if (bp->fw_reset_state != BNXT_FW_RESET_STATE_POLL_VF) { 12210 bnxt_ulp_start(bp, rc); 12211 bnxt_dl_health_fw_status_update(bp, false); 12212 } 12213 bp->fw_reset_state = 0; 12214 dev_close(bp->dev); 12215 } 12216 12217 static void bnxt_fw_reset_task(struct work_struct *work) 12218 { 12219 struct bnxt *bp = container_of(work, struct bnxt, fw_reset_task.work); 12220 int rc = 0; 12221 12222 if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) { 12223 netdev_err(bp->dev, "bnxt_fw_reset_task() called when not in fw reset mode!\n"); 12224 return; 12225 } 12226 12227 switch (bp->fw_reset_state) { 12228 case BNXT_FW_RESET_STATE_POLL_VF: { 12229 int n = bnxt_get_registered_vfs(bp); 12230 int tmo; 12231 12232 if (n < 0) { 12233 netdev_err(bp->dev, "Firmware reset aborted, subsequent func_qcfg cmd failed, rc = %d, %d msecs since reset timestamp\n", 12234 n, jiffies_to_msecs(jiffies - 12235 bp->fw_reset_timestamp)); 12236 goto fw_reset_abort; 12237 } else if (n > 0) { 12238 if (bnxt_fw_reset_timeout(bp)) { 12239 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 12240 bp->fw_reset_state = 0; 12241 netdev_err(bp->dev, "Firmware reset aborted, bnxt_get_registered_vfs() returns %d\n", 12242 n); 12243 return; 12244 } 12245 bnxt_queue_fw_reset_work(bp, HZ / 10); 12246 return; 12247 } 12248 bp->fw_reset_timestamp = jiffies; 12249 rtnl_lock(); 12250 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) { 12251 bnxt_fw_reset_abort(bp, rc); 12252 rtnl_unlock(); 12253 return; 12254 } 12255 bnxt_fw_reset_close(bp); 12256 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) { 12257 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN; 12258 tmo = HZ / 10; 12259 } else { 12260 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 12261 tmo = bp->fw_reset_min_dsecs * HZ / 10; 12262 } 12263 rtnl_unlock(); 12264 bnxt_queue_fw_reset_work(bp, tmo); 12265 return; 12266 } 12267 case BNXT_FW_RESET_STATE_POLL_FW_DOWN: { 12268 u32 val; 12269 12270 val = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG); 12271 if (!(val & BNXT_FW_STATUS_SHUTDOWN) && 12272 !bnxt_fw_reset_timeout(bp)) { 12273 bnxt_queue_fw_reset_work(bp, HZ / 5); 12274 return; 12275 } 12276 12277 if (!bp->fw_health->primary) { 12278 u32 wait_dsecs = bp->fw_health->normal_func_wait_dsecs; 12279 12280 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 12281 bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10); 12282 return; 12283 } 12284 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW; 12285 } 12286 fallthrough; 12287 case BNXT_FW_RESET_STATE_RESET_FW: 12288 bnxt_reset_all(bp); 12289 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 12290 bnxt_queue_fw_reset_work(bp, bp->fw_reset_min_dsecs * HZ / 10); 12291 return; 12292 case BNXT_FW_RESET_STATE_ENABLE_DEV: 12293 bnxt_inv_fw_health_reg(bp); 12294 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state) && 12295 !bp->fw_reset_min_dsecs) { 12296 u16 val; 12297 12298 pci_read_config_word(bp->pdev, PCI_SUBSYSTEM_ID, &val); 12299 if (val == 0xffff) { 12300 if (bnxt_fw_reset_timeout(bp)) { 12301 netdev_err(bp->dev, "Firmware reset aborted, PCI config space invalid\n"); 12302 rc = -ETIMEDOUT; 12303 goto fw_reset_abort; 12304 } 12305 bnxt_queue_fw_reset_work(bp, HZ / 1000); 12306 return; 12307 } 12308 } 12309 clear_bit(BNXT_STATE_FW_FATAL_COND, &bp->state); 12310 clear_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state); 12311 if (test_and_clear_bit(BNXT_STATE_FW_ACTIVATE_RESET, &bp->state) && 12312 !test_bit(BNXT_STATE_FW_ACTIVATE, &bp->state)) 12313 bnxt_dl_remote_reload(bp); 12314 if (pci_enable_device(bp->pdev)) { 12315 netdev_err(bp->dev, "Cannot re-enable PCI device\n"); 12316 rc = -ENODEV; 12317 goto fw_reset_abort; 12318 } 12319 pci_set_master(bp->pdev); 12320 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW; 12321 fallthrough; 12322 case BNXT_FW_RESET_STATE_POLL_FW: 12323 bp->hwrm_cmd_timeout = SHORT_HWRM_CMD_TIMEOUT; 12324 rc = bnxt_hwrm_poll(bp); 12325 if (rc) { 12326 if (bnxt_fw_reset_timeout(bp)) { 12327 netdev_err(bp->dev, "Firmware reset aborted\n"); 12328 goto fw_reset_abort_status; 12329 } 12330 bnxt_queue_fw_reset_work(bp, HZ / 5); 12331 return; 12332 } 12333 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT; 12334 bp->fw_reset_state = BNXT_FW_RESET_STATE_OPENING; 12335 fallthrough; 12336 case BNXT_FW_RESET_STATE_OPENING: 12337 while (!rtnl_trylock()) { 12338 bnxt_queue_fw_reset_work(bp, HZ / 10); 12339 return; 12340 } 12341 rc = bnxt_open(bp->dev); 12342 if (rc) { 12343 netdev_err(bp->dev, "bnxt_open() failed during FW reset\n"); 12344 bnxt_fw_reset_abort(bp, rc); 12345 rtnl_unlock(); 12346 return; 12347 } 12348 12349 if ((bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) && 12350 bp->fw_health->enabled) { 12351 bp->fw_health->last_fw_reset_cnt = 12352 bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG); 12353 } 12354 bp->fw_reset_state = 0; 12355 /* Make sure fw_reset_state is 0 before clearing the flag */ 12356 smp_mb__before_atomic(); 12357 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 12358 bnxt_ulp_start(bp, 0); 12359 bnxt_reenable_sriov(bp); 12360 bnxt_vf_reps_alloc(bp); 12361 bnxt_vf_reps_open(bp); 12362 bnxt_ptp_reapply_pps(bp); 12363 clear_bit(BNXT_STATE_FW_ACTIVATE, &bp->state); 12364 if (test_and_clear_bit(BNXT_STATE_RECOVER, &bp->state)) { 12365 bnxt_dl_health_fw_recovery_done(bp); 12366 bnxt_dl_health_fw_status_update(bp, true); 12367 } 12368 rtnl_unlock(); 12369 break; 12370 } 12371 return; 12372 12373 fw_reset_abort_status: 12374 if (bp->fw_health->status_reliable || 12375 (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) { 12376 u32 sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG); 12377 12378 netdev_err(bp->dev, "fw_health_status 0x%x\n", sts); 12379 } 12380 fw_reset_abort: 12381 rtnl_lock(); 12382 bnxt_fw_reset_abort(bp, rc); 12383 rtnl_unlock(); 12384 } 12385 12386 static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev) 12387 { 12388 int rc; 12389 struct bnxt *bp = netdev_priv(dev); 12390 12391 SET_NETDEV_DEV(dev, &pdev->dev); 12392 12393 /* enable device (incl. PCI PM wakeup), and bus-mastering */ 12394 rc = pci_enable_device(pdev); 12395 if (rc) { 12396 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n"); 12397 goto init_err; 12398 } 12399 12400 if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) { 12401 dev_err(&pdev->dev, 12402 "Cannot find PCI device base address, aborting\n"); 12403 rc = -ENODEV; 12404 goto init_err_disable; 12405 } 12406 12407 rc = pci_request_regions(pdev, DRV_MODULE_NAME); 12408 if (rc) { 12409 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n"); 12410 goto init_err_disable; 12411 } 12412 12413 if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 && 12414 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) { 12415 dev_err(&pdev->dev, "System does not support DMA, aborting\n"); 12416 rc = -EIO; 12417 goto init_err_release; 12418 } 12419 12420 pci_set_master(pdev); 12421 12422 bp->dev = dev; 12423 bp->pdev = pdev; 12424 12425 /* Doorbell BAR bp->bar1 is mapped after bnxt_fw_init_one_p2() 12426 * determines the BAR size. 12427 */ 12428 bp->bar0 = pci_ioremap_bar(pdev, 0); 12429 if (!bp->bar0) { 12430 dev_err(&pdev->dev, "Cannot map device registers, aborting\n"); 12431 rc = -ENOMEM; 12432 goto init_err_release; 12433 } 12434 12435 bp->bar2 = pci_ioremap_bar(pdev, 4); 12436 if (!bp->bar2) { 12437 dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n"); 12438 rc = -ENOMEM; 12439 goto init_err_release; 12440 } 12441 12442 pci_enable_pcie_error_reporting(pdev); 12443 12444 INIT_WORK(&bp->sp_task, bnxt_sp_task); 12445 INIT_DELAYED_WORK(&bp->fw_reset_task, bnxt_fw_reset_task); 12446 12447 spin_lock_init(&bp->ntp_fltr_lock); 12448 #if BITS_PER_LONG == 32 12449 spin_lock_init(&bp->db_lock); 12450 #endif 12451 12452 bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE; 12453 bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE; 12454 12455 timer_setup(&bp->timer, bnxt_timer, 0); 12456 bp->current_interval = BNXT_TIMER_INTERVAL; 12457 12458 bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID; 12459 bp->nge_fw_dst_port_id = INVALID_HW_RING_ID; 12460 12461 clear_bit(BNXT_STATE_OPEN, &bp->state); 12462 return 0; 12463 12464 init_err_release: 12465 bnxt_unmap_bars(bp, pdev); 12466 pci_release_regions(pdev); 12467 12468 init_err_disable: 12469 pci_disable_device(pdev); 12470 12471 init_err: 12472 return rc; 12473 } 12474 12475 /* rtnl_lock held */ 12476 static int bnxt_change_mac_addr(struct net_device *dev, void *p) 12477 { 12478 struct sockaddr *addr = p; 12479 struct bnxt *bp = netdev_priv(dev); 12480 int rc = 0; 12481 12482 if (!is_valid_ether_addr(addr->sa_data)) 12483 return -EADDRNOTAVAIL; 12484 12485 if (ether_addr_equal(addr->sa_data, dev->dev_addr)) 12486 return 0; 12487 12488 rc = bnxt_approve_mac(bp, addr->sa_data, true); 12489 if (rc) 12490 return rc; 12491 12492 eth_hw_addr_set(dev, addr->sa_data); 12493 if (netif_running(dev)) { 12494 bnxt_close_nic(bp, false, false); 12495 rc = bnxt_open_nic(bp, false, false); 12496 } 12497 12498 return rc; 12499 } 12500 12501 /* rtnl_lock held */ 12502 static int bnxt_change_mtu(struct net_device *dev, int new_mtu) 12503 { 12504 struct bnxt *bp = netdev_priv(dev); 12505 12506 if (netif_running(dev)) 12507 bnxt_close_nic(bp, true, false); 12508 12509 dev->mtu = new_mtu; 12510 bnxt_set_ring_params(bp); 12511 12512 if (netif_running(dev)) 12513 return bnxt_open_nic(bp, true, false); 12514 12515 return 0; 12516 } 12517 12518 int bnxt_setup_mq_tc(struct net_device *dev, u8 tc) 12519 { 12520 struct bnxt *bp = netdev_priv(dev); 12521 bool sh = false; 12522 int rc; 12523 12524 if (tc > bp->max_tc) { 12525 netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n", 12526 tc, bp->max_tc); 12527 return -EINVAL; 12528 } 12529 12530 if (netdev_get_num_tc(dev) == tc) 12531 return 0; 12532 12533 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 12534 sh = true; 12535 12536 rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings, 12537 sh, tc, bp->tx_nr_rings_xdp); 12538 if (rc) 12539 return rc; 12540 12541 /* Needs to close the device and do hw resource re-allocations */ 12542 if (netif_running(bp->dev)) 12543 bnxt_close_nic(bp, true, false); 12544 12545 if (tc) { 12546 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc; 12547 netdev_set_num_tc(dev, tc); 12548 } else { 12549 bp->tx_nr_rings = bp->tx_nr_rings_per_tc; 12550 netdev_reset_tc(dev); 12551 } 12552 bp->tx_nr_rings += bp->tx_nr_rings_xdp; 12553 bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) : 12554 bp->tx_nr_rings + bp->rx_nr_rings; 12555 12556 if (netif_running(bp->dev)) 12557 return bnxt_open_nic(bp, true, false); 12558 12559 return 0; 12560 } 12561 12562 static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 12563 void *cb_priv) 12564 { 12565 struct bnxt *bp = cb_priv; 12566 12567 if (!bnxt_tc_flower_enabled(bp) || 12568 !tc_cls_can_offload_and_chain0(bp->dev, type_data)) 12569 return -EOPNOTSUPP; 12570 12571 switch (type) { 12572 case TC_SETUP_CLSFLOWER: 12573 return bnxt_tc_setup_flower(bp, bp->pf.fw_fid, type_data); 12574 default: 12575 return -EOPNOTSUPP; 12576 } 12577 } 12578 12579 LIST_HEAD(bnxt_block_cb_list); 12580 12581 static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type, 12582 void *type_data) 12583 { 12584 struct bnxt *bp = netdev_priv(dev); 12585 12586 switch (type) { 12587 case TC_SETUP_BLOCK: 12588 return flow_block_cb_setup_simple(type_data, 12589 &bnxt_block_cb_list, 12590 bnxt_setup_tc_block_cb, 12591 bp, bp, true); 12592 case TC_SETUP_QDISC_MQPRIO: { 12593 struct tc_mqprio_qopt *mqprio = type_data; 12594 12595 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 12596 12597 return bnxt_setup_mq_tc(dev, mqprio->num_tc); 12598 } 12599 default: 12600 return -EOPNOTSUPP; 12601 } 12602 } 12603 12604 #ifdef CONFIG_RFS_ACCEL 12605 static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1, 12606 struct bnxt_ntuple_filter *f2) 12607 { 12608 struct flow_keys *keys1 = &f1->fkeys; 12609 struct flow_keys *keys2 = &f2->fkeys; 12610 12611 if (keys1->basic.n_proto != keys2->basic.n_proto || 12612 keys1->basic.ip_proto != keys2->basic.ip_proto) 12613 return false; 12614 12615 if (keys1->basic.n_proto == htons(ETH_P_IP)) { 12616 if (keys1->addrs.v4addrs.src != keys2->addrs.v4addrs.src || 12617 keys1->addrs.v4addrs.dst != keys2->addrs.v4addrs.dst) 12618 return false; 12619 } else { 12620 if (memcmp(&keys1->addrs.v6addrs.src, &keys2->addrs.v6addrs.src, 12621 sizeof(keys1->addrs.v6addrs.src)) || 12622 memcmp(&keys1->addrs.v6addrs.dst, &keys2->addrs.v6addrs.dst, 12623 sizeof(keys1->addrs.v6addrs.dst))) 12624 return false; 12625 } 12626 12627 if (keys1->ports.ports == keys2->ports.ports && 12628 keys1->control.flags == keys2->control.flags && 12629 ether_addr_equal(f1->src_mac_addr, f2->src_mac_addr) && 12630 ether_addr_equal(f1->dst_mac_addr, f2->dst_mac_addr)) 12631 return true; 12632 12633 return false; 12634 } 12635 12636 static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb, 12637 u16 rxq_index, u32 flow_id) 12638 { 12639 struct bnxt *bp = netdev_priv(dev); 12640 struct bnxt_ntuple_filter *fltr, *new_fltr; 12641 struct flow_keys *fkeys; 12642 struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb); 12643 int rc = 0, idx, bit_id, l2_idx = 0; 12644 struct hlist_head *head; 12645 u32 flags; 12646 12647 if (!ether_addr_equal(dev->dev_addr, eth->h_dest)) { 12648 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 12649 int off = 0, j; 12650 12651 netif_addr_lock_bh(dev); 12652 for (j = 0; j < vnic->uc_filter_count; j++, off += ETH_ALEN) { 12653 if (ether_addr_equal(eth->h_dest, 12654 vnic->uc_list + off)) { 12655 l2_idx = j + 1; 12656 break; 12657 } 12658 } 12659 netif_addr_unlock_bh(dev); 12660 if (!l2_idx) 12661 return -EINVAL; 12662 } 12663 new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC); 12664 if (!new_fltr) 12665 return -ENOMEM; 12666 12667 fkeys = &new_fltr->fkeys; 12668 if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) { 12669 rc = -EPROTONOSUPPORT; 12670 goto err_free; 12671 } 12672 12673 if ((fkeys->basic.n_proto != htons(ETH_P_IP) && 12674 fkeys->basic.n_proto != htons(ETH_P_IPV6)) || 12675 ((fkeys->basic.ip_proto != IPPROTO_TCP) && 12676 (fkeys->basic.ip_proto != IPPROTO_UDP))) { 12677 rc = -EPROTONOSUPPORT; 12678 goto err_free; 12679 } 12680 if (fkeys->basic.n_proto == htons(ETH_P_IPV6) && 12681 bp->hwrm_spec_code < 0x10601) { 12682 rc = -EPROTONOSUPPORT; 12683 goto err_free; 12684 } 12685 flags = fkeys->control.flags; 12686 if (((flags & FLOW_DIS_ENCAPSULATION) && 12687 bp->hwrm_spec_code < 0x10601) || (flags & FLOW_DIS_IS_FRAGMENT)) { 12688 rc = -EPROTONOSUPPORT; 12689 goto err_free; 12690 } 12691 12692 memcpy(new_fltr->dst_mac_addr, eth->h_dest, ETH_ALEN); 12693 memcpy(new_fltr->src_mac_addr, eth->h_source, ETH_ALEN); 12694 12695 idx = skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK; 12696 head = &bp->ntp_fltr_hash_tbl[idx]; 12697 rcu_read_lock(); 12698 hlist_for_each_entry_rcu(fltr, head, hash) { 12699 if (bnxt_fltr_match(fltr, new_fltr)) { 12700 rcu_read_unlock(); 12701 rc = 0; 12702 goto err_free; 12703 } 12704 } 12705 rcu_read_unlock(); 12706 12707 spin_lock_bh(&bp->ntp_fltr_lock); 12708 bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap, 12709 BNXT_NTP_FLTR_MAX_FLTR, 0); 12710 if (bit_id < 0) { 12711 spin_unlock_bh(&bp->ntp_fltr_lock); 12712 rc = -ENOMEM; 12713 goto err_free; 12714 } 12715 12716 new_fltr->sw_id = (u16)bit_id; 12717 new_fltr->flow_id = flow_id; 12718 new_fltr->l2_fltr_idx = l2_idx; 12719 new_fltr->rxq = rxq_index; 12720 hlist_add_head_rcu(&new_fltr->hash, head); 12721 bp->ntp_fltr_count++; 12722 spin_unlock_bh(&bp->ntp_fltr_lock); 12723 12724 set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event); 12725 bnxt_queue_sp_work(bp); 12726 12727 return new_fltr->sw_id; 12728 12729 err_free: 12730 kfree(new_fltr); 12731 return rc; 12732 } 12733 12734 static void bnxt_cfg_ntp_filters(struct bnxt *bp) 12735 { 12736 int i; 12737 12738 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) { 12739 struct hlist_head *head; 12740 struct hlist_node *tmp; 12741 struct bnxt_ntuple_filter *fltr; 12742 int rc; 12743 12744 head = &bp->ntp_fltr_hash_tbl[i]; 12745 hlist_for_each_entry_safe(fltr, tmp, head, hash) { 12746 bool del = false; 12747 12748 if (test_bit(BNXT_FLTR_VALID, &fltr->state)) { 12749 if (rps_may_expire_flow(bp->dev, fltr->rxq, 12750 fltr->flow_id, 12751 fltr->sw_id)) { 12752 bnxt_hwrm_cfa_ntuple_filter_free(bp, 12753 fltr); 12754 del = true; 12755 } 12756 } else { 12757 rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp, 12758 fltr); 12759 if (rc) 12760 del = true; 12761 else 12762 set_bit(BNXT_FLTR_VALID, &fltr->state); 12763 } 12764 12765 if (del) { 12766 spin_lock_bh(&bp->ntp_fltr_lock); 12767 hlist_del_rcu(&fltr->hash); 12768 bp->ntp_fltr_count--; 12769 spin_unlock_bh(&bp->ntp_fltr_lock); 12770 synchronize_rcu(); 12771 clear_bit(fltr->sw_id, bp->ntp_fltr_bmap); 12772 kfree(fltr); 12773 } 12774 } 12775 } 12776 if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event)) 12777 netdev_info(bp->dev, "Receive PF driver unload event!\n"); 12778 } 12779 12780 #else 12781 12782 static void bnxt_cfg_ntp_filters(struct bnxt *bp) 12783 { 12784 } 12785 12786 #endif /* CONFIG_RFS_ACCEL */ 12787 12788 static int bnxt_udp_tunnel_sync(struct net_device *netdev, unsigned int table) 12789 { 12790 struct bnxt *bp = netdev_priv(netdev); 12791 struct udp_tunnel_info ti; 12792 unsigned int cmd; 12793 12794 udp_tunnel_nic_get_port(netdev, table, 0, &ti); 12795 if (ti.type == UDP_TUNNEL_TYPE_VXLAN) 12796 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN; 12797 else 12798 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE; 12799 12800 if (ti.port) 12801 return bnxt_hwrm_tunnel_dst_port_alloc(bp, ti.port, cmd); 12802 12803 return bnxt_hwrm_tunnel_dst_port_free(bp, cmd); 12804 } 12805 12806 static const struct udp_tunnel_nic_info bnxt_udp_tunnels = { 12807 .sync_table = bnxt_udp_tunnel_sync, 12808 .flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP | 12809 UDP_TUNNEL_NIC_INFO_OPEN_ONLY, 12810 .tables = { 12811 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN, }, 12812 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_GENEVE, }, 12813 }, 12814 }; 12815 12816 static int bnxt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 12817 struct net_device *dev, u32 filter_mask, 12818 int nlflags) 12819 { 12820 struct bnxt *bp = netdev_priv(dev); 12821 12822 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bp->br_mode, 0, 0, 12823 nlflags, filter_mask, NULL); 12824 } 12825 12826 static int bnxt_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh, 12827 u16 flags, struct netlink_ext_ack *extack) 12828 { 12829 struct bnxt *bp = netdev_priv(dev); 12830 struct nlattr *attr, *br_spec; 12831 int rem, rc = 0; 12832 12833 if (bp->hwrm_spec_code < 0x10708 || !BNXT_SINGLE_PF(bp)) 12834 return -EOPNOTSUPP; 12835 12836 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 12837 if (!br_spec) 12838 return -EINVAL; 12839 12840 nla_for_each_nested(attr, br_spec, rem) { 12841 u16 mode; 12842 12843 if (nla_type(attr) != IFLA_BRIDGE_MODE) 12844 continue; 12845 12846 if (nla_len(attr) < sizeof(mode)) 12847 return -EINVAL; 12848 12849 mode = nla_get_u16(attr); 12850 if (mode == bp->br_mode) 12851 break; 12852 12853 rc = bnxt_hwrm_set_br_mode(bp, mode); 12854 if (!rc) 12855 bp->br_mode = mode; 12856 break; 12857 } 12858 return rc; 12859 } 12860 12861 int bnxt_get_port_parent_id(struct net_device *dev, 12862 struct netdev_phys_item_id *ppid) 12863 { 12864 struct bnxt *bp = netdev_priv(dev); 12865 12866 if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV) 12867 return -EOPNOTSUPP; 12868 12869 /* The PF and it's VF-reps only support the switchdev framework */ 12870 if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_DSN_VALID)) 12871 return -EOPNOTSUPP; 12872 12873 ppid->id_len = sizeof(bp->dsn); 12874 memcpy(ppid->id, bp->dsn, ppid->id_len); 12875 12876 return 0; 12877 } 12878 12879 static struct devlink_port *bnxt_get_devlink_port(struct net_device *dev) 12880 { 12881 struct bnxt *bp = netdev_priv(dev); 12882 12883 return &bp->dl_port; 12884 } 12885 12886 static const struct net_device_ops bnxt_netdev_ops = { 12887 .ndo_open = bnxt_open, 12888 .ndo_start_xmit = bnxt_start_xmit, 12889 .ndo_stop = bnxt_close, 12890 .ndo_get_stats64 = bnxt_get_stats64, 12891 .ndo_set_rx_mode = bnxt_set_rx_mode, 12892 .ndo_eth_ioctl = bnxt_ioctl, 12893 .ndo_validate_addr = eth_validate_addr, 12894 .ndo_set_mac_address = bnxt_change_mac_addr, 12895 .ndo_change_mtu = bnxt_change_mtu, 12896 .ndo_fix_features = bnxt_fix_features, 12897 .ndo_set_features = bnxt_set_features, 12898 .ndo_features_check = bnxt_features_check, 12899 .ndo_tx_timeout = bnxt_tx_timeout, 12900 #ifdef CONFIG_BNXT_SRIOV 12901 .ndo_get_vf_config = bnxt_get_vf_config, 12902 .ndo_set_vf_mac = bnxt_set_vf_mac, 12903 .ndo_set_vf_vlan = bnxt_set_vf_vlan, 12904 .ndo_set_vf_rate = bnxt_set_vf_bw, 12905 .ndo_set_vf_link_state = bnxt_set_vf_link_state, 12906 .ndo_set_vf_spoofchk = bnxt_set_vf_spoofchk, 12907 .ndo_set_vf_trust = bnxt_set_vf_trust, 12908 #endif 12909 .ndo_setup_tc = bnxt_setup_tc, 12910 #ifdef CONFIG_RFS_ACCEL 12911 .ndo_rx_flow_steer = bnxt_rx_flow_steer, 12912 #endif 12913 .ndo_bpf = bnxt_xdp, 12914 .ndo_xdp_xmit = bnxt_xdp_xmit, 12915 .ndo_bridge_getlink = bnxt_bridge_getlink, 12916 .ndo_bridge_setlink = bnxt_bridge_setlink, 12917 .ndo_get_devlink_port = bnxt_get_devlink_port, 12918 }; 12919 12920 static void bnxt_remove_one(struct pci_dev *pdev) 12921 { 12922 struct net_device *dev = pci_get_drvdata(pdev); 12923 struct bnxt *bp = netdev_priv(dev); 12924 12925 if (BNXT_PF(bp)) 12926 bnxt_sriov_disable(bp); 12927 12928 if (BNXT_PF(bp)) 12929 devlink_port_type_clear(&bp->dl_port); 12930 12931 bnxt_ptp_clear(bp); 12932 pci_disable_pcie_error_reporting(pdev); 12933 unregister_netdev(dev); 12934 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 12935 /* Flush any pending tasks */ 12936 cancel_work_sync(&bp->sp_task); 12937 cancel_delayed_work_sync(&bp->fw_reset_task); 12938 bp->sp_event = 0; 12939 12940 bnxt_dl_fw_reporters_destroy(bp, true); 12941 bnxt_dl_unregister(bp); 12942 bnxt_shutdown_tc(bp); 12943 12944 bnxt_clear_int_mode(bp); 12945 bnxt_hwrm_func_drv_unrgtr(bp); 12946 bnxt_free_hwrm_resources(bp); 12947 bnxt_ethtool_free(bp); 12948 bnxt_dcb_free(bp); 12949 kfree(bp->edev); 12950 bp->edev = NULL; 12951 kfree(bp->ptp_cfg); 12952 bp->ptp_cfg = NULL; 12953 kfree(bp->fw_health); 12954 bp->fw_health = NULL; 12955 bnxt_cleanup_pci(bp); 12956 bnxt_free_ctx_mem(bp); 12957 kfree(bp->ctx); 12958 bp->ctx = NULL; 12959 kfree(bp->rss_indir_tbl); 12960 bp->rss_indir_tbl = NULL; 12961 bnxt_free_port_stats(bp); 12962 free_netdev(dev); 12963 } 12964 12965 static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt) 12966 { 12967 int rc = 0; 12968 struct bnxt_link_info *link_info = &bp->link_info; 12969 12970 bp->phy_flags = 0; 12971 rc = bnxt_hwrm_phy_qcaps(bp); 12972 if (rc) { 12973 netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n", 12974 rc); 12975 return rc; 12976 } 12977 if (bp->phy_flags & BNXT_PHY_FL_NO_FCS) 12978 bp->dev->priv_flags |= IFF_SUPP_NOFCS; 12979 else 12980 bp->dev->priv_flags &= ~IFF_SUPP_NOFCS; 12981 if (!fw_dflt) 12982 return 0; 12983 12984 mutex_lock(&bp->link_lock); 12985 rc = bnxt_update_link(bp, false); 12986 if (rc) { 12987 mutex_unlock(&bp->link_lock); 12988 netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n", 12989 rc); 12990 return rc; 12991 } 12992 12993 /* Older firmware does not have supported_auto_speeds, so assume 12994 * that all supported speeds can be autonegotiated. 12995 */ 12996 if (link_info->auto_link_speeds && !link_info->support_auto_speeds) 12997 link_info->support_auto_speeds = link_info->support_speeds; 12998 12999 bnxt_init_ethtool_link_settings(bp); 13000 mutex_unlock(&bp->link_lock); 13001 return 0; 13002 } 13003 13004 static int bnxt_get_max_irq(struct pci_dev *pdev) 13005 { 13006 u16 ctrl; 13007 13008 if (!pdev->msix_cap) 13009 return 1; 13010 13011 pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl); 13012 return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1; 13013 } 13014 13015 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, 13016 int *max_cp) 13017 { 13018 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 13019 int max_ring_grps = 0, max_irq; 13020 13021 *max_tx = hw_resc->max_tx_rings; 13022 *max_rx = hw_resc->max_rx_rings; 13023 *max_cp = bnxt_get_max_func_cp_rings_for_en(bp); 13024 max_irq = min_t(int, bnxt_get_max_func_irqs(bp) - 13025 bnxt_get_ulp_msix_num(bp), 13026 hw_resc->max_stat_ctxs - bnxt_get_ulp_stat_ctxs(bp)); 13027 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 13028 *max_cp = min_t(int, *max_cp, max_irq); 13029 max_ring_grps = hw_resc->max_hw_ring_grps; 13030 if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) { 13031 *max_cp -= 1; 13032 *max_rx -= 2; 13033 } 13034 if (bp->flags & BNXT_FLAG_AGG_RINGS) 13035 *max_rx >>= 1; 13036 if (bp->flags & BNXT_FLAG_CHIP_P5) { 13037 bnxt_trim_rings(bp, max_rx, max_tx, *max_cp, false); 13038 /* On P5 chips, max_cp output param should be available NQs */ 13039 *max_cp = max_irq; 13040 } 13041 *max_rx = min_t(int, *max_rx, max_ring_grps); 13042 } 13043 13044 int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared) 13045 { 13046 int rx, tx, cp; 13047 13048 _bnxt_get_max_rings(bp, &rx, &tx, &cp); 13049 *max_rx = rx; 13050 *max_tx = tx; 13051 if (!rx || !tx || !cp) 13052 return -ENOMEM; 13053 13054 return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared); 13055 } 13056 13057 static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx, 13058 bool shared) 13059 { 13060 int rc; 13061 13062 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared); 13063 if (rc && (bp->flags & BNXT_FLAG_AGG_RINGS)) { 13064 /* Not enough rings, try disabling agg rings. */ 13065 bp->flags &= ~BNXT_FLAG_AGG_RINGS; 13066 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared); 13067 if (rc) { 13068 /* set BNXT_FLAG_AGG_RINGS back for consistency */ 13069 bp->flags |= BNXT_FLAG_AGG_RINGS; 13070 return rc; 13071 } 13072 bp->flags |= BNXT_FLAG_NO_AGG_RINGS; 13073 bp->dev->hw_features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); 13074 bp->dev->features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); 13075 bnxt_set_ring_params(bp); 13076 } 13077 13078 if (bp->flags & BNXT_FLAG_ROCE_CAP) { 13079 int max_cp, max_stat, max_irq; 13080 13081 /* Reserve minimum resources for RoCE */ 13082 max_cp = bnxt_get_max_func_cp_rings(bp); 13083 max_stat = bnxt_get_max_func_stat_ctxs(bp); 13084 max_irq = bnxt_get_max_func_irqs(bp); 13085 if (max_cp <= BNXT_MIN_ROCE_CP_RINGS || 13086 max_irq <= BNXT_MIN_ROCE_CP_RINGS || 13087 max_stat <= BNXT_MIN_ROCE_STAT_CTXS) 13088 return 0; 13089 13090 max_cp -= BNXT_MIN_ROCE_CP_RINGS; 13091 max_irq -= BNXT_MIN_ROCE_CP_RINGS; 13092 max_stat -= BNXT_MIN_ROCE_STAT_CTXS; 13093 max_cp = min_t(int, max_cp, max_irq); 13094 max_cp = min_t(int, max_cp, max_stat); 13095 rc = bnxt_trim_rings(bp, max_rx, max_tx, max_cp, shared); 13096 if (rc) 13097 rc = 0; 13098 } 13099 return rc; 13100 } 13101 13102 /* In initial default shared ring setting, each shared ring must have a 13103 * RX/TX ring pair. 13104 */ 13105 static void bnxt_trim_dflt_sh_rings(struct bnxt *bp) 13106 { 13107 bp->cp_nr_rings = min_t(int, bp->tx_nr_rings_per_tc, bp->rx_nr_rings); 13108 bp->rx_nr_rings = bp->cp_nr_rings; 13109 bp->tx_nr_rings_per_tc = bp->cp_nr_rings; 13110 bp->tx_nr_rings = bp->tx_nr_rings_per_tc; 13111 } 13112 13113 static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh) 13114 { 13115 int dflt_rings, max_rx_rings, max_tx_rings, rc; 13116 13117 if (!bnxt_can_reserve_rings(bp)) 13118 return 0; 13119 13120 if (sh) 13121 bp->flags |= BNXT_FLAG_SHARED_RINGS; 13122 dflt_rings = is_kdump_kernel() ? 1 : netif_get_num_default_rss_queues(); 13123 /* Reduce default rings on multi-port cards so that total default 13124 * rings do not exceed CPU count. 13125 */ 13126 if (bp->port_count > 1) { 13127 int max_rings = 13128 max_t(int, num_online_cpus() / bp->port_count, 1); 13129 13130 dflt_rings = min_t(int, dflt_rings, max_rings); 13131 } 13132 rc = bnxt_get_dflt_rings(bp, &max_rx_rings, &max_tx_rings, sh); 13133 if (rc) 13134 return rc; 13135 bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings); 13136 bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings); 13137 if (sh) 13138 bnxt_trim_dflt_sh_rings(bp); 13139 else 13140 bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings; 13141 bp->tx_nr_rings = bp->tx_nr_rings_per_tc; 13142 13143 rc = __bnxt_reserve_rings(bp); 13144 if (rc && rc != -ENODEV) 13145 netdev_warn(bp->dev, "Unable to reserve tx rings\n"); 13146 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 13147 if (sh) 13148 bnxt_trim_dflt_sh_rings(bp); 13149 13150 /* Rings may have been trimmed, re-reserve the trimmed rings. */ 13151 if (bnxt_need_reserve_rings(bp)) { 13152 rc = __bnxt_reserve_rings(bp); 13153 if (rc && rc != -ENODEV) 13154 netdev_warn(bp->dev, "2nd rings reservation failed.\n"); 13155 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 13156 } 13157 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 13158 bp->rx_nr_rings++; 13159 bp->cp_nr_rings++; 13160 } 13161 if (rc) { 13162 bp->tx_nr_rings = 0; 13163 bp->rx_nr_rings = 0; 13164 } 13165 return rc; 13166 } 13167 13168 static int bnxt_init_dflt_ring_mode(struct bnxt *bp) 13169 { 13170 int rc; 13171 13172 if (bp->tx_nr_rings) 13173 return 0; 13174 13175 bnxt_ulp_irq_stop(bp); 13176 bnxt_clear_int_mode(bp); 13177 rc = bnxt_set_dflt_rings(bp, true); 13178 if (rc) { 13179 if (BNXT_VF(bp) && rc == -ENODEV) 13180 netdev_err(bp->dev, "Cannot configure VF rings while PF is unavailable.\n"); 13181 else 13182 netdev_err(bp->dev, "Not enough rings available.\n"); 13183 goto init_dflt_ring_err; 13184 } 13185 rc = bnxt_init_int_mode(bp); 13186 if (rc) 13187 goto init_dflt_ring_err; 13188 13189 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 13190 if (bnxt_rfs_supported(bp) && bnxt_rfs_capable(bp)) { 13191 bp->flags |= BNXT_FLAG_RFS; 13192 bp->dev->features |= NETIF_F_NTUPLE; 13193 } 13194 init_dflt_ring_err: 13195 bnxt_ulp_irq_restart(bp, rc); 13196 return rc; 13197 } 13198 13199 int bnxt_restore_pf_fw_resources(struct bnxt *bp) 13200 { 13201 int rc; 13202 13203 ASSERT_RTNL(); 13204 bnxt_hwrm_func_qcaps(bp); 13205 13206 if (netif_running(bp->dev)) 13207 __bnxt_close_nic(bp, true, false); 13208 13209 bnxt_ulp_irq_stop(bp); 13210 bnxt_clear_int_mode(bp); 13211 rc = bnxt_init_int_mode(bp); 13212 bnxt_ulp_irq_restart(bp, rc); 13213 13214 if (netif_running(bp->dev)) { 13215 if (rc) 13216 dev_close(bp->dev); 13217 else 13218 rc = bnxt_open_nic(bp, true, false); 13219 } 13220 13221 return rc; 13222 } 13223 13224 static int bnxt_init_mac_addr(struct bnxt *bp) 13225 { 13226 int rc = 0; 13227 13228 if (BNXT_PF(bp)) { 13229 eth_hw_addr_set(bp->dev, bp->pf.mac_addr); 13230 } else { 13231 #ifdef CONFIG_BNXT_SRIOV 13232 struct bnxt_vf_info *vf = &bp->vf; 13233 bool strict_approval = true; 13234 13235 if (is_valid_ether_addr(vf->mac_addr)) { 13236 /* overwrite netdev dev_addr with admin VF MAC */ 13237 eth_hw_addr_set(bp->dev, vf->mac_addr); 13238 /* Older PF driver or firmware may not approve this 13239 * correctly. 13240 */ 13241 strict_approval = false; 13242 } else { 13243 eth_hw_addr_random(bp->dev); 13244 } 13245 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, strict_approval); 13246 #endif 13247 } 13248 return rc; 13249 } 13250 13251 static void bnxt_vpd_read_info(struct bnxt *bp) 13252 { 13253 struct pci_dev *pdev = bp->pdev; 13254 unsigned int vpd_size, kw_len; 13255 int pos, size; 13256 u8 *vpd_data; 13257 13258 vpd_data = pci_vpd_alloc(pdev, &vpd_size); 13259 if (IS_ERR(vpd_data)) { 13260 pci_warn(pdev, "Unable to read VPD\n"); 13261 return; 13262 } 13263 13264 pos = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size, 13265 PCI_VPD_RO_KEYWORD_PARTNO, &kw_len); 13266 if (pos < 0) 13267 goto read_sn; 13268 13269 size = min_t(int, kw_len, BNXT_VPD_FLD_LEN - 1); 13270 memcpy(bp->board_partno, &vpd_data[pos], size); 13271 13272 read_sn: 13273 pos = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size, 13274 PCI_VPD_RO_KEYWORD_SERIALNO, 13275 &kw_len); 13276 if (pos < 0) 13277 goto exit; 13278 13279 size = min_t(int, kw_len, BNXT_VPD_FLD_LEN - 1); 13280 memcpy(bp->board_serialno, &vpd_data[pos], size); 13281 exit: 13282 kfree(vpd_data); 13283 } 13284 13285 static int bnxt_pcie_dsn_get(struct bnxt *bp, u8 dsn[]) 13286 { 13287 struct pci_dev *pdev = bp->pdev; 13288 u64 qword; 13289 13290 qword = pci_get_dsn(pdev); 13291 if (!qword) { 13292 netdev_info(bp->dev, "Unable to read adapter's DSN\n"); 13293 return -EOPNOTSUPP; 13294 } 13295 13296 put_unaligned_le64(qword, dsn); 13297 13298 bp->flags |= BNXT_FLAG_DSN_VALID; 13299 return 0; 13300 } 13301 13302 static int bnxt_map_db_bar(struct bnxt *bp) 13303 { 13304 if (!bp->db_size) 13305 return -ENODEV; 13306 bp->bar1 = pci_iomap(bp->pdev, 2, bp->db_size); 13307 if (!bp->bar1) 13308 return -ENOMEM; 13309 return 0; 13310 } 13311 13312 void bnxt_print_device_info(struct bnxt *bp) 13313 { 13314 netdev_info(bp->dev, "%s found at mem %lx, node addr %pM\n", 13315 board_info[bp->board_idx].name, 13316 (long)pci_resource_start(bp->pdev, 0), bp->dev->dev_addr); 13317 13318 pcie_print_link_status(bp->pdev); 13319 } 13320 13321 static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) 13322 { 13323 struct net_device *dev; 13324 struct bnxt *bp; 13325 int rc, max_irqs; 13326 13327 if (pci_is_bridge(pdev)) 13328 return -ENODEV; 13329 13330 /* Clear any pending DMA transactions from crash kernel 13331 * while loading driver in capture kernel. 13332 */ 13333 if (is_kdump_kernel()) { 13334 pci_clear_master(pdev); 13335 pcie_flr(pdev); 13336 } 13337 13338 max_irqs = bnxt_get_max_irq(pdev); 13339 dev = alloc_etherdev_mq(sizeof(*bp), max_irqs); 13340 if (!dev) 13341 return -ENOMEM; 13342 13343 bp = netdev_priv(dev); 13344 bp->board_idx = ent->driver_data; 13345 bp->msg_enable = BNXT_DEF_MSG_ENABLE; 13346 bnxt_set_max_func_irqs(bp, max_irqs); 13347 13348 if (bnxt_vf_pciid(bp->board_idx)) 13349 bp->flags |= BNXT_FLAG_VF; 13350 13351 if (pdev->msix_cap) 13352 bp->flags |= BNXT_FLAG_MSIX_CAP; 13353 13354 rc = bnxt_init_board(pdev, dev); 13355 if (rc < 0) 13356 goto init_err_free; 13357 13358 dev->netdev_ops = &bnxt_netdev_ops; 13359 dev->watchdog_timeo = BNXT_TX_TIMEOUT; 13360 dev->ethtool_ops = &bnxt_ethtool_ops; 13361 pci_set_drvdata(pdev, dev); 13362 13363 rc = bnxt_alloc_hwrm_resources(bp); 13364 if (rc) 13365 goto init_err_pci_clean; 13366 13367 mutex_init(&bp->hwrm_cmd_lock); 13368 mutex_init(&bp->link_lock); 13369 13370 rc = bnxt_fw_init_one_p1(bp); 13371 if (rc) 13372 goto init_err_pci_clean; 13373 13374 if (BNXT_PF(bp)) 13375 bnxt_vpd_read_info(bp); 13376 13377 if (BNXT_CHIP_P5(bp)) { 13378 bp->flags |= BNXT_FLAG_CHIP_P5; 13379 if (BNXT_CHIP_SR2(bp)) 13380 bp->flags |= BNXT_FLAG_CHIP_SR2; 13381 } 13382 13383 rc = bnxt_alloc_rss_indir_tbl(bp); 13384 if (rc) 13385 goto init_err_pci_clean; 13386 13387 rc = bnxt_fw_init_one_p2(bp); 13388 if (rc) 13389 goto init_err_pci_clean; 13390 13391 rc = bnxt_map_db_bar(bp); 13392 if (rc) { 13393 dev_err(&pdev->dev, "Cannot map doorbell BAR rc = %d, aborting\n", 13394 rc); 13395 goto init_err_pci_clean; 13396 } 13397 13398 dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG | 13399 NETIF_F_TSO | NETIF_F_TSO6 | 13400 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE | 13401 NETIF_F_GSO_IPXIP4 | 13402 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM | 13403 NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH | 13404 NETIF_F_RXCSUM | NETIF_F_GRO; 13405 13406 if (BNXT_SUPPORTS_TPA(bp)) 13407 dev->hw_features |= NETIF_F_LRO; 13408 13409 dev->hw_enc_features = 13410 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG | 13411 NETIF_F_TSO | NETIF_F_TSO6 | 13412 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE | 13413 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM | 13414 NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL; 13415 dev->udp_tunnel_nic_info = &bnxt_udp_tunnels; 13416 13417 dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM | 13418 NETIF_F_GSO_GRE_CSUM; 13419 dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA; 13420 if (bp->fw_cap & BNXT_FW_CAP_VLAN_RX_STRIP) 13421 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_RX; 13422 if (bp->fw_cap & BNXT_FW_CAP_VLAN_TX_INSERT) 13423 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_TX; 13424 if (BNXT_SUPPORTS_TPA(bp)) 13425 dev->hw_features |= NETIF_F_GRO_HW; 13426 dev->features |= dev->hw_features | NETIF_F_HIGHDMA; 13427 if (dev->features & NETIF_F_GRO_HW) 13428 dev->features &= ~NETIF_F_LRO; 13429 dev->priv_flags |= IFF_UNICAST_FLT; 13430 13431 #ifdef CONFIG_BNXT_SRIOV 13432 init_waitqueue_head(&bp->sriov_cfg_wait); 13433 mutex_init(&bp->sriov_lock); 13434 #endif 13435 if (BNXT_SUPPORTS_TPA(bp)) { 13436 bp->gro_func = bnxt_gro_func_5730x; 13437 if (BNXT_CHIP_P4(bp)) 13438 bp->gro_func = bnxt_gro_func_5731x; 13439 else if (BNXT_CHIP_P5(bp)) 13440 bp->gro_func = bnxt_gro_func_5750x; 13441 } 13442 if (!BNXT_CHIP_P4_PLUS(bp)) 13443 bp->flags |= BNXT_FLAG_DOUBLE_DB; 13444 13445 rc = bnxt_init_mac_addr(bp); 13446 if (rc) { 13447 dev_err(&pdev->dev, "Unable to initialize mac address.\n"); 13448 rc = -EADDRNOTAVAIL; 13449 goto init_err_pci_clean; 13450 } 13451 13452 if (BNXT_PF(bp)) { 13453 /* Read the adapter's DSN to use as the eswitch switch_id */ 13454 rc = bnxt_pcie_dsn_get(bp, bp->dsn); 13455 } 13456 13457 /* MTU range: 60 - FW defined max */ 13458 dev->min_mtu = ETH_ZLEN; 13459 dev->max_mtu = bp->max_mtu; 13460 13461 rc = bnxt_probe_phy(bp, true); 13462 if (rc) 13463 goto init_err_pci_clean; 13464 13465 bnxt_set_rx_skb_mode(bp, false); 13466 bnxt_set_tpa_flags(bp); 13467 bnxt_set_ring_params(bp); 13468 rc = bnxt_set_dflt_rings(bp, true); 13469 if (rc) { 13470 if (BNXT_VF(bp) && rc == -ENODEV) { 13471 netdev_err(bp->dev, "Cannot configure VF rings while PF is unavailable.\n"); 13472 } else { 13473 netdev_err(bp->dev, "Not enough rings available.\n"); 13474 rc = -ENOMEM; 13475 } 13476 goto init_err_pci_clean; 13477 } 13478 13479 bnxt_fw_init_one_p3(bp); 13480 13481 bnxt_init_dflt_coal(bp); 13482 13483 if (dev->hw_features & BNXT_HW_FEATURE_VLAN_ALL_RX) 13484 bp->flags |= BNXT_FLAG_STRIP_VLAN; 13485 13486 rc = bnxt_init_int_mode(bp); 13487 if (rc) 13488 goto init_err_pci_clean; 13489 13490 /* No TC has been set yet and rings may have been trimmed due to 13491 * limited MSIX, so we re-initialize the TX rings per TC. 13492 */ 13493 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 13494 13495 if (BNXT_PF(bp)) { 13496 if (!bnxt_pf_wq) { 13497 bnxt_pf_wq = 13498 create_singlethread_workqueue("bnxt_pf_wq"); 13499 if (!bnxt_pf_wq) { 13500 dev_err(&pdev->dev, "Unable to create workqueue.\n"); 13501 rc = -ENOMEM; 13502 goto init_err_pci_clean; 13503 } 13504 } 13505 rc = bnxt_init_tc(bp); 13506 if (rc) 13507 netdev_err(dev, "Failed to initialize TC flower offload, err = %d.\n", 13508 rc); 13509 } 13510 13511 bnxt_inv_fw_health_reg(bp); 13512 rc = bnxt_dl_register(bp); 13513 if (rc) 13514 goto init_err_dl; 13515 13516 rc = register_netdev(dev); 13517 if (rc) 13518 goto init_err_cleanup; 13519 13520 if (BNXT_PF(bp)) 13521 devlink_port_type_eth_set(&bp->dl_port, bp->dev); 13522 bnxt_dl_fw_reporters_create(bp); 13523 13524 bnxt_print_device_info(bp); 13525 13526 pci_save_state(pdev); 13527 return 0; 13528 13529 init_err_cleanup: 13530 bnxt_dl_unregister(bp); 13531 init_err_dl: 13532 bnxt_shutdown_tc(bp); 13533 bnxt_clear_int_mode(bp); 13534 13535 init_err_pci_clean: 13536 bnxt_hwrm_func_drv_unrgtr(bp); 13537 bnxt_free_hwrm_resources(bp); 13538 bnxt_ethtool_free(bp); 13539 bnxt_ptp_clear(bp); 13540 kfree(bp->ptp_cfg); 13541 bp->ptp_cfg = NULL; 13542 kfree(bp->fw_health); 13543 bp->fw_health = NULL; 13544 bnxt_cleanup_pci(bp); 13545 bnxt_free_ctx_mem(bp); 13546 kfree(bp->ctx); 13547 bp->ctx = NULL; 13548 kfree(bp->rss_indir_tbl); 13549 bp->rss_indir_tbl = NULL; 13550 13551 init_err_free: 13552 free_netdev(dev); 13553 return rc; 13554 } 13555 13556 static void bnxt_shutdown(struct pci_dev *pdev) 13557 { 13558 struct net_device *dev = pci_get_drvdata(pdev); 13559 struct bnxt *bp; 13560 13561 if (!dev) 13562 return; 13563 13564 rtnl_lock(); 13565 bp = netdev_priv(dev); 13566 if (!bp) 13567 goto shutdown_exit; 13568 13569 if (netif_running(dev)) 13570 dev_close(dev); 13571 13572 bnxt_ulp_shutdown(bp); 13573 bnxt_clear_int_mode(bp); 13574 pci_disable_device(pdev); 13575 13576 if (system_state == SYSTEM_POWER_OFF) { 13577 pci_wake_from_d3(pdev, bp->wol); 13578 pci_set_power_state(pdev, PCI_D3hot); 13579 } 13580 13581 shutdown_exit: 13582 rtnl_unlock(); 13583 } 13584 13585 #ifdef CONFIG_PM_SLEEP 13586 static int bnxt_suspend(struct device *device) 13587 { 13588 struct net_device *dev = dev_get_drvdata(device); 13589 struct bnxt *bp = netdev_priv(dev); 13590 int rc = 0; 13591 13592 rtnl_lock(); 13593 bnxt_ulp_stop(bp); 13594 if (netif_running(dev)) { 13595 netif_device_detach(dev); 13596 rc = bnxt_close(dev); 13597 } 13598 bnxt_hwrm_func_drv_unrgtr(bp); 13599 pci_disable_device(bp->pdev); 13600 bnxt_free_ctx_mem(bp); 13601 kfree(bp->ctx); 13602 bp->ctx = NULL; 13603 rtnl_unlock(); 13604 return rc; 13605 } 13606 13607 static int bnxt_resume(struct device *device) 13608 { 13609 struct net_device *dev = dev_get_drvdata(device); 13610 struct bnxt *bp = netdev_priv(dev); 13611 int rc = 0; 13612 13613 rtnl_lock(); 13614 rc = pci_enable_device(bp->pdev); 13615 if (rc) { 13616 netdev_err(dev, "Cannot re-enable PCI device during resume, err = %d\n", 13617 rc); 13618 goto resume_exit; 13619 } 13620 pci_set_master(bp->pdev); 13621 if (bnxt_hwrm_ver_get(bp)) { 13622 rc = -ENODEV; 13623 goto resume_exit; 13624 } 13625 rc = bnxt_hwrm_func_reset(bp); 13626 if (rc) { 13627 rc = -EBUSY; 13628 goto resume_exit; 13629 } 13630 13631 rc = bnxt_hwrm_func_qcaps(bp); 13632 if (rc) 13633 goto resume_exit; 13634 13635 if (bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false)) { 13636 rc = -ENODEV; 13637 goto resume_exit; 13638 } 13639 13640 bnxt_get_wol_settings(bp); 13641 if (netif_running(dev)) { 13642 rc = bnxt_open(dev); 13643 if (!rc) 13644 netif_device_attach(dev); 13645 } 13646 13647 resume_exit: 13648 bnxt_ulp_start(bp, rc); 13649 if (!rc) 13650 bnxt_reenable_sriov(bp); 13651 rtnl_unlock(); 13652 return rc; 13653 } 13654 13655 static SIMPLE_DEV_PM_OPS(bnxt_pm_ops, bnxt_suspend, bnxt_resume); 13656 #define BNXT_PM_OPS (&bnxt_pm_ops) 13657 13658 #else 13659 13660 #define BNXT_PM_OPS NULL 13661 13662 #endif /* CONFIG_PM_SLEEP */ 13663 13664 /** 13665 * bnxt_io_error_detected - called when PCI error is detected 13666 * @pdev: Pointer to PCI device 13667 * @state: The current pci connection state 13668 * 13669 * This function is called after a PCI bus error affecting 13670 * this device has been detected. 13671 */ 13672 static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev, 13673 pci_channel_state_t state) 13674 { 13675 struct net_device *netdev = pci_get_drvdata(pdev); 13676 struct bnxt *bp = netdev_priv(netdev); 13677 13678 netdev_info(netdev, "PCI I/O error detected\n"); 13679 13680 rtnl_lock(); 13681 netif_device_detach(netdev); 13682 13683 bnxt_ulp_stop(bp); 13684 13685 if (state == pci_channel_io_perm_failure) { 13686 rtnl_unlock(); 13687 return PCI_ERS_RESULT_DISCONNECT; 13688 } 13689 13690 if (state == pci_channel_io_frozen) 13691 set_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, &bp->state); 13692 13693 if (netif_running(netdev)) 13694 bnxt_close(netdev); 13695 13696 if (pci_is_enabled(pdev)) 13697 pci_disable_device(pdev); 13698 bnxt_free_ctx_mem(bp); 13699 kfree(bp->ctx); 13700 bp->ctx = NULL; 13701 rtnl_unlock(); 13702 13703 /* Request a slot slot reset. */ 13704 return PCI_ERS_RESULT_NEED_RESET; 13705 } 13706 13707 /** 13708 * bnxt_io_slot_reset - called after the pci bus has been reset. 13709 * @pdev: Pointer to PCI device 13710 * 13711 * Restart the card from scratch, as if from a cold-boot. 13712 * At this point, the card has exprienced a hard reset, 13713 * followed by fixups by BIOS, and has its config space 13714 * set up identically to what it was at cold boot. 13715 */ 13716 static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev) 13717 { 13718 pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT; 13719 struct net_device *netdev = pci_get_drvdata(pdev); 13720 struct bnxt *bp = netdev_priv(netdev); 13721 int err = 0, off; 13722 13723 netdev_info(bp->dev, "PCI Slot Reset\n"); 13724 13725 rtnl_lock(); 13726 13727 if (pci_enable_device(pdev)) { 13728 dev_err(&pdev->dev, 13729 "Cannot re-enable PCI device after reset.\n"); 13730 } else { 13731 pci_set_master(pdev); 13732 /* Upon fatal error, our device internal logic that latches to 13733 * BAR value is getting reset and will restore only upon 13734 * rewritting the BARs. 13735 * 13736 * As pci_restore_state() does not re-write the BARs if the 13737 * value is same as saved value earlier, driver needs to 13738 * write the BARs to 0 to force restore, in case of fatal error. 13739 */ 13740 if (test_and_clear_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, 13741 &bp->state)) { 13742 for (off = PCI_BASE_ADDRESS_0; 13743 off <= PCI_BASE_ADDRESS_5; off += 4) 13744 pci_write_config_dword(bp->pdev, off, 0); 13745 } 13746 pci_restore_state(pdev); 13747 pci_save_state(pdev); 13748 13749 err = bnxt_hwrm_func_reset(bp); 13750 if (!err) 13751 result = PCI_ERS_RESULT_RECOVERED; 13752 } 13753 13754 rtnl_unlock(); 13755 13756 return result; 13757 } 13758 13759 /** 13760 * bnxt_io_resume - called when traffic can start flowing again. 13761 * @pdev: Pointer to PCI device 13762 * 13763 * This callback is called when the error recovery driver tells 13764 * us that its OK to resume normal operation. 13765 */ 13766 static void bnxt_io_resume(struct pci_dev *pdev) 13767 { 13768 struct net_device *netdev = pci_get_drvdata(pdev); 13769 struct bnxt *bp = netdev_priv(netdev); 13770 int err; 13771 13772 netdev_info(bp->dev, "PCI Slot Resume\n"); 13773 rtnl_lock(); 13774 13775 err = bnxt_hwrm_func_qcaps(bp); 13776 if (!err && netif_running(netdev)) 13777 err = bnxt_open(netdev); 13778 13779 bnxt_ulp_start(bp, err); 13780 if (!err) { 13781 bnxt_reenable_sriov(bp); 13782 netif_device_attach(netdev); 13783 } 13784 13785 rtnl_unlock(); 13786 } 13787 13788 static const struct pci_error_handlers bnxt_err_handler = { 13789 .error_detected = bnxt_io_error_detected, 13790 .slot_reset = bnxt_io_slot_reset, 13791 .resume = bnxt_io_resume 13792 }; 13793 13794 static struct pci_driver bnxt_pci_driver = { 13795 .name = DRV_MODULE_NAME, 13796 .id_table = bnxt_pci_tbl, 13797 .probe = bnxt_init_one, 13798 .remove = bnxt_remove_one, 13799 .shutdown = bnxt_shutdown, 13800 .driver.pm = BNXT_PM_OPS, 13801 .err_handler = &bnxt_err_handler, 13802 #if defined(CONFIG_BNXT_SRIOV) 13803 .sriov_configure = bnxt_sriov_configure, 13804 #endif 13805 }; 13806 13807 static int __init bnxt_init(void) 13808 { 13809 bnxt_debug_init(); 13810 return pci_register_driver(&bnxt_pci_driver); 13811 } 13812 13813 static void __exit bnxt_exit(void) 13814 { 13815 pci_unregister_driver(&bnxt_pci_driver); 13816 if (bnxt_pf_wq) 13817 destroy_workqueue(bnxt_pf_wq); 13818 bnxt_debug_exit(); 13819 } 13820 13821 module_init(bnxt_init); 13822 module_exit(bnxt_exit); 13823