1 /* Broadcom NetXtreme-C/E network driver. 2 * 3 * Copyright (c) 2014-2016 Broadcom Corporation 4 * Copyright (c) 2016-2019 Broadcom Limited 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License as published by 8 * the Free Software Foundation. 9 */ 10 11 #include <linux/module.h> 12 13 #include <linux/stringify.h> 14 #include <linux/kernel.h> 15 #include <linux/timer.h> 16 #include <linux/errno.h> 17 #include <linux/ioport.h> 18 #include <linux/slab.h> 19 #include <linux/vmalloc.h> 20 #include <linux/interrupt.h> 21 #include <linux/pci.h> 22 #include <linux/netdevice.h> 23 #include <linux/etherdevice.h> 24 #include <linux/skbuff.h> 25 #include <linux/dma-mapping.h> 26 #include <linux/bitops.h> 27 #include <linux/io.h> 28 #include <linux/irq.h> 29 #include <linux/delay.h> 30 #include <asm/byteorder.h> 31 #include <asm/page.h> 32 #include <linux/time.h> 33 #include <linux/mii.h> 34 #include <linux/mdio.h> 35 #include <linux/if.h> 36 #include <linux/if_vlan.h> 37 #include <linux/if_bridge.h> 38 #include <linux/rtc.h> 39 #include <linux/bpf.h> 40 #include <net/gro.h> 41 #include <net/ip.h> 42 #include <net/tcp.h> 43 #include <net/udp.h> 44 #include <net/checksum.h> 45 #include <net/ip6_checksum.h> 46 #include <net/udp_tunnel.h> 47 #include <linux/workqueue.h> 48 #include <linux/prefetch.h> 49 #include <linux/cache.h> 50 #include <linux/log2.h> 51 #include <linux/aer.h> 52 #include <linux/bitmap.h> 53 #include <linux/cpu_rmap.h> 54 #include <linux/cpumask.h> 55 #include <net/pkt_cls.h> 56 #include <linux/hwmon.h> 57 #include <linux/hwmon-sysfs.h> 58 #include <net/page_pool.h> 59 #include <linux/align.h> 60 61 #include "bnxt_hsi.h" 62 #include "bnxt.h" 63 #include "bnxt_hwrm.h" 64 #include "bnxt_ulp.h" 65 #include "bnxt_sriov.h" 66 #include "bnxt_ethtool.h" 67 #include "bnxt_dcb.h" 68 #include "bnxt_xdp.h" 69 #include "bnxt_ptp.h" 70 #include "bnxt_vfr.h" 71 #include "bnxt_tc.h" 72 #include "bnxt_devlink.h" 73 #include "bnxt_debugfs.h" 74 75 #define BNXT_TX_TIMEOUT (5 * HZ) 76 #define BNXT_DEF_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_HW | \ 77 NETIF_MSG_TX_ERR) 78 79 MODULE_LICENSE("GPL"); 80 MODULE_DESCRIPTION("Broadcom BCM573xx network driver"); 81 82 #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN) 83 #define BNXT_RX_DMA_OFFSET NET_SKB_PAD 84 #define BNXT_RX_COPY_THRESH 256 85 86 #define BNXT_TX_PUSH_THRESH 164 87 88 /* indexed by enum board_idx */ 89 static const struct { 90 char *name; 91 } board_info[] = { 92 [BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" }, 93 [BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" }, 94 [BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" }, 95 [BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" }, 96 [BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" }, 97 [BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" }, 98 [BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" }, 99 [BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" }, 100 [BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" }, 101 [BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" }, 102 [BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" }, 103 [BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" }, 104 [BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" }, 105 [BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" }, 106 [BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" }, 107 [BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" }, 108 [BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" }, 109 [BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" }, 110 [BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" }, 111 [BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" }, 112 [BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" }, 113 [BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" }, 114 [BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" }, 115 [BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" }, 116 [BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" }, 117 [BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" }, 118 [BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" }, 119 [BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" }, 120 [BCM5745x_NPAR] = { "Broadcom BCM5745x NetXtreme-E Ethernet Partition" }, 121 [BCM57508] = { "Broadcom BCM57508 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" }, 122 [BCM57504] = { "Broadcom BCM57504 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" }, 123 [BCM57502] = { "Broadcom BCM57502 NetXtreme-E 10Gb/25Gb/50Gb Ethernet" }, 124 [BCM57508_NPAR] = { "Broadcom BCM57508 NetXtreme-E Ethernet Partition" }, 125 [BCM57504_NPAR] = { "Broadcom BCM57504 NetXtreme-E Ethernet Partition" }, 126 [BCM57502_NPAR] = { "Broadcom BCM57502 NetXtreme-E Ethernet Partition" }, 127 [BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" }, 128 [BCM58804] = { "Broadcom BCM58804 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" }, 129 [BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" }, 130 [NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" }, 131 [NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" }, 132 [NETXTREME_S_VF] = { "Broadcom NetXtreme-S Ethernet Virtual Function" }, 133 [NETXTREME_C_VF_HV] = { "Broadcom NetXtreme-C Virtual Function for Hyper-V" }, 134 [NETXTREME_E_VF_HV] = { "Broadcom NetXtreme-E Virtual Function for Hyper-V" }, 135 [NETXTREME_E_P5_VF] = { "Broadcom BCM5750X NetXtreme-E Ethernet Virtual Function" }, 136 [NETXTREME_E_P5_VF_HV] = { "Broadcom BCM5750X NetXtreme-E Virtual Function for Hyper-V" }, 137 }; 138 139 static const struct pci_device_id bnxt_pci_tbl[] = { 140 { PCI_VDEVICE(BROADCOM, 0x1604), .driver_data = BCM5745x_NPAR }, 141 { PCI_VDEVICE(BROADCOM, 0x1605), .driver_data = BCM5745x_NPAR }, 142 { PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 }, 143 { PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR }, 144 { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 }, 145 { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 }, 146 { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 }, 147 { PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR }, 148 { PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 }, 149 { PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 }, 150 { PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 }, 151 { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 }, 152 { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 }, 153 { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 }, 154 { PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR }, 155 { PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 }, 156 { PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 }, 157 { PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 }, 158 { PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 }, 159 { PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 }, 160 { PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR }, 161 { PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 }, 162 { PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP }, 163 { PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP }, 164 { PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR }, 165 { PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR }, 166 { PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP }, 167 { PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR }, 168 { PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR }, 169 { PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR }, 170 { PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR }, 171 { PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR }, 172 { PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR }, 173 { PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 }, 174 { PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 }, 175 { PCI_VDEVICE(BROADCOM, 0x1750), .driver_data = BCM57508 }, 176 { PCI_VDEVICE(BROADCOM, 0x1751), .driver_data = BCM57504 }, 177 { PCI_VDEVICE(BROADCOM, 0x1752), .driver_data = BCM57502 }, 178 { PCI_VDEVICE(BROADCOM, 0x1800), .driver_data = BCM57502_NPAR }, 179 { PCI_VDEVICE(BROADCOM, 0x1801), .driver_data = BCM57504_NPAR }, 180 { PCI_VDEVICE(BROADCOM, 0x1802), .driver_data = BCM57508_NPAR }, 181 { PCI_VDEVICE(BROADCOM, 0x1803), .driver_data = BCM57502_NPAR }, 182 { PCI_VDEVICE(BROADCOM, 0x1804), .driver_data = BCM57504_NPAR }, 183 { PCI_VDEVICE(BROADCOM, 0x1805), .driver_data = BCM57508_NPAR }, 184 { PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 }, 185 { PCI_VDEVICE(BROADCOM, 0xd804), .driver_data = BCM58804 }, 186 #ifdef CONFIG_BNXT_SRIOV 187 { PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF }, 188 { PCI_VDEVICE(BROADCOM, 0x1607), .driver_data = NETXTREME_E_VF_HV }, 189 { PCI_VDEVICE(BROADCOM, 0x1608), .driver_data = NETXTREME_E_VF_HV }, 190 { PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF }, 191 { PCI_VDEVICE(BROADCOM, 0x16bd), .driver_data = NETXTREME_E_VF_HV }, 192 { PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF }, 193 { PCI_VDEVICE(BROADCOM, 0x16c2), .driver_data = NETXTREME_C_VF_HV }, 194 { PCI_VDEVICE(BROADCOM, 0x16c3), .driver_data = NETXTREME_C_VF_HV }, 195 { PCI_VDEVICE(BROADCOM, 0x16c4), .driver_data = NETXTREME_E_VF_HV }, 196 { PCI_VDEVICE(BROADCOM, 0x16c5), .driver_data = NETXTREME_E_VF_HV }, 197 { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF }, 198 { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF }, 199 { PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF }, 200 { PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF }, 201 { PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF }, 202 { PCI_VDEVICE(BROADCOM, 0x16e6), .driver_data = NETXTREME_C_VF_HV }, 203 { PCI_VDEVICE(BROADCOM, 0x1806), .driver_data = NETXTREME_E_P5_VF }, 204 { PCI_VDEVICE(BROADCOM, 0x1807), .driver_data = NETXTREME_E_P5_VF }, 205 { PCI_VDEVICE(BROADCOM, 0x1808), .driver_data = NETXTREME_E_P5_VF_HV }, 206 { PCI_VDEVICE(BROADCOM, 0x1809), .driver_data = NETXTREME_E_P5_VF_HV }, 207 { PCI_VDEVICE(BROADCOM, 0xd800), .driver_data = NETXTREME_S_VF }, 208 #endif 209 { 0 } 210 }; 211 212 MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl); 213 214 static const u16 bnxt_vf_req_snif[] = { 215 HWRM_FUNC_CFG, 216 HWRM_FUNC_VF_CFG, 217 HWRM_PORT_PHY_QCFG, 218 HWRM_CFA_L2_FILTER_ALLOC, 219 }; 220 221 static const u16 bnxt_async_events_arr[] = { 222 ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE, 223 ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE, 224 ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD, 225 ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED, 226 ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE, 227 ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE, 228 ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE, 229 ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY, 230 ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY, 231 ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION, 232 ASYNC_EVENT_CMPL_EVENT_ID_DEFERRED_RESPONSE, 233 ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG, 234 ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST, 235 ASYNC_EVENT_CMPL_EVENT_ID_PPS_TIMESTAMP, 236 ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT, 237 ASYNC_EVENT_CMPL_EVENT_ID_PHC_UPDATE, 238 }; 239 240 static struct workqueue_struct *bnxt_pf_wq; 241 242 static bool bnxt_vf_pciid(enum board_idx idx) 243 { 244 return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF || 245 idx == NETXTREME_S_VF || idx == NETXTREME_C_VF_HV || 246 idx == NETXTREME_E_VF_HV || idx == NETXTREME_E_P5_VF || 247 idx == NETXTREME_E_P5_VF_HV); 248 } 249 250 #define DB_CP_REARM_FLAGS (DB_KEY_CP | DB_IDX_VALID) 251 #define DB_CP_FLAGS (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS) 252 #define DB_CP_IRQ_DIS_FLAGS (DB_KEY_CP | DB_IRQ_DIS) 253 254 #define BNXT_CP_DB_IRQ_DIS(db) \ 255 writel(DB_CP_IRQ_DIS_FLAGS, db) 256 257 #define BNXT_DB_CQ(db, idx) \ 258 writel(DB_CP_FLAGS | RING_CMP(idx), (db)->doorbell) 259 260 #define BNXT_DB_NQ_P5(db, idx) \ 261 bnxt_writeq(bp, (db)->db_key64 | DBR_TYPE_NQ | RING_CMP(idx), \ 262 (db)->doorbell) 263 264 #define BNXT_DB_CQ_ARM(db, idx) \ 265 writel(DB_CP_REARM_FLAGS | RING_CMP(idx), (db)->doorbell) 266 267 #define BNXT_DB_NQ_ARM_P5(db, idx) \ 268 bnxt_writeq(bp, (db)->db_key64 | DBR_TYPE_NQ_ARM | RING_CMP(idx),\ 269 (db)->doorbell) 270 271 static void bnxt_db_nq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx) 272 { 273 if (bp->flags & BNXT_FLAG_CHIP_P5) 274 BNXT_DB_NQ_P5(db, idx); 275 else 276 BNXT_DB_CQ(db, idx); 277 } 278 279 static void bnxt_db_nq_arm(struct bnxt *bp, struct bnxt_db_info *db, u32 idx) 280 { 281 if (bp->flags & BNXT_FLAG_CHIP_P5) 282 BNXT_DB_NQ_ARM_P5(db, idx); 283 else 284 BNXT_DB_CQ_ARM(db, idx); 285 } 286 287 static void bnxt_db_cq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx) 288 { 289 if (bp->flags & BNXT_FLAG_CHIP_P5) 290 bnxt_writeq(bp, db->db_key64 | DBR_TYPE_CQ_ARMALL | 291 RING_CMP(idx), db->doorbell); 292 else 293 BNXT_DB_CQ(db, idx); 294 } 295 296 const u16 bnxt_lhint_arr[] = { 297 TX_BD_FLAGS_LHINT_512_AND_SMALLER, 298 TX_BD_FLAGS_LHINT_512_TO_1023, 299 TX_BD_FLAGS_LHINT_1024_TO_2047, 300 TX_BD_FLAGS_LHINT_1024_TO_2047, 301 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 302 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 303 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 304 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 305 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 306 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 307 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 308 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 309 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 310 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 311 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 312 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 313 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 314 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 315 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 316 }; 317 318 static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb) 319 { 320 struct metadata_dst *md_dst = skb_metadata_dst(skb); 321 322 if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX) 323 return 0; 324 325 return md_dst->u.port_info.port_id; 326 } 327 328 static void bnxt_txr_db_kick(struct bnxt *bp, struct bnxt_tx_ring_info *txr, 329 u16 prod) 330 { 331 bnxt_db_write(bp, &txr->tx_db, prod); 332 txr->kick_pending = 0; 333 } 334 335 static bool bnxt_txr_netif_try_stop_queue(struct bnxt *bp, 336 struct bnxt_tx_ring_info *txr, 337 struct netdev_queue *txq) 338 { 339 netif_tx_stop_queue(txq); 340 341 /* netif_tx_stop_queue() must be done before checking 342 * tx index in bnxt_tx_avail() below, because in 343 * bnxt_tx_int(), we update tx index before checking for 344 * netif_tx_queue_stopped(). 345 */ 346 smp_mb(); 347 if (bnxt_tx_avail(bp, txr) >= bp->tx_wake_thresh) { 348 netif_tx_wake_queue(txq); 349 return false; 350 } 351 352 return true; 353 } 354 355 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev) 356 { 357 struct bnxt *bp = netdev_priv(dev); 358 struct tx_bd *txbd; 359 struct tx_bd_ext *txbd1; 360 struct netdev_queue *txq; 361 int i; 362 dma_addr_t mapping; 363 unsigned int length, pad = 0; 364 u32 len, free_size, vlan_tag_flags, cfa_action, flags; 365 u16 prod, last_frag; 366 struct pci_dev *pdev = bp->pdev; 367 struct bnxt_tx_ring_info *txr; 368 struct bnxt_sw_tx_bd *tx_buf; 369 __le32 lflags = 0; 370 371 i = skb_get_queue_mapping(skb); 372 if (unlikely(i >= bp->tx_nr_rings)) { 373 dev_kfree_skb_any(skb); 374 dev_core_stats_tx_dropped_inc(dev); 375 return NETDEV_TX_OK; 376 } 377 378 txq = netdev_get_tx_queue(dev, i); 379 txr = &bp->tx_ring[bp->tx_ring_map[i]]; 380 prod = txr->tx_prod; 381 382 free_size = bnxt_tx_avail(bp, txr); 383 if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) { 384 /* We must have raced with NAPI cleanup */ 385 if (net_ratelimit() && txr->kick_pending) 386 netif_warn(bp, tx_err, dev, 387 "bnxt: ring busy w/ flush pending!\n"); 388 if (bnxt_txr_netif_try_stop_queue(bp, txr, txq)) 389 return NETDEV_TX_BUSY; 390 } 391 392 if (unlikely(ipv6_hopopt_jumbo_remove(skb))) 393 goto tx_free; 394 395 length = skb->len; 396 len = skb_headlen(skb); 397 last_frag = skb_shinfo(skb)->nr_frags; 398 399 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 400 401 txbd->tx_bd_opaque = prod; 402 403 tx_buf = &txr->tx_buf_ring[prod]; 404 tx_buf->skb = skb; 405 tx_buf->nr_frags = last_frag; 406 407 vlan_tag_flags = 0; 408 cfa_action = bnxt_xmit_get_cfa_action(skb); 409 if (skb_vlan_tag_present(skb)) { 410 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN | 411 skb_vlan_tag_get(skb); 412 /* Currently supports 8021Q, 8021AD vlan offloads 413 * QINQ1, QINQ2, QINQ3 vlan headers are deprecated 414 */ 415 if (skb->vlan_proto == htons(ETH_P_8021Q)) 416 vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT; 417 } 418 419 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) { 420 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; 421 422 if (ptp && ptp->tx_tstamp_en && !skb_is_gso(skb) && 423 atomic_dec_if_positive(&ptp->tx_avail) >= 0) { 424 if (!bnxt_ptp_parse(skb, &ptp->tx_seqid, 425 &ptp->tx_hdr_off)) { 426 if (vlan_tag_flags) 427 ptp->tx_hdr_off += VLAN_HLEN; 428 lflags |= cpu_to_le32(TX_BD_FLAGS_STAMP); 429 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 430 } else { 431 atomic_inc(&bp->ptp_cfg->tx_avail); 432 } 433 } 434 } 435 436 if (unlikely(skb->no_fcs)) 437 lflags |= cpu_to_le32(TX_BD_FLAGS_NO_CRC); 438 439 if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh && 440 !lflags) { 441 struct tx_push_buffer *tx_push_buf = txr->tx_push; 442 struct tx_push_bd *tx_push = &tx_push_buf->push_bd; 443 struct tx_bd_ext *tx_push1 = &tx_push->txbd2; 444 void __iomem *db = txr->tx_db.doorbell; 445 void *pdata = tx_push_buf->data; 446 u64 *end; 447 int j, push_len; 448 449 /* Set COAL_NOW to be ready quickly for the next push */ 450 tx_push->tx_bd_len_flags_type = 451 cpu_to_le32((length << TX_BD_LEN_SHIFT) | 452 TX_BD_TYPE_LONG_TX_BD | 453 TX_BD_FLAGS_LHINT_512_AND_SMALLER | 454 TX_BD_FLAGS_COAL_NOW | 455 TX_BD_FLAGS_PACKET_END | 456 (2 << TX_BD_FLAGS_BD_CNT_SHIFT)); 457 458 if (skb->ip_summed == CHECKSUM_PARTIAL) 459 tx_push1->tx_bd_hsize_lflags = 460 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM); 461 else 462 tx_push1->tx_bd_hsize_lflags = 0; 463 464 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags); 465 tx_push1->tx_bd_cfa_action = 466 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT); 467 468 end = pdata + length; 469 end = PTR_ALIGN(end, 8) - 1; 470 *end = 0; 471 472 skb_copy_from_linear_data(skb, pdata, len); 473 pdata += len; 474 for (j = 0; j < last_frag; j++) { 475 skb_frag_t *frag = &skb_shinfo(skb)->frags[j]; 476 void *fptr; 477 478 fptr = skb_frag_address_safe(frag); 479 if (!fptr) 480 goto normal_tx; 481 482 memcpy(pdata, fptr, skb_frag_size(frag)); 483 pdata += skb_frag_size(frag); 484 } 485 486 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type; 487 txbd->tx_bd_haddr = txr->data_mapping; 488 prod = NEXT_TX(prod); 489 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 490 memcpy(txbd, tx_push1, sizeof(*txbd)); 491 prod = NEXT_TX(prod); 492 tx_push->doorbell = 493 cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | prod); 494 txr->tx_prod = prod; 495 496 tx_buf->is_push = 1; 497 netdev_tx_sent_queue(txq, skb->len); 498 wmb(); /* Sync is_push and byte queue before pushing data */ 499 500 push_len = (length + sizeof(*tx_push) + 7) / 8; 501 if (push_len > 16) { 502 __iowrite64_copy(db, tx_push_buf, 16); 503 __iowrite32_copy(db + 4, tx_push_buf + 1, 504 (push_len - 16) << 1); 505 } else { 506 __iowrite64_copy(db, tx_push_buf, push_len); 507 } 508 509 goto tx_done; 510 } 511 512 normal_tx: 513 if (length < BNXT_MIN_PKT_SIZE) { 514 pad = BNXT_MIN_PKT_SIZE - length; 515 if (skb_pad(skb, pad)) 516 /* SKB already freed. */ 517 goto tx_kick_pending; 518 length = BNXT_MIN_PKT_SIZE; 519 } 520 521 mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE); 522 523 if (unlikely(dma_mapping_error(&pdev->dev, mapping))) 524 goto tx_free; 525 526 dma_unmap_addr_set(tx_buf, mapping, mapping); 527 flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD | 528 ((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT); 529 530 txbd->tx_bd_haddr = cpu_to_le64(mapping); 531 532 prod = NEXT_TX(prod); 533 txbd1 = (struct tx_bd_ext *) 534 &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 535 536 txbd1->tx_bd_hsize_lflags = lflags; 537 if (skb_is_gso(skb)) { 538 u32 hdr_len; 539 540 if (skb->encapsulation) 541 hdr_len = skb_inner_tcp_all_headers(skb); 542 else 543 hdr_len = skb_tcp_all_headers(skb); 544 545 txbd1->tx_bd_hsize_lflags |= cpu_to_le32(TX_BD_FLAGS_LSO | 546 TX_BD_FLAGS_T_IPID | 547 (hdr_len << (TX_BD_HSIZE_SHIFT - 1))); 548 length = skb_shinfo(skb)->gso_size; 549 txbd1->tx_bd_mss = cpu_to_le32(length); 550 length += hdr_len; 551 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { 552 txbd1->tx_bd_hsize_lflags |= 553 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM); 554 txbd1->tx_bd_mss = 0; 555 } 556 557 length >>= 9; 558 if (unlikely(length >= ARRAY_SIZE(bnxt_lhint_arr))) { 559 dev_warn_ratelimited(&pdev->dev, "Dropped oversize %d bytes TX packet.\n", 560 skb->len); 561 i = 0; 562 goto tx_dma_error; 563 } 564 flags |= bnxt_lhint_arr[length]; 565 txbd->tx_bd_len_flags_type = cpu_to_le32(flags); 566 567 txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags); 568 txbd1->tx_bd_cfa_action = 569 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT); 570 for (i = 0; i < last_frag; i++) { 571 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 572 573 prod = NEXT_TX(prod); 574 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 575 576 len = skb_frag_size(frag); 577 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len, 578 DMA_TO_DEVICE); 579 580 if (unlikely(dma_mapping_error(&pdev->dev, mapping))) 581 goto tx_dma_error; 582 583 tx_buf = &txr->tx_buf_ring[prod]; 584 dma_unmap_addr_set(tx_buf, mapping, mapping); 585 586 txbd->tx_bd_haddr = cpu_to_le64(mapping); 587 588 flags = len << TX_BD_LEN_SHIFT; 589 txbd->tx_bd_len_flags_type = cpu_to_le32(flags); 590 } 591 592 flags &= ~TX_BD_LEN; 593 txbd->tx_bd_len_flags_type = 594 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags | 595 TX_BD_FLAGS_PACKET_END); 596 597 netdev_tx_sent_queue(txq, skb->len); 598 599 skb_tx_timestamp(skb); 600 601 /* Sync BD data before updating doorbell */ 602 wmb(); 603 604 prod = NEXT_TX(prod); 605 txr->tx_prod = prod; 606 607 if (!netdev_xmit_more() || netif_xmit_stopped(txq)) 608 bnxt_txr_db_kick(bp, txr, prod); 609 else 610 txr->kick_pending = 1; 611 612 tx_done: 613 614 if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) { 615 if (netdev_xmit_more() && !tx_buf->is_push) 616 bnxt_txr_db_kick(bp, txr, prod); 617 618 bnxt_txr_netif_try_stop_queue(bp, txr, txq); 619 } 620 return NETDEV_TX_OK; 621 622 tx_dma_error: 623 if (BNXT_TX_PTP_IS_SET(lflags)) 624 atomic_inc(&bp->ptp_cfg->tx_avail); 625 626 last_frag = i; 627 628 /* start back at beginning and unmap skb */ 629 prod = txr->tx_prod; 630 tx_buf = &txr->tx_buf_ring[prod]; 631 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping), 632 skb_headlen(skb), DMA_TO_DEVICE); 633 prod = NEXT_TX(prod); 634 635 /* unmap remaining mapped pages */ 636 for (i = 0; i < last_frag; i++) { 637 prod = NEXT_TX(prod); 638 tx_buf = &txr->tx_buf_ring[prod]; 639 dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping), 640 skb_frag_size(&skb_shinfo(skb)->frags[i]), 641 DMA_TO_DEVICE); 642 } 643 644 tx_free: 645 dev_kfree_skb_any(skb); 646 tx_kick_pending: 647 if (txr->kick_pending) 648 bnxt_txr_db_kick(bp, txr, txr->tx_prod); 649 txr->tx_buf_ring[txr->tx_prod].skb = NULL; 650 dev_core_stats_tx_dropped_inc(dev); 651 return NETDEV_TX_OK; 652 } 653 654 static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts) 655 { 656 struct bnxt_tx_ring_info *txr = bnapi->tx_ring; 657 struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index); 658 u16 cons = txr->tx_cons; 659 struct pci_dev *pdev = bp->pdev; 660 int i; 661 unsigned int tx_bytes = 0; 662 663 for (i = 0; i < nr_pkts; i++) { 664 struct bnxt_sw_tx_bd *tx_buf; 665 struct sk_buff *skb; 666 int j, last; 667 668 tx_buf = &txr->tx_buf_ring[cons]; 669 cons = NEXT_TX(cons); 670 skb = tx_buf->skb; 671 tx_buf->skb = NULL; 672 673 tx_bytes += skb->len; 674 675 if (tx_buf->is_push) { 676 tx_buf->is_push = 0; 677 goto next_tx_int; 678 } 679 680 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping), 681 skb_headlen(skb), DMA_TO_DEVICE); 682 last = tx_buf->nr_frags; 683 684 for (j = 0; j < last; j++) { 685 cons = NEXT_TX(cons); 686 tx_buf = &txr->tx_buf_ring[cons]; 687 dma_unmap_page( 688 &pdev->dev, 689 dma_unmap_addr(tx_buf, mapping), 690 skb_frag_size(&skb_shinfo(skb)->frags[j]), 691 DMA_TO_DEVICE); 692 } 693 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)) { 694 if (bp->flags & BNXT_FLAG_CHIP_P5) { 695 /* PTP worker takes ownership of the skb */ 696 if (!bnxt_get_tx_ts_p5(bp, skb)) 697 skb = NULL; 698 else 699 atomic_inc(&bp->ptp_cfg->tx_avail); 700 } 701 } 702 703 next_tx_int: 704 cons = NEXT_TX(cons); 705 706 dev_kfree_skb_any(skb); 707 } 708 709 netdev_tx_completed_queue(txq, nr_pkts, tx_bytes); 710 txr->tx_cons = cons; 711 712 /* Need to make the tx_cons update visible to bnxt_start_xmit() 713 * before checking for netif_tx_queue_stopped(). Without the 714 * memory barrier, there is a small possibility that bnxt_start_xmit() 715 * will miss it and cause the queue to be stopped forever. 716 */ 717 smp_mb(); 718 719 if (unlikely(netif_tx_queue_stopped(txq)) && 720 bnxt_tx_avail(bp, txr) >= bp->tx_wake_thresh && 721 READ_ONCE(txr->dev_state) != BNXT_DEV_STATE_CLOSING) 722 netif_tx_wake_queue(txq); 723 } 724 725 static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping, 726 struct bnxt_rx_ring_info *rxr, 727 gfp_t gfp) 728 { 729 struct device *dev = &bp->pdev->dev; 730 struct page *page; 731 732 page = page_pool_dev_alloc_pages(rxr->page_pool); 733 if (!page) 734 return NULL; 735 736 *mapping = dma_map_page_attrs(dev, page, 0, PAGE_SIZE, bp->rx_dir, 737 DMA_ATTR_WEAK_ORDERING); 738 if (dma_mapping_error(dev, *mapping)) { 739 page_pool_recycle_direct(rxr->page_pool, page); 740 return NULL; 741 } 742 return page; 743 } 744 745 static inline u8 *__bnxt_alloc_rx_frag(struct bnxt *bp, dma_addr_t *mapping, 746 gfp_t gfp) 747 { 748 u8 *data; 749 struct pci_dev *pdev = bp->pdev; 750 751 if (gfp == GFP_ATOMIC) 752 data = napi_alloc_frag(bp->rx_buf_size); 753 else 754 data = netdev_alloc_frag(bp->rx_buf_size); 755 if (!data) 756 return NULL; 757 758 *mapping = dma_map_single_attrs(&pdev->dev, data + bp->rx_dma_offset, 759 bp->rx_buf_use_size, bp->rx_dir, 760 DMA_ATTR_WEAK_ORDERING); 761 762 if (dma_mapping_error(&pdev->dev, *mapping)) { 763 skb_free_frag(data); 764 data = NULL; 765 } 766 return data; 767 } 768 769 int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, 770 u16 prod, gfp_t gfp) 771 { 772 struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 773 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod]; 774 dma_addr_t mapping; 775 776 if (BNXT_RX_PAGE_MODE(bp)) { 777 struct page *page = 778 __bnxt_alloc_rx_page(bp, &mapping, rxr, gfp); 779 780 if (!page) 781 return -ENOMEM; 782 783 mapping += bp->rx_dma_offset; 784 rx_buf->data = page; 785 rx_buf->data_ptr = page_address(page) + bp->rx_offset; 786 } else { 787 u8 *data = __bnxt_alloc_rx_frag(bp, &mapping, gfp); 788 789 if (!data) 790 return -ENOMEM; 791 792 rx_buf->data = data; 793 rx_buf->data_ptr = data + bp->rx_offset; 794 } 795 rx_buf->mapping = mapping; 796 797 rxbd->rx_bd_haddr = cpu_to_le64(mapping); 798 return 0; 799 } 800 801 void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data) 802 { 803 u16 prod = rxr->rx_prod; 804 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf; 805 struct rx_bd *cons_bd, *prod_bd; 806 807 prod_rx_buf = &rxr->rx_buf_ring[prod]; 808 cons_rx_buf = &rxr->rx_buf_ring[cons]; 809 810 prod_rx_buf->data = data; 811 prod_rx_buf->data_ptr = cons_rx_buf->data_ptr; 812 813 prod_rx_buf->mapping = cons_rx_buf->mapping; 814 815 prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 816 cons_bd = &rxr->rx_desc_ring[RX_RING(cons)][RX_IDX(cons)]; 817 818 prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr; 819 } 820 821 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx) 822 { 823 u16 next, max = rxr->rx_agg_bmap_size; 824 825 next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx); 826 if (next >= max) 827 next = find_first_zero_bit(rxr->rx_agg_bmap, max); 828 return next; 829 } 830 831 static inline int bnxt_alloc_rx_page(struct bnxt *bp, 832 struct bnxt_rx_ring_info *rxr, 833 u16 prod, gfp_t gfp) 834 { 835 struct rx_bd *rxbd = 836 &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 837 struct bnxt_sw_rx_agg_bd *rx_agg_buf; 838 struct pci_dev *pdev = bp->pdev; 839 struct page *page; 840 dma_addr_t mapping; 841 u16 sw_prod = rxr->rx_sw_agg_prod; 842 unsigned int offset = 0; 843 844 if (BNXT_RX_PAGE_MODE(bp)) { 845 page = __bnxt_alloc_rx_page(bp, &mapping, rxr, gfp); 846 847 if (!page) 848 return -ENOMEM; 849 850 } else { 851 if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) { 852 page = rxr->rx_page; 853 if (!page) { 854 page = alloc_page(gfp); 855 if (!page) 856 return -ENOMEM; 857 rxr->rx_page = page; 858 rxr->rx_page_offset = 0; 859 } 860 offset = rxr->rx_page_offset; 861 rxr->rx_page_offset += BNXT_RX_PAGE_SIZE; 862 if (rxr->rx_page_offset == PAGE_SIZE) 863 rxr->rx_page = NULL; 864 else 865 get_page(page); 866 } else { 867 page = alloc_page(gfp); 868 if (!page) 869 return -ENOMEM; 870 } 871 872 mapping = dma_map_page_attrs(&pdev->dev, page, offset, 873 BNXT_RX_PAGE_SIZE, DMA_FROM_DEVICE, 874 DMA_ATTR_WEAK_ORDERING); 875 if (dma_mapping_error(&pdev->dev, mapping)) { 876 __free_page(page); 877 return -EIO; 878 } 879 } 880 881 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap))) 882 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod); 883 884 __set_bit(sw_prod, rxr->rx_agg_bmap); 885 rx_agg_buf = &rxr->rx_agg_ring[sw_prod]; 886 rxr->rx_sw_agg_prod = NEXT_RX_AGG(sw_prod); 887 888 rx_agg_buf->page = page; 889 rx_agg_buf->offset = offset; 890 rx_agg_buf->mapping = mapping; 891 rxbd->rx_bd_haddr = cpu_to_le64(mapping); 892 rxbd->rx_bd_opaque = sw_prod; 893 return 0; 894 } 895 896 static struct rx_agg_cmp *bnxt_get_agg(struct bnxt *bp, 897 struct bnxt_cp_ring_info *cpr, 898 u16 cp_cons, u16 curr) 899 { 900 struct rx_agg_cmp *agg; 901 902 cp_cons = RING_CMP(ADV_RAW_CMP(cp_cons, curr)); 903 agg = (struct rx_agg_cmp *) 904 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 905 return agg; 906 } 907 908 static struct rx_agg_cmp *bnxt_get_tpa_agg_p5(struct bnxt *bp, 909 struct bnxt_rx_ring_info *rxr, 910 u16 agg_id, u16 curr) 911 { 912 struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[agg_id]; 913 914 return &tpa_info->agg_arr[curr]; 915 } 916 917 static void bnxt_reuse_rx_agg_bufs(struct bnxt_cp_ring_info *cpr, u16 idx, 918 u16 start, u32 agg_bufs, bool tpa) 919 { 920 struct bnxt_napi *bnapi = cpr->bnapi; 921 struct bnxt *bp = bnapi->bp; 922 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 923 u16 prod = rxr->rx_agg_prod; 924 u16 sw_prod = rxr->rx_sw_agg_prod; 925 bool p5_tpa = false; 926 u32 i; 927 928 if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa) 929 p5_tpa = true; 930 931 for (i = 0; i < agg_bufs; i++) { 932 u16 cons; 933 struct rx_agg_cmp *agg; 934 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf; 935 struct rx_bd *prod_bd; 936 struct page *page; 937 938 if (p5_tpa) 939 agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, start + i); 940 else 941 agg = bnxt_get_agg(bp, cpr, idx, start + i); 942 cons = agg->rx_agg_cmp_opaque; 943 __clear_bit(cons, rxr->rx_agg_bmap); 944 945 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap))) 946 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod); 947 948 __set_bit(sw_prod, rxr->rx_agg_bmap); 949 prod_rx_buf = &rxr->rx_agg_ring[sw_prod]; 950 cons_rx_buf = &rxr->rx_agg_ring[cons]; 951 952 /* It is possible for sw_prod to be equal to cons, so 953 * set cons_rx_buf->page to NULL first. 954 */ 955 page = cons_rx_buf->page; 956 cons_rx_buf->page = NULL; 957 prod_rx_buf->page = page; 958 prod_rx_buf->offset = cons_rx_buf->offset; 959 960 prod_rx_buf->mapping = cons_rx_buf->mapping; 961 962 prod_bd = &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 963 964 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping); 965 prod_bd->rx_bd_opaque = sw_prod; 966 967 prod = NEXT_RX_AGG(prod); 968 sw_prod = NEXT_RX_AGG(sw_prod); 969 } 970 rxr->rx_agg_prod = prod; 971 rxr->rx_sw_agg_prod = sw_prod; 972 } 973 974 static struct sk_buff *bnxt_rx_multi_page_skb(struct bnxt *bp, 975 struct bnxt_rx_ring_info *rxr, 976 u16 cons, void *data, u8 *data_ptr, 977 dma_addr_t dma_addr, 978 unsigned int offset_and_len) 979 { 980 unsigned int len = offset_and_len & 0xffff; 981 struct page *page = data; 982 u16 prod = rxr->rx_prod; 983 struct sk_buff *skb; 984 int err; 985 986 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC); 987 if (unlikely(err)) { 988 bnxt_reuse_rx_data(rxr, cons, data); 989 return NULL; 990 } 991 dma_addr -= bp->rx_dma_offset; 992 dma_unmap_page_attrs(&bp->pdev->dev, dma_addr, PAGE_SIZE, bp->rx_dir, 993 DMA_ATTR_WEAK_ORDERING); 994 skb = build_skb(page_address(page), PAGE_SIZE); 995 if (!skb) { 996 page_pool_recycle_direct(rxr->page_pool, page); 997 return NULL; 998 } 999 skb_mark_for_recycle(skb); 1000 skb_reserve(skb, bp->rx_dma_offset); 1001 __skb_put(skb, len); 1002 1003 return skb; 1004 } 1005 1006 static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp, 1007 struct bnxt_rx_ring_info *rxr, 1008 u16 cons, void *data, u8 *data_ptr, 1009 dma_addr_t dma_addr, 1010 unsigned int offset_and_len) 1011 { 1012 unsigned int payload = offset_and_len >> 16; 1013 unsigned int len = offset_and_len & 0xffff; 1014 skb_frag_t *frag; 1015 struct page *page = data; 1016 u16 prod = rxr->rx_prod; 1017 struct sk_buff *skb; 1018 int off, err; 1019 1020 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC); 1021 if (unlikely(err)) { 1022 bnxt_reuse_rx_data(rxr, cons, data); 1023 return NULL; 1024 } 1025 dma_addr -= bp->rx_dma_offset; 1026 dma_unmap_page_attrs(&bp->pdev->dev, dma_addr, PAGE_SIZE, bp->rx_dir, 1027 DMA_ATTR_WEAK_ORDERING); 1028 1029 if (unlikely(!payload)) 1030 payload = eth_get_headlen(bp->dev, data_ptr, len); 1031 1032 skb = napi_alloc_skb(&rxr->bnapi->napi, payload); 1033 if (!skb) { 1034 page_pool_recycle_direct(rxr->page_pool, page); 1035 return NULL; 1036 } 1037 1038 skb_mark_for_recycle(skb); 1039 off = (void *)data_ptr - page_address(page); 1040 skb_add_rx_frag(skb, 0, page, off, len, PAGE_SIZE); 1041 memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN, 1042 payload + NET_IP_ALIGN); 1043 1044 frag = &skb_shinfo(skb)->frags[0]; 1045 skb_frag_size_sub(frag, payload); 1046 skb_frag_off_add(frag, payload); 1047 skb->data_len -= payload; 1048 skb->tail += payload; 1049 1050 return skb; 1051 } 1052 1053 static struct sk_buff *bnxt_rx_skb(struct bnxt *bp, 1054 struct bnxt_rx_ring_info *rxr, u16 cons, 1055 void *data, u8 *data_ptr, 1056 dma_addr_t dma_addr, 1057 unsigned int offset_and_len) 1058 { 1059 u16 prod = rxr->rx_prod; 1060 struct sk_buff *skb; 1061 int err; 1062 1063 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC); 1064 if (unlikely(err)) { 1065 bnxt_reuse_rx_data(rxr, cons, data); 1066 return NULL; 1067 } 1068 1069 skb = build_skb(data, bp->rx_buf_size); 1070 dma_unmap_single_attrs(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size, 1071 bp->rx_dir, DMA_ATTR_WEAK_ORDERING); 1072 if (!skb) { 1073 skb_free_frag(data); 1074 return NULL; 1075 } 1076 1077 skb_reserve(skb, bp->rx_offset); 1078 skb_put(skb, offset_and_len & 0xffff); 1079 return skb; 1080 } 1081 1082 static u32 __bnxt_rx_agg_pages(struct bnxt *bp, 1083 struct bnxt_cp_ring_info *cpr, 1084 struct skb_shared_info *shinfo, 1085 u16 idx, u32 agg_bufs, bool tpa, 1086 struct xdp_buff *xdp) 1087 { 1088 struct bnxt_napi *bnapi = cpr->bnapi; 1089 struct pci_dev *pdev = bp->pdev; 1090 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1091 u16 prod = rxr->rx_agg_prod; 1092 u32 i, total_frag_len = 0; 1093 bool p5_tpa = false; 1094 1095 if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa) 1096 p5_tpa = true; 1097 1098 for (i = 0; i < agg_bufs; i++) { 1099 skb_frag_t *frag = &shinfo->frags[i]; 1100 u16 cons, frag_len; 1101 struct rx_agg_cmp *agg; 1102 struct bnxt_sw_rx_agg_bd *cons_rx_buf; 1103 struct page *page; 1104 dma_addr_t mapping; 1105 1106 if (p5_tpa) 1107 agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, i); 1108 else 1109 agg = bnxt_get_agg(bp, cpr, idx, i); 1110 cons = agg->rx_agg_cmp_opaque; 1111 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) & 1112 RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT; 1113 1114 cons_rx_buf = &rxr->rx_agg_ring[cons]; 1115 skb_frag_off_set(frag, cons_rx_buf->offset); 1116 skb_frag_size_set(frag, frag_len); 1117 __skb_frag_set_page(frag, cons_rx_buf->page); 1118 shinfo->nr_frags = i + 1; 1119 __clear_bit(cons, rxr->rx_agg_bmap); 1120 1121 /* It is possible for bnxt_alloc_rx_page() to allocate 1122 * a sw_prod index that equals the cons index, so we 1123 * need to clear the cons entry now. 1124 */ 1125 mapping = cons_rx_buf->mapping; 1126 page = cons_rx_buf->page; 1127 cons_rx_buf->page = NULL; 1128 1129 if (xdp && page_is_pfmemalloc(page)) 1130 xdp_buff_set_frag_pfmemalloc(xdp); 1131 1132 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) { 1133 unsigned int nr_frags; 1134 1135 nr_frags = --shinfo->nr_frags; 1136 __skb_frag_set_page(&shinfo->frags[nr_frags], NULL); 1137 cons_rx_buf->page = page; 1138 1139 /* Update prod since possibly some pages have been 1140 * allocated already. 1141 */ 1142 rxr->rx_agg_prod = prod; 1143 bnxt_reuse_rx_agg_bufs(cpr, idx, i, agg_bufs - i, tpa); 1144 return 0; 1145 } 1146 1147 dma_unmap_page_attrs(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE, 1148 bp->rx_dir, 1149 DMA_ATTR_WEAK_ORDERING); 1150 1151 total_frag_len += frag_len; 1152 prod = NEXT_RX_AGG(prod); 1153 } 1154 rxr->rx_agg_prod = prod; 1155 return total_frag_len; 1156 } 1157 1158 static struct sk_buff *bnxt_rx_agg_pages_skb(struct bnxt *bp, 1159 struct bnxt_cp_ring_info *cpr, 1160 struct sk_buff *skb, u16 idx, 1161 u32 agg_bufs, bool tpa) 1162 { 1163 struct skb_shared_info *shinfo = skb_shinfo(skb); 1164 u32 total_frag_len = 0; 1165 1166 total_frag_len = __bnxt_rx_agg_pages(bp, cpr, shinfo, idx, 1167 agg_bufs, tpa, NULL); 1168 if (!total_frag_len) { 1169 dev_kfree_skb(skb); 1170 return NULL; 1171 } 1172 1173 skb->data_len += total_frag_len; 1174 skb->len += total_frag_len; 1175 skb->truesize += PAGE_SIZE * agg_bufs; 1176 return skb; 1177 } 1178 1179 static u32 bnxt_rx_agg_pages_xdp(struct bnxt *bp, 1180 struct bnxt_cp_ring_info *cpr, 1181 struct xdp_buff *xdp, u16 idx, 1182 u32 agg_bufs, bool tpa) 1183 { 1184 struct skb_shared_info *shinfo = xdp_get_shared_info_from_buff(xdp); 1185 u32 total_frag_len = 0; 1186 1187 if (!xdp_buff_has_frags(xdp)) 1188 shinfo->nr_frags = 0; 1189 1190 total_frag_len = __bnxt_rx_agg_pages(bp, cpr, shinfo, 1191 idx, agg_bufs, tpa, xdp); 1192 if (total_frag_len) { 1193 xdp_buff_set_frags_flag(xdp); 1194 shinfo->nr_frags = agg_bufs; 1195 shinfo->xdp_frags_size = total_frag_len; 1196 } 1197 return total_frag_len; 1198 } 1199 1200 static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 1201 u8 agg_bufs, u32 *raw_cons) 1202 { 1203 u16 last; 1204 struct rx_agg_cmp *agg; 1205 1206 *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs); 1207 last = RING_CMP(*raw_cons); 1208 agg = (struct rx_agg_cmp *) 1209 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)]; 1210 return RX_AGG_CMP_VALID(agg, *raw_cons); 1211 } 1212 1213 static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data, 1214 unsigned int len, 1215 dma_addr_t mapping) 1216 { 1217 struct bnxt *bp = bnapi->bp; 1218 struct pci_dev *pdev = bp->pdev; 1219 struct sk_buff *skb; 1220 1221 skb = napi_alloc_skb(&bnapi->napi, len); 1222 if (!skb) 1223 return NULL; 1224 1225 dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copy_thresh, 1226 bp->rx_dir); 1227 1228 memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN, 1229 len + NET_IP_ALIGN); 1230 1231 dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copy_thresh, 1232 bp->rx_dir); 1233 1234 skb_put(skb, len); 1235 return skb; 1236 } 1237 1238 static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 1239 u32 *raw_cons, void *cmp) 1240 { 1241 struct rx_cmp *rxcmp = cmp; 1242 u32 tmp_raw_cons = *raw_cons; 1243 u8 cmp_type, agg_bufs = 0; 1244 1245 cmp_type = RX_CMP_TYPE(rxcmp); 1246 1247 if (cmp_type == CMP_TYPE_RX_L2_CMP) { 1248 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) & 1249 RX_CMP_AGG_BUFS) >> 1250 RX_CMP_AGG_BUFS_SHIFT; 1251 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) { 1252 struct rx_tpa_end_cmp *tpa_end = cmp; 1253 1254 if (bp->flags & BNXT_FLAG_CHIP_P5) 1255 return 0; 1256 1257 agg_bufs = TPA_END_AGG_BUFS(tpa_end); 1258 } 1259 1260 if (agg_bufs) { 1261 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons)) 1262 return -EBUSY; 1263 } 1264 *raw_cons = tmp_raw_cons; 1265 return 0; 1266 } 1267 1268 static void bnxt_queue_fw_reset_work(struct bnxt *bp, unsigned long delay) 1269 { 1270 if (!(test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))) 1271 return; 1272 1273 if (BNXT_PF(bp)) 1274 queue_delayed_work(bnxt_pf_wq, &bp->fw_reset_task, delay); 1275 else 1276 schedule_delayed_work(&bp->fw_reset_task, delay); 1277 } 1278 1279 static void bnxt_queue_sp_work(struct bnxt *bp) 1280 { 1281 if (BNXT_PF(bp)) 1282 queue_work(bnxt_pf_wq, &bp->sp_task); 1283 else 1284 schedule_work(&bp->sp_task); 1285 } 1286 1287 static void bnxt_sched_reset(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) 1288 { 1289 if (!rxr->bnapi->in_reset) { 1290 rxr->bnapi->in_reset = true; 1291 if (bp->flags & BNXT_FLAG_CHIP_P5) 1292 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event); 1293 else 1294 set_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event); 1295 bnxt_queue_sp_work(bp); 1296 } 1297 rxr->rx_next_cons = 0xffff; 1298 } 1299 1300 static u16 bnxt_alloc_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id) 1301 { 1302 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map; 1303 u16 idx = agg_id & MAX_TPA_P5_MASK; 1304 1305 if (test_bit(idx, map->agg_idx_bmap)) 1306 idx = find_first_zero_bit(map->agg_idx_bmap, 1307 BNXT_AGG_IDX_BMAP_SIZE); 1308 __set_bit(idx, map->agg_idx_bmap); 1309 map->agg_id_tbl[agg_id] = idx; 1310 return idx; 1311 } 1312 1313 static void bnxt_free_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx) 1314 { 1315 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map; 1316 1317 __clear_bit(idx, map->agg_idx_bmap); 1318 } 1319 1320 static u16 bnxt_lookup_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id) 1321 { 1322 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map; 1323 1324 return map->agg_id_tbl[agg_id]; 1325 } 1326 1327 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, 1328 struct rx_tpa_start_cmp *tpa_start, 1329 struct rx_tpa_start_cmp_ext *tpa_start1) 1330 { 1331 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf; 1332 struct bnxt_tpa_info *tpa_info; 1333 u16 cons, prod, agg_id; 1334 struct rx_bd *prod_bd; 1335 dma_addr_t mapping; 1336 1337 if (bp->flags & BNXT_FLAG_CHIP_P5) { 1338 agg_id = TPA_START_AGG_ID_P5(tpa_start); 1339 agg_id = bnxt_alloc_agg_idx(rxr, agg_id); 1340 } else { 1341 agg_id = TPA_START_AGG_ID(tpa_start); 1342 } 1343 cons = tpa_start->rx_tpa_start_cmp_opaque; 1344 prod = rxr->rx_prod; 1345 cons_rx_buf = &rxr->rx_buf_ring[cons]; 1346 prod_rx_buf = &rxr->rx_buf_ring[prod]; 1347 tpa_info = &rxr->rx_tpa[agg_id]; 1348 1349 if (unlikely(cons != rxr->rx_next_cons || 1350 TPA_START_ERROR(tpa_start))) { 1351 netdev_warn(bp->dev, "TPA cons %x, expected cons %x, error code %x\n", 1352 cons, rxr->rx_next_cons, 1353 TPA_START_ERROR_CODE(tpa_start1)); 1354 bnxt_sched_reset(bp, rxr); 1355 return; 1356 } 1357 /* Store cfa_code in tpa_info to use in tpa_end 1358 * completion processing. 1359 */ 1360 tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1); 1361 prod_rx_buf->data = tpa_info->data; 1362 prod_rx_buf->data_ptr = tpa_info->data_ptr; 1363 1364 mapping = tpa_info->mapping; 1365 prod_rx_buf->mapping = mapping; 1366 1367 prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 1368 1369 prod_bd->rx_bd_haddr = cpu_to_le64(mapping); 1370 1371 tpa_info->data = cons_rx_buf->data; 1372 tpa_info->data_ptr = cons_rx_buf->data_ptr; 1373 cons_rx_buf->data = NULL; 1374 tpa_info->mapping = cons_rx_buf->mapping; 1375 1376 tpa_info->len = 1377 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >> 1378 RX_TPA_START_CMP_LEN_SHIFT; 1379 if (likely(TPA_START_HASH_VALID(tpa_start))) { 1380 u32 hash_type = TPA_START_HASH_TYPE(tpa_start); 1381 1382 tpa_info->hash_type = PKT_HASH_TYPE_L4; 1383 tpa_info->gso_type = SKB_GSO_TCPV4; 1384 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */ 1385 if (hash_type == 3 || TPA_START_IS_IPV6(tpa_start1)) 1386 tpa_info->gso_type = SKB_GSO_TCPV6; 1387 tpa_info->rss_hash = 1388 le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash); 1389 } else { 1390 tpa_info->hash_type = PKT_HASH_TYPE_NONE; 1391 tpa_info->gso_type = 0; 1392 netif_warn(bp, rx_err, bp->dev, "TPA packet without valid hash\n"); 1393 } 1394 tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2); 1395 tpa_info->metadata = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata); 1396 tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info); 1397 tpa_info->agg_count = 0; 1398 1399 rxr->rx_prod = NEXT_RX(prod); 1400 cons = NEXT_RX(cons); 1401 rxr->rx_next_cons = NEXT_RX(cons); 1402 cons_rx_buf = &rxr->rx_buf_ring[cons]; 1403 1404 bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data); 1405 rxr->rx_prod = NEXT_RX(rxr->rx_prod); 1406 cons_rx_buf->data = NULL; 1407 } 1408 1409 static void bnxt_abort_tpa(struct bnxt_cp_ring_info *cpr, u16 idx, u32 agg_bufs) 1410 { 1411 if (agg_bufs) 1412 bnxt_reuse_rx_agg_bufs(cpr, idx, 0, agg_bufs, true); 1413 } 1414 1415 #ifdef CONFIG_INET 1416 static void bnxt_gro_tunnel(struct sk_buff *skb, __be16 ip_proto) 1417 { 1418 struct udphdr *uh = NULL; 1419 1420 if (ip_proto == htons(ETH_P_IP)) { 1421 struct iphdr *iph = (struct iphdr *)skb->data; 1422 1423 if (iph->protocol == IPPROTO_UDP) 1424 uh = (struct udphdr *)(iph + 1); 1425 } else { 1426 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data; 1427 1428 if (iph->nexthdr == IPPROTO_UDP) 1429 uh = (struct udphdr *)(iph + 1); 1430 } 1431 if (uh) { 1432 if (uh->check) 1433 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM; 1434 else 1435 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL; 1436 } 1437 } 1438 #endif 1439 1440 static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info, 1441 int payload_off, int tcp_ts, 1442 struct sk_buff *skb) 1443 { 1444 #ifdef CONFIG_INET 1445 struct tcphdr *th; 1446 int len, nw_off; 1447 u16 outer_ip_off, inner_ip_off, inner_mac_off; 1448 u32 hdr_info = tpa_info->hdr_info; 1449 bool loopback = false; 1450 1451 inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info); 1452 inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info); 1453 outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info); 1454 1455 /* If the packet is an internal loopback packet, the offsets will 1456 * have an extra 4 bytes. 1457 */ 1458 if (inner_mac_off == 4) { 1459 loopback = true; 1460 } else if (inner_mac_off > 4) { 1461 __be16 proto = *((__be16 *)(skb->data + inner_ip_off - 1462 ETH_HLEN - 2)); 1463 1464 /* We only support inner iPv4/ipv6. If we don't see the 1465 * correct protocol ID, it must be a loopback packet where 1466 * the offsets are off by 4. 1467 */ 1468 if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6)) 1469 loopback = true; 1470 } 1471 if (loopback) { 1472 /* internal loopback packet, subtract all offsets by 4 */ 1473 inner_ip_off -= 4; 1474 inner_mac_off -= 4; 1475 outer_ip_off -= 4; 1476 } 1477 1478 nw_off = inner_ip_off - ETH_HLEN; 1479 skb_set_network_header(skb, nw_off); 1480 if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) { 1481 struct ipv6hdr *iph = ipv6_hdr(skb); 1482 1483 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr)); 1484 len = skb->len - skb_transport_offset(skb); 1485 th = tcp_hdr(skb); 1486 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0); 1487 } else { 1488 struct iphdr *iph = ip_hdr(skb); 1489 1490 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr)); 1491 len = skb->len - skb_transport_offset(skb); 1492 th = tcp_hdr(skb); 1493 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0); 1494 } 1495 1496 if (inner_mac_off) { /* tunnel */ 1497 __be16 proto = *((__be16 *)(skb->data + outer_ip_off - 1498 ETH_HLEN - 2)); 1499 1500 bnxt_gro_tunnel(skb, proto); 1501 } 1502 #endif 1503 return skb; 1504 } 1505 1506 static struct sk_buff *bnxt_gro_func_5750x(struct bnxt_tpa_info *tpa_info, 1507 int payload_off, int tcp_ts, 1508 struct sk_buff *skb) 1509 { 1510 #ifdef CONFIG_INET 1511 u16 outer_ip_off, inner_ip_off, inner_mac_off; 1512 u32 hdr_info = tpa_info->hdr_info; 1513 int iphdr_len, nw_off; 1514 1515 inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info); 1516 inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info); 1517 outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info); 1518 1519 nw_off = inner_ip_off - ETH_HLEN; 1520 skb_set_network_header(skb, nw_off); 1521 iphdr_len = (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) ? 1522 sizeof(struct ipv6hdr) : sizeof(struct iphdr); 1523 skb_set_transport_header(skb, nw_off + iphdr_len); 1524 1525 if (inner_mac_off) { /* tunnel */ 1526 __be16 proto = *((__be16 *)(skb->data + outer_ip_off - 1527 ETH_HLEN - 2)); 1528 1529 bnxt_gro_tunnel(skb, proto); 1530 } 1531 #endif 1532 return skb; 1533 } 1534 1535 #define BNXT_IPV4_HDR_SIZE (sizeof(struct iphdr) + sizeof(struct tcphdr)) 1536 #define BNXT_IPV6_HDR_SIZE (sizeof(struct ipv6hdr) + sizeof(struct tcphdr)) 1537 1538 static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info, 1539 int payload_off, int tcp_ts, 1540 struct sk_buff *skb) 1541 { 1542 #ifdef CONFIG_INET 1543 struct tcphdr *th; 1544 int len, nw_off, tcp_opt_len = 0; 1545 1546 if (tcp_ts) 1547 tcp_opt_len = 12; 1548 1549 if (tpa_info->gso_type == SKB_GSO_TCPV4) { 1550 struct iphdr *iph; 1551 1552 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len - 1553 ETH_HLEN; 1554 skb_set_network_header(skb, nw_off); 1555 iph = ip_hdr(skb); 1556 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr)); 1557 len = skb->len - skb_transport_offset(skb); 1558 th = tcp_hdr(skb); 1559 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0); 1560 } else if (tpa_info->gso_type == SKB_GSO_TCPV6) { 1561 struct ipv6hdr *iph; 1562 1563 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len - 1564 ETH_HLEN; 1565 skb_set_network_header(skb, nw_off); 1566 iph = ipv6_hdr(skb); 1567 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr)); 1568 len = skb->len - skb_transport_offset(skb); 1569 th = tcp_hdr(skb); 1570 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0); 1571 } else { 1572 dev_kfree_skb_any(skb); 1573 return NULL; 1574 } 1575 1576 if (nw_off) /* tunnel */ 1577 bnxt_gro_tunnel(skb, skb->protocol); 1578 #endif 1579 return skb; 1580 } 1581 1582 static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp, 1583 struct bnxt_tpa_info *tpa_info, 1584 struct rx_tpa_end_cmp *tpa_end, 1585 struct rx_tpa_end_cmp_ext *tpa_end1, 1586 struct sk_buff *skb) 1587 { 1588 #ifdef CONFIG_INET 1589 int payload_off; 1590 u16 segs; 1591 1592 segs = TPA_END_TPA_SEGS(tpa_end); 1593 if (segs == 1) 1594 return skb; 1595 1596 NAPI_GRO_CB(skb)->count = segs; 1597 skb_shinfo(skb)->gso_size = 1598 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len); 1599 skb_shinfo(skb)->gso_type = tpa_info->gso_type; 1600 if (bp->flags & BNXT_FLAG_CHIP_P5) 1601 payload_off = TPA_END_PAYLOAD_OFF_P5(tpa_end1); 1602 else 1603 payload_off = TPA_END_PAYLOAD_OFF(tpa_end); 1604 skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb); 1605 if (likely(skb)) 1606 tcp_gro_complete(skb); 1607 #endif 1608 return skb; 1609 } 1610 1611 /* Given the cfa_code of a received packet determine which 1612 * netdev (vf-rep or PF) the packet is destined to. 1613 */ 1614 static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code) 1615 { 1616 struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code); 1617 1618 /* if vf-rep dev is NULL, the must belongs to the PF */ 1619 return dev ? dev : bp->dev; 1620 } 1621 1622 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp, 1623 struct bnxt_cp_ring_info *cpr, 1624 u32 *raw_cons, 1625 struct rx_tpa_end_cmp *tpa_end, 1626 struct rx_tpa_end_cmp_ext *tpa_end1, 1627 u8 *event) 1628 { 1629 struct bnxt_napi *bnapi = cpr->bnapi; 1630 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1631 u8 *data_ptr, agg_bufs; 1632 unsigned int len; 1633 struct bnxt_tpa_info *tpa_info; 1634 dma_addr_t mapping; 1635 struct sk_buff *skb; 1636 u16 idx = 0, agg_id; 1637 void *data; 1638 bool gro; 1639 1640 if (unlikely(bnapi->in_reset)) { 1641 int rc = bnxt_discard_rx(bp, cpr, raw_cons, tpa_end); 1642 1643 if (rc < 0) 1644 return ERR_PTR(-EBUSY); 1645 return NULL; 1646 } 1647 1648 if (bp->flags & BNXT_FLAG_CHIP_P5) { 1649 agg_id = TPA_END_AGG_ID_P5(tpa_end); 1650 agg_id = bnxt_lookup_agg_idx(rxr, agg_id); 1651 agg_bufs = TPA_END_AGG_BUFS_P5(tpa_end1); 1652 tpa_info = &rxr->rx_tpa[agg_id]; 1653 if (unlikely(agg_bufs != tpa_info->agg_count)) { 1654 netdev_warn(bp->dev, "TPA end agg_buf %d != expected agg_bufs %d\n", 1655 agg_bufs, tpa_info->agg_count); 1656 agg_bufs = tpa_info->agg_count; 1657 } 1658 tpa_info->agg_count = 0; 1659 *event |= BNXT_AGG_EVENT; 1660 bnxt_free_agg_idx(rxr, agg_id); 1661 idx = agg_id; 1662 gro = !!(bp->flags & BNXT_FLAG_GRO); 1663 } else { 1664 agg_id = TPA_END_AGG_ID(tpa_end); 1665 agg_bufs = TPA_END_AGG_BUFS(tpa_end); 1666 tpa_info = &rxr->rx_tpa[agg_id]; 1667 idx = RING_CMP(*raw_cons); 1668 if (agg_bufs) { 1669 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons)) 1670 return ERR_PTR(-EBUSY); 1671 1672 *event |= BNXT_AGG_EVENT; 1673 idx = NEXT_CMP(idx); 1674 } 1675 gro = !!TPA_END_GRO(tpa_end); 1676 } 1677 data = tpa_info->data; 1678 data_ptr = tpa_info->data_ptr; 1679 prefetch(data_ptr); 1680 len = tpa_info->len; 1681 mapping = tpa_info->mapping; 1682 1683 if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) { 1684 bnxt_abort_tpa(cpr, idx, agg_bufs); 1685 if (agg_bufs > MAX_SKB_FRAGS) 1686 netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n", 1687 agg_bufs, (int)MAX_SKB_FRAGS); 1688 return NULL; 1689 } 1690 1691 if (len <= bp->rx_copy_thresh) { 1692 skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping); 1693 if (!skb) { 1694 bnxt_abort_tpa(cpr, idx, agg_bufs); 1695 cpr->sw_stats.rx.rx_oom_discards += 1; 1696 return NULL; 1697 } 1698 } else { 1699 u8 *new_data; 1700 dma_addr_t new_mapping; 1701 1702 new_data = __bnxt_alloc_rx_frag(bp, &new_mapping, GFP_ATOMIC); 1703 if (!new_data) { 1704 bnxt_abort_tpa(cpr, idx, agg_bufs); 1705 cpr->sw_stats.rx.rx_oom_discards += 1; 1706 return NULL; 1707 } 1708 1709 tpa_info->data = new_data; 1710 tpa_info->data_ptr = new_data + bp->rx_offset; 1711 tpa_info->mapping = new_mapping; 1712 1713 skb = build_skb(data, bp->rx_buf_size); 1714 dma_unmap_single_attrs(&bp->pdev->dev, mapping, 1715 bp->rx_buf_use_size, bp->rx_dir, 1716 DMA_ATTR_WEAK_ORDERING); 1717 1718 if (!skb) { 1719 skb_free_frag(data); 1720 bnxt_abort_tpa(cpr, idx, agg_bufs); 1721 cpr->sw_stats.rx.rx_oom_discards += 1; 1722 return NULL; 1723 } 1724 skb_reserve(skb, bp->rx_offset); 1725 skb_put(skb, len); 1726 } 1727 1728 if (agg_bufs) { 1729 skb = bnxt_rx_agg_pages_skb(bp, cpr, skb, idx, agg_bufs, true); 1730 if (!skb) { 1731 /* Page reuse already handled by bnxt_rx_pages(). */ 1732 cpr->sw_stats.rx.rx_oom_discards += 1; 1733 return NULL; 1734 } 1735 } 1736 1737 skb->protocol = 1738 eth_type_trans(skb, bnxt_get_pkt_dev(bp, tpa_info->cfa_code)); 1739 1740 if (tpa_info->hash_type != PKT_HASH_TYPE_NONE) 1741 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type); 1742 1743 if ((tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) && 1744 (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) { 1745 __be16 vlan_proto = htons(tpa_info->metadata >> 1746 RX_CMP_FLAGS2_METADATA_TPID_SFT); 1747 u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_TCI_MASK; 1748 1749 if (eth_type_vlan(vlan_proto)) { 1750 __vlan_hwaccel_put_tag(skb, vlan_proto, vtag); 1751 } else { 1752 dev_kfree_skb(skb); 1753 return NULL; 1754 } 1755 } 1756 1757 skb_checksum_none_assert(skb); 1758 if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) { 1759 skb->ip_summed = CHECKSUM_UNNECESSARY; 1760 skb->csum_level = 1761 (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3; 1762 } 1763 1764 if (gro) 1765 skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb); 1766 1767 return skb; 1768 } 1769 1770 static void bnxt_tpa_agg(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, 1771 struct rx_agg_cmp *rx_agg) 1772 { 1773 u16 agg_id = TPA_AGG_AGG_ID(rx_agg); 1774 struct bnxt_tpa_info *tpa_info; 1775 1776 agg_id = bnxt_lookup_agg_idx(rxr, agg_id); 1777 tpa_info = &rxr->rx_tpa[agg_id]; 1778 BUG_ON(tpa_info->agg_count >= MAX_SKB_FRAGS); 1779 tpa_info->agg_arr[tpa_info->agg_count++] = *rx_agg; 1780 } 1781 1782 static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi, 1783 struct sk_buff *skb) 1784 { 1785 if (skb->dev != bp->dev) { 1786 /* this packet belongs to a vf-rep */ 1787 bnxt_vf_rep_rx(bp, skb); 1788 return; 1789 } 1790 skb_record_rx_queue(skb, bnapi->index); 1791 napi_gro_receive(&bnapi->napi, skb); 1792 } 1793 1794 /* returns the following: 1795 * 1 - 1 packet successfully received 1796 * 0 - successful TPA_START, packet not completed yet 1797 * -EBUSY - completion ring does not have all the agg buffers yet 1798 * -ENOMEM - packet aborted due to out of memory 1799 * -EIO - packet aborted due to hw error indicated in BD 1800 */ 1801 static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 1802 u32 *raw_cons, u8 *event) 1803 { 1804 struct bnxt_napi *bnapi = cpr->bnapi; 1805 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1806 struct net_device *dev = bp->dev; 1807 struct rx_cmp *rxcmp; 1808 struct rx_cmp_ext *rxcmp1; 1809 u32 tmp_raw_cons = *raw_cons; 1810 u16 cfa_code, cons, prod, cp_cons = RING_CMP(tmp_raw_cons); 1811 struct bnxt_sw_rx_bd *rx_buf; 1812 unsigned int len; 1813 u8 *data_ptr, agg_bufs, cmp_type; 1814 bool xdp_active = false; 1815 dma_addr_t dma_addr; 1816 struct sk_buff *skb; 1817 struct xdp_buff xdp; 1818 u32 flags, misc; 1819 void *data; 1820 int rc = 0; 1821 1822 rxcmp = (struct rx_cmp *) 1823 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1824 1825 cmp_type = RX_CMP_TYPE(rxcmp); 1826 1827 if (cmp_type == CMP_TYPE_RX_TPA_AGG_CMP) { 1828 bnxt_tpa_agg(bp, rxr, (struct rx_agg_cmp *)rxcmp); 1829 goto next_rx_no_prod_no_len; 1830 } 1831 1832 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons); 1833 cp_cons = RING_CMP(tmp_raw_cons); 1834 rxcmp1 = (struct rx_cmp_ext *) 1835 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1836 1837 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons)) 1838 return -EBUSY; 1839 1840 /* The valid test of the entry must be done first before 1841 * reading any further. 1842 */ 1843 dma_rmb(); 1844 prod = rxr->rx_prod; 1845 1846 if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) { 1847 bnxt_tpa_start(bp, rxr, (struct rx_tpa_start_cmp *)rxcmp, 1848 (struct rx_tpa_start_cmp_ext *)rxcmp1); 1849 1850 *event |= BNXT_RX_EVENT; 1851 goto next_rx_no_prod_no_len; 1852 1853 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) { 1854 skb = bnxt_tpa_end(bp, cpr, &tmp_raw_cons, 1855 (struct rx_tpa_end_cmp *)rxcmp, 1856 (struct rx_tpa_end_cmp_ext *)rxcmp1, event); 1857 1858 if (IS_ERR(skb)) 1859 return -EBUSY; 1860 1861 rc = -ENOMEM; 1862 if (likely(skb)) { 1863 bnxt_deliver_skb(bp, bnapi, skb); 1864 rc = 1; 1865 } 1866 *event |= BNXT_RX_EVENT; 1867 goto next_rx_no_prod_no_len; 1868 } 1869 1870 cons = rxcmp->rx_cmp_opaque; 1871 if (unlikely(cons != rxr->rx_next_cons)) { 1872 int rc1 = bnxt_discard_rx(bp, cpr, &tmp_raw_cons, rxcmp); 1873 1874 /* 0xffff is forced error, don't print it */ 1875 if (rxr->rx_next_cons != 0xffff) 1876 netdev_warn(bp->dev, "RX cons %x != expected cons %x\n", 1877 cons, rxr->rx_next_cons); 1878 bnxt_sched_reset(bp, rxr); 1879 if (rc1) 1880 return rc1; 1881 goto next_rx_no_prod_no_len; 1882 } 1883 rx_buf = &rxr->rx_buf_ring[cons]; 1884 data = rx_buf->data; 1885 data_ptr = rx_buf->data_ptr; 1886 prefetch(data_ptr); 1887 1888 misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1); 1889 agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT; 1890 1891 if (agg_bufs) { 1892 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons)) 1893 return -EBUSY; 1894 1895 cp_cons = NEXT_CMP(cp_cons); 1896 *event |= BNXT_AGG_EVENT; 1897 } 1898 *event |= BNXT_RX_EVENT; 1899 1900 rx_buf->data = NULL; 1901 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) { 1902 u32 rx_err = le32_to_cpu(rxcmp1->rx_cmp_cfa_code_errors_v2); 1903 1904 bnxt_reuse_rx_data(rxr, cons, data); 1905 if (agg_bufs) 1906 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, agg_bufs, 1907 false); 1908 1909 rc = -EIO; 1910 if (rx_err & RX_CMPL_ERRORS_BUFFER_ERROR_MASK) { 1911 bnapi->cp_ring.sw_stats.rx.rx_buf_errors++; 1912 if (!(bp->flags & BNXT_FLAG_CHIP_P5) && 1913 !(bp->fw_cap & BNXT_FW_CAP_RING_MONITOR)) { 1914 netdev_warn_once(bp->dev, "RX buffer error %x\n", 1915 rx_err); 1916 bnxt_sched_reset(bp, rxr); 1917 } 1918 } 1919 goto next_rx_no_len; 1920 } 1921 1922 flags = le32_to_cpu(rxcmp->rx_cmp_len_flags_type); 1923 len = flags >> RX_CMP_LEN_SHIFT; 1924 dma_addr = rx_buf->mapping; 1925 1926 if (bnxt_xdp_attached(bp, rxr)) { 1927 bnxt_xdp_buff_init(bp, rxr, cons, data_ptr, len, &xdp); 1928 if (agg_bufs) { 1929 u32 frag_len = bnxt_rx_agg_pages_xdp(bp, cpr, &xdp, 1930 cp_cons, agg_bufs, 1931 false); 1932 if (!frag_len) { 1933 cpr->sw_stats.rx.rx_oom_discards += 1; 1934 rc = -ENOMEM; 1935 goto next_rx; 1936 } 1937 } 1938 xdp_active = true; 1939 } 1940 1941 if (xdp_active) { 1942 if (bnxt_rx_xdp(bp, rxr, cons, xdp, data, &data_ptr, &len, event)) { 1943 rc = 1; 1944 goto next_rx; 1945 } 1946 } 1947 1948 if (len <= bp->rx_copy_thresh) { 1949 skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr); 1950 bnxt_reuse_rx_data(rxr, cons, data); 1951 if (!skb) { 1952 if (agg_bufs) { 1953 if (!xdp_active) 1954 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, 1955 agg_bufs, false); 1956 else 1957 bnxt_xdp_buff_frags_free(rxr, &xdp); 1958 } 1959 cpr->sw_stats.rx.rx_oom_discards += 1; 1960 rc = -ENOMEM; 1961 goto next_rx; 1962 } 1963 } else { 1964 u32 payload; 1965 1966 if (rx_buf->data_ptr == data_ptr) 1967 payload = misc & RX_CMP_PAYLOAD_OFFSET; 1968 else 1969 payload = 0; 1970 skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr, 1971 payload | len); 1972 if (!skb) { 1973 cpr->sw_stats.rx.rx_oom_discards += 1; 1974 rc = -ENOMEM; 1975 goto next_rx; 1976 } 1977 } 1978 1979 if (agg_bufs) { 1980 if (!xdp_active) { 1981 skb = bnxt_rx_agg_pages_skb(bp, cpr, skb, cp_cons, agg_bufs, false); 1982 if (!skb) { 1983 cpr->sw_stats.rx.rx_oom_discards += 1; 1984 rc = -ENOMEM; 1985 goto next_rx; 1986 } 1987 } else { 1988 skb = bnxt_xdp_build_skb(bp, skb, agg_bufs, rxr->page_pool, &xdp, rxcmp1); 1989 if (!skb) { 1990 /* we should be able to free the old skb here */ 1991 bnxt_xdp_buff_frags_free(rxr, &xdp); 1992 cpr->sw_stats.rx.rx_oom_discards += 1; 1993 rc = -ENOMEM; 1994 goto next_rx; 1995 } 1996 } 1997 } 1998 1999 if (RX_CMP_HASH_VALID(rxcmp)) { 2000 u32 hash_type = RX_CMP_HASH_TYPE(rxcmp); 2001 enum pkt_hash_types type = PKT_HASH_TYPE_L4; 2002 2003 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */ 2004 if (hash_type != 1 && hash_type != 3) 2005 type = PKT_HASH_TYPE_L3; 2006 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type); 2007 } 2008 2009 cfa_code = RX_CMP_CFA_CODE(rxcmp1); 2010 skb->protocol = eth_type_trans(skb, bnxt_get_pkt_dev(bp, cfa_code)); 2011 2012 if ((rxcmp1->rx_cmp_flags2 & 2013 cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) && 2014 (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) { 2015 u32 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data); 2016 u16 vtag = meta_data & RX_CMP_FLAGS2_METADATA_TCI_MASK; 2017 __be16 vlan_proto = htons(meta_data >> 2018 RX_CMP_FLAGS2_METADATA_TPID_SFT); 2019 2020 if (eth_type_vlan(vlan_proto)) { 2021 __vlan_hwaccel_put_tag(skb, vlan_proto, vtag); 2022 } else { 2023 dev_kfree_skb(skb); 2024 goto next_rx; 2025 } 2026 } 2027 2028 skb_checksum_none_assert(skb); 2029 if (RX_CMP_L4_CS_OK(rxcmp1)) { 2030 if (dev->features & NETIF_F_RXCSUM) { 2031 skb->ip_summed = CHECKSUM_UNNECESSARY; 2032 skb->csum_level = RX_CMP_ENCAP(rxcmp1); 2033 } 2034 } else { 2035 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) { 2036 if (dev->features & NETIF_F_RXCSUM) 2037 bnapi->cp_ring.sw_stats.rx.rx_l4_csum_errors++; 2038 } 2039 } 2040 2041 if (unlikely((flags & RX_CMP_FLAGS_ITYPES_MASK) == 2042 RX_CMP_FLAGS_ITYPE_PTP_W_TS) || bp->ptp_all_rx_tstamp) { 2043 if (bp->flags & BNXT_FLAG_CHIP_P5) { 2044 u32 cmpl_ts = le32_to_cpu(rxcmp1->rx_cmp_timestamp); 2045 u64 ns, ts; 2046 2047 if (!bnxt_get_rx_ts_p5(bp, &ts, cmpl_ts)) { 2048 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; 2049 2050 spin_lock_bh(&ptp->ptp_lock); 2051 ns = timecounter_cyc2time(&ptp->tc, ts); 2052 spin_unlock_bh(&ptp->ptp_lock); 2053 memset(skb_hwtstamps(skb), 0, 2054 sizeof(*skb_hwtstamps(skb))); 2055 skb_hwtstamps(skb)->hwtstamp = ns_to_ktime(ns); 2056 } 2057 } 2058 } 2059 bnxt_deliver_skb(bp, bnapi, skb); 2060 rc = 1; 2061 2062 next_rx: 2063 cpr->rx_packets += 1; 2064 cpr->rx_bytes += len; 2065 2066 next_rx_no_len: 2067 rxr->rx_prod = NEXT_RX(prod); 2068 rxr->rx_next_cons = NEXT_RX(cons); 2069 2070 next_rx_no_prod_no_len: 2071 *raw_cons = tmp_raw_cons; 2072 2073 return rc; 2074 } 2075 2076 /* In netpoll mode, if we are using a combined completion ring, we need to 2077 * discard the rx packets and recycle the buffers. 2078 */ 2079 static int bnxt_force_rx_discard(struct bnxt *bp, 2080 struct bnxt_cp_ring_info *cpr, 2081 u32 *raw_cons, u8 *event) 2082 { 2083 u32 tmp_raw_cons = *raw_cons; 2084 struct rx_cmp_ext *rxcmp1; 2085 struct rx_cmp *rxcmp; 2086 u16 cp_cons; 2087 u8 cmp_type; 2088 int rc; 2089 2090 cp_cons = RING_CMP(tmp_raw_cons); 2091 rxcmp = (struct rx_cmp *) 2092 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 2093 2094 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons); 2095 cp_cons = RING_CMP(tmp_raw_cons); 2096 rxcmp1 = (struct rx_cmp_ext *) 2097 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 2098 2099 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons)) 2100 return -EBUSY; 2101 2102 /* The valid test of the entry must be done first before 2103 * reading any further. 2104 */ 2105 dma_rmb(); 2106 cmp_type = RX_CMP_TYPE(rxcmp); 2107 if (cmp_type == CMP_TYPE_RX_L2_CMP) { 2108 rxcmp1->rx_cmp_cfa_code_errors_v2 |= 2109 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR); 2110 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) { 2111 struct rx_tpa_end_cmp_ext *tpa_end1; 2112 2113 tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1; 2114 tpa_end1->rx_tpa_end_cmp_errors_v2 |= 2115 cpu_to_le32(RX_TPA_END_CMP_ERRORS); 2116 } 2117 rc = bnxt_rx_pkt(bp, cpr, raw_cons, event); 2118 if (rc && rc != -EBUSY) 2119 cpr->sw_stats.rx.rx_netpoll_discards += 1; 2120 return rc; 2121 } 2122 2123 u32 bnxt_fw_health_readl(struct bnxt *bp, int reg_idx) 2124 { 2125 struct bnxt_fw_health *fw_health = bp->fw_health; 2126 u32 reg = fw_health->regs[reg_idx]; 2127 u32 reg_type, reg_off, val = 0; 2128 2129 reg_type = BNXT_FW_HEALTH_REG_TYPE(reg); 2130 reg_off = BNXT_FW_HEALTH_REG_OFF(reg); 2131 switch (reg_type) { 2132 case BNXT_FW_HEALTH_REG_TYPE_CFG: 2133 pci_read_config_dword(bp->pdev, reg_off, &val); 2134 break; 2135 case BNXT_FW_HEALTH_REG_TYPE_GRC: 2136 reg_off = fw_health->mapped_regs[reg_idx]; 2137 fallthrough; 2138 case BNXT_FW_HEALTH_REG_TYPE_BAR0: 2139 val = readl(bp->bar0 + reg_off); 2140 break; 2141 case BNXT_FW_HEALTH_REG_TYPE_BAR1: 2142 val = readl(bp->bar1 + reg_off); 2143 break; 2144 } 2145 if (reg_idx == BNXT_FW_RESET_INPROG_REG) 2146 val &= fw_health->fw_reset_inprog_reg_mask; 2147 return val; 2148 } 2149 2150 static u16 bnxt_agg_ring_id_to_grp_idx(struct bnxt *bp, u16 ring_id) 2151 { 2152 int i; 2153 2154 for (i = 0; i < bp->rx_nr_rings; i++) { 2155 u16 grp_idx = bp->rx_ring[i].bnapi->index; 2156 struct bnxt_ring_grp_info *grp_info; 2157 2158 grp_info = &bp->grp_info[grp_idx]; 2159 if (grp_info->agg_fw_ring_id == ring_id) 2160 return grp_idx; 2161 } 2162 return INVALID_HW_RING_ID; 2163 } 2164 2165 static void bnxt_event_error_report(struct bnxt *bp, u32 data1, u32 data2) 2166 { 2167 u32 err_type = BNXT_EVENT_ERROR_REPORT_TYPE(data1); 2168 2169 switch (err_type) { 2170 case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_INVALID_SIGNAL: 2171 netdev_err(bp->dev, "1PPS: Received invalid signal on pin%lu from the external source. Please fix the signal and reconfigure the pin\n", 2172 BNXT_EVENT_INVALID_SIGNAL_DATA(data2)); 2173 break; 2174 case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_PAUSE_STORM: 2175 netdev_warn(bp->dev, "Pause Storm detected!\n"); 2176 break; 2177 case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_DOORBELL_DROP_THRESHOLD: 2178 netdev_warn(bp->dev, "One or more MMIO doorbells dropped by the device!\n"); 2179 break; 2180 default: 2181 netdev_err(bp->dev, "FW reported unknown error type %u\n", 2182 err_type); 2183 break; 2184 } 2185 } 2186 2187 #define BNXT_GET_EVENT_PORT(data) \ 2188 ((data) & \ 2189 ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK) 2190 2191 #define BNXT_EVENT_RING_TYPE(data2) \ 2192 ((data2) & \ 2193 ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_MASK) 2194 2195 #define BNXT_EVENT_RING_TYPE_RX(data2) \ 2196 (BNXT_EVENT_RING_TYPE(data2) == \ 2197 ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_RX) 2198 2199 #define BNXT_EVENT_PHC_EVENT_TYPE(data1) \ 2200 (((data1) & ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_MASK) >>\ 2201 ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_SFT) 2202 2203 #define BNXT_EVENT_PHC_RTC_UPDATE(data1) \ 2204 (((data1) & ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_PHC_TIME_MSB_MASK) >>\ 2205 ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_PHC_TIME_MSB_SFT) 2206 2207 #define BNXT_PHC_BITS 48 2208 2209 static int bnxt_async_event_process(struct bnxt *bp, 2210 struct hwrm_async_event_cmpl *cmpl) 2211 { 2212 u16 event_id = le16_to_cpu(cmpl->event_id); 2213 u32 data1 = le32_to_cpu(cmpl->event_data1); 2214 u32 data2 = le32_to_cpu(cmpl->event_data2); 2215 2216 netdev_dbg(bp->dev, "hwrm event 0x%x {0x%x, 0x%x}\n", 2217 event_id, data1, data2); 2218 2219 /* TODO CHIMP_FW: Define event id's for link change, error etc */ 2220 switch (event_id) { 2221 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: { 2222 struct bnxt_link_info *link_info = &bp->link_info; 2223 2224 if (BNXT_VF(bp)) 2225 goto async_event_process_exit; 2226 2227 /* print unsupported speed warning in forced speed mode only */ 2228 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED) && 2229 (data1 & 0x20000)) { 2230 u16 fw_speed = link_info->force_link_speed; 2231 u32 speed = bnxt_fw_to_ethtool_speed(fw_speed); 2232 2233 if (speed != SPEED_UNKNOWN) 2234 netdev_warn(bp->dev, "Link speed %d no longer supported\n", 2235 speed); 2236 } 2237 set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event); 2238 } 2239 fallthrough; 2240 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE: 2241 case ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE: 2242 set_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT, &bp->sp_event); 2243 fallthrough; 2244 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE: 2245 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event); 2246 break; 2247 case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD: 2248 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event); 2249 break; 2250 case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: { 2251 u16 port_id = BNXT_GET_EVENT_PORT(data1); 2252 2253 if (BNXT_VF(bp)) 2254 break; 2255 2256 if (bp->pf.port_id != port_id) 2257 break; 2258 2259 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event); 2260 break; 2261 } 2262 case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE: 2263 if (BNXT_PF(bp)) 2264 goto async_event_process_exit; 2265 set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event); 2266 break; 2267 case ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY: { 2268 char *type_str = "Solicited"; 2269 2270 if (!bp->fw_health) 2271 goto async_event_process_exit; 2272 2273 bp->fw_reset_timestamp = jiffies; 2274 bp->fw_reset_min_dsecs = cmpl->timestamp_lo; 2275 if (!bp->fw_reset_min_dsecs) 2276 bp->fw_reset_min_dsecs = BNXT_DFLT_FW_RST_MIN_DSECS; 2277 bp->fw_reset_max_dsecs = le16_to_cpu(cmpl->timestamp_hi); 2278 if (!bp->fw_reset_max_dsecs) 2279 bp->fw_reset_max_dsecs = BNXT_DFLT_FW_RST_MAX_DSECS; 2280 if (EVENT_DATA1_RESET_NOTIFY_FW_ACTIVATION(data1)) { 2281 set_bit(BNXT_STATE_FW_ACTIVATE_RESET, &bp->state); 2282 } else if (EVENT_DATA1_RESET_NOTIFY_FATAL(data1)) { 2283 type_str = "Fatal"; 2284 bp->fw_health->fatalities++; 2285 set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state); 2286 } else if (data2 && BNXT_FW_STATUS_HEALTHY != 2287 EVENT_DATA2_RESET_NOTIFY_FW_STATUS_CODE(data2)) { 2288 type_str = "Non-fatal"; 2289 bp->fw_health->survivals++; 2290 set_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state); 2291 } 2292 netif_warn(bp, hw, bp->dev, 2293 "%s firmware reset event, data1: 0x%x, data2: 0x%x, min wait %u ms, max wait %u ms\n", 2294 type_str, data1, data2, 2295 bp->fw_reset_min_dsecs * 100, 2296 bp->fw_reset_max_dsecs * 100); 2297 set_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event); 2298 break; 2299 } 2300 case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY: { 2301 struct bnxt_fw_health *fw_health = bp->fw_health; 2302 char *status_desc = "healthy"; 2303 u32 status; 2304 2305 if (!fw_health) 2306 goto async_event_process_exit; 2307 2308 if (!EVENT_DATA1_RECOVERY_ENABLED(data1)) { 2309 fw_health->enabled = false; 2310 netif_info(bp, drv, bp->dev, "Driver recovery watchdog is disabled\n"); 2311 break; 2312 } 2313 fw_health->primary = EVENT_DATA1_RECOVERY_MASTER_FUNC(data1); 2314 fw_health->tmr_multiplier = 2315 DIV_ROUND_UP(fw_health->polling_dsecs * HZ, 2316 bp->current_interval * 10); 2317 fw_health->tmr_counter = fw_health->tmr_multiplier; 2318 if (!fw_health->enabled) 2319 fw_health->last_fw_heartbeat = 2320 bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG); 2321 fw_health->last_fw_reset_cnt = 2322 bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG); 2323 status = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG); 2324 if (status != BNXT_FW_STATUS_HEALTHY) 2325 status_desc = "unhealthy"; 2326 netif_info(bp, drv, bp->dev, 2327 "Driver recovery watchdog, role: %s, firmware status: 0x%x (%s), resets: %u\n", 2328 fw_health->primary ? "primary" : "backup", status, 2329 status_desc, fw_health->last_fw_reset_cnt); 2330 if (!fw_health->enabled) { 2331 /* Make sure tmr_counter is set and visible to 2332 * bnxt_health_check() before setting enabled to true. 2333 */ 2334 smp_wmb(); 2335 fw_health->enabled = true; 2336 } 2337 goto async_event_process_exit; 2338 } 2339 case ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION: 2340 netif_notice(bp, hw, bp->dev, 2341 "Received firmware debug notification, data1: 0x%x, data2: 0x%x\n", 2342 data1, data2); 2343 goto async_event_process_exit; 2344 case ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG: { 2345 struct bnxt_rx_ring_info *rxr; 2346 u16 grp_idx; 2347 2348 if (bp->flags & BNXT_FLAG_CHIP_P5) 2349 goto async_event_process_exit; 2350 2351 netdev_warn(bp->dev, "Ring monitor event, ring type %lu id 0x%x\n", 2352 BNXT_EVENT_RING_TYPE(data2), data1); 2353 if (!BNXT_EVENT_RING_TYPE_RX(data2)) 2354 goto async_event_process_exit; 2355 2356 grp_idx = bnxt_agg_ring_id_to_grp_idx(bp, data1); 2357 if (grp_idx == INVALID_HW_RING_ID) { 2358 netdev_warn(bp->dev, "Unknown RX agg ring id 0x%x\n", 2359 data1); 2360 goto async_event_process_exit; 2361 } 2362 rxr = bp->bnapi[grp_idx]->rx_ring; 2363 bnxt_sched_reset(bp, rxr); 2364 goto async_event_process_exit; 2365 } 2366 case ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST: { 2367 struct bnxt_fw_health *fw_health = bp->fw_health; 2368 2369 netif_notice(bp, hw, bp->dev, 2370 "Received firmware echo request, data1: 0x%x, data2: 0x%x\n", 2371 data1, data2); 2372 if (fw_health) { 2373 fw_health->echo_req_data1 = data1; 2374 fw_health->echo_req_data2 = data2; 2375 set_bit(BNXT_FW_ECHO_REQUEST_SP_EVENT, &bp->sp_event); 2376 break; 2377 } 2378 goto async_event_process_exit; 2379 } 2380 case ASYNC_EVENT_CMPL_EVENT_ID_PPS_TIMESTAMP: { 2381 bnxt_ptp_pps_event(bp, data1, data2); 2382 goto async_event_process_exit; 2383 } 2384 case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT: { 2385 bnxt_event_error_report(bp, data1, data2); 2386 goto async_event_process_exit; 2387 } 2388 case ASYNC_EVENT_CMPL_EVENT_ID_PHC_UPDATE: { 2389 switch (BNXT_EVENT_PHC_EVENT_TYPE(data1)) { 2390 case ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_PHC_RTC_UPDATE: 2391 if (BNXT_PTP_USE_RTC(bp)) { 2392 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; 2393 u64 ns; 2394 2395 spin_lock_bh(&ptp->ptp_lock); 2396 bnxt_ptp_update_current_time(bp); 2397 ns = (((u64)BNXT_EVENT_PHC_RTC_UPDATE(data1) << 2398 BNXT_PHC_BITS) | ptp->current_time); 2399 bnxt_ptp_rtc_timecounter_init(ptp, ns); 2400 spin_unlock_bh(&ptp->ptp_lock); 2401 } 2402 break; 2403 } 2404 goto async_event_process_exit; 2405 } 2406 case ASYNC_EVENT_CMPL_EVENT_ID_DEFERRED_RESPONSE: { 2407 u16 seq_id = le32_to_cpu(cmpl->event_data2) & 0xffff; 2408 2409 hwrm_update_token(bp, seq_id, BNXT_HWRM_DEFERRED); 2410 goto async_event_process_exit; 2411 } 2412 default: 2413 goto async_event_process_exit; 2414 } 2415 bnxt_queue_sp_work(bp); 2416 async_event_process_exit: 2417 return 0; 2418 } 2419 2420 static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp) 2421 { 2422 u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id; 2423 struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp; 2424 struct hwrm_fwd_req_cmpl *fwd_req_cmpl = 2425 (struct hwrm_fwd_req_cmpl *)txcmp; 2426 2427 switch (cmpl_type) { 2428 case CMPL_BASE_TYPE_HWRM_DONE: 2429 seq_id = le16_to_cpu(h_cmpl->sequence_id); 2430 hwrm_update_token(bp, seq_id, BNXT_HWRM_COMPLETE); 2431 break; 2432 2433 case CMPL_BASE_TYPE_HWRM_FWD_REQ: 2434 vf_id = le16_to_cpu(fwd_req_cmpl->source_id); 2435 2436 if ((vf_id < bp->pf.first_vf_id) || 2437 (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) { 2438 netdev_err(bp->dev, "Msg contains invalid VF id %x\n", 2439 vf_id); 2440 return -EINVAL; 2441 } 2442 2443 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap); 2444 set_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event); 2445 bnxt_queue_sp_work(bp); 2446 break; 2447 2448 case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT: 2449 bnxt_async_event_process(bp, 2450 (struct hwrm_async_event_cmpl *)txcmp); 2451 break; 2452 2453 default: 2454 break; 2455 } 2456 2457 return 0; 2458 } 2459 2460 static irqreturn_t bnxt_msix(int irq, void *dev_instance) 2461 { 2462 struct bnxt_napi *bnapi = dev_instance; 2463 struct bnxt *bp = bnapi->bp; 2464 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2465 u32 cons = RING_CMP(cpr->cp_raw_cons); 2466 2467 cpr->event_ctr++; 2468 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]); 2469 napi_schedule(&bnapi->napi); 2470 return IRQ_HANDLED; 2471 } 2472 2473 static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr) 2474 { 2475 u32 raw_cons = cpr->cp_raw_cons; 2476 u16 cons = RING_CMP(raw_cons); 2477 struct tx_cmp *txcmp; 2478 2479 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]; 2480 2481 return TX_CMP_VALID(txcmp, raw_cons); 2482 } 2483 2484 static irqreturn_t bnxt_inta(int irq, void *dev_instance) 2485 { 2486 struct bnxt_napi *bnapi = dev_instance; 2487 struct bnxt *bp = bnapi->bp; 2488 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2489 u32 cons = RING_CMP(cpr->cp_raw_cons); 2490 u32 int_status; 2491 2492 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]); 2493 2494 if (!bnxt_has_work(bp, cpr)) { 2495 int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS); 2496 /* return if erroneous interrupt */ 2497 if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id))) 2498 return IRQ_NONE; 2499 } 2500 2501 /* disable ring IRQ */ 2502 BNXT_CP_DB_IRQ_DIS(cpr->cp_db.doorbell); 2503 2504 /* Return here if interrupt is shared and is disabled. */ 2505 if (unlikely(atomic_read(&bp->intr_sem) != 0)) 2506 return IRQ_HANDLED; 2507 2508 napi_schedule(&bnapi->napi); 2509 return IRQ_HANDLED; 2510 } 2511 2512 static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 2513 int budget) 2514 { 2515 struct bnxt_napi *bnapi = cpr->bnapi; 2516 u32 raw_cons = cpr->cp_raw_cons; 2517 u32 cons; 2518 int tx_pkts = 0; 2519 int rx_pkts = 0; 2520 u8 event = 0; 2521 struct tx_cmp *txcmp; 2522 2523 cpr->has_more_work = 0; 2524 cpr->had_work_done = 1; 2525 while (1) { 2526 int rc; 2527 2528 cons = RING_CMP(raw_cons); 2529 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]; 2530 2531 if (!TX_CMP_VALID(txcmp, raw_cons)) 2532 break; 2533 2534 /* The valid test of the entry must be done first before 2535 * reading any further. 2536 */ 2537 dma_rmb(); 2538 if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) { 2539 tx_pkts++; 2540 /* return full budget so NAPI will complete. */ 2541 if (unlikely(tx_pkts >= bp->tx_wake_thresh)) { 2542 rx_pkts = budget; 2543 raw_cons = NEXT_RAW_CMP(raw_cons); 2544 if (budget) 2545 cpr->has_more_work = 1; 2546 break; 2547 } 2548 } else if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) { 2549 if (likely(budget)) 2550 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event); 2551 else 2552 rc = bnxt_force_rx_discard(bp, cpr, &raw_cons, 2553 &event); 2554 if (likely(rc >= 0)) 2555 rx_pkts += rc; 2556 /* Increment rx_pkts when rc is -ENOMEM to count towards 2557 * the NAPI budget. Otherwise, we may potentially loop 2558 * here forever if we consistently cannot allocate 2559 * buffers. 2560 */ 2561 else if (rc == -ENOMEM && budget) 2562 rx_pkts++; 2563 else if (rc == -EBUSY) /* partial completion */ 2564 break; 2565 } else if (unlikely((TX_CMP_TYPE(txcmp) == 2566 CMPL_BASE_TYPE_HWRM_DONE) || 2567 (TX_CMP_TYPE(txcmp) == 2568 CMPL_BASE_TYPE_HWRM_FWD_REQ) || 2569 (TX_CMP_TYPE(txcmp) == 2570 CMPL_BASE_TYPE_HWRM_ASYNC_EVENT))) { 2571 bnxt_hwrm_handler(bp, txcmp); 2572 } 2573 raw_cons = NEXT_RAW_CMP(raw_cons); 2574 2575 if (rx_pkts && rx_pkts == budget) { 2576 cpr->has_more_work = 1; 2577 break; 2578 } 2579 } 2580 2581 if (event & BNXT_REDIRECT_EVENT) 2582 xdp_do_flush(); 2583 2584 if (event & BNXT_TX_EVENT) { 2585 struct bnxt_tx_ring_info *txr = bnapi->tx_ring; 2586 u16 prod = txr->tx_prod; 2587 2588 /* Sync BD data before updating doorbell */ 2589 wmb(); 2590 2591 bnxt_db_write_relaxed(bp, &txr->tx_db, prod); 2592 } 2593 2594 cpr->cp_raw_cons = raw_cons; 2595 bnapi->tx_pkts += tx_pkts; 2596 bnapi->events |= event; 2597 return rx_pkts; 2598 } 2599 2600 static void __bnxt_poll_work_done(struct bnxt *bp, struct bnxt_napi *bnapi) 2601 { 2602 if (bnapi->tx_pkts) { 2603 bnapi->tx_int(bp, bnapi, bnapi->tx_pkts); 2604 bnapi->tx_pkts = 0; 2605 } 2606 2607 if ((bnapi->events & BNXT_RX_EVENT) && !(bnapi->in_reset)) { 2608 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 2609 2610 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 2611 } 2612 if (bnapi->events & BNXT_AGG_EVENT) { 2613 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 2614 2615 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); 2616 } 2617 bnapi->events = 0; 2618 } 2619 2620 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 2621 int budget) 2622 { 2623 struct bnxt_napi *bnapi = cpr->bnapi; 2624 int rx_pkts; 2625 2626 rx_pkts = __bnxt_poll_work(bp, cpr, budget); 2627 2628 /* ACK completion ring before freeing tx ring and producing new 2629 * buffers in rx/agg rings to prevent overflowing the completion 2630 * ring. 2631 */ 2632 bnxt_db_cq(bp, &cpr->cp_db, cpr->cp_raw_cons); 2633 2634 __bnxt_poll_work_done(bp, bnapi); 2635 return rx_pkts; 2636 } 2637 2638 static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget) 2639 { 2640 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi); 2641 struct bnxt *bp = bnapi->bp; 2642 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2643 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 2644 struct tx_cmp *txcmp; 2645 struct rx_cmp_ext *rxcmp1; 2646 u32 cp_cons, tmp_raw_cons; 2647 u32 raw_cons = cpr->cp_raw_cons; 2648 u32 rx_pkts = 0; 2649 u8 event = 0; 2650 2651 while (1) { 2652 int rc; 2653 2654 cp_cons = RING_CMP(raw_cons); 2655 txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 2656 2657 if (!TX_CMP_VALID(txcmp, raw_cons)) 2658 break; 2659 2660 /* The valid test of the entry must be done first before 2661 * reading any further. 2662 */ 2663 dma_rmb(); 2664 if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) { 2665 tmp_raw_cons = NEXT_RAW_CMP(raw_cons); 2666 cp_cons = RING_CMP(tmp_raw_cons); 2667 rxcmp1 = (struct rx_cmp_ext *) 2668 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 2669 2670 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons)) 2671 break; 2672 2673 /* force an error to recycle the buffer */ 2674 rxcmp1->rx_cmp_cfa_code_errors_v2 |= 2675 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR); 2676 2677 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event); 2678 if (likely(rc == -EIO) && budget) 2679 rx_pkts++; 2680 else if (rc == -EBUSY) /* partial completion */ 2681 break; 2682 } else if (unlikely(TX_CMP_TYPE(txcmp) == 2683 CMPL_BASE_TYPE_HWRM_DONE)) { 2684 bnxt_hwrm_handler(bp, txcmp); 2685 } else { 2686 netdev_err(bp->dev, 2687 "Invalid completion received on special ring\n"); 2688 } 2689 raw_cons = NEXT_RAW_CMP(raw_cons); 2690 2691 if (rx_pkts == budget) 2692 break; 2693 } 2694 2695 cpr->cp_raw_cons = raw_cons; 2696 BNXT_DB_CQ(&cpr->cp_db, cpr->cp_raw_cons); 2697 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 2698 2699 if (event & BNXT_AGG_EVENT) 2700 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); 2701 2702 if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) { 2703 napi_complete_done(napi, rx_pkts); 2704 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons); 2705 } 2706 return rx_pkts; 2707 } 2708 2709 static int bnxt_poll(struct napi_struct *napi, int budget) 2710 { 2711 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi); 2712 struct bnxt *bp = bnapi->bp; 2713 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2714 int work_done = 0; 2715 2716 if (unlikely(test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))) { 2717 napi_complete(napi); 2718 return 0; 2719 } 2720 while (1) { 2721 work_done += bnxt_poll_work(bp, cpr, budget - work_done); 2722 2723 if (work_done >= budget) { 2724 if (!budget) 2725 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons); 2726 break; 2727 } 2728 2729 if (!bnxt_has_work(bp, cpr)) { 2730 if (napi_complete_done(napi, work_done)) 2731 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons); 2732 break; 2733 } 2734 } 2735 if (bp->flags & BNXT_FLAG_DIM) { 2736 struct dim_sample dim_sample = {}; 2737 2738 dim_update_sample(cpr->event_ctr, 2739 cpr->rx_packets, 2740 cpr->rx_bytes, 2741 &dim_sample); 2742 net_dim(&cpr->dim, dim_sample); 2743 } 2744 return work_done; 2745 } 2746 2747 static int __bnxt_poll_cqs(struct bnxt *bp, struct bnxt_napi *bnapi, int budget) 2748 { 2749 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2750 int i, work_done = 0; 2751 2752 for (i = 0; i < 2; i++) { 2753 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i]; 2754 2755 if (cpr2) { 2756 work_done += __bnxt_poll_work(bp, cpr2, 2757 budget - work_done); 2758 cpr->has_more_work |= cpr2->has_more_work; 2759 } 2760 } 2761 return work_done; 2762 } 2763 2764 static void __bnxt_poll_cqs_done(struct bnxt *bp, struct bnxt_napi *bnapi, 2765 u64 dbr_type) 2766 { 2767 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2768 int i; 2769 2770 for (i = 0; i < 2; i++) { 2771 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i]; 2772 struct bnxt_db_info *db; 2773 2774 if (cpr2 && cpr2->had_work_done) { 2775 db = &cpr2->cp_db; 2776 bnxt_writeq(bp, db->db_key64 | dbr_type | 2777 RING_CMP(cpr2->cp_raw_cons), db->doorbell); 2778 cpr2->had_work_done = 0; 2779 } 2780 } 2781 __bnxt_poll_work_done(bp, bnapi); 2782 } 2783 2784 static int bnxt_poll_p5(struct napi_struct *napi, int budget) 2785 { 2786 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi); 2787 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2788 struct bnxt_cp_ring_info *cpr_rx; 2789 u32 raw_cons = cpr->cp_raw_cons; 2790 struct bnxt *bp = bnapi->bp; 2791 struct nqe_cn *nqcmp; 2792 int work_done = 0; 2793 u32 cons; 2794 2795 if (unlikely(test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))) { 2796 napi_complete(napi); 2797 return 0; 2798 } 2799 if (cpr->has_more_work) { 2800 cpr->has_more_work = 0; 2801 work_done = __bnxt_poll_cqs(bp, bnapi, budget); 2802 } 2803 while (1) { 2804 cons = RING_CMP(raw_cons); 2805 nqcmp = &cpr->nq_desc_ring[CP_RING(cons)][CP_IDX(cons)]; 2806 2807 if (!NQ_CMP_VALID(nqcmp, raw_cons)) { 2808 if (cpr->has_more_work) 2809 break; 2810 2811 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ_ARMALL); 2812 cpr->cp_raw_cons = raw_cons; 2813 if (napi_complete_done(napi, work_done)) 2814 BNXT_DB_NQ_ARM_P5(&cpr->cp_db, 2815 cpr->cp_raw_cons); 2816 goto poll_done; 2817 } 2818 2819 /* The valid test of the entry must be done first before 2820 * reading any further. 2821 */ 2822 dma_rmb(); 2823 2824 if (nqcmp->type == cpu_to_le16(NQ_CN_TYPE_CQ_NOTIFICATION)) { 2825 u32 idx = le32_to_cpu(nqcmp->cq_handle_low); 2826 struct bnxt_cp_ring_info *cpr2; 2827 2828 /* No more budget for RX work */ 2829 if (budget && work_done >= budget && idx == BNXT_RX_HDL) 2830 break; 2831 2832 cpr2 = cpr->cp_ring_arr[idx]; 2833 work_done += __bnxt_poll_work(bp, cpr2, 2834 budget - work_done); 2835 cpr->has_more_work |= cpr2->has_more_work; 2836 } else { 2837 bnxt_hwrm_handler(bp, (struct tx_cmp *)nqcmp); 2838 } 2839 raw_cons = NEXT_RAW_CMP(raw_cons); 2840 } 2841 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ); 2842 if (raw_cons != cpr->cp_raw_cons) { 2843 cpr->cp_raw_cons = raw_cons; 2844 BNXT_DB_NQ_P5(&cpr->cp_db, raw_cons); 2845 } 2846 poll_done: 2847 cpr_rx = cpr->cp_ring_arr[BNXT_RX_HDL]; 2848 if (cpr_rx && (bp->flags & BNXT_FLAG_DIM)) { 2849 struct dim_sample dim_sample = {}; 2850 2851 dim_update_sample(cpr->event_ctr, 2852 cpr_rx->rx_packets, 2853 cpr_rx->rx_bytes, 2854 &dim_sample); 2855 net_dim(&cpr->dim, dim_sample); 2856 } 2857 return work_done; 2858 } 2859 2860 static void bnxt_free_tx_skbs(struct bnxt *bp) 2861 { 2862 int i, max_idx; 2863 struct pci_dev *pdev = bp->pdev; 2864 2865 if (!bp->tx_ring) 2866 return; 2867 2868 max_idx = bp->tx_nr_pages * TX_DESC_CNT; 2869 for (i = 0; i < bp->tx_nr_rings; i++) { 2870 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 2871 int j; 2872 2873 if (!txr->tx_buf_ring) 2874 continue; 2875 2876 for (j = 0; j < max_idx;) { 2877 struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j]; 2878 struct sk_buff *skb; 2879 int k, last; 2880 2881 if (i < bp->tx_nr_rings_xdp && 2882 tx_buf->action == XDP_REDIRECT) { 2883 dma_unmap_single(&pdev->dev, 2884 dma_unmap_addr(tx_buf, mapping), 2885 dma_unmap_len(tx_buf, len), 2886 DMA_TO_DEVICE); 2887 xdp_return_frame(tx_buf->xdpf); 2888 tx_buf->action = 0; 2889 tx_buf->xdpf = NULL; 2890 j++; 2891 continue; 2892 } 2893 2894 skb = tx_buf->skb; 2895 if (!skb) { 2896 j++; 2897 continue; 2898 } 2899 2900 tx_buf->skb = NULL; 2901 2902 if (tx_buf->is_push) { 2903 dev_kfree_skb(skb); 2904 j += 2; 2905 continue; 2906 } 2907 2908 dma_unmap_single(&pdev->dev, 2909 dma_unmap_addr(tx_buf, mapping), 2910 skb_headlen(skb), 2911 DMA_TO_DEVICE); 2912 2913 last = tx_buf->nr_frags; 2914 j += 2; 2915 for (k = 0; k < last; k++, j++) { 2916 int ring_idx = j & bp->tx_ring_mask; 2917 skb_frag_t *frag = &skb_shinfo(skb)->frags[k]; 2918 2919 tx_buf = &txr->tx_buf_ring[ring_idx]; 2920 dma_unmap_page( 2921 &pdev->dev, 2922 dma_unmap_addr(tx_buf, mapping), 2923 skb_frag_size(frag), DMA_TO_DEVICE); 2924 } 2925 dev_kfree_skb(skb); 2926 } 2927 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i)); 2928 } 2929 } 2930 2931 static void bnxt_free_one_rx_ring_skbs(struct bnxt *bp, int ring_nr) 2932 { 2933 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr]; 2934 struct pci_dev *pdev = bp->pdev; 2935 struct bnxt_tpa_idx_map *map; 2936 int i, max_idx, max_agg_idx; 2937 2938 max_idx = bp->rx_nr_pages * RX_DESC_CNT; 2939 max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT; 2940 if (!rxr->rx_tpa) 2941 goto skip_rx_tpa_free; 2942 2943 for (i = 0; i < bp->max_tpa; i++) { 2944 struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[i]; 2945 u8 *data = tpa_info->data; 2946 2947 if (!data) 2948 continue; 2949 2950 dma_unmap_single_attrs(&pdev->dev, tpa_info->mapping, 2951 bp->rx_buf_use_size, bp->rx_dir, 2952 DMA_ATTR_WEAK_ORDERING); 2953 2954 tpa_info->data = NULL; 2955 2956 skb_free_frag(data); 2957 } 2958 2959 skip_rx_tpa_free: 2960 if (!rxr->rx_buf_ring) 2961 goto skip_rx_buf_free; 2962 2963 for (i = 0; i < max_idx; i++) { 2964 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[i]; 2965 dma_addr_t mapping = rx_buf->mapping; 2966 void *data = rx_buf->data; 2967 2968 if (!data) 2969 continue; 2970 2971 rx_buf->data = NULL; 2972 if (BNXT_RX_PAGE_MODE(bp)) { 2973 mapping -= bp->rx_dma_offset; 2974 dma_unmap_page_attrs(&pdev->dev, mapping, PAGE_SIZE, 2975 bp->rx_dir, 2976 DMA_ATTR_WEAK_ORDERING); 2977 page_pool_recycle_direct(rxr->page_pool, data); 2978 } else { 2979 dma_unmap_single_attrs(&pdev->dev, mapping, 2980 bp->rx_buf_use_size, bp->rx_dir, 2981 DMA_ATTR_WEAK_ORDERING); 2982 skb_free_frag(data); 2983 } 2984 } 2985 2986 skip_rx_buf_free: 2987 if (!rxr->rx_agg_ring) 2988 goto skip_rx_agg_free; 2989 2990 for (i = 0; i < max_agg_idx; i++) { 2991 struct bnxt_sw_rx_agg_bd *rx_agg_buf = &rxr->rx_agg_ring[i]; 2992 struct page *page = rx_agg_buf->page; 2993 2994 if (!page) 2995 continue; 2996 2997 if (BNXT_RX_PAGE_MODE(bp)) { 2998 dma_unmap_page_attrs(&pdev->dev, rx_agg_buf->mapping, 2999 BNXT_RX_PAGE_SIZE, bp->rx_dir, 3000 DMA_ATTR_WEAK_ORDERING); 3001 rx_agg_buf->page = NULL; 3002 __clear_bit(i, rxr->rx_agg_bmap); 3003 3004 page_pool_recycle_direct(rxr->page_pool, page); 3005 } else { 3006 dma_unmap_page_attrs(&pdev->dev, rx_agg_buf->mapping, 3007 BNXT_RX_PAGE_SIZE, DMA_FROM_DEVICE, 3008 DMA_ATTR_WEAK_ORDERING); 3009 rx_agg_buf->page = NULL; 3010 __clear_bit(i, rxr->rx_agg_bmap); 3011 3012 __free_page(page); 3013 } 3014 } 3015 3016 skip_rx_agg_free: 3017 if (rxr->rx_page) { 3018 __free_page(rxr->rx_page); 3019 rxr->rx_page = NULL; 3020 } 3021 map = rxr->rx_tpa_idx_map; 3022 if (map) 3023 memset(map->agg_idx_bmap, 0, sizeof(map->agg_idx_bmap)); 3024 } 3025 3026 static void bnxt_free_rx_skbs(struct bnxt *bp) 3027 { 3028 int i; 3029 3030 if (!bp->rx_ring) 3031 return; 3032 3033 for (i = 0; i < bp->rx_nr_rings; i++) 3034 bnxt_free_one_rx_ring_skbs(bp, i); 3035 } 3036 3037 static void bnxt_free_skbs(struct bnxt *bp) 3038 { 3039 bnxt_free_tx_skbs(bp); 3040 bnxt_free_rx_skbs(bp); 3041 } 3042 3043 static void bnxt_init_ctx_mem(struct bnxt_mem_init *mem_init, void *p, int len) 3044 { 3045 u8 init_val = mem_init->init_val; 3046 u16 offset = mem_init->offset; 3047 u8 *p2 = p; 3048 int i; 3049 3050 if (!init_val) 3051 return; 3052 if (offset == BNXT_MEM_INVALID_OFFSET) { 3053 memset(p, init_val, len); 3054 return; 3055 } 3056 for (i = 0; i < len; i += mem_init->size) 3057 *(p2 + i + offset) = init_val; 3058 } 3059 3060 static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem) 3061 { 3062 struct pci_dev *pdev = bp->pdev; 3063 int i; 3064 3065 if (!rmem->pg_arr) 3066 goto skip_pages; 3067 3068 for (i = 0; i < rmem->nr_pages; i++) { 3069 if (!rmem->pg_arr[i]) 3070 continue; 3071 3072 dma_free_coherent(&pdev->dev, rmem->page_size, 3073 rmem->pg_arr[i], rmem->dma_arr[i]); 3074 3075 rmem->pg_arr[i] = NULL; 3076 } 3077 skip_pages: 3078 if (rmem->pg_tbl) { 3079 size_t pg_tbl_size = rmem->nr_pages * 8; 3080 3081 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG) 3082 pg_tbl_size = rmem->page_size; 3083 dma_free_coherent(&pdev->dev, pg_tbl_size, 3084 rmem->pg_tbl, rmem->pg_tbl_map); 3085 rmem->pg_tbl = NULL; 3086 } 3087 if (rmem->vmem_size && *rmem->vmem) { 3088 vfree(*rmem->vmem); 3089 *rmem->vmem = NULL; 3090 } 3091 } 3092 3093 static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem) 3094 { 3095 struct pci_dev *pdev = bp->pdev; 3096 u64 valid_bit = 0; 3097 int i; 3098 3099 if (rmem->flags & (BNXT_RMEM_VALID_PTE_FLAG | BNXT_RMEM_RING_PTE_FLAG)) 3100 valid_bit = PTU_PTE_VALID; 3101 if ((rmem->nr_pages > 1 || rmem->depth > 0) && !rmem->pg_tbl) { 3102 size_t pg_tbl_size = rmem->nr_pages * 8; 3103 3104 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG) 3105 pg_tbl_size = rmem->page_size; 3106 rmem->pg_tbl = dma_alloc_coherent(&pdev->dev, pg_tbl_size, 3107 &rmem->pg_tbl_map, 3108 GFP_KERNEL); 3109 if (!rmem->pg_tbl) 3110 return -ENOMEM; 3111 } 3112 3113 for (i = 0; i < rmem->nr_pages; i++) { 3114 u64 extra_bits = valid_bit; 3115 3116 rmem->pg_arr[i] = dma_alloc_coherent(&pdev->dev, 3117 rmem->page_size, 3118 &rmem->dma_arr[i], 3119 GFP_KERNEL); 3120 if (!rmem->pg_arr[i]) 3121 return -ENOMEM; 3122 3123 if (rmem->mem_init) 3124 bnxt_init_ctx_mem(rmem->mem_init, rmem->pg_arr[i], 3125 rmem->page_size); 3126 if (rmem->nr_pages > 1 || rmem->depth > 0) { 3127 if (i == rmem->nr_pages - 2 && 3128 (rmem->flags & BNXT_RMEM_RING_PTE_FLAG)) 3129 extra_bits |= PTU_PTE_NEXT_TO_LAST; 3130 else if (i == rmem->nr_pages - 1 && 3131 (rmem->flags & BNXT_RMEM_RING_PTE_FLAG)) 3132 extra_bits |= PTU_PTE_LAST; 3133 rmem->pg_tbl[i] = 3134 cpu_to_le64(rmem->dma_arr[i] | extra_bits); 3135 } 3136 } 3137 3138 if (rmem->vmem_size) { 3139 *rmem->vmem = vzalloc(rmem->vmem_size); 3140 if (!(*rmem->vmem)) 3141 return -ENOMEM; 3142 } 3143 return 0; 3144 } 3145 3146 static void bnxt_free_tpa_info(struct bnxt *bp) 3147 { 3148 int i, j; 3149 3150 for (i = 0; i < bp->rx_nr_rings; i++) { 3151 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 3152 3153 kfree(rxr->rx_tpa_idx_map); 3154 rxr->rx_tpa_idx_map = NULL; 3155 if (rxr->rx_tpa) { 3156 for (j = 0; j < bp->max_tpa; j++) { 3157 kfree(rxr->rx_tpa[j].agg_arr); 3158 rxr->rx_tpa[j].agg_arr = NULL; 3159 } 3160 } 3161 kfree(rxr->rx_tpa); 3162 rxr->rx_tpa = NULL; 3163 } 3164 } 3165 3166 static int bnxt_alloc_tpa_info(struct bnxt *bp) 3167 { 3168 int i, j; 3169 3170 bp->max_tpa = MAX_TPA; 3171 if (bp->flags & BNXT_FLAG_CHIP_P5) { 3172 if (!bp->max_tpa_v2) 3173 return 0; 3174 bp->max_tpa = max_t(u16, bp->max_tpa_v2, MAX_TPA_P5); 3175 } 3176 3177 for (i = 0; i < bp->rx_nr_rings; i++) { 3178 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 3179 struct rx_agg_cmp *agg; 3180 3181 rxr->rx_tpa = kcalloc(bp->max_tpa, sizeof(struct bnxt_tpa_info), 3182 GFP_KERNEL); 3183 if (!rxr->rx_tpa) 3184 return -ENOMEM; 3185 3186 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 3187 continue; 3188 for (j = 0; j < bp->max_tpa; j++) { 3189 agg = kcalloc(MAX_SKB_FRAGS, sizeof(*agg), GFP_KERNEL); 3190 if (!agg) 3191 return -ENOMEM; 3192 rxr->rx_tpa[j].agg_arr = agg; 3193 } 3194 rxr->rx_tpa_idx_map = kzalloc(sizeof(*rxr->rx_tpa_idx_map), 3195 GFP_KERNEL); 3196 if (!rxr->rx_tpa_idx_map) 3197 return -ENOMEM; 3198 } 3199 return 0; 3200 } 3201 3202 static void bnxt_free_rx_rings(struct bnxt *bp) 3203 { 3204 int i; 3205 3206 if (!bp->rx_ring) 3207 return; 3208 3209 bnxt_free_tpa_info(bp); 3210 for (i = 0; i < bp->rx_nr_rings; i++) { 3211 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 3212 struct bnxt_ring_struct *ring; 3213 3214 if (rxr->xdp_prog) 3215 bpf_prog_put(rxr->xdp_prog); 3216 3217 if (xdp_rxq_info_is_reg(&rxr->xdp_rxq)) 3218 xdp_rxq_info_unreg(&rxr->xdp_rxq); 3219 3220 page_pool_destroy(rxr->page_pool); 3221 rxr->page_pool = NULL; 3222 3223 kfree(rxr->rx_agg_bmap); 3224 rxr->rx_agg_bmap = NULL; 3225 3226 ring = &rxr->rx_ring_struct; 3227 bnxt_free_ring(bp, &ring->ring_mem); 3228 3229 ring = &rxr->rx_agg_ring_struct; 3230 bnxt_free_ring(bp, &ring->ring_mem); 3231 } 3232 } 3233 3234 static int bnxt_alloc_rx_page_pool(struct bnxt *bp, 3235 struct bnxt_rx_ring_info *rxr) 3236 { 3237 struct page_pool_params pp = { 0 }; 3238 3239 pp.pool_size = bp->rx_ring_size; 3240 pp.nid = dev_to_node(&bp->pdev->dev); 3241 pp.dev = &bp->pdev->dev; 3242 pp.dma_dir = DMA_BIDIRECTIONAL; 3243 3244 rxr->page_pool = page_pool_create(&pp); 3245 if (IS_ERR(rxr->page_pool)) { 3246 int err = PTR_ERR(rxr->page_pool); 3247 3248 rxr->page_pool = NULL; 3249 return err; 3250 } 3251 return 0; 3252 } 3253 3254 static int bnxt_alloc_rx_rings(struct bnxt *bp) 3255 { 3256 int i, rc = 0, agg_rings = 0; 3257 3258 if (!bp->rx_ring) 3259 return -ENOMEM; 3260 3261 if (bp->flags & BNXT_FLAG_AGG_RINGS) 3262 agg_rings = 1; 3263 3264 for (i = 0; i < bp->rx_nr_rings; i++) { 3265 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 3266 struct bnxt_ring_struct *ring; 3267 3268 ring = &rxr->rx_ring_struct; 3269 3270 rc = bnxt_alloc_rx_page_pool(bp, rxr); 3271 if (rc) 3272 return rc; 3273 3274 rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i, 0); 3275 if (rc < 0) 3276 return rc; 3277 3278 rc = xdp_rxq_info_reg_mem_model(&rxr->xdp_rxq, 3279 MEM_TYPE_PAGE_POOL, 3280 rxr->page_pool); 3281 if (rc) { 3282 xdp_rxq_info_unreg(&rxr->xdp_rxq); 3283 return rc; 3284 } 3285 3286 rc = bnxt_alloc_ring(bp, &ring->ring_mem); 3287 if (rc) 3288 return rc; 3289 3290 ring->grp_idx = i; 3291 if (agg_rings) { 3292 u16 mem_size; 3293 3294 ring = &rxr->rx_agg_ring_struct; 3295 rc = bnxt_alloc_ring(bp, &ring->ring_mem); 3296 if (rc) 3297 return rc; 3298 3299 ring->grp_idx = i; 3300 rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1; 3301 mem_size = rxr->rx_agg_bmap_size / 8; 3302 rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL); 3303 if (!rxr->rx_agg_bmap) 3304 return -ENOMEM; 3305 } 3306 } 3307 if (bp->flags & BNXT_FLAG_TPA) 3308 rc = bnxt_alloc_tpa_info(bp); 3309 return rc; 3310 } 3311 3312 static void bnxt_free_tx_rings(struct bnxt *bp) 3313 { 3314 int i; 3315 struct pci_dev *pdev = bp->pdev; 3316 3317 if (!bp->tx_ring) 3318 return; 3319 3320 for (i = 0; i < bp->tx_nr_rings; i++) { 3321 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 3322 struct bnxt_ring_struct *ring; 3323 3324 if (txr->tx_push) { 3325 dma_free_coherent(&pdev->dev, bp->tx_push_size, 3326 txr->tx_push, txr->tx_push_mapping); 3327 txr->tx_push = NULL; 3328 } 3329 3330 ring = &txr->tx_ring_struct; 3331 3332 bnxt_free_ring(bp, &ring->ring_mem); 3333 } 3334 } 3335 3336 static int bnxt_alloc_tx_rings(struct bnxt *bp) 3337 { 3338 int i, j, rc; 3339 struct pci_dev *pdev = bp->pdev; 3340 3341 bp->tx_push_size = 0; 3342 if (bp->tx_push_thresh) { 3343 int push_size; 3344 3345 push_size = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) + 3346 bp->tx_push_thresh); 3347 3348 if (push_size > 256) { 3349 push_size = 0; 3350 bp->tx_push_thresh = 0; 3351 } 3352 3353 bp->tx_push_size = push_size; 3354 } 3355 3356 for (i = 0, j = 0; i < bp->tx_nr_rings; i++) { 3357 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 3358 struct bnxt_ring_struct *ring; 3359 u8 qidx; 3360 3361 ring = &txr->tx_ring_struct; 3362 3363 rc = bnxt_alloc_ring(bp, &ring->ring_mem); 3364 if (rc) 3365 return rc; 3366 3367 ring->grp_idx = txr->bnapi->index; 3368 if (bp->tx_push_size) { 3369 dma_addr_t mapping; 3370 3371 /* One pre-allocated DMA buffer to backup 3372 * TX push operation 3373 */ 3374 txr->tx_push = dma_alloc_coherent(&pdev->dev, 3375 bp->tx_push_size, 3376 &txr->tx_push_mapping, 3377 GFP_KERNEL); 3378 3379 if (!txr->tx_push) 3380 return -ENOMEM; 3381 3382 mapping = txr->tx_push_mapping + 3383 sizeof(struct tx_push_bd); 3384 txr->data_mapping = cpu_to_le64(mapping); 3385 } 3386 qidx = bp->tc_to_qidx[j]; 3387 ring->queue_id = bp->q_info[qidx].queue_id; 3388 spin_lock_init(&txr->xdp_tx_lock); 3389 if (i < bp->tx_nr_rings_xdp) 3390 continue; 3391 if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1)) 3392 j++; 3393 } 3394 return 0; 3395 } 3396 3397 static void bnxt_free_cp_arrays(struct bnxt_cp_ring_info *cpr) 3398 { 3399 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 3400 3401 kfree(cpr->cp_desc_ring); 3402 cpr->cp_desc_ring = NULL; 3403 ring->ring_mem.pg_arr = NULL; 3404 kfree(cpr->cp_desc_mapping); 3405 cpr->cp_desc_mapping = NULL; 3406 ring->ring_mem.dma_arr = NULL; 3407 } 3408 3409 static int bnxt_alloc_cp_arrays(struct bnxt_cp_ring_info *cpr, int n) 3410 { 3411 cpr->cp_desc_ring = kcalloc(n, sizeof(*cpr->cp_desc_ring), GFP_KERNEL); 3412 if (!cpr->cp_desc_ring) 3413 return -ENOMEM; 3414 cpr->cp_desc_mapping = kcalloc(n, sizeof(*cpr->cp_desc_mapping), 3415 GFP_KERNEL); 3416 if (!cpr->cp_desc_mapping) 3417 return -ENOMEM; 3418 return 0; 3419 } 3420 3421 static void bnxt_free_all_cp_arrays(struct bnxt *bp) 3422 { 3423 int i; 3424 3425 if (!bp->bnapi) 3426 return; 3427 for (i = 0; i < bp->cp_nr_rings; i++) { 3428 struct bnxt_napi *bnapi = bp->bnapi[i]; 3429 3430 if (!bnapi) 3431 continue; 3432 bnxt_free_cp_arrays(&bnapi->cp_ring); 3433 } 3434 } 3435 3436 static int bnxt_alloc_all_cp_arrays(struct bnxt *bp) 3437 { 3438 int i, n = bp->cp_nr_pages; 3439 3440 for (i = 0; i < bp->cp_nr_rings; i++) { 3441 struct bnxt_napi *bnapi = bp->bnapi[i]; 3442 int rc; 3443 3444 if (!bnapi) 3445 continue; 3446 rc = bnxt_alloc_cp_arrays(&bnapi->cp_ring, n); 3447 if (rc) 3448 return rc; 3449 } 3450 return 0; 3451 } 3452 3453 static void bnxt_free_cp_rings(struct bnxt *bp) 3454 { 3455 int i; 3456 3457 if (!bp->bnapi) 3458 return; 3459 3460 for (i = 0; i < bp->cp_nr_rings; i++) { 3461 struct bnxt_napi *bnapi = bp->bnapi[i]; 3462 struct bnxt_cp_ring_info *cpr; 3463 struct bnxt_ring_struct *ring; 3464 int j; 3465 3466 if (!bnapi) 3467 continue; 3468 3469 cpr = &bnapi->cp_ring; 3470 ring = &cpr->cp_ring_struct; 3471 3472 bnxt_free_ring(bp, &ring->ring_mem); 3473 3474 for (j = 0; j < 2; j++) { 3475 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j]; 3476 3477 if (cpr2) { 3478 ring = &cpr2->cp_ring_struct; 3479 bnxt_free_ring(bp, &ring->ring_mem); 3480 bnxt_free_cp_arrays(cpr2); 3481 kfree(cpr2); 3482 cpr->cp_ring_arr[j] = NULL; 3483 } 3484 } 3485 } 3486 } 3487 3488 static struct bnxt_cp_ring_info *bnxt_alloc_cp_sub_ring(struct bnxt *bp) 3489 { 3490 struct bnxt_ring_mem_info *rmem; 3491 struct bnxt_ring_struct *ring; 3492 struct bnxt_cp_ring_info *cpr; 3493 int rc; 3494 3495 cpr = kzalloc(sizeof(*cpr), GFP_KERNEL); 3496 if (!cpr) 3497 return NULL; 3498 3499 rc = bnxt_alloc_cp_arrays(cpr, bp->cp_nr_pages); 3500 if (rc) { 3501 bnxt_free_cp_arrays(cpr); 3502 kfree(cpr); 3503 return NULL; 3504 } 3505 ring = &cpr->cp_ring_struct; 3506 rmem = &ring->ring_mem; 3507 rmem->nr_pages = bp->cp_nr_pages; 3508 rmem->page_size = HW_CMPD_RING_SIZE; 3509 rmem->pg_arr = (void **)cpr->cp_desc_ring; 3510 rmem->dma_arr = cpr->cp_desc_mapping; 3511 rmem->flags = BNXT_RMEM_RING_PTE_FLAG; 3512 rc = bnxt_alloc_ring(bp, rmem); 3513 if (rc) { 3514 bnxt_free_ring(bp, rmem); 3515 bnxt_free_cp_arrays(cpr); 3516 kfree(cpr); 3517 cpr = NULL; 3518 } 3519 return cpr; 3520 } 3521 3522 static int bnxt_alloc_cp_rings(struct bnxt *bp) 3523 { 3524 bool sh = !!(bp->flags & BNXT_FLAG_SHARED_RINGS); 3525 int i, rc, ulp_base_vec, ulp_msix; 3526 3527 ulp_msix = bnxt_get_ulp_msix_num(bp); 3528 ulp_base_vec = bnxt_get_ulp_msix_base(bp); 3529 for (i = 0; i < bp->cp_nr_rings; i++) { 3530 struct bnxt_napi *bnapi = bp->bnapi[i]; 3531 struct bnxt_cp_ring_info *cpr; 3532 struct bnxt_ring_struct *ring; 3533 3534 if (!bnapi) 3535 continue; 3536 3537 cpr = &bnapi->cp_ring; 3538 cpr->bnapi = bnapi; 3539 ring = &cpr->cp_ring_struct; 3540 3541 rc = bnxt_alloc_ring(bp, &ring->ring_mem); 3542 if (rc) 3543 return rc; 3544 3545 if (ulp_msix && i >= ulp_base_vec) 3546 ring->map_idx = i + ulp_msix; 3547 else 3548 ring->map_idx = i; 3549 3550 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 3551 continue; 3552 3553 if (i < bp->rx_nr_rings) { 3554 struct bnxt_cp_ring_info *cpr2 = 3555 bnxt_alloc_cp_sub_ring(bp); 3556 3557 cpr->cp_ring_arr[BNXT_RX_HDL] = cpr2; 3558 if (!cpr2) 3559 return -ENOMEM; 3560 cpr2->bnapi = bnapi; 3561 } 3562 if ((sh && i < bp->tx_nr_rings) || 3563 (!sh && i >= bp->rx_nr_rings)) { 3564 struct bnxt_cp_ring_info *cpr2 = 3565 bnxt_alloc_cp_sub_ring(bp); 3566 3567 cpr->cp_ring_arr[BNXT_TX_HDL] = cpr2; 3568 if (!cpr2) 3569 return -ENOMEM; 3570 cpr2->bnapi = bnapi; 3571 } 3572 } 3573 return 0; 3574 } 3575 3576 static void bnxt_init_ring_struct(struct bnxt *bp) 3577 { 3578 int i; 3579 3580 for (i = 0; i < bp->cp_nr_rings; i++) { 3581 struct bnxt_napi *bnapi = bp->bnapi[i]; 3582 struct bnxt_ring_mem_info *rmem; 3583 struct bnxt_cp_ring_info *cpr; 3584 struct bnxt_rx_ring_info *rxr; 3585 struct bnxt_tx_ring_info *txr; 3586 struct bnxt_ring_struct *ring; 3587 3588 if (!bnapi) 3589 continue; 3590 3591 cpr = &bnapi->cp_ring; 3592 ring = &cpr->cp_ring_struct; 3593 rmem = &ring->ring_mem; 3594 rmem->nr_pages = bp->cp_nr_pages; 3595 rmem->page_size = HW_CMPD_RING_SIZE; 3596 rmem->pg_arr = (void **)cpr->cp_desc_ring; 3597 rmem->dma_arr = cpr->cp_desc_mapping; 3598 rmem->vmem_size = 0; 3599 3600 rxr = bnapi->rx_ring; 3601 if (!rxr) 3602 goto skip_rx; 3603 3604 ring = &rxr->rx_ring_struct; 3605 rmem = &ring->ring_mem; 3606 rmem->nr_pages = bp->rx_nr_pages; 3607 rmem->page_size = HW_RXBD_RING_SIZE; 3608 rmem->pg_arr = (void **)rxr->rx_desc_ring; 3609 rmem->dma_arr = rxr->rx_desc_mapping; 3610 rmem->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages; 3611 rmem->vmem = (void **)&rxr->rx_buf_ring; 3612 3613 ring = &rxr->rx_agg_ring_struct; 3614 rmem = &ring->ring_mem; 3615 rmem->nr_pages = bp->rx_agg_nr_pages; 3616 rmem->page_size = HW_RXBD_RING_SIZE; 3617 rmem->pg_arr = (void **)rxr->rx_agg_desc_ring; 3618 rmem->dma_arr = rxr->rx_agg_desc_mapping; 3619 rmem->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages; 3620 rmem->vmem = (void **)&rxr->rx_agg_ring; 3621 3622 skip_rx: 3623 txr = bnapi->tx_ring; 3624 if (!txr) 3625 continue; 3626 3627 ring = &txr->tx_ring_struct; 3628 rmem = &ring->ring_mem; 3629 rmem->nr_pages = bp->tx_nr_pages; 3630 rmem->page_size = HW_RXBD_RING_SIZE; 3631 rmem->pg_arr = (void **)txr->tx_desc_ring; 3632 rmem->dma_arr = txr->tx_desc_mapping; 3633 rmem->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages; 3634 rmem->vmem = (void **)&txr->tx_buf_ring; 3635 } 3636 } 3637 3638 static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type) 3639 { 3640 int i; 3641 u32 prod; 3642 struct rx_bd **rx_buf_ring; 3643 3644 rx_buf_ring = (struct rx_bd **)ring->ring_mem.pg_arr; 3645 for (i = 0, prod = 0; i < ring->ring_mem.nr_pages; i++) { 3646 int j; 3647 struct rx_bd *rxbd; 3648 3649 rxbd = rx_buf_ring[i]; 3650 if (!rxbd) 3651 continue; 3652 3653 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) { 3654 rxbd->rx_bd_len_flags_type = cpu_to_le32(type); 3655 rxbd->rx_bd_opaque = prod; 3656 } 3657 } 3658 } 3659 3660 static int bnxt_alloc_one_rx_ring(struct bnxt *bp, int ring_nr) 3661 { 3662 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr]; 3663 struct net_device *dev = bp->dev; 3664 u32 prod; 3665 int i; 3666 3667 prod = rxr->rx_prod; 3668 for (i = 0; i < bp->rx_ring_size; i++) { 3669 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL)) { 3670 netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n", 3671 ring_nr, i, bp->rx_ring_size); 3672 break; 3673 } 3674 prod = NEXT_RX(prod); 3675 } 3676 rxr->rx_prod = prod; 3677 3678 if (!(bp->flags & BNXT_FLAG_AGG_RINGS)) 3679 return 0; 3680 3681 prod = rxr->rx_agg_prod; 3682 for (i = 0; i < bp->rx_agg_ring_size; i++) { 3683 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL)) { 3684 netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n", 3685 ring_nr, i, bp->rx_ring_size); 3686 break; 3687 } 3688 prod = NEXT_RX_AGG(prod); 3689 } 3690 rxr->rx_agg_prod = prod; 3691 3692 if (rxr->rx_tpa) { 3693 dma_addr_t mapping; 3694 u8 *data; 3695 3696 for (i = 0; i < bp->max_tpa; i++) { 3697 data = __bnxt_alloc_rx_frag(bp, &mapping, GFP_KERNEL); 3698 if (!data) 3699 return -ENOMEM; 3700 3701 rxr->rx_tpa[i].data = data; 3702 rxr->rx_tpa[i].data_ptr = data + bp->rx_offset; 3703 rxr->rx_tpa[i].mapping = mapping; 3704 } 3705 } 3706 return 0; 3707 } 3708 3709 static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr) 3710 { 3711 struct bnxt_rx_ring_info *rxr; 3712 struct bnxt_ring_struct *ring; 3713 u32 type; 3714 3715 type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) | 3716 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP; 3717 3718 if (NET_IP_ALIGN == 2) 3719 type |= RX_BD_FLAGS_SOP; 3720 3721 rxr = &bp->rx_ring[ring_nr]; 3722 ring = &rxr->rx_ring_struct; 3723 bnxt_init_rxbd_pages(ring, type); 3724 3725 if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) { 3726 bpf_prog_add(bp->xdp_prog, 1); 3727 rxr->xdp_prog = bp->xdp_prog; 3728 } 3729 ring->fw_ring_id = INVALID_HW_RING_ID; 3730 3731 ring = &rxr->rx_agg_ring_struct; 3732 ring->fw_ring_id = INVALID_HW_RING_ID; 3733 3734 if ((bp->flags & BNXT_FLAG_AGG_RINGS)) { 3735 type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) | 3736 RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP; 3737 3738 bnxt_init_rxbd_pages(ring, type); 3739 } 3740 3741 return bnxt_alloc_one_rx_ring(bp, ring_nr); 3742 } 3743 3744 static void bnxt_init_cp_rings(struct bnxt *bp) 3745 { 3746 int i, j; 3747 3748 for (i = 0; i < bp->cp_nr_rings; i++) { 3749 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring; 3750 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 3751 3752 ring->fw_ring_id = INVALID_HW_RING_ID; 3753 cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks; 3754 cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs; 3755 for (j = 0; j < 2; j++) { 3756 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j]; 3757 3758 if (!cpr2) 3759 continue; 3760 3761 ring = &cpr2->cp_ring_struct; 3762 ring->fw_ring_id = INVALID_HW_RING_ID; 3763 cpr2->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks; 3764 cpr2->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs; 3765 } 3766 } 3767 } 3768 3769 static int bnxt_init_rx_rings(struct bnxt *bp) 3770 { 3771 int i, rc = 0; 3772 3773 if (BNXT_RX_PAGE_MODE(bp)) { 3774 bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM; 3775 bp->rx_dma_offset = XDP_PACKET_HEADROOM; 3776 } else { 3777 bp->rx_offset = BNXT_RX_OFFSET; 3778 bp->rx_dma_offset = BNXT_RX_DMA_OFFSET; 3779 } 3780 3781 for (i = 0; i < bp->rx_nr_rings; i++) { 3782 rc = bnxt_init_one_rx_ring(bp, i); 3783 if (rc) 3784 break; 3785 } 3786 3787 return rc; 3788 } 3789 3790 static int bnxt_init_tx_rings(struct bnxt *bp) 3791 { 3792 u16 i; 3793 3794 bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2, 3795 BNXT_MIN_TX_DESC_CNT); 3796 3797 for (i = 0; i < bp->tx_nr_rings; i++) { 3798 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 3799 struct bnxt_ring_struct *ring = &txr->tx_ring_struct; 3800 3801 ring->fw_ring_id = INVALID_HW_RING_ID; 3802 } 3803 3804 return 0; 3805 } 3806 3807 static void bnxt_free_ring_grps(struct bnxt *bp) 3808 { 3809 kfree(bp->grp_info); 3810 bp->grp_info = NULL; 3811 } 3812 3813 static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init) 3814 { 3815 int i; 3816 3817 if (irq_re_init) { 3818 bp->grp_info = kcalloc(bp->cp_nr_rings, 3819 sizeof(struct bnxt_ring_grp_info), 3820 GFP_KERNEL); 3821 if (!bp->grp_info) 3822 return -ENOMEM; 3823 } 3824 for (i = 0; i < bp->cp_nr_rings; i++) { 3825 if (irq_re_init) 3826 bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID; 3827 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID; 3828 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID; 3829 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID; 3830 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID; 3831 } 3832 return 0; 3833 } 3834 3835 static void bnxt_free_vnics(struct bnxt *bp) 3836 { 3837 kfree(bp->vnic_info); 3838 bp->vnic_info = NULL; 3839 bp->nr_vnics = 0; 3840 } 3841 3842 static int bnxt_alloc_vnics(struct bnxt *bp) 3843 { 3844 int num_vnics = 1; 3845 3846 #ifdef CONFIG_RFS_ACCEL 3847 if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS) 3848 num_vnics += bp->rx_nr_rings; 3849 #endif 3850 3851 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 3852 num_vnics++; 3853 3854 bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info), 3855 GFP_KERNEL); 3856 if (!bp->vnic_info) 3857 return -ENOMEM; 3858 3859 bp->nr_vnics = num_vnics; 3860 return 0; 3861 } 3862 3863 static void bnxt_init_vnics(struct bnxt *bp) 3864 { 3865 int i; 3866 3867 for (i = 0; i < bp->nr_vnics; i++) { 3868 struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; 3869 int j; 3870 3871 vnic->fw_vnic_id = INVALID_HW_RING_ID; 3872 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) 3873 vnic->fw_rss_cos_lb_ctx[j] = INVALID_HW_RING_ID; 3874 3875 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID; 3876 3877 if (bp->vnic_info[i].rss_hash_key) { 3878 if (i == 0) 3879 get_random_bytes(vnic->rss_hash_key, 3880 HW_HASH_KEY_SIZE); 3881 else 3882 memcpy(vnic->rss_hash_key, 3883 bp->vnic_info[0].rss_hash_key, 3884 HW_HASH_KEY_SIZE); 3885 } 3886 } 3887 } 3888 3889 static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg) 3890 { 3891 int pages; 3892 3893 pages = ring_size / desc_per_pg; 3894 3895 if (!pages) 3896 return 1; 3897 3898 pages++; 3899 3900 while (pages & (pages - 1)) 3901 pages++; 3902 3903 return pages; 3904 } 3905 3906 void bnxt_set_tpa_flags(struct bnxt *bp) 3907 { 3908 bp->flags &= ~BNXT_FLAG_TPA; 3909 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS) 3910 return; 3911 if (bp->dev->features & NETIF_F_LRO) 3912 bp->flags |= BNXT_FLAG_LRO; 3913 else if (bp->dev->features & NETIF_F_GRO_HW) 3914 bp->flags |= BNXT_FLAG_GRO; 3915 } 3916 3917 /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must 3918 * be set on entry. 3919 */ 3920 void bnxt_set_ring_params(struct bnxt *bp) 3921 { 3922 u32 ring_size, rx_size, rx_space, max_rx_cmpl; 3923 u32 agg_factor = 0, agg_ring_size = 0; 3924 3925 /* 8 for CRC and VLAN */ 3926 rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8); 3927 3928 rx_space = rx_size + ALIGN(max(NET_SKB_PAD, XDP_PACKET_HEADROOM), 8) + 3929 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 3930 3931 bp->rx_copy_thresh = BNXT_RX_COPY_THRESH; 3932 ring_size = bp->rx_ring_size; 3933 bp->rx_agg_ring_size = 0; 3934 bp->rx_agg_nr_pages = 0; 3935 3936 if (bp->flags & BNXT_FLAG_TPA) 3937 agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE); 3938 3939 bp->flags &= ~BNXT_FLAG_JUMBO; 3940 if (rx_space > PAGE_SIZE && !(bp->flags & BNXT_FLAG_NO_AGG_RINGS)) { 3941 u32 jumbo_factor; 3942 3943 bp->flags |= BNXT_FLAG_JUMBO; 3944 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT; 3945 if (jumbo_factor > agg_factor) 3946 agg_factor = jumbo_factor; 3947 } 3948 if (agg_factor) { 3949 if (ring_size > BNXT_MAX_RX_DESC_CNT_JUM_ENA) { 3950 ring_size = BNXT_MAX_RX_DESC_CNT_JUM_ENA; 3951 netdev_warn(bp->dev, "RX ring size reduced from %d to %d because the jumbo ring is now enabled\n", 3952 bp->rx_ring_size, ring_size); 3953 bp->rx_ring_size = ring_size; 3954 } 3955 agg_ring_size = ring_size * agg_factor; 3956 3957 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size, 3958 RX_DESC_CNT); 3959 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) { 3960 u32 tmp = agg_ring_size; 3961 3962 bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES; 3963 agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1; 3964 netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n", 3965 tmp, agg_ring_size); 3966 } 3967 bp->rx_agg_ring_size = agg_ring_size; 3968 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1; 3969 3970 if (BNXT_RX_PAGE_MODE(bp)) { 3971 rx_space = PAGE_SIZE; 3972 rx_size = PAGE_SIZE - 3973 ALIGN(max(NET_SKB_PAD, XDP_PACKET_HEADROOM), 8) - 3974 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 3975 } else { 3976 rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN); 3977 rx_space = rx_size + NET_SKB_PAD + 3978 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 3979 } 3980 } 3981 3982 bp->rx_buf_use_size = rx_size; 3983 bp->rx_buf_size = rx_space; 3984 3985 bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT); 3986 bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1; 3987 3988 ring_size = bp->tx_ring_size; 3989 bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT); 3990 bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1; 3991 3992 max_rx_cmpl = bp->rx_ring_size; 3993 /* MAX TPA needs to be added because TPA_START completions are 3994 * immediately recycled, so the TPA completions are not bound by 3995 * the RX ring size. 3996 */ 3997 if (bp->flags & BNXT_FLAG_TPA) 3998 max_rx_cmpl += bp->max_tpa; 3999 /* RX and TPA completions are 32-byte, all others are 16-byte */ 4000 ring_size = max_rx_cmpl * 2 + agg_ring_size + bp->tx_ring_size; 4001 bp->cp_ring_size = ring_size; 4002 4003 bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT); 4004 if (bp->cp_nr_pages > MAX_CP_PAGES) { 4005 bp->cp_nr_pages = MAX_CP_PAGES; 4006 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1; 4007 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n", 4008 ring_size, bp->cp_ring_size); 4009 } 4010 bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT; 4011 bp->cp_ring_mask = bp->cp_bit - 1; 4012 } 4013 4014 /* Changing allocation mode of RX rings. 4015 * TODO: Update when extending xdp_rxq_info to support allocation modes. 4016 */ 4017 int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode) 4018 { 4019 if (page_mode) { 4020 bp->flags &= ~BNXT_FLAG_AGG_RINGS; 4021 bp->flags |= BNXT_FLAG_RX_PAGE_MODE; 4022 4023 if (bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU) { 4024 bp->flags |= BNXT_FLAG_JUMBO; 4025 bp->rx_skb_func = bnxt_rx_multi_page_skb; 4026 bp->dev->max_mtu = 4027 min_t(u16, bp->max_mtu, BNXT_MAX_MTU); 4028 } else { 4029 bp->flags |= BNXT_FLAG_NO_AGG_RINGS; 4030 bp->rx_skb_func = bnxt_rx_page_skb; 4031 bp->dev->max_mtu = 4032 min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU); 4033 } 4034 bp->rx_dir = DMA_BIDIRECTIONAL; 4035 /* Disable LRO or GRO_HW */ 4036 netdev_update_features(bp->dev); 4037 } else { 4038 bp->dev->max_mtu = bp->max_mtu; 4039 bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE; 4040 bp->rx_dir = DMA_FROM_DEVICE; 4041 bp->rx_skb_func = bnxt_rx_skb; 4042 } 4043 return 0; 4044 } 4045 4046 static void bnxt_free_vnic_attributes(struct bnxt *bp) 4047 { 4048 int i; 4049 struct bnxt_vnic_info *vnic; 4050 struct pci_dev *pdev = bp->pdev; 4051 4052 if (!bp->vnic_info) 4053 return; 4054 4055 for (i = 0; i < bp->nr_vnics; i++) { 4056 vnic = &bp->vnic_info[i]; 4057 4058 kfree(vnic->fw_grp_ids); 4059 vnic->fw_grp_ids = NULL; 4060 4061 kfree(vnic->uc_list); 4062 vnic->uc_list = NULL; 4063 4064 if (vnic->mc_list) { 4065 dma_free_coherent(&pdev->dev, vnic->mc_list_size, 4066 vnic->mc_list, vnic->mc_list_mapping); 4067 vnic->mc_list = NULL; 4068 } 4069 4070 if (vnic->rss_table) { 4071 dma_free_coherent(&pdev->dev, vnic->rss_table_size, 4072 vnic->rss_table, 4073 vnic->rss_table_dma_addr); 4074 vnic->rss_table = NULL; 4075 } 4076 4077 vnic->rss_hash_key = NULL; 4078 vnic->flags = 0; 4079 } 4080 } 4081 4082 static int bnxt_alloc_vnic_attributes(struct bnxt *bp) 4083 { 4084 int i, rc = 0, size; 4085 struct bnxt_vnic_info *vnic; 4086 struct pci_dev *pdev = bp->pdev; 4087 int max_rings; 4088 4089 for (i = 0; i < bp->nr_vnics; i++) { 4090 vnic = &bp->vnic_info[i]; 4091 4092 if (vnic->flags & BNXT_VNIC_UCAST_FLAG) { 4093 int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN; 4094 4095 if (mem_size > 0) { 4096 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL); 4097 if (!vnic->uc_list) { 4098 rc = -ENOMEM; 4099 goto out; 4100 } 4101 } 4102 } 4103 4104 if (vnic->flags & BNXT_VNIC_MCAST_FLAG) { 4105 vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN; 4106 vnic->mc_list = 4107 dma_alloc_coherent(&pdev->dev, 4108 vnic->mc_list_size, 4109 &vnic->mc_list_mapping, 4110 GFP_KERNEL); 4111 if (!vnic->mc_list) { 4112 rc = -ENOMEM; 4113 goto out; 4114 } 4115 } 4116 4117 if (bp->flags & BNXT_FLAG_CHIP_P5) 4118 goto vnic_skip_grps; 4119 4120 if (vnic->flags & BNXT_VNIC_RSS_FLAG) 4121 max_rings = bp->rx_nr_rings; 4122 else 4123 max_rings = 1; 4124 4125 vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL); 4126 if (!vnic->fw_grp_ids) { 4127 rc = -ENOMEM; 4128 goto out; 4129 } 4130 vnic_skip_grps: 4131 if ((bp->flags & BNXT_FLAG_NEW_RSS_CAP) && 4132 !(vnic->flags & BNXT_VNIC_RSS_FLAG)) 4133 continue; 4134 4135 /* Allocate rss table and hash key */ 4136 size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16)); 4137 if (bp->flags & BNXT_FLAG_CHIP_P5) 4138 size = L1_CACHE_ALIGN(BNXT_MAX_RSS_TABLE_SIZE_P5); 4139 4140 vnic->rss_table_size = size + HW_HASH_KEY_SIZE; 4141 vnic->rss_table = dma_alloc_coherent(&pdev->dev, 4142 vnic->rss_table_size, 4143 &vnic->rss_table_dma_addr, 4144 GFP_KERNEL); 4145 if (!vnic->rss_table) { 4146 rc = -ENOMEM; 4147 goto out; 4148 } 4149 4150 vnic->rss_hash_key = ((void *)vnic->rss_table) + size; 4151 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size; 4152 } 4153 return 0; 4154 4155 out: 4156 return rc; 4157 } 4158 4159 static void bnxt_free_hwrm_resources(struct bnxt *bp) 4160 { 4161 struct bnxt_hwrm_wait_token *token; 4162 4163 dma_pool_destroy(bp->hwrm_dma_pool); 4164 bp->hwrm_dma_pool = NULL; 4165 4166 rcu_read_lock(); 4167 hlist_for_each_entry_rcu(token, &bp->hwrm_pending_list, node) 4168 WRITE_ONCE(token->state, BNXT_HWRM_CANCELLED); 4169 rcu_read_unlock(); 4170 } 4171 4172 static int bnxt_alloc_hwrm_resources(struct bnxt *bp) 4173 { 4174 bp->hwrm_dma_pool = dma_pool_create("bnxt_hwrm", &bp->pdev->dev, 4175 BNXT_HWRM_DMA_SIZE, 4176 BNXT_HWRM_DMA_ALIGN, 0); 4177 if (!bp->hwrm_dma_pool) 4178 return -ENOMEM; 4179 4180 INIT_HLIST_HEAD(&bp->hwrm_pending_list); 4181 4182 return 0; 4183 } 4184 4185 static void bnxt_free_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats) 4186 { 4187 kfree(stats->hw_masks); 4188 stats->hw_masks = NULL; 4189 kfree(stats->sw_stats); 4190 stats->sw_stats = NULL; 4191 if (stats->hw_stats) { 4192 dma_free_coherent(&bp->pdev->dev, stats->len, stats->hw_stats, 4193 stats->hw_stats_map); 4194 stats->hw_stats = NULL; 4195 } 4196 } 4197 4198 static int bnxt_alloc_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats, 4199 bool alloc_masks) 4200 { 4201 stats->hw_stats = dma_alloc_coherent(&bp->pdev->dev, stats->len, 4202 &stats->hw_stats_map, GFP_KERNEL); 4203 if (!stats->hw_stats) 4204 return -ENOMEM; 4205 4206 stats->sw_stats = kzalloc(stats->len, GFP_KERNEL); 4207 if (!stats->sw_stats) 4208 goto stats_mem_err; 4209 4210 if (alloc_masks) { 4211 stats->hw_masks = kzalloc(stats->len, GFP_KERNEL); 4212 if (!stats->hw_masks) 4213 goto stats_mem_err; 4214 } 4215 return 0; 4216 4217 stats_mem_err: 4218 bnxt_free_stats_mem(bp, stats); 4219 return -ENOMEM; 4220 } 4221 4222 static void bnxt_fill_masks(u64 *mask_arr, u64 mask, int count) 4223 { 4224 int i; 4225 4226 for (i = 0; i < count; i++) 4227 mask_arr[i] = mask; 4228 } 4229 4230 static void bnxt_copy_hw_masks(u64 *mask_arr, __le64 *hw_mask_arr, int count) 4231 { 4232 int i; 4233 4234 for (i = 0; i < count; i++) 4235 mask_arr[i] = le64_to_cpu(hw_mask_arr[i]); 4236 } 4237 4238 static int bnxt_hwrm_func_qstat_ext(struct bnxt *bp, 4239 struct bnxt_stats_mem *stats) 4240 { 4241 struct hwrm_func_qstats_ext_output *resp; 4242 struct hwrm_func_qstats_ext_input *req; 4243 __le64 *hw_masks; 4244 int rc; 4245 4246 if (!(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED) || 4247 !(bp->flags & BNXT_FLAG_CHIP_P5)) 4248 return -EOPNOTSUPP; 4249 4250 rc = hwrm_req_init(bp, req, HWRM_FUNC_QSTATS_EXT); 4251 if (rc) 4252 return rc; 4253 4254 req->fid = cpu_to_le16(0xffff); 4255 req->flags = FUNC_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK; 4256 4257 resp = hwrm_req_hold(bp, req); 4258 rc = hwrm_req_send(bp, req); 4259 if (!rc) { 4260 hw_masks = &resp->rx_ucast_pkts; 4261 bnxt_copy_hw_masks(stats->hw_masks, hw_masks, stats->len / 8); 4262 } 4263 hwrm_req_drop(bp, req); 4264 return rc; 4265 } 4266 4267 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags); 4268 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags); 4269 4270 static void bnxt_init_stats(struct bnxt *bp) 4271 { 4272 struct bnxt_napi *bnapi = bp->bnapi[0]; 4273 struct bnxt_cp_ring_info *cpr; 4274 struct bnxt_stats_mem *stats; 4275 __le64 *rx_stats, *tx_stats; 4276 int rc, rx_count, tx_count; 4277 u64 *rx_masks, *tx_masks; 4278 u64 mask; 4279 u8 flags; 4280 4281 cpr = &bnapi->cp_ring; 4282 stats = &cpr->stats; 4283 rc = bnxt_hwrm_func_qstat_ext(bp, stats); 4284 if (rc) { 4285 if (bp->flags & BNXT_FLAG_CHIP_P5) 4286 mask = (1ULL << 48) - 1; 4287 else 4288 mask = -1ULL; 4289 bnxt_fill_masks(stats->hw_masks, mask, stats->len / 8); 4290 } 4291 if (bp->flags & BNXT_FLAG_PORT_STATS) { 4292 stats = &bp->port_stats; 4293 rx_stats = stats->hw_stats; 4294 rx_masks = stats->hw_masks; 4295 rx_count = sizeof(struct rx_port_stats) / 8; 4296 tx_stats = rx_stats + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 4297 tx_masks = rx_masks + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 4298 tx_count = sizeof(struct tx_port_stats) / 8; 4299 4300 flags = PORT_QSTATS_REQ_FLAGS_COUNTER_MASK; 4301 rc = bnxt_hwrm_port_qstats(bp, flags); 4302 if (rc) { 4303 mask = (1ULL << 40) - 1; 4304 4305 bnxt_fill_masks(rx_masks, mask, rx_count); 4306 bnxt_fill_masks(tx_masks, mask, tx_count); 4307 } else { 4308 bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count); 4309 bnxt_copy_hw_masks(tx_masks, tx_stats, tx_count); 4310 bnxt_hwrm_port_qstats(bp, 0); 4311 } 4312 } 4313 if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) { 4314 stats = &bp->rx_port_stats_ext; 4315 rx_stats = stats->hw_stats; 4316 rx_masks = stats->hw_masks; 4317 rx_count = sizeof(struct rx_port_stats_ext) / 8; 4318 stats = &bp->tx_port_stats_ext; 4319 tx_stats = stats->hw_stats; 4320 tx_masks = stats->hw_masks; 4321 tx_count = sizeof(struct tx_port_stats_ext) / 8; 4322 4323 flags = PORT_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK; 4324 rc = bnxt_hwrm_port_qstats_ext(bp, flags); 4325 if (rc) { 4326 mask = (1ULL << 40) - 1; 4327 4328 bnxt_fill_masks(rx_masks, mask, rx_count); 4329 if (tx_stats) 4330 bnxt_fill_masks(tx_masks, mask, tx_count); 4331 } else { 4332 bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count); 4333 if (tx_stats) 4334 bnxt_copy_hw_masks(tx_masks, tx_stats, 4335 tx_count); 4336 bnxt_hwrm_port_qstats_ext(bp, 0); 4337 } 4338 } 4339 } 4340 4341 static void bnxt_free_port_stats(struct bnxt *bp) 4342 { 4343 bp->flags &= ~BNXT_FLAG_PORT_STATS; 4344 bp->flags &= ~BNXT_FLAG_PORT_STATS_EXT; 4345 4346 bnxt_free_stats_mem(bp, &bp->port_stats); 4347 bnxt_free_stats_mem(bp, &bp->rx_port_stats_ext); 4348 bnxt_free_stats_mem(bp, &bp->tx_port_stats_ext); 4349 } 4350 4351 static void bnxt_free_ring_stats(struct bnxt *bp) 4352 { 4353 int i; 4354 4355 if (!bp->bnapi) 4356 return; 4357 4358 for (i = 0; i < bp->cp_nr_rings; i++) { 4359 struct bnxt_napi *bnapi = bp->bnapi[i]; 4360 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 4361 4362 bnxt_free_stats_mem(bp, &cpr->stats); 4363 } 4364 } 4365 4366 static int bnxt_alloc_stats(struct bnxt *bp) 4367 { 4368 u32 size, i; 4369 int rc; 4370 4371 size = bp->hw_ring_stats_size; 4372 4373 for (i = 0; i < bp->cp_nr_rings; i++) { 4374 struct bnxt_napi *bnapi = bp->bnapi[i]; 4375 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 4376 4377 cpr->stats.len = size; 4378 rc = bnxt_alloc_stats_mem(bp, &cpr->stats, !i); 4379 if (rc) 4380 return rc; 4381 4382 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID; 4383 } 4384 4385 if (BNXT_VF(bp) || bp->chip_num == CHIP_NUM_58700) 4386 return 0; 4387 4388 if (bp->port_stats.hw_stats) 4389 goto alloc_ext_stats; 4390 4391 bp->port_stats.len = BNXT_PORT_STATS_SIZE; 4392 rc = bnxt_alloc_stats_mem(bp, &bp->port_stats, true); 4393 if (rc) 4394 return rc; 4395 4396 bp->flags |= BNXT_FLAG_PORT_STATS; 4397 4398 alloc_ext_stats: 4399 /* Display extended statistics only if FW supports it */ 4400 if (bp->hwrm_spec_code < 0x10804 || bp->hwrm_spec_code == 0x10900) 4401 if (!(bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED)) 4402 return 0; 4403 4404 if (bp->rx_port_stats_ext.hw_stats) 4405 goto alloc_tx_ext_stats; 4406 4407 bp->rx_port_stats_ext.len = sizeof(struct rx_port_stats_ext); 4408 rc = bnxt_alloc_stats_mem(bp, &bp->rx_port_stats_ext, true); 4409 /* Extended stats are optional */ 4410 if (rc) 4411 return 0; 4412 4413 alloc_tx_ext_stats: 4414 if (bp->tx_port_stats_ext.hw_stats) 4415 return 0; 4416 4417 if (bp->hwrm_spec_code >= 0x10902 || 4418 (bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED)) { 4419 bp->tx_port_stats_ext.len = sizeof(struct tx_port_stats_ext); 4420 rc = bnxt_alloc_stats_mem(bp, &bp->tx_port_stats_ext, true); 4421 /* Extended stats are optional */ 4422 if (rc) 4423 return 0; 4424 } 4425 bp->flags |= BNXT_FLAG_PORT_STATS_EXT; 4426 return 0; 4427 } 4428 4429 static void bnxt_clear_ring_indices(struct bnxt *bp) 4430 { 4431 int i; 4432 4433 if (!bp->bnapi) 4434 return; 4435 4436 for (i = 0; i < bp->cp_nr_rings; i++) { 4437 struct bnxt_napi *bnapi = bp->bnapi[i]; 4438 struct bnxt_cp_ring_info *cpr; 4439 struct bnxt_rx_ring_info *rxr; 4440 struct bnxt_tx_ring_info *txr; 4441 4442 if (!bnapi) 4443 continue; 4444 4445 cpr = &bnapi->cp_ring; 4446 cpr->cp_raw_cons = 0; 4447 4448 txr = bnapi->tx_ring; 4449 if (txr) { 4450 txr->tx_prod = 0; 4451 txr->tx_cons = 0; 4452 } 4453 4454 rxr = bnapi->rx_ring; 4455 if (rxr) { 4456 rxr->rx_prod = 0; 4457 rxr->rx_agg_prod = 0; 4458 rxr->rx_sw_agg_prod = 0; 4459 rxr->rx_next_cons = 0; 4460 } 4461 } 4462 } 4463 4464 static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool irq_reinit) 4465 { 4466 #ifdef CONFIG_RFS_ACCEL 4467 int i; 4468 4469 /* Under rtnl_lock and all our NAPIs have been disabled. It's 4470 * safe to delete the hash table. 4471 */ 4472 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) { 4473 struct hlist_head *head; 4474 struct hlist_node *tmp; 4475 struct bnxt_ntuple_filter *fltr; 4476 4477 head = &bp->ntp_fltr_hash_tbl[i]; 4478 hlist_for_each_entry_safe(fltr, tmp, head, hash) { 4479 hlist_del(&fltr->hash); 4480 kfree(fltr); 4481 } 4482 } 4483 if (irq_reinit) { 4484 bitmap_free(bp->ntp_fltr_bmap); 4485 bp->ntp_fltr_bmap = NULL; 4486 } 4487 bp->ntp_fltr_count = 0; 4488 #endif 4489 } 4490 4491 static int bnxt_alloc_ntp_fltrs(struct bnxt *bp) 4492 { 4493 #ifdef CONFIG_RFS_ACCEL 4494 int i, rc = 0; 4495 4496 if (!(bp->flags & BNXT_FLAG_RFS)) 4497 return 0; 4498 4499 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) 4500 INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]); 4501 4502 bp->ntp_fltr_count = 0; 4503 bp->ntp_fltr_bmap = bitmap_zalloc(BNXT_NTP_FLTR_MAX_FLTR, GFP_KERNEL); 4504 4505 if (!bp->ntp_fltr_bmap) 4506 rc = -ENOMEM; 4507 4508 return rc; 4509 #else 4510 return 0; 4511 #endif 4512 } 4513 4514 static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init) 4515 { 4516 bnxt_free_vnic_attributes(bp); 4517 bnxt_free_tx_rings(bp); 4518 bnxt_free_rx_rings(bp); 4519 bnxt_free_cp_rings(bp); 4520 bnxt_free_all_cp_arrays(bp); 4521 bnxt_free_ntp_fltrs(bp, irq_re_init); 4522 if (irq_re_init) { 4523 bnxt_free_ring_stats(bp); 4524 if (!(bp->phy_flags & BNXT_PHY_FL_PORT_STATS_NO_RESET) || 4525 test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 4526 bnxt_free_port_stats(bp); 4527 bnxt_free_ring_grps(bp); 4528 bnxt_free_vnics(bp); 4529 kfree(bp->tx_ring_map); 4530 bp->tx_ring_map = NULL; 4531 kfree(bp->tx_ring); 4532 bp->tx_ring = NULL; 4533 kfree(bp->rx_ring); 4534 bp->rx_ring = NULL; 4535 kfree(bp->bnapi); 4536 bp->bnapi = NULL; 4537 } else { 4538 bnxt_clear_ring_indices(bp); 4539 } 4540 } 4541 4542 static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init) 4543 { 4544 int i, j, rc, size, arr_size; 4545 void *bnapi; 4546 4547 if (irq_re_init) { 4548 /* Allocate bnapi mem pointer array and mem block for 4549 * all queues 4550 */ 4551 arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) * 4552 bp->cp_nr_rings); 4553 size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi)); 4554 bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL); 4555 if (!bnapi) 4556 return -ENOMEM; 4557 4558 bp->bnapi = bnapi; 4559 bnapi += arr_size; 4560 for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) { 4561 bp->bnapi[i] = bnapi; 4562 bp->bnapi[i]->index = i; 4563 bp->bnapi[i]->bp = bp; 4564 if (bp->flags & BNXT_FLAG_CHIP_P5) { 4565 struct bnxt_cp_ring_info *cpr = 4566 &bp->bnapi[i]->cp_ring; 4567 4568 cpr->cp_ring_struct.ring_mem.flags = 4569 BNXT_RMEM_RING_PTE_FLAG; 4570 } 4571 } 4572 4573 bp->rx_ring = kcalloc(bp->rx_nr_rings, 4574 sizeof(struct bnxt_rx_ring_info), 4575 GFP_KERNEL); 4576 if (!bp->rx_ring) 4577 return -ENOMEM; 4578 4579 for (i = 0; i < bp->rx_nr_rings; i++) { 4580 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 4581 4582 if (bp->flags & BNXT_FLAG_CHIP_P5) { 4583 rxr->rx_ring_struct.ring_mem.flags = 4584 BNXT_RMEM_RING_PTE_FLAG; 4585 rxr->rx_agg_ring_struct.ring_mem.flags = 4586 BNXT_RMEM_RING_PTE_FLAG; 4587 } 4588 rxr->bnapi = bp->bnapi[i]; 4589 bp->bnapi[i]->rx_ring = &bp->rx_ring[i]; 4590 } 4591 4592 bp->tx_ring = kcalloc(bp->tx_nr_rings, 4593 sizeof(struct bnxt_tx_ring_info), 4594 GFP_KERNEL); 4595 if (!bp->tx_ring) 4596 return -ENOMEM; 4597 4598 bp->tx_ring_map = kcalloc(bp->tx_nr_rings, sizeof(u16), 4599 GFP_KERNEL); 4600 4601 if (!bp->tx_ring_map) 4602 return -ENOMEM; 4603 4604 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 4605 j = 0; 4606 else 4607 j = bp->rx_nr_rings; 4608 4609 for (i = 0; i < bp->tx_nr_rings; i++, j++) { 4610 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 4611 4612 if (bp->flags & BNXT_FLAG_CHIP_P5) 4613 txr->tx_ring_struct.ring_mem.flags = 4614 BNXT_RMEM_RING_PTE_FLAG; 4615 txr->bnapi = bp->bnapi[j]; 4616 bp->bnapi[j]->tx_ring = txr; 4617 bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i; 4618 if (i >= bp->tx_nr_rings_xdp) { 4619 txr->txq_index = i - bp->tx_nr_rings_xdp; 4620 bp->bnapi[j]->tx_int = bnxt_tx_int; 4621 } else { 4622 bp->bnapi[j]->flags |= BNXT_NAPI_FLAG_XDP; 4623 bp->bnapi[j]->tx_int = bnxt_tx_int_xdp; 4624 } 4625 } 4626 4627 rc = bnxt_alloc_stats(bp); 4628 if (rc) 4629 goto alloc_mem_err; 4630 bnxt_init_stats(bp); 4631 4632 rc = bnxt_alloc_ntp_fltrs(bp); 4633 if (rc) 4634 goto alloc_mem_err; 4635 4636 rc = bnxt_alloc_vnics(bp); 4637 if (rc) 4638 goto alloc_mem_err; 4639 } 4640 4641 rc = bnxt_alloc_all_cp_arrays(bp); 4642 if (rc) 4643 goto alloc_mem_err; 4644 4645 bnxt_init_ring_struct(bp); 4646 4647 rc = bnxt_alloc_rx_rings(bp); 4648 if (rc) 4649 goto alloc_mem_err; 4650 4651 rc = bnxt_alloc_tx_rings(bp); 4652 if (rc) 4653 goto alloc_mem_err; 4654 4655 rc = bnxt_alloc_cp_rings(bp); 4656 if (rc) 4657 goto alloc_mem_err; 4658 4659 bp->vnic_info[0].flags |= BNXT_VNIC_RSS_FLAG | BNXT_VNIC_MCAST_FLAG | 4660 BNXT_VNIC_UCAST_FLAG; 4661 rc = bnxt_alloc_vnic_attributes(bp); 4662 if (rc) 4663 goto alloc_mem_err; 4664 return 0; 4665 4666 alloc_mem_err: 4667 bnxt_free_mem(bp, true); 4668 return rc; 4669 } 4670 4671 static void bnxt_disable_int(struct bnxt *bp) 4672 { 4673 int i; 4674 4675 if (!bp->bnapi) 4676 return; 4677 4678 for (i = 0; i < bp->cp_nr_rings; i++) { 4679 struct bnxt_napi *bnapi = bp->bnapi[i]; 4680 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 4681 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 4682 4683 if (ring->fw_ring_id != INVALID_HW_RING_ID) 4684 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons); 4685 } 4686 } 4687 4688 static int bnxt_cp_num_to_irq_num(struct bnxt *bp, int n) 4689 { 4690 struct bnxt_napi *bnapi = bp->bnapi[n]; 4691 struct bnxt_cp_ring_info *cpr; 4692 4693 cpr = &bnapi->cp_ring; 4694 return cpr->cp_ring_struct.map_idx; 4695 } 4696 4697 static void bnxt_disable_int_sync(struct bnxt *bp) 4698 { 4699 int i; 4700 4701 if (!bp->irq_tbl) 4702 return; 4703 4704 atomic_inc(&bp->intr_sem); 4705 4706 bnxt_disable_int(bp); 4707 for (i = 0; i < bp->cp_nr_rings; i++) { 4708 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 4709 4710 synchronize_irq(bp->irq_tbl[map_idx].vector); 4711 } 4712 } 4713 4714 static void bnxt_enable_int(struct bnxt *bp) 4715 { 4716 int i; 4717 4718 atomic_set(&bp->intr_sem, 0); 4719 for (i = 0; i < bp->cp_nr_rings; i++) { 4720 struct bnxt_napi *bnapi = bp->bnapi[i]; 4721 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 4722 4723 bnxt_db_nq_arm(bp, &cpr->cp_db, cpr->cp_raw_cons); 4724 } 4725 } 4726 4727 int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp, unsigned long *bmap, int bmap_size, 4728 bool async_only) 4729 { 4730 DECLARE_BITMAP(async_events_bmap, 256); 4731 u32 *events = (u32 *)async_events_bmap; 4732 struct hwrm_func_drv_rgtr_output *resp; 4733 struct hwrm_func_drv_rgtr_input *req; 4734 u32 flags; 4735 int rc, i; 4736 4737 rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_RGTR); 4738 if (rc) 4739 return rc; 4740 4741 req->enables = cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE | 4742 FUNC_DRV_RGTR_REQ_ENABLES_VER | 4743 FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD); 4744 4745 req->os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX); 4746 flags = FUNC_DRV_RGTR_REQ_FLAGS_16BIT_VER_MODE; 4747 if (bp->fw_cap & BNXT_FW_CAP_HOT_RESET) 4748 flags |= FUNC_DRV_RGTR_REQ_FLAGS_HOT_RESET_SUPPORT; 4749 if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) 4750 flags |= FUNC_DRV_RGTR_REQ_FLAGS_ERROR_RECOVERY_SUPPORT | 4751 FUNC_DRV_RGTR_REQ_FLAGS_MASTER_SUPPORT; 4752 req->flags = cpu_to_le32(flags); 4753 req->ver_maj_8b = DRV_VER_MAJ; 4754 req->ver_min_8b = DRV_VER_MIN; 4755 req->ver_upd_8b = DRV_VER_UPD; 4756 req->ver_maj = cpu_to_le16(DRV_VER_MAJ); 4757 req->ver_min = cpu_to_le16(DRV_VER_MIN); 4758 req->ver_upd = cpu_to_le16(DRV_VER_UPD); 4759 4760 if (BNXT_PF(bp)) { 4761 u32 data[8]; 4762 int i; 4763 4764 memset(data, 0, sizeof(data)); 4765 for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) { 4766 u16 cmd = bnxt_vf_req_snif[i]; 4767 unsigned int bit, idx; 4768 4769 idx = cmd / 32; 4770 bit = cmd % 32; 4771 data[idx] |= 1 << bit; 4772 } 4773 4774 for (i = 0; i < 8; i++) 4775 req->vf_req_fwd[i] = cpu_to_le32(data[i]); 4776 4777 req->enables |= 4778 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD); 4779 } 4780 4781 if (bp->fw_cap & BNXT_FW_CAP_OVS_64BIT_HANDLE) 4782 req->flags |= cpu_to_le32( 4783 FUNC_DRV_RGTR_REQ_FLAGS_FLOW_HANDLE_64BIT_MODE); 4784 4785 memset(async_events_bmap, 0, sizeof(async_events_bmap)); 4786 for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++) { 4787 u16 event_id = bnxt_async_events_arr[i]; 4788 4789 if (event_id == ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY && 4790 !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) 4791 continue; 4792 __set_bit(bnxt_async_events_arr[i], async_events_bmap); 4793 } 4794 if (bmap && bmap_size) { 4795 for (i = 0; i < bmap_size; i++) { 4796 if (test_bit(i, bmap)) 4797 __set_bit(i, async_events_bmap); 4798 } 4799 } 4800 for (i = 0; i < 8; i++) 4801 req->async_event_fwd[i] |= cpu_to_le32(events[i]); 4802 4803 if (async_only) 4804 req->enables = 4805 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD); 4806 4807 resp = hwrm_req_hold(bp, req); 4808 rc = hwrm_req_send(bp, req); 4809 if (!rc) { 4810 set_bit(BNXT_STATE_DRV_REGISTERED, &bp->state); 4811 if (resp->flags & 4812 cpu_to_le32(FUNC_DRV_RGTR_RESP_FLAGS_IF_CHANGE_SUPPORTED)) 4813 bp->fw_cap |= BNXT_FW_CAP_IF_CHANGE; 4814 } 4815 hwrm_req_drop(bp, req); 4816 return rc; 4817 } 4818 4819 int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp) 4820 { 4821 struct hwrm_func_drv_unrgtr_input *req; 4822 int rc; 4823 4824 if (!test_and_clear_bit(BNXT_STATE_DRV_REGISTERED, &bp->state)) 4825 return 0; 4826 4827 rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_UNRGTR); 4828 if (rc) 4829 return rc; 4830 return hwrm_req_send(bp, req); 4831 } 4832 4833 static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type) 4834 { 4835 struct hwrm_tunnel_dst_port_free_input *req; 4836 int rc; 4837 4838 if (tunnel_type == TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN && 4839 bp->vxlan_fw_dst_port_id == INVALID_HW_RING_ID) 4840 return 0; 4841 if (tunnel_type == TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE && 4842 bp->nge_fw_dst_port_id == INVALID_HW_RING_ID) 4843 return 0; 4844 4845 rc = hwrm_req_init(bp, req, HWRM_TUNNEL_DST_PORT_FREE); 4846 if (rc) 4847 return rc; 4848 4849 req->tunnel_type = tunnel_type; 4850 4851 switch (tunnel_type) { 4852 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN: 4853 req->tunnel_dst_port_id = cpu_to_le16(bp->vxlan_fw_dst_port_id); 4854 bp->vxlan_port = 0; 4855 bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID; 4856 break; 4857 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE: 4858 req->tunnel_dst_port_id = cpu_to_le16(bp->nge_fw_dst_port_id); 4859 bp->nge_port = 0; 4860 bp->nge_fw_dst_port_id = INVALID_HW_RING_ID; 4861 break; 4862 default: 4863 break; 4864 } 4865 4866 rc = hwrm_req_send(bp, req); 4867 if (rc) 4868 netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n", 4869 rc); 4870 return rc; 4871 } 4872 4873 static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port, 4874 u8 tunnel_type) 4875 { 4876 struct hwrm_tunnel_dst_port_alloc_output *resp; 4877 struct hwrm_tunnel_dst_port_alloc_input *req; 4878 int rc; 4879 4880 rc = hwrm_req_init(bp, req, HWRM_TUNNEL_DST_PORT_ALLOC); 4881 if (rc) 4882 return rc; 4883 4884 req->tunnel_type = tunnel_type; 4885 req->tunnel_dst_port_val = port; 4886 4887 resp = hwrm_req_hold(bp, req); 4888 rc = hwrm_req_send(bp, req); 4889 if (rc) { 4890 netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n", 4891 rc); 4892 goto err_out; 4893 } 4894 4895 switch (tunnel_type) { 4896 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN: 4897 bp->vxlan_port = port; 4898 bp->vxlan_fw_dst_port_id = 4899 le16_to_cpu(resp->tunnel_dst_port_id); 4900 break; 4901 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE: 4902 bp->nge_port = port; 4903 bp->nge_fw_dst_port_id = le16_to_cpu(resp->tunnel_dst_port_id); 4904 break; 4905 default: 4906 break; 4907 } 4908 4909 err_out: 4910 hwrm_req_drop(bp, req); 4911 return rc; 4912 } 4913 4914 static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id) 4915 { 4916 struct hwrm_cfa_l2_set_rx_mask_input *req; 4917 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 4918 int rc; 4919 4920 rc = hwrm_req_init(bp, req, HWRM_CFA_L2_SET_RX_MASK); 4921 if (rc) 4922 return rc; 4923 4924 req->vnic_id = cpu_to_le32(vnic->fw_vnic_id); 4925 if (vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_MCAST) { 4926 req->num_mc_entries = cpu_to_le32(vnic->mc_list_count); 4927 req->mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping); 4928 } 4929 req->mask = cpu_to_le32(vnic->rx_mask); 4930 return hwrm_req_send_silent(bp, req); 4931 } 4932 4933 #ifdef CONFIG_RFS_ACCEL 4934 static int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp, 4935 struct bnxt_ntuple_filter *fltr) 4936 { 4937 struct hwrm_cfa_ntuple_filter_free_input *req; 4938 int rc; 4939 4940 rc = hwrm_req_init(bp, req, HWRM_CFA_NTUPLE_FILTER_FREE); 4941 if (rc) 4942 return rc; 4943 4944 req->ntuple_filter_id = fltr->filter_id; 4945 return hwrm_req_send(bp, req); 4946 } 4947 4948 #define BNXT_NTP_FLTR_FLAGS \ 4949 (CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID | \ 4950 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE | \ 4951 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR | \ 4952 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE | \ 4953 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR | \ 4954 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK | \ 4955 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR | \ 4956 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK | \ 4957 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL | \ 4958 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT | \ 4959 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK | \ 4960 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT | \ 4961 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK | \ 4962 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID) 4963 4964 #define BNXT_NTP_TUNNEL_FLTR_FLAG \ 4965 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE 4966 4967 static int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp, 4968 struct bnxt_ntuple_filter *fltr) 4969 { 4970 struct hwrm_cfa_ntuple_filter_alloc_output *resp; 4971 struct hwrm_cfa_ntuple_filter_alloc_input *req; 4972 struct flow_keys *keys = &fltr->fkeys; 4973 struct bnxt_vnic_info *vnic; 4974 u32 flags = 0; 4975 int rc; 4976 4977 rc = hwrm_req_init(bp, req, HWRM_CFA_NTUPLE_FILTER_ALLOC); 4978 if (rc) 4979 return rc; 4980 4981 req->l2_filter_id = bp->vnic_info[0].fw_l2_filter_id[fltr->l2_fltr_idx]; 4982 4983 if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2) { 4984 flags = CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DEST_RFS_RING_IDX; 4985 req->dst_id = cpu_to_le16(fltr->rxq); 4986 } else { 4987 vnic = &bp->vnic_info[fltr->rxq + 1]; 4988 req->dst_id = cpu_to_le16(vnic->fw_vnic_id); 4989 } 4990 req->flags = cpu_to_le32(flags); 4991 req->enables = cpu_to_le32(BNXT_NTP_FLTR_FLAGS); 4992 4993 req->ethertype = htons(ETH_P_IP); 4994 memcpy(req->src_macaddr, fltr->src_mac_addr, ETH_ALEN); 4995 req->ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4; 4996 req->ip_protocol = keys->basic.ip_proto; 4997 4998 if (keys->basic.n_proto == htons(ETH_P_IPV6)) { 4999 int i; 5000 5001 req->ethertype = htons(ETH_P_IPV6); 5002 req->ip_addr_type = 5003 CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6; 5004 *(struct in6_addr *)&req->src_ipaddr[0] = 5005 keys->addrs.v6addrs.src; 5006 *(struct in6_addr *)&req->dst_ipaddr[0] = 5007 keys->addrs.v6addrs.dst; 5008 for (i = 0; i < 4; i++) { 5009 req->src_ipaddr_mask[i] = cpu_to_be32(0xffffffff); 5010 req->dst_ipaddr_mask[i] = cpu_to_be32(0xffffffff); 5011 } 5012 } else { 5013 req->src_ipaddr[0] = keys->addrs.v4addrs.src; 5014 req->src_ipaddr_mask[0] = cpu_to_be32(0xffffffff); 5015 req->dst_ipaddr[0] = keys->addrs.v4addrs.dst; 5016 req->dst_ipaddr_mask[0] = cpu_to_be32(0xffffffff); 5017 } 5018 if (keys->control.flags & FLOW_DIS_ENCAPSULATION) { 5019 req->enables |= cpu_to_le32(BNXT_NTP_TUNNEL_FLTR_FLAG); 5020 req->tunnel_type = 5021 CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL; 5022 } 5023 5024 req->src_port = keys->ports.src; 5025 req->src_port_mask = cpu_to_be16(0xffff); 5026 req->dst_port = keys->ports.dst; 5027 req->dst_port_mask = cpu_to_be16(0xffff); 5028 5029 resp = hwrm_req_hold(bp, req); 5030 rc = hwrm_req_send(bp, req); 5031 if (!rc) 5032 fltr->filter_id = resp->ntuple_filter_id; 5033 hwrm_req_drop(bp, req); 5034 return rc; 5035 } 5036 #endif 5037 5038 static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx, 5039 const u8 *mac_addr) 5040 { 5041 struct hwrm_cfa_l2_filter_alloc_output *resp; 5042 struct hwrm_cfa_l2_filter_alloc_input *req; 5043 int rc; 5044 5045 rc = hwrm_req_init(bp, req, HWRM_CFA_L2_FILTER_ALLOC); 5046 if (rc) 5047 return rc; 5048 5049 req->flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX); 5050 if (!BNXT_CHIP_TYPE_NITRO_A0(bp)) 5051 req->flags |= 5052 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST); 5053 req->dst_id = cpu_to_le16(bp->vnic_info[vnic_id].fw_vnic_id); 5054 req->enables = 5055 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR | 5056 CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID | 5057 CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK); 5058 memcpy(req->l2_addr, mac_addr, ETH_ALEN); 5059 req->l2_addr_mask[0] = 0xff; 5060 req->l2_addr_mask[1] = 0xff; 5061 req->l2_addr_mask[2] = 0xff; 5062 req->l2_addr_mask[3] = 0xff; 5063 req->l2_addr_mask[4] = 0xff; 5064 req->l2_addr_mask[5] = 0xff; 5065 5066 resp = hwrm_req_hold(bp, req); 5067 rc = hwrm_req_send(bp, req); 5068 if (!rc) 5069 bp->vnic_info[vnic_id].fw_l2_filter_id[idx] = 5070 resp->l2_filter_id; 5071 hwrm_req_drop(bp, req); 5072 return rc; 5073 } 5074 5075 static int bnxt_hwrm_clear_vnic_filter(struct bnxt *bp) 5076 { 5077 struct hwrm_cfa_l2_filter_free_input *req; 5078 u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */ 5079 int rc; 5080 5081 /* Any associated ntuple filters will also be cleared by firmware. */ 5082 rc = hwrm_req_init(bp, req, HWRM_CFA_L2_FILTER_FREE); 5083 if (rc) 5084 return rc; 5085 hwrm_req_hold(bp, req); 5086 for (i = 0; i < num_of_vnics; i++) { 5087 struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; 5088 5089 for (j = 0; j < vnic->uc_filter_count; j++) { 5090 req->l2_filter_id = vnic->fw_l2_filter_id[j]; 5091 5092 rc = hwrm_req_send(bp, req); 5093 } 5094 vnic->uc_filter_count = 0; 5095 } 5096 hwrm_req_drop(bp, req); 5097 return rc; 5098 } 5099 5100 static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags) 5101 { 5102 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5103 u16 max_aggs = VNIC_TPA_CFG_REQ_MAX_AGGS_MAX; 5104 struct hwrm_vnic_tpa_cfg_input *req; 5105 int rc; 5106 5107 if (vnic->fw_vnic_id == INVALID_HW_RING_ID) 5108 return 0; 5109 5110 rc = hwrm_req_init(bp, req, HWRM_VNIC_TPA_CFG); 5111 if (rc) 5112 return rc; 5113 5114 if (tpa_flags) { 5115 u16 mss = bp->dev->mtu - 40; 5116 u32 nsegs, n, segs = 0, flags; 5117 5118 flags = VNIC_TPA_CFG_REQ_FLAGS_TPA | 5119 VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA | 5120 VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE | 5121 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN | 5122 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ; 5123 if (tpa_flags & BNXT_FLAG_GRO) 5124 flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO; 5125 5126 req->flags = cpu_to_le32(flags); 5127 5128 req->enables = 5129 cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS | 5130 VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS | 5131 VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN); 5132 5133 /* Number of segs are log2 units, and first packet is not 5134 * included as part of this units. 5135 */ 5136 if (mss <= BNXT_RX_PAGE_SIZE) { 5137 n = BNXT_RX_PAGE_SIZE / mss; 5138 nsegs = (MAX_SKB_FRAGS - 1) * n; 5139 } else { 5140 n = mss / BNXT_RX_PAGE_SIZE; 5141 if (mss & (BNXT_RX_PAGE_SIZE - 1)) 5142 n++; 5143 nsegs = (MAX_SKB_FRAGS - n) / n; 5144 } 5145 5146 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5147 segs = MAX_TPA_SEGS_P5; 5148 max_aggs = bp->max_tpa; 5149 } else { 5150 segs = ilog2(nsegs); 5151 } 5152 req->max_agg_segs = cpu_to_le16(segs); 5153 req->max_aggs = cpu_to_le16(max_aggs); 5154 5155 req->min_agg_len = cpu_to_le32(512); 5156 } 5157 req->vnic_id = cpu_to_le16(vnic->fw_vnic_id); 5158 5159 return hwrm_req_send(bp, req); 5160 } 5161 5162 static u16 bnxt_cp_ring_from_grp(struct bnxt *bp, struct bnxt_ring_struct *ring) 5163 { 5164 struct bnxt_ring_grp_info *grp_info; 5165 5166 grp_info = &bp->grp_info[ring->grp_idx]; 5167 return grp_info->cp_fw_ring_id; 5168 } 5169 5170 static u16 bnxt_cp_ring_for_rx(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) 5171 { 5172 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5173 struct bnxt_napi *bnapi = rxr->bnapi; 5174 struct bnxt_cp_ring_info *cpr; 5175 5176 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_RX_HDL]; 5177 return cpr->cp_ring_struct.fw_ring_id; 5178 } else { 5179 return bnxt_cp_ring_from_grp(bp, &rxr->rx_ring_struct); 5180 } 5181 } 5182 5183 static u16 bnxt_cp_ring_for_tx(struct bnxt *bp, struct bnxt_tx_ring_info *txr) 5184 { 5185 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5186 struct bnxt_napi *bnapi = txr->bnapi; 5187 struct bnxt_cp_ring_info *cpr; 5188 5189 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_TX_HDL]; 5190 return cpr->cp_ring_struct.fw_ring_id; 5191 } else { 5192 return bnxt_cp_ring_from_grp(bp, &txr->tx_ring_struct); 5193 } 5194 } 5195 5196 static int bnxt_alloc_rss_indir_tbl(struct bnxt *bp) 5197 { 5198 int entries; 5199 5200 if (bp->flags & BNXT_FLAG_CHIP_P5) 5201 entries = BNXT_MAX_RSS_TABLE_ENTRIES_P5; 5202 else 5203 entries = HW_HASH_INDEX_SIZE; 5204 5205 bp->rss_indir_tbl_entries = entries; 5206 bp->rss_indir_tbl = kmalloc_array(entries, sizeof(*bp->rss_indir_tbl), 5207 GFP_KERNEL); 5208 if (!bp->rss_indir_tbl) 5209 return -ENOMEM; 5210 return 0; 5211 } 5212 5213 static void bnxt_set_dflt_rss_indir_tbl(struct bnxt *bp) 5214 { 5215 u16 max_rings, max_entries, pad, i; 5216 5217 if (!bp->rx_nr_rings) 5218 return; 5219 5220 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 5221 max_rings = bp->rx_nr_rings - 1; 5222 else 5223 max_rings = bp->rx_nr_rings; 5224 5225 max_entries = bnxt_get_rxfh_indir_size(bp->dev); 5226 5227 for (i = 0; i < max_entries; i++) 5228 bp->rss_indir_tbl[i] = ethtool_rxfh_indir_default(i, max_rings); 5229 5230 pad = bp->rss_indir_tbl_entries - max_entries; 5231 if (pad) 5232 memset(&bp->rss_indir_tbl[i], 0, pad * sizeof(u16)); 5233 } 5234 5235 static u16 bnxt_get_max_rss_ring(struct bnxt *bp) 5236 { 5237 u16 i, tbl_size, max_ring = 0; 5238 5239 if (!bp->rss_indir_tbl) 5240 return 0; 5241 5242 tbl_size = bnxt_get_rxfh_indir_size(bp->dev); 5243 for (i = 0; i < tbl_size; i++) 5244 max_ring = max(max_ring, bp->rss_indir_tbl[i]); 5245 return max_ring; 5246 } 5247 5248 int bnxt_get_nr_rss_ctxs(struct bnxt *bp, int rx_rings) 5249 { 5250 if (bp->flags & BNXT_FLAG_CHIP_P5) 5251 return DIV_ROUND_UP(rx_rings, BNXT_RSS_TABLE_ENTRIES_P5); 5252 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 5253 return 2; 5254 return 1; 5255 } 5256 5257 static void bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic) 5258 { 5259 bool no_rss = !(vnic->flags & BNXT_VNIC_RSS_FLAG); 5260 u16 i, j; 5261 5262 /* Fill the RSS indirection table with ring group ids */ 5263 for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++) { 5264 if (!no_rss) 5265 j = bp->rss_indir_tbl[i]; 5266 vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]); 5267 } 5268 } 5269 5270 static void bnxt_fill_hw_rss_tbl_p5(struct bnxt *bp, 5271 struct bnxt_vnic_info *vnic) 5272 { 5273 __le16 *ring_tbl = vnic->rss_table; 5274 struct bnxt_rx_ring_info *rxr; 5275 u16 tbl_size, i; 5276 5277 tbl_size = bnxt_get_rxfh_indir_size(bp->dev); 5278 5279 for (i = 0; i < tbl_size; i++) { 5280 u16 ring_id, j; 5281 5282 j = bp->rss_indir_tbl[i]; 5283 rxr = &bp->rx_ring[j]; 5284 5285 ring_id = rxr->rx_ring_struct.fw_ring_id; 5286 *ring_tbl++ = cpu_to_le16(ring_id); 5287 ring_id = bnxt_cp_ring_for_rx(bp, rxr); 5288 *ring_tbl++ = cpu_to_le16(ring_id); 5289 } 5290 } 5291 5292 static void 5293 __bnxt_hwrm_vnic_set_rss(struct bnxt *bp, struct hwrm_vnic_rss_cfg_input *req, 5294 struct bnxt_vnic_info *vnic) 5295 { 5296 if (bp->flags & BNXT_FLAG_CHIP_P5) 5297 bnxt_fill_hw_rss_tbl_p5(bp, vnic); 5298 else 5299 bnxt_fill_hw_rss_tbl(bp, vnic); 5300 5301 if (bp->rss_hash_delta) { 5302 req->hash_type = cpu_to_le32(bp->rss_hash_delta); 5303 if (bp->rss_hash_cfg & bp->rss_hash_delta) 5304 req->flags |= VNIC_RSS_CFG_REQ_FLAGS_HASH_TYPE_INCLUDE; 5305 else 5306 req->flags |= VNIC_RSS_CFG_REQ_FLAGS_HASH_TYPE_EXCLUDE; 5307 } else { 5308 req->hash_type = cpu_to_le32(bp->rss_hash_cfg); 5309 } 5310 req->hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT; 5311 req->ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr); 5312 req->hash_key_tbl_addr = cpu_to_le64(vnic->rss_hash_key_dma_addr); 5313 } 5314 5315 static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss) 5316 { 5317 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5318 struct hwrm_vnic_rss_cfg_input *req; 5319 int rc; 5320 5321 if ((bp->flags & BNXT_FLAG_CHIP_P5) || 5322 vnic->fw_rss_cos_lb_ctx[0] == INVALID_HW_RING_ID) 5323 return 0; 5324 5325 rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_CFG); 5326 if (rc) 5327 return rc; 5328 5329 if (set_rss) 5330 __bnxt_hwrm_vnic_set_rss(bp, req, vnic); 5331 req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]); 5332 return hwrm_req_send(bp, req); 5333 } 5334 5335 static int bnxt_hwrm_vnic_set_rss_p5(struct bnxt *bp, u16 vnic_id, bool set_rss) 5336 { 5337 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5338 struct hwrm_vnic_rss_cfg_input *req; 5339 dma_addr_t ring_tbl_map; 5340 u32 i, nr_ctxs; 5341 int rc; 5342 5343 rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_CFG); 5344 if (rc) 5345 return rc; 5346 5347 req->vnic_id = cpu_to_le16(vnic->fw_vnic_id); 5348 if (!set_rss) 5349 return hwrm_req_send(bp, req); 5350 5351 __bnxt_hwrm_vnic_set_rss(bp, req, vnic); 5352 ring_tbl_map = vnic->rss_table_dma_addr; 5353 nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings); 5354 5355 hwrm_req_hold(bp, req); 5356 for (i = 0; i < nr_ctxs; ring_tbl_map += BNXT_RSS_TABLE_SIZE_P5, i++) { 5357 req->ring_grp_tbl_addr = cpu_to_le64(ring_tbl_map); 5358 req->ring_table_pair_index = i; 5359 req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[i]); 5360 rc = hwrm_req_send(bp, req); 5361 if (rc) 5362 goto exit; 5363 } 5364 5365 exit: 5366 hwrm_req_drop(bp, req); 5367 return rc; 5368 } 5369 5370 static void bnxt_hwrm_update_rss_hash_cfg(struct bnxt *bp) 5371 { 5372 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 5373 struct hwrm_vnic_rss_qcfg_output *resp; 5374 struct hwrm_vnic_rss_qcfg_input *req; 5375 5376 if (hwrm_req_init(bp, req, HWRM_VNIC_RSS_QCFG)) 5377 return; 5378 5379 /* all contexts configured to same hash_type, zero always exists */ 5380 req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]); 5381 resp = hwrm_req_hold(bp, req); 5382 if (!hwrm_req_send(bp, req)) { 5383 bp->rss_hash_cfg = le32_to_cpu(resp->hash_type) ?: bp->rss_hash_cfg; 5384 bp->rss_hash_delta = 0; 5385 } 5386 hwrm_req_drop(bp, req); 5387 } 5388 5389 static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id) 5390 { 5391 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5392 struct hwrm_vnic_plcmodes_cfg_input *req; 5393 int rc; 5394 5395 rc = hwrm_req_init(bp, req, HWRM_VNIC_PLCMODES_CFG); 5396 if (rc) 5397 return rc; 5398 5399 req->flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT); 5400 req->enables = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID); 5401 5402 if (BNXT_RX_PAGE_MODE(bp)) { 5403 req->jumbo_thresh = cpu_to_le16(bp->rx_buf_use_size); 5404 } else { 5405 req->flags |= cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 | 5406 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6); 5407 req->enables |= 5408 cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID); 5409 req->jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh); 5410 req->hds_threshold = cpu_to_le16(bp->rx_copy_thresh); 5411 } 5412 req->vnic_id = cpu_to_le32(vnic->fw_vnic_id); 5413 return hwrm_req_send(bp, req); 5414 } 5415 5416 static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id, 5417 u16 ctx_idx) 5418 { 5419 struct hwrm_vnic_rss_cos_lb_ctx_free_input *req; 5420 5421 if (hwrm_req_init(bp, req, HWRM_VNIC_RSS_COS_LB_CTX_FREE)) 5422 return; 5423 5424 req->rss_cos_lb_ctx_id = 5425 cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx]); 5426 5427 hwrm_req_send(bp, req); 5428 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID; 5429 } 5430 5431 static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp) 5432 { 5433 int i, j; 5434 5435 for (i = 0; i < bp->nr_vnics; i++) { 5436 struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; 5437 5438 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) { 5439 if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID) 5440 bnxt_hwrm_vnic_ctx_free_one(bp, i, j); 5441 } 5442 } 5443 bp->rsscos_nr_ctxs = 0; 5444 } 5445 5446 static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id, u16 ctx_idx) 5447 { 5448 struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp; 5449 struct hwrm_vnic_rss_cos_lb_ctx_alloc_input *req; 5450 int rc; 5451 5452 rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC); 5453 if (rc) 5454 return rc; 5455 5456 resp = hwrm_req_hold(bp, req); 5457 rc = hwrm_req_send(bp, req); 5458 if (!rc) 5459 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = 5460 le16_to_cpu(resp->rss_cos_lb_ctx_id); 5461 hwrm_req_drop(bp, req); 5462 5463 return rc; 5464 } 5465 5466 static u32 bnxt_get_roce_vnic_mode(struct bnxt *bp) 5467 { 5468 if (bp->flags & BNXT_FLAG_ROCE_MIRROR_CAP) 5469 return VNIC_CFG_REQ_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE; 5470 return VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE; 5471 } 5472 5473 int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id) 5474 { 5475 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5476 struct hwrm_vnic_cfg_input *req; 5477 unsigned int ring = 0, grp_idx; 5478 u16 def_vlan = 0; 5479 int rc; 5480 5481 rc = hwrm_req_init(bp, req, HWRM_VNIC_CFG); 5482 if (rc) 5483 return rc; 5484 5485 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5486 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0]; 5487 5488 req->default_rx_ring_id = 5489 cpu_to_le16(rxr->rx_ring_struct.fw_ring_id); 5490 req->default_cmpl_ring_id = 5491 cpu_to_le16(bnxt_cp_ring_for_rx(bp, rxr)); 5492 req->enables = 5493 cpu_to_le32(VNIC_CFG_REQ_ENABLES_DEFAULT_RX_RING_ID | 5494 VNIC_CFG_REQ_ENABLES_DEFAULT_CMPL_RING_ID); 5495 goto vnic_mru; 5496 } 5497 req->enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP); 5498 /* Only RSS support for now TBD: COS & LB */ 5499 if (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID) { 5500 req->rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]); 5501 req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE | 5502 VNIC_CFG_REQ_ENABLES_MRU); 5503 } else if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) { 5504 req->rss_rule = 5505 cpu_to_le16(bp->vnic_info[0].fw_rss_cos_lb_ctx[0]); 5506 req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE | 5507 VNIC_CFG_REQ_ENABLES_MRU); 5508 req->flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE); 5509 } else { 5510 req->rss_rule = cpu_to_le16(0xffff); 5511 } 5512 5513 if (BNXT_CHIP_TYPE_NITRO_A0(bp) && 5514 (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID)) { 5515 req->cos_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[1]); 5516 req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_COS_RULE); 5517 } else { 5518 req->cos_rule = cpu_to_le16(0xffff); 5519 } 5520 5521 if (vnic->flags & BNXT_VNIC_RSS_FLAG) 5522 ring = 0; 5523 else if (vnic->flags & BNXT_VNIC_RFS_FLAG) 5524 ring = vnic_id - 1; 5525 else if ((vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp)) 5526 ring = bp->rx_nr_rings - 1; 5527 5528 grp_idx = bp->rx_ring[ring].bnapi->index; 5529 req->dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id); 5530 req->lb_rule = cpu_to_le16(0xffff); 5531 vnic_mru: 5532 req->mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + VLAN_HLEN); 5533 5534 req->vnic_id = cpu_to_le16(vnic->fw_vnic_id); 5535 #ifdef CONFIG_BNXT_SRIOV 5536 if (BNXT_VF(bp)) 5537 def_vlan = bp->vf.vlan; 5538 #endif 5539 if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan) 5540 req->flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE); 5541 if (!vnic_id && bnxt_ulp_registered(bp->edev)) 5542 req->flags |= cpu_to_le32(bnxt_get_roce_vnic_mode(bp)); 5543 5544 return hwrm_req_send(bp, req); 5545 } 5546 5547 static void bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id) 5548 { 5549 if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) { 5550 struct hwrm_vnic_free_input *req; 5551 5552 if (hwrm_req_init(bp, req, HWRM_VNIC_FREE)) 5553 return; 5554 5555 req->vnic_id = 5556 cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id); 5557 5558 hwrm_req_send(bp, req); 5559 bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID; 5560 } 5561 } 5562 5563 static void bnxt_hwrm_vnic_free(struct bnxt *bp) 5564 { 5565 u16 i; 5566 5567 for (i = 0; i < bp->nr_vnics; i++) 5568 bnxt_hwrm_vnic_free_one(bp, i); 5569 } 5570 5571 static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id, 5572 unsigned int start_rx_ring_idx, 5573 unsigned int nr_rings) 5574 { 5575 unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings; 5576 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5577 struct hwrm_vnic_alloc_output *resp; 5578 struct hwrm_vnic_alloc_input *req; 5579 int rc; 5580 5581 rc = hwrm_req_init(bp, req, HWRM_VNIC_ALLOC); 5582 if (rc) 5583 return rc; 5584 5585 if (bp->flags & BNXT_FLAG_CHIP_P5) 5586 goto vnic_no_ring_grps; 5587 5588 /* map ring groups to this vnic */ 5589 for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) { 5590 grp_idx = bp->rx_ring[i].bnapi->index; 5591 if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) { 5592 netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n", 5593 j, nr_rings); 5594 break; 5595 } 5596 vnic->fw_grp_ids[j] = bp->grp_info[grp_idx].fw_grp_id; 5597 } 5598 5599 vnic_no_ring_grps: 5600 for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++) 5601 vnic->fw_rss_cos_lb_ctx[i] = INVALID_HW_RING_ID; 5602 if (vnic_id == 0) 5603 req->flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT); 5604 5605 resp = hwrm_req_hold(bp, req); 5606 rc = hwrm_req_send(bp, req); 5607 if (!rc) 5608 vnic->fw_vnic_id = le32_to_cpu(resp->vnic_id); 5609 hwrm_req_drop(bp, req); 5610 return rc; 5611 } 5612 5613 static int bnxt_hwrm_vnic_qcaps(struct bnxt *bp) 5614 { 5615 struct hwrm_vnic_qcaps_output *resp; 5616 struct hwrm_vnic_qcaps_input *req; 5617 int rc; 5618 5619 bp->hw_ring_stats_size = sizeof(struct ctx_hw_stats); 5620 bp->flags &= ~(BNXT_FLAG_NEW_RSS_CAP | BNXT_FLAG_ROCE_MIRROR_CAP); 5621 if (bp->hwrm_spec_code < 0x10600) 5622 return 0; 5623 5624 rc = hwrm_req_init(bp, req, HWRM_VNIC_QCAPS); 5625 if (rc) 5626 return rc; 5627 5628 resp = hwrm_req_hold(bp, req); 5629 rc = hwrm_req_send(bp, req); 5630 if (!rc) { 5631 u32 flags = le32_to_cpu(resp->flags); 5632 5633 if (!(bp->flags & BNXT_FLAG_CHIP_P5) && 5634 (flags & VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP)) 5635 bp->flags |= BNXT_FLAG_NEW_RSS_CAP; 5636 if (flags & 5637 VNIC_QCAPS_RESP_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_CAP) 5638 bp->flags |= BNXT_FLAG_ROCE_MIRROR_CAP; 5639 5640 /* Older P5 fw before EXT_HW_STATS support did not set 5641 * VLAN_STRIP_CAP properly. 5642 */ 5643 if ((flags & VNIC_QCAPS_RESP_FLAGS_VLAN_STRIP_CAP) || 5644 (BNXT_CHIP_P5_THOR(bp) && 5645 !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))) 5646 bp->fw_cap |= BNXT_FW_CAP_VLAN_RX_STRIP; 5647 if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_HASH_TYPE_DELTA_CAP) 5648 bp->fw_cap |= BNXT_FW_CAP_RSS_HASH_TYPE_DELTA; 5649 bp->max_tpa_v2 = le16_to_cpu(resp->max_aggs_supported); 5650 if (bp->max_tpa_v2) { 5651 if (BNXT_CHIP_P5_THOR(bp)) 5652 bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P5; 5653 else 5654 bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P5_SR2; 5655 } 5656 } 5657 hwrm_req_drop(bp, req); 5658 return rc; 5659 } 5660 5661 static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp) 5662 { 5663 struct hwrm_ring_grp_alloc_output *resp; 5664 struct hwrm_ring_grp_alloc_input *req; 5665 int rc; 5666 u16 i; 5667 5668 if (bp->flags & BNXT_FLAG_CHIP_P5) 5669 return 0; 5670 5671 rc = hwrm_req_init(bp, req, HWRM_RING_GRP_ALLOC); 5672 if (rc) 5673 return rc; 5674 5675 resp = hwrm_req_hold(bp, req); 5676 for (i = 0; i < bp->rx_nr_rings; i++) { 5677 unsigned int grp_idx = bp->rx_ring[i].bnapi->index; 5678 5679 req->cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id); 5680 req->rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id); 5681 req->ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id); 5682 req->sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx); 5683 5684 rc = hwrm_req_send(bp, req); 5685 5686 if (rc) 5687 break; 5688 5689 bp->grp_info[grp_idx].fw_grp_id = 5690 le32_to_cpu(resp->ring_group_id); 5691 } 5692 hwrm_req_drop(bp, req); 5693 return rc; 5694 } 5695 5696 static void bnxt_hwrm_ring_grp_free(struct bnxt *bp) 5697 { 5698 struct hwrm_ring_grp_free_input *req; 5699 u16 i; 5700 5701 if (!bp->grp_info || (bp->flags & BNXT_FLAG_CHIP_P5)) 5702 return; 5703 5704 if (hwrm_req_init(bp, req, HWRM_RING_GRP_FREE)) 5705 return; 5706 5707 hwrm_req_hold(bp, req); 5708 for (i = 0; i < bp->cp_nr_rings; i++) { 5709 if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID) 5710 continue; 5711 req->ring_group_id = 5712 cpu_to_le32(bp->grp_info[i].fw_grp_id); 5713 5714 hwrm_req_send(bp, req); 5715 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID; 5716 } 5717 hwrm_req_drop(bp, req); 5718 } 5719 5720 static int hwrm_ring_alloc_send_msg(struct bnxt *bp, 5721 struct bnxt_ring_struct *ring, 5722 u32 ring_type, u32 map_index) 5723 { 5724 struct hwrm_ring_alloc_output *resp; 5725 struct hwrm_ring_alloc_input *req; 5726 struct bnxt_ring_mem_info *rmem = &ring->ring_mem; 5727 struct bnxt_ring_grp_info *grp_info; 5728 int rc, err = 0; 5729 u16 ring_id; 5730 5731 rc = hwrm_req_init(bp, req, HWRM_RING_ALLOC); 5732 if (rc) 5733 goto exit; 5734 5735 req->enables = 0; 5736 if (rmem->nr_pages > 1) { 5737 req->page_tbl_addr = cpu_to_le64(rmem->pg_tbl_map); 5738 /* Page size is in log2 units */ 5739 req->page_size = BNXT_PAGE_SHIFT; 5740 req->page_tbl_depth = 1; 5741 } else { 5742 req->page_tbl_addr = cpu_to_le64(rmem->dma_arr[0]); 5743 } 5744 req->fbo = 0; 5745 /* Association of ring index with doorbell index and MSIX number */ 5746 req->logical_id = cpu_to_le16(map_index); 5747 5748 switch (ring_type) { 5749 case HWRM_RING_ALLOC_TX: { 5750 struct bnxt_tx_ring_info *txr; 5751 5752 txr = container_of(ring, struct bnxt_tx_ring_info, 5753 tx_ring_struct); 5754 req->ring_type = RING_ALLOC_REQ_RING_TYPE_TX; 5755 /* Association of transmit ring with completion ring */ 5756 grp_info = &bp->grp_info[ring->grp_idx]; 5757 req->cmpl_ring_id = cpu_to_le16(bnxt_cp_ring_for_tx(bp, txr)); 5758 req->length = cpu_to_le32(bp->tx_ring_mask + 1); 5759 req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx); 5760 req->queue_id = cpu_to_le16(ring->queue_id); 5761 break; 5762 } 5763 case HWRM_RING_ALLOC_RX: 5764 req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX; 5765 req->length = cpu_to_le32(bp->rx_ring_mask + 1); 5766 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5767 u16 flags = 0; 5768 5769 /* Association of rx ring with stats context */ 5770 grp_info = &bp->grp_info[ring->grp_idx]; 5771 req->rx_buf_size = cpu_to_le16(bp->rx_buf_use_size); 5772 req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx); 5773 req->enables |= cpu_to_le32( 5774 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID); 5775 if (NET_IP_ALIGN == 2) 5776 flags = RING_ALLOC_REQ_FLAGS_RX_SOP_PAD; 5777 req->flags = cpu_to_le16(flags); 5778 } 5779 break; 5780 case HWRM_RING_ALLOC_AGG: 5781 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5782 req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX_AGG; 5783 /* Association of agg ring with rx ring */ 5784 grp_info = &bp->grp_info[ring->grp_idx]; 5785 req->rx_ring_id = cpu_to_le16(grp_info->rx_fw_ring_id); 5786 req->rx_buf_size = cpu_to_le16(BNXT_RX_PAGE_SIZE); 5787 req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx); 5788 req->enables |= cpu_to_le32( 5789 RING_ALLOC_REQ_ENABLES_RX_RING_ID_VALID | 5790 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID); 5791 } else { 5792 req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX; 5793 } 5794 req->length = cpu_to_le32(bp->rx_agg_ring_mask + 1); 5795 break; 5796 case HWRM_RING_ALLOC_CMPL: 5797 req->ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL; 5798 req->length = cpu_to_le32(bp->cp_ring_mask + 1); 5799 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5800 /* Association of cp ring with nq */ 5801 grp_info = &bp->grp_info[map_index]; 5802 req->nq_ring_id = cpu_to_le16(grp_info->cp_fw_ring_id); 5803 req->cq_handle = cpu_to_le64(ring->handle); 5804 req->enables |= cpu_to_le32( 5805 RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID); 5806 } else if (bp->flags & BNXT_FLAG_USING_MSIX) { 5807 req->int_mode = RING_ALLOC_REQ_INT_MODE_MSIX; 5808 } 5809 break; 5810 case HWRM_RING_ALLOC_NQ: 5811 req->ring_type = RING_ALLOC_REQ_RING_TYPE_NQ; 5812 req->length = cpu_to_le32(bp->cp_ring_mask + 1); 5813 if (bp->flags & BNXT_FLAG_USING_MSIX) 5814 req->int_mode = RING_ALLOC_REQ_INT_MODE_MSIX; 5815 break; 5816 default: 5817 netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n", 5818 ring_type); 5819 return -1; 5820 } 5821 5822 resp = hwrm_req_hold(bp, req); 5823 rc = hwrm_req_send(bp, req); 5824 err = le16_to_cpu(resp->error_code); 5825 ring_id = le16_to_cpu(resp->ring_id); 5826 hwrm_req_drop(bp, req); 5827 5828 exit: 5829 if (rc || err) { 5830 netdev_err(bp->dev, "hwrm_ring_alloc type %d failed. rc:%x err:%x\n", 5831 ring_type, rc, err); 5832 return -EIO; 5833 } 5834 ring->fw_ring_id = ring_id; 5835 return rc; 5836 } 5837 5838 static int bnxt_hwrm_set_async_event_cr(struct bnxt *bp, int idx) 5839 { 5840 int rc; 5841 5842 if (BNXT_PF(bp)) { 5843 struct hwrm_func_cfg_input *req; 5844 5845 rc = hwrm_req_init(bp, req, HWRM_FUNC_CFG); 5846 if (rc) 5847 return rc; 5848 5849 req->fid = cpu_to_le16(0xffff); 5850 req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR); 5851 req->async_event_cr = cpu_to_le16(idx); 5852 return hwrm_req_send(bp, req); 5853 } else { 5854 struct hwrm_func_vf_cfg_input *req; 5855 5856 rc = hwrm_req_init(bp, req, HWRM_FUNC_VF_CFG); 5857 if (rc) 5858 return rc; 5859 5860 req->enables = 5861 cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR); 5862 req->async_event_cr = cpu_to_le16(idx); 5863 return hwrm_req_send(bp, req); 5864 } 5865 } 5866 5867 static void bnxt_set_db(struct bnxt *bp, struct bnxt_db_info *db, u32 ring_type, 5868 u32 map_idx, u32 xid) 5869 { 5870 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5871 if (BNXT_PF(bp)) 5872 db->doorbell = bp->bar1 + DB_PF_OFFSET_P5; 5873 else 5874 db->doorbell = bp->bar1 + DB_VF_OFFSET_P5; 5875 switch (ring_type) { 5876 case HWRM_RING_ALLOC_TX: 5877 db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SQ; 5878 break; 5879 case HWRM_RING_ALLOC_RX: 5880 case HWRM_RING_ALLOC_AGG: 5881 db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SRQ; 5882 break; 5883 case HWRM_RING_ALLOC_CMPL: 5884 db->db_key64 = DBR_PATH_L2; 5885 break; 5886 case HWRM_RING_ALLOC_NQ: 5887 db->db_key64 = DBR_PATH_L2; 5888 break; 5889 } 5890 db->db_key64 |= (u64)xid << DBR_XID_SFT; 5891 } else { 5892 db->doorbell = bp->bar1 + map_idx * 0x80; 5893 switch (ring_type) { 5894 case HWRM_RING_ALLOC_TX: 5895 db->db_key32 = DB_KEY_TX; 5896 break; 5897 case HWRM_RING_ALLOC_RX: 5898 case HWRM_RING_ALLOC_AGG: 5899 db->db_key32 = DB_KEY_RX; 5900 break; 5901 case HWRM_RING_ALLOC_CMPL: 5902 db->db_key32 = DB_KEY_CP; 5903 break; 5904 } 5905 } 5906 } 5907 5908 static int bnxt_hwrm_ring_alloc(struct bnxt *bp) 5909 { 5910 bool agg_rings = !!(bp->flags & BNXT_FLAG_AGG_RINGS); 5911 int i, rc = 0; 5912 u32 type; 5913 5914 if (bp->flags & BNXT_FLAG_CHIP_P5) 5915 type = HWRM_RING_ALLOC_NQ; 5916 else 5917 type = HWRM_RING_ALLOC_CMPL; 5918 for (i = 0; i < bp->cp_nr_rings; i++) { 5919 struct bnxt_napi *bnapi = bp->bnapi[i]; 5920 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 5921 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 5922 u32 map_idx = ring->map_idx; 5923 unsigned int vector; 5924 5925 vector = bp->irq_tbl[map_idx].vector; 5926 disable_irq_nosync(vector); 5927 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 5928 if (rc) { 5929 enable_irq(vector); 5930 goto err_out; 5931 } 5932 bnxt_set_db(bp, &cpr->cp_db, type, map_idx, ring->fw_ring_id); 5933 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons); 5934 enable_irq(vector); 5935 bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id; 5936 5937 if (!i) { 5938 rc = bnxt_hwrm_set_async_event_cr(bp, ring->fw_ring_id); 5939 if (rc) 5940 netdev_warn(bp->dev, "Failed to set async event completion ring.\n"); 5941 } 5942 } 5943 5944 type = HWRM_RING_ALLOC_TX; 5945 for (i = 0; i < bp->tx_nr_rings; i++) { 5946 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 5947 struct bnxt_ring_struct *ring; 5948 u32 map_idx; 5949 5950 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5951 struct bnxt_napi *bnapi = txr->bnapi; 5952 struct bnxt_cp_ring_info *cpr, *cpr2; 5953 u32 type2 = HWRM_RING_ALLOC_CMPL; 5954 5955 cpr = &bnapi->cp_ring; 5956 cpr2 = cpr->cp_ring_arr[BNXT_TX_HDL]; 5957 ring = &cpr2->cp_ring_struct; 5958 ring->handle = BNXT_TX_HDL; 5959 map_idx = bnapi->index; 5960 rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx); 5961 if (rc) 5962 goto err_out; 5963 bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx, 5964 ring->fw_ring_id); 5965 bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons); 5966 } 5967 ring = &txr->tx_ring_struct; 5968 map_idx = i; 5969 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 5970 if (rc) 5971 goto err_out; 5972 bnxt_set_db(bp, &txr->tx_db, type, map_idx, ring->fw_ring_id); 5973 } 5974 5975 type = HWRM_RING_ALLOC_RX; 5976 for (i = 0; i < bp->rx_nr_rings; i++) { 5977 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 5978 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct; 5979 struct bnxt_napi *bnapi = rxr->bnapi; 5980 u32 map_idx = bnapi->index; 5981 5982 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 5983 if (rc) 5984 goto err_out; 5985 bnxt_set_db(bp, &rxr->rx_db, type, map_idx, ring->fw_ring_id); 5986 /* If we have agg rings, post agg buffers first. */ 5987 if (!agg_rings) 5988 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 5989 bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id; 5990 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5991 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 5992 u32 type2 = HWRM_RING_ALLOC_CMPL; 5993 struct bnxt_cp_ring_info *cpr2; 5994 5995 cpr2 = cpr->cp_ring_arr[BNXT_RX_HDL]; 5996 ring = &cpr2->cp_ring_struct; 5997 ring->handle = BNXT_RX_HDL; 5998 rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx); 5999 if (rc) 6000 goto err_out; 6001 bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx, 6002 ring->fw_ring_id); 6003 bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons); 6004 } 6005 } 6006 6007 if (agg_rings) { 6008 type = HWRM_RING_ALLOC_AGG; 6009 for (i = 0; i < bp->rx_nr_rings; i++) { 6010 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 6011 struct bnxt_ring_struct *ring = 6012 &rxr->rx_agg_ring_struct; 6013 u32 grp_idx = ring->grp_idx; 6014 u32 map_idx = grp_idx + bp->rx_nr_rings; 6015 6016 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 6017 if (rc) 6018 goto err_out; 6019 6020 bnxt_set_db(bp, &rxr->rx_agg_db, type, map_idx, 6021 ring->fw_ring_id); 6022 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); 6023 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 6024 bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id; 6025 } 6026 } 6027 err_out: 6028 return rc; 6029 } 6030 6031 static int hwrm_ring_free_send_msg(struct bnxt *bp, 6032 struct bnxt_ring_struct *ring, 6033 u32 ring_type, int cmpl_ring_id) 6034 { 6035 struct hwrm_ring_free_output *resp; 6036 struct hwrm_ring_free_input *req; 6037 u16 error_code = 0; 6038 int rc; 6039 6040 if (BNXT_NO_FW_ACCESS(bp)) 6041 return 0; 6042 6043 rc = hwrm_req_init(bp, req, HWRM_RING_FREE); 6044 if (rc) 6045 goto exit; 6046 6047 req->cmpl_ring = cpu_to_le16(cmpl_ring_id); 6048 req->ring_type = ring_type; 6049 req->ring_id = cpu_to_le16(ring->fw_ring_id); 6050 6051 resp = hwrm_req_hold(bp, req); 6052 rc = hwrm_req_send(bp, req); 6053 error_code = le16_to_cpu(resp->error_code); 6054 hwrm_req_drop(bp, req); 6055 exit: 6056 if (rc || error_code) { 6057 netdev_err(bp->dev, "hwrm_ring_free type %d failed. rc:%x err:%x\n", 6058 ring_type, rc, error_code); 6059 return -EIO; 6060 } 6061 return 0; 6062 } 6063 6064 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path) 6065 { 6066 u32 type; 6067 int i; 6068 6069 if (!bp->bnapi) 6070 return; 6071 6072 for (i = 0; i < bp->tx_nr_rings; i++) { 6073 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 6074 struct bnxt_ring_struct *ring = &txr->tx_ring_struct; 6075 6076 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 6077 u32 cmpl_ring_id = bnxt_cp_ring_for_tx(bp, txr); 6078 6079 hwrm_ring_free_send_msg(bp, ring, 6080 RING_FREE_REQ_RING_TYPE_TX, 6081 close_path ? cmpl_ring_id : 6082 INVALID_HW_RING_ID); 6083 ring->fw_ring_id = INVALID_HW_RING_ID; 6084 } 6085 } 6086 6087 for (i = 0; i < bp->rx_nr_rings; i++) { 6088 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 6089 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct; 6090 u32 grp_idx = rxr->bnapi->index; 6091 6092 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 6093 u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr); 6094 6095 hwrm_ring_free_send_msg(bp, ring, 6096 RING_FREE_REQ_RING_TYPE_RX, 6097 close_path ? cmpl_ring_id : 6098 INVALID_HW_RING_ID); 6099 ring->fw_ring_id = INVALID_HW_RING_ID; 6100 bp->grp_info[grp_idx].rx_fw_ring_id = 6101 INVALID_HW_RING_ID; 6102 } 6103 } 6104 6105 if (bp->flags & BNXT_FLAG_CHIP_P5) 6106 type = RING_FREE_REQ_RING_TYPE_RX_AGG; 6107 else 6108 type = RING_FREE_REQ_RING_TYPE_RX; 6109 for (i = 0; i < bp->rx_nr_rings; i++) { 6110 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 6111 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct; 6112 u32 grp_idx = rxr->bnapi->index; 6113 6114 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 6115 u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr); 6116 6117 hwrm_ring_free_send_msg(bp, ring, type, 6118 close_path ? cmpl_ring_id : 6119 INVALID_HW_RING_ID); 6120 ring->fw_ring_id = INVALID_HW_RING_ID; 6121 bp->grp_info[grp_idx].agg_fw_ring_id = 6122 INVALID_HW_RING_ID; 6123 } 6124 } 6125 6126 /* The completion rings are about to be freed. After that the 6127 * IRQ doorbell will not work anymore. So we need to disable 6128 * IRQ here. 6129 */ 6130 bnxt_disable_int_sync(bp); 6131 6132 if (bp->flags & BNXT_FLAG_CHIP_P5) 6133 type = RING_FREE_REQ_RING_TYPE_NQ; 6134 else 6135 type = RING_FREE_REQ_RING_TYPE_L2_CMPL; 6136 for (i = 0; i < bp->cp_nr_rings; i++) { 6137 struct bnxt_napi *bnapi = bp->bnapi[i]; 6138 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 6139 struct bnxt_ring_struct *ring; 6140 int j; 6141 6142 for (j = 0; j < 2; j++) { 6143 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j]; 6144 6145 if (cpr2) { 6146 ring = &cpr2->cp_ring_struct; 6147 if (ring->fw_ring_id == INVALID_HW_RING_ID) 6148 continue; 6149 hwrm_ring_free_send_msg(bp, ring, 6150 RING_FREE_REQ_RING_TYPE_L2_CMPL, 6151 INVALID_HW_RING_ID); 6152 ring->fw_ring_id = INVALID_HW_RING_ID; 6153 } 6154 } 6155 ring = &cpr->cp_ring_struct; 6156 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 6157 hwrm_ring_free_send_msg(bp, ring, type, 6158 INVALID_HW_RING_ID); 6159 ring->fw_ring_id = INVALID_HW_RING_ID; 6160 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID; 6161 } 6162 } 6163 } 6164 6165 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max, 6166 bool shared); 6167 6168 static int bnxt_hwrm_get_rings(struct bnxt *bp) 6169 { 6170 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 6171 struct hwrm_func_qcfg_output *resp; 6172 struct hwrm_func_qcfg_input *req; 6173 int rc; 6174 6175 if (bp->hwrm_spec_code < 0x10601) 6176 return 0; 6177 6178 rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG); 6179 if (rc) 6180 return rc; 6181 6182 req->fid = cpu_to_le16(0xffff); 6183 resp = hwrm_req_hold(bp, req); 6184 rc = hwrm_req_send(bp, req); 6185 if (rc) { 6186 hwrm_req_drop(bp, req); 6187 return rc; 6188 } 6189 6190 hw_resc->resv_tx_rings = le16_to_cpu(resp->alloc_tx_rings); 6191 if (BNXT_NEW_RM(bp)) { 6192 u16 cp, stats; 6193 6194 hw_resc->resv_rx_rings = le16_to_cpu(resp->alloc_rx_rings); 6195 hw_resc->resv_hw_ring_grps = 6196 le32_to_cpu(resp->alloc_hw_ring_grps); 6197 hw_resc->resv_vnics = le16_to_cpu(resp->alloc_vnics); 6198 cp = le16_to_cpu(resp->alloc_cmpl_rings); 6199 stats = le16_to_cpu(resp->alloc_stat_ctx); 6200 hw_resc->resv_irqs = cp; 6201 if (bp->flags & BNXT_FLAG_CHIP_P5) { 6202 int rx = hw_resc->resv_rx_rings; 6203 int tx = hw_resc->resv_tx_rings; 6204 6205 if (bp->flags & BNXT_FLAG_AGG_RINGS) 6206 rx >>= 1; 6207 if (cp < (rx + tx)) { 6208 bnxt_trim_rings(bp, &rx, &tx, cp, false); 6209 if (bp->flags & BNXT_FLAG_AGG_RINGS) 6210 rx <<= 1; 6211 hw_resc->resv_rx_rings = rx; 6212 hw_resc->resv_tx_rings = tx; 6213 } 6214 hw_resc->resv_irqs = le16_to_cpu(resp->alloc_msix); 6215 hw_resc->resv_hw_ring_grps = rx; 6216 } 6217 hw_resc->resv_cp_rings = cp; 6218 hw_resc->resv_stat_ctxs = stats; 6219 } 6220 hwrm_req_drop(bp, req); 6221 return 0; 6222 } 6223 6224 int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings) 6225 { 6226 struct hwrm_func_qcfg_output *resp; 6227 struct hwrm_func_qcfg_input *req; 6228 int rc; 6229 6230 if (bp->hwrm_spec_code < 0x10601) 6231 return 0; 6232 6233 rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG); 6234 if (rc) 6235 return rc; 6236 6237 req->fid = cpu_to_le16(fid); 6238 resp = hwrm_req_hold(bp, req); 6239 rc = hwrm_req_send(bp, req); 6240 if (!rc) 6241 *tx_rings = le16_to_cpu(resp->alloc_tx_rings); 6242 6243 hwrm_req_drop(bp, req); 6244 return rc; 6245 } 6246 6247 static bool bnxt_rfs_supported(struct bnxt *bp); 6248 6249 static struct hwrm_func_cfg_input * 6250 __bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6251 int ring_grps, int cp_rings, int stats, int vnics) 6252 { 6253 struct hwrm_func_cfg_input *req; 6254 u32 enables = 0; 6255 6256 if (hwrm_req_init(bp, req, HWRM_FUNC_CFG)) 6257 return NULL; 6258 6259 req->fid = cpu_to_le16(0xffff); 6260 enables |= tx_rings ? FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS : 0; 6261 req->num_tx_rings = cpu_to_le16(tx_rings); 6262 if (BNXT_NEW_RM(bp)) { 6263 enables |= rx_rings ? FUNC_CFG_REQ_ENABLES_NUM_RX_RINGS : 0; 6264 enables |= stats ? FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0; 6265 if (bp->flags & BNXT_FLAG_CHIP_P5) { 6266 enables |= cp_rings ? FUNC_CFG_REQ_ENABLES_NUM_MSIX : 0; 6267 enables |= tx_rings + ring_grps ? 6268 FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0; 6269 enables |= rx_rings ? 6270 FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0; 6271 } else { 6272 enables |= cp_rings ? 6273 FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0; 6274 enables |= ring_grps ? 6275 FUNC_CFG_REQ_ENABLES_NUM_HW_RING_GRPS | 6276 FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0; 6277 } 6278 enables |= vnics ? FUNC_CFG_REQ_ENABLES_NUM_VNICS : 0; 6279 6280 req->num_rx_rings = cpu_to_le16(rx_rings); 6281 if (bp->flags & BNXT_FLAG_CHIP_P5) { 6282 req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps); 6283 req->num_msix = cpu_to_le16(cp_rings); 6284 req->num_rsscos_ctxs = 6285 cpu_to_le16(DIV_ROUND_UP(ring_grps, 64)); 6286 } else { 6287 req->num_cmpl_rings = cpu_to_le16(cp_rings); 6288 req->num_hw_ring_grps = cpu_to_le16(ring_grps); 6289 req->num_rsscos_ctxs = cpu_to_le16(1); 6290 if (!(bp->flags & BNXT_FLAG_NEW_RSS_CAP) && 6291 bnxt_rfs_supported(bp)) 6292 req->num_rsscos_ctxs = 6293 cpu_to_le16(ring_grps + 1); 6294 } 6295 req->num_stat_ctxs = cpu_to_le16(stats); 6296 req->num_vnics = cpu_to_le16(vnics); 6297 } 6298 req->enables = cpu_to_le32(enables); 6299 return req; 6300 } 6301 6302 static struct hwrm_func_vf_cfg_input * 6303 __bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6304 int ring_grps, int cp_rings, int stats, int vnics) 6305 { 6306 struct hwrm_func_vf_cfg_input *req; 6307 u32 enables = 0; 6308 6309 if (hwrm_req_init(bp, req, HWRM_FUNC_VF_CFG)) 6310 return NULL; 6311 6312 enables |= tx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_TX_RINGS : 0; 6313 enables |= rx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_RX_RINGS | 6314 FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0; 6315 enables |= stats ? FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0; 6316 if (bp->flags & BNXT_FLAG_CHIP_P5) { 6317 enables |= tx_rings + ring_grps ? 6318 FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0; 6319 } else { 6320 enables |= cp_rings ? 6321 FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0; 6322 enables |= ring_grps ? 6323 FUNC_VF_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0; 6324 } 6325 enables |= vnics ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0; 6326 enables |= FUNC_VF_CFG_REQ_ENABLES_NUM_L2_CTXS; 6327 6328 req->num_l2_ctxs = cpu_to_le16(BNXT_VF_MAX_L2_CTX); 6329 req->num_tx_rings = cpu_to_le16(tx_rings); 6330 req->num_rx_rings = cpu_to_le16(rx_rings); 6331 if (bp->flags & BNXT_FLAG_CHIP_P5) { 6332 req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps); 6333 req->num_rsscos_ctxs = cpu_to_le16(DIV_ROUND_UP(ring_grps, 64)); 6334 } else { 6335 req->num_cmpl_rings = cpu_to_le16(cp_rings); 6336 req->num_hw_ring_grps = cpu_to_le16(ring_grps); 6337 req->num_rsscos_ctxs = cpu_to_le16(BNXT_VF_MAX_RSS_CTX); 6338 } 6339 req->num_stat_ctxs = cpu_to_le16(stats); 6340 req->num_vnics = cpu_to_le16(vnics); 6341 6342 req->enables = cpu_to_le32(enables); 6343 return req; 6344 } 6345 6346 static int 6347 bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6348 int ring_grps, int cp_rings, int stats, int vnics) 6349 { 6350 struct hwrm_func_cfg_input *req; 6351 int rc; 6352 6353 req = __bnxt_hwrm_reserve_pf_rings(bp, tx_rings, rx_rings, ring_grps, 6354 cp_rings, stats, vnics); 6355 if (!req) 6356 return -ENOMEM; 6357 6358 if (!req->enables) { 6359 hwrm_req_drop(bp, req); 6360 return 0; 6361 } 6362 6363 rc = hwrm_req_send(bp, req); 6364 if (rc) 6365 return rc; 6366 6367 if (bp->hwrm_spec_code < 0x10601) 6368 bp->hw_resc.resv_tx_rings = tx_rings; 6369 6370 return bnxt_hwrm_get_rings(bp); 6371 } 6372 6373 static int 6374 bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6375 int ring_grps, int cp_rings, int stats, int vnics) 6376 { 6377 struct hwrm_func_vf_cfg_input *req; 6378 int rc; 6379 6380 if (!BNXT_NEW_RM(bp)) { 6381 bp->hw_resc.resv_tx_rings = tx_rings; 6382 return 0; 6383 } 6384 6385 req = __bnxt_hwrm_reserve_vf_rings(bp, tx_rings, rx_rings, ring_grps, 6386 cp_rings, stats, vnics); 6387 if (!req) 6388 return -ENOMEM; 6389 6390 rc = hwrm_req_send(bp, req); 6391 if (rc) 6392 return rc; 6393 6394 return bnxt_hwrm_get_rings(bp); 6395 } 6396 6397 static int bnxt_hwrm_reserve_rings(struct bnxt *bp, int tx, int rx, int grp, 6398 int cp, int stat, int vnic) 6399 { 6400 if (BNXT_PF(bp)) 6401 return bnxt_hwrm_reserve_pf_rings(bp, tx, rx, grp, cp, stat, 6402 vnic); 6403 else 6404 return bnxt_hwrm_reserve_vf_rings(bp, tx, rx, grp, cp, stat, 6405 vnic); 6406 } 6407 6408 int bnxt_nq_rings_in_use(struct bnxt *bp) 6409 { 6410 int cp = bp->cp_nr_rings; 6411 int ulp_msix, ulp_base; 6412 6413 ulp_msix = bnxt_get_ulp_msix_num(bp); 6414 if (ulp_msix) { 6415 ulp_base = bnxt_get_ulp_msix_base(bp); 6416 cp += ulp_msix; 6417 if ((ulp_base + ulp_msix) > cp) 6418 cp = ulp_base + ulp_msix; 6419 } 6420 return cp; 6421 } 6422 6423 static int bnxt_cp_rings_in_use(struct bnxt *bp) 6424 { 6425 int cp; 6426 6427 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 6428 return bnxt_nq_rings_in_use(bp); 6429 6430 cp = bp->tx_nr_rings + bp->rx_nr_rings; 6431 return cp; 6432 } 6433 6434 static int bnxt_get_func_stat_ctxs(struct bnxt *bp) 6435 { 6436 int ulp_stat = bnxt_get_ulp_stat_ctxs(bp); 6437 int cp = bp->cp_nr_rings; 6438 6439 if (!ulp_stat) 6440 return cp; 6441 6442 if (bnxt_nq_rings_in_use(bp) > cp + bnxt_get_ulp_msix_num(bp)) 6443 return bnxt_get_ulp_msix_base(bp) + ulp_stat; 6444 6445 return cp + ulp_stat; 6446 } 6447 6448 /* Check if a default RSS map needs to be setup. This function is only 6449 * used on older firmware that does not require reserving RX rings. 6450 */ 6451 static void bnxt_check_rss_tbl_no_rmgr(struct bnxt *bp) 6452 { 6453 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 6454 6455 /* The RSS map is valid for RX rings set to resv_rx_rings */ 6456 if (hw_resc->resv_rx_rings != bp->rx_nr_rings) { 6457 hw_resc->resv_rx_rings = bp->rx_nr_rings; 6458 if (!netif_is_rxfh_configured(bp->dev)) 6459 bnxt_set_dflt_rss_indir_tbl(bp); 6460 } 6461 } 6462 6463 static bool bnxt_need_reserve_rings(struct bnxt *bp) 6464 { 6465 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 6466 int cp = bnxt_cp_rings_in_use(bp); 6467 int nq = bnxt_nq_rings_in_use(bp); 6468 int rx = bp->rx_nr_rings, stat; 6469 int vnic = 1, grp = rx; 6470 6471 if (hw_resc->resv_tx_rings != bp->tx_nr_rings && 6472 bp->hwrm_spec_code >= 0x10601) 6473 return true; 6474 6475 /* Old firmware does not need RX ring reservations but we still 6476 * need to setup a default RSS map when needed. With new firmware 6477 * we go through RX ring reservations first and then set up the 6478 * RSS map for the successfully reserved RX rings when needed. 6479 */ 6480 if (!BNXT_NEW_RM(bp)) { 6481 bnxt_check_rss_tbl_no_rmgr(bp); 6482 return false; 6483 } 6484 if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5)) 6485 vnic = rx + 1; 6486 if (bp->flags & BNXT_FLAG_AGG_RINGS) 6487 rx <<= 1; 6488 stat = bnxt_get_func_stat_ctxs(bp); 6489 if (hw_resc->resv_rx_rings != rx || hw_resc->resv_cp_rings != cp || 6490 hw_resc->resv_vnics != vnic || hw_resc->resv_stat_ctxs != stat || 6491 (hw_resc->resv_hw_ring_grps != grp && 6492 !(bp->flags & BNXT_FLAG_CHIP_P5))) 6493 return true; 6494 if ((bp->flags & BNXT_FLAG_CHIP_P5) && BNXT_PF(bp) && 6495 hw_resc->resv_irqs != nq) 6496 return true; 6497 return false; 6498 } 6499 6500 static int __bnxt_reserve_rings(struct bnxt *bp) 6501 { 6502 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 6503 int cp = bnxt_nq_rings_in_use(bp); 6504 int tx = bp->tx_nr_rings; 6505 int rx = bp->rx_nr_rings; 6506 int grp, rx_rings, rc; 6507 int vnic = 1, stat; 6508 bool sh = false; 6509 6510 if (!bnxt_need_reserve_rings(bp)) 6511 return 0; 6512 6513 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 6514 sh = true; 6515 if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5)) 6516 vnic = rx + 1; 6517 if (bp->flags & BNXT_FLAG_AGG_RINGS) 6518 rx <<= 1; 6519 grp = bp->rx_nr_rings; 6520 stat = bnxt_get_func_stat_ctxs(bp); 6521 6522 rc = bnxt_hwrm_reserve_rings(bp, tx, rx, grp, cp, stat, vnic); 6523 if (rc) 6524 return rc; 6525 6526 tx = hw_resc->resv_tx_rings; 6527 if (BNXT_NEW_RM(bp)) { 6528 rx = hw_resc->resv_rx_rings; 6529 cp = hw_resc->resv_irqs; 6530 grp = hw_resc->resv_hw_ring_grps; 6531 vnic = hw_resc->resv_vnics; 6532 stat = hw_resc->resv_stat_ctxs; 6533 } 6534 6535 rx_rings = rx; 6536 if (bp->flags & BNXT_FLAG_AGG_RINGS) { 6537 if (rx >= 2) { 6538 rx_rings = rx >> 1; 6539 } else { 6540 if (netif_running(bp->dev)) 6541 return -ENOMEM; 6542 6543 bp->flags &= ~BNXT_FLAG_AGG_RINGS; 6544 bp->flags |= BNXT_FLAG_NO_AGG_RINGS; 6545 bp->dev->hw_features &= ~NETIF_F_LRO; 6546 bp->dev->features &= ~NETIF_F_LRO; 6547 bnxt_set_ring_params(bp); 6548 } 6549 } 6550 rx_rings = min_t(int, rx_rings, grp); 6551 cp = min_t(int, cp, bp->cp_nr_rings); 6552 if (stat > bnxt_get_ulp_stat_ctxs(bp)) 6553 stat -= bnxt_get_ulp_stat_ctxs(bp); 6554 cp = min_t(int, cp, stat); 6555 rc = bnxt_trim_rings(bp, &rx_rings, &tx, cp, sh); 6556 if (bp->flags & BNXT_FLAG_AGG_RINGS) 6557 rx = rx_rings << 1; 6558 cp = sh ? max_t(int, tx, rx_rings) : tx + rx_rings; 6559 bp->tx_nr_rings = tx; 6560 6561 /* If we cannot reserve all the RX rings, reset the RSS map only 6562 * if absolutely necessary 6563 */ 6564 if (rx_rings != bp->rx_nr_rings) { 6565 netdev_warn(bp->dev, "Able to reserve only %d out of %d requested RX rings\n", 6566 rx_rings, bp->rx_nr_rings); 6567 if (netif_is_rxfh_configured(bp->dev) && 6568 (bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings) != 6569 bnxt_get_nr_rss_ctxs(bp, rx_rings) || 6570 bnxt_get_max_rss_ring(bp) >= rx_rings)) { 6571 netdev_warn(bp->dev, "RSS table entries reverting to default\n"); 6572 bp->dev->priv_flags &= ~IFF_RXFH_CONFIGURED; 6573 } 6574 } 6575 bp->rx_nr_rings = rx_rings; 6576 bp->cp_nr_rings = cp; 6577 6578 if (!tx || !rx || !cp || !grp || !vnic || !stat) 6579 return -ENOMEM; 6580 6581 if (!netif_is_rxfh_configured(bp->dev)) 6582 bnxt_set_dflt_rss_indir_tbl(bp); 6583 6584 return rc; 6585 } 6586 6587 static int bnxt_hwrm_check_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6588 int ring_grps, int cp_rings, int stats, 6589 int vnics) 6590 { 6591 struct hwrm_func_vf_cfg_input *req; 6592 u32 flags; 6593 6594 if (!BNXT_NEW_RM(bp)) 6595 return 0; 6596 6597 req = __bnxt_hwrm_reserve_vf_rings(bp, tx_rings, rx_rings, ring_grps, 6598 cp_rings, stats, vnics); 6599 flags = FUNC_VF_CFG_REQ_FLAGS_TX_ASSETS_TEST | 6600 FUNC_VF_CFG_REQ_FLAGS_RX_ASSETS_TEST | 6601 FUNC_VF_CFG_REQ_FLAGS_CMPL_ASSETS_TEST | 6602 FUNC_VF_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST | 6603 FUNC_VF_CFG_REQ_FLAGS_VNIC_ASSETS_TEST | 6604 FUNC_VF_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST; 6605 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 6606 flags |= FUNC_VF_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST; 6607 6608 req->flags = cpu_to_le32(flags); 6609 return hwrm_req_send_silent(bp, req); 6610 } 6611 6612 static int bnxt_hwrm_check_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6613 int ring_grps, int cp_rings, int stats, 6614 int vnics) 6615 { 6616 struct hwrm_func_cfg_input *req; 6617 u32 flags; 6618 6619 req = __bnxt_hwrm_reserve_pf_rings(bp, tx_rings, rx_rings, ring_grps, 6620 cp_rings, stats, vnics); 6621 flags = FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST; 6622 if (BNXT_NEW_RM(bp)) { 6623 flags |= FUNC_CFG_REQ_FLAGS_RX_ASSETS_TEST | 6624 FUNC_CFG_REQ_FLAGS_CMPL_ASSETS_TEST | 6625 FUNC_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST | 6626 FUNC_CFG_REQ_FLAGS_VNIC_ASSETS_TEST; 6627 if (bp->flags & BNXT_FLAG_CHIP_P5) 6628 flags |= FUNC_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST | 6629 FUNC_CFG_REQ_FLAGS_NQ_ASSETS_TEST; 6630 else 6631 flags |= FUNC_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST; 6632 } 6633 6634 req->flags = cpu_to_le32(flags); 6635 return hwrm_req_send_silent(bp, req); 6636 } 6637 6638 static int bnxt_hwrm_check_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6639 int ring_grps, int cp_rings, int stats, 6640 int vnics) 6641 { 6642 if (bp->hwrm_spec_code < 0x10801) 6643 return 0; 6644 6645 if (BNXT_PF(bp)) 6646 return bnxt_hwrm_check_pf_rings(bp, tx_rings, rx_rings, 6647 ring_grps, cp_rings, stats, 6648 vnics); 6649 6650 return bnxt_hwrm_check_vf_rings(bp, tx_rings, rx_rings, ring_grps, 6651 cp_rings, stats, vnics); 6652 } 6653 6654 static void bnxt_hwrm_coal_params_qcaps(struct bnxt *bp) 6655 { 6656 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 6657 struct hwrm_ring_aggint_qcaps_output *resp; 6658 struct hwrm_ring_aggint_qcaps_input *req; 6659 int rc; 6660 6661 coal_cap->cmpl_params = BNXT_LEGACY_COAL_CMPL_PARAMS; 6662 coal_cap->num_cmpl_dma_aggr_max = 63; 6663 coal_cap->num_cmpl_dma_aggr_during_int_max = 63; 6664 coal_cap->cmpl_aggr_dma_tmr_max = 65535; 6665 coal_cap->cmpl_aggr_dma_tmr_during_int_max = 65535; 6666 coal_cap->int_lat_tmr_min_max = 65535; 6667 coal_cap->int_lat_tmr_max_max = 65535; 6668 coal_cap->num_cmpl_aggr_int_max = 65535; 6669 coal_cap->timer_units = 80; 6670 6671 if (bp->hwrm_spec_code < 0x10902) 6672 return; 6673 6674 if (hwrm_req_init(bp, req, HWRM_RING_AGGINT_QCAPS)) 6675 return; 6676 6677 resp = hwrm_req_hold(bp, req); 6678 rc = hwrm_req_send_silent(bp, req); 6679 if (!rc) { 6680 coal_cap->cmpl_params = le32_to_cpu(resp->cmpl_params); 6681 coal_cap->nq_params = le32_to_cpu(resp->nq_params); 6682 coal_cap->num_cmpl_dma_aggr_max = 6683 le16_to_cpu(resp->num_cmpl_dma_aggr_max); 6684 coal_cap->num_cmpl_dma_aggr_during_int_max = 6685 le16_to_cpu(resp->num_cmpl_dma_aggr_during_int_max); 6686 coal_cap->cmpl_aggr_dma_tmr_max = 6687 le16_to_cpu(resp->cmpl_aggr_dma_tmr_max); 6688 coal_cap->cmpl_aggr_dma_tmr_during_int_max = 6689 le16_to_cpu(resp->cmpl_aggr_dma_tmr_during_int_max); 6690 coal_cap->int_lat_tmr_min_max = 6691 le16_to_cpu(resp->int_lat_tmr_min_max); 6692 coal_cap->int_lat_tmr_max_max = 6693 le16_to_cpu(resp->int_lat_tmr_max_max); 6694 coal_cap->num_cmpl_aggr_int_max = 6695 le16_to_cpu(resp->num_cmpl_aggr_int_max); 6696 coal_cap->timer_units = le16_to_cpu(resp->timer_units); 6697 } 6698 hwrm_req_drop(bp, req); 6699 } 6700 6701 static u16 bnxt_usec_to_coal_tmr(struct bnxt *bp, u16 usec) 6702 { 6703 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 6704 6705 return usec * 1000 / coal_cap->timer_units; 6706 } 6707 6708 static void bnxt_hwrm_set_coal_params(struct bnxt *bp, 6709 struct bnxt_coal *hw_coal, 6710 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req) 6711 { 6712 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 6713 u16 val, tmr, max, flags = hw_coal->flags; 6714 u32 cmpl_params = coal_cap->cmpl_params; 6715 6716 max = hw_coal->bufs_per_record * 128; 6717 if (hw_coal->budget) 6718 max = hw_coal->bufs_per_record * hw_coal->budget; 6719 max = min_t(u16, max, coal_cap->num_cmpl_aggr_int_max); 6720 6721 val = clamp_t(u16, hw_coal->coal_bufs, 1, max); 6722 req->num_cmpl_aggr_int = cpu_to_le16(val); 6723 6724 val = min_t(u16, val, coal_cap->num_cmpl_dma_aggr_max); 6725 req->num_cmpl_dma_aggr = cpu_to_le16(val); 6726 6727 val = clamp_t(u16, hw_coal->coal_bufs_irq, 1, 6728 coal_cap->num_cmpl_dma_aggr_during_int_max); 6729 req->num_cmpl_dma_aggr_during_int = cpu_to_le16(val); 6730 6731 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks); 6732 tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_max_max); 6733 req->int_lat_tmr_max = cpu_to_le16(tmr); 6734 6735 /* min timer set to 1/2 of interrupt timer */ 6736 if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_INT_LAT_TMR_MIN) { 6737 val = tmr / 2; 6738 val = clamp_t(u16, val, 1, coal_cap->int_lat_tmr_min_max); 6739 req->int_lat_tmr_min = cpu_to_le16(val); 6740 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE); 6741 } 6742 6743 /* buf timer set to 1/4 of interrupt timer */ 6744 val = clamp_t(u16, tmr / 4, 1, coal_cap->cmpl_aggr_dma_tmr_max); 6745 req->cmpl_aggr_dma_tmr = cpu_to_le16(val); 6746 6747 if (cmpl_params & 6748 RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_NUM_CMPL_DMA_AGGR_DURING_INT) { 6749 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks_irq); 6750 val = clamp_t(u16, tmr, 1, 6751 coal_cap->cmpl_aggr_dma_tmr_during_int_max); 6752 req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(val); 6753 req->enables |= 6754 cpu_to_le16(BNXT_COAL_CMPL_AGGR_TMR_DURING_INT_ENABLE); 6755 } 6756 6757 if ((cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_RING_IDLE) && 6758 hw_coal->idle_thresh && hw_coal->coal_ticks < hw_coal->idle_thresh) 6759 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE; 6760 req->flags = cpu_to_le16(flags); 6761 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_ENABLES); 6762 } 6763 6764 static int __bnxt_hwrm_set_coal_nq(struct bnxt *bp, struct bnxt_napi *bnapi, 6765 struct bnxt_coal *hw_coal) 6766 { 6767 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req; 6768 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 6769 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 6770 u32 nq_params = coal_cap->nq_params; 6771 u16 tmr; 6772 int rc; 6773 6774 if (!(nq_params & RING_AGGINT_QCAPS_RESP_NQ_PARAMS_INT_LAT_TMR_MIN)) 6775 return 0; 6776 6777 rc = hwrm_req_init(bp, req, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS); 6778 if (rc) 6779 return rc; 6780 6781 req->ring_id = cpu_to_le16(cpr->cp_ring_struct.fw_ring_id); 6782 req->flags = 6783 cpu_to_le16(RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_IS_NQ); 6784 6785 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks) / 2; 6786 tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_min_max); 6787 req->int_lat_tmr_min = cpu_to_le16(tmr); 6788 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE); 6789 return hwrm_req_send(bp, req); 6790 } 6791 6792 int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi) 6793 { 6794 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req_rx; 6795 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 6796 struct bnxt_coal coal; 6797 int rc; 6798 6799 /* Tick values in micro seconds. 6800 * 1 coal_buf x bufs_per_record = 1 completion record. 6801 */ 6802 memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal)); 6803 6804 coal.coal_ticks = cpr->rx_ring_coal.coal_ticks; 6805 coal.coal_bufs = cpr->rx_ring_coal.coal_bufs; 6806 6807 if (!bnapi->rx_ring) 6808 return -ENODEV; 6809 6810 rc = hwrm_req_init(bp, req_rx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS); 6811 if (rc) 6812 return rc; 6813 6814 bnxt_hwrm_set_coal_params(bp, &coal, req_rx); 6815 6816 req_rx->ring_id = cpu_to_le16(bnxt_cp_ring_for_rx(bp, bnapi->rx_ring)); 6817 6818 return hwrm_req_send(bp, req_rx); 6819 } 6820 6821 int bnxt_hwrm_set_coal(struct bnxt *bp) 6822 { 6823 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req_rx, *req_tx, 6824 *req; 6825 int i, rc; 6826 6827 rc = hwrm_req_init(bp, req_rx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS); 6828 if (rc) 6829 return rc; 6830 6831 rc = hwrm_req_init(bp, req_tx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS); 6832 if (rc) { 6833 hwrm_req_drop(bp, req_rx); 6834 return rc; 6835 } 6836 6837 bnxt_hwrm_set_coal_params(bp, &bp->rx_coal, req_rx); 6838 bnxt_hwrm_set_coal_params(bp, &bp->tx_coal, req_tx); 6839 6840 hwrm_req_hold(bp, req_rx); 6841 hwrm_req_hold(bp, req_tx); 6842 for (i = 0; i < bp->cp_nr_rings; i++) { 6843 struct bnxt_napi *bnapi = bp->bnapi[i]; 6844 struct bnxt_coal *hw_coal; 6845 u16 ring_id; 6846 6847 req = req_rx; 6848 if (!bnapi->rx_ring) { 6849 ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring); 6850 req = req_tx; 6851 } else { 6852 ring_id = bnxt_cp_ring_for_rx(bp, bnapi->rx_ring); 6853 } 6854 req->ring_id = cpu_to_le16(ring_id); 6855 6856 rc = hwrm_req_send(bp, req); 6857 if (rc) 6858 break; 6859 6860 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 6861 continue; 6862 6863 if (bnapi->rx_ring && bnapi->tx_ring) { 6864 req = req_tx; 6865 ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring); 6866 req->ring_id = cpu_to_le16(ring_id); 6867 rc = hwrm_req_send(bp, req); 6868 if (rc) 6869 break; 6870 } 6871 if (bnapi->rx_ring) 6872 hw_coal = &bp->rx_coal; 6873 else 6874 hw_coal = &bp->tx_coal; 6875 __bnxt_hwrm_set_coal_nq(bp, bnapi, hw_coal); 6876 } 6877 hwrm_req_drop(bp, req_rx); 6878 hwrm_req_drop(bp, req_tx); 6879 return rc; 6880 } 6881 6882 static void bnxt_hwrm_stat_ctx_free(struct bnxt *bp) 6883 { 6884 struct hwrm_stat_ctx_clr_stats_input *req0 = NULL; 6885 struct hwrm_stat_ctx_free_input *req; 6886 int i; 6887 6888 if (!bp->bnapi) 6889 return; 6890 6891 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 6892 return; 6893 6894 if (hwrm_req_init(bp, req, HWRM_STAT_CTX_FREE)) 6895 return; 6896 if (BNXT_FW_MAJ(bp) <= 20) { 6897 if (hwrm_req_init(bp, req0, HWRM_STAT_CTX_CLR_STATS)) { 6898 hwrm_req_drop(bp, req); 6899 return; 6900 } 6901 hwrm_req_hold(bp, req0); 6902 } 6903 hwrm_req_hold(bp, req); 6904 for (i = 0; i < bp->cp_nr_rings; i++) { 6905 struct bnxt_napi *bnapi = bp->bnapi[i]; 6906 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 6907 6908 if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) { 6909 req->stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id); 6910 if (req0) { 6911 req0->stat_ctx_id = req->stat_ctx_id; 6912 hwrm_req_send(bp, req0); 6913 } 6914 hwrm_req_send(bp, req); 6915 6916 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID; 6917 } 6918 } 6919 hwrm_req_drop(bp, req); 6920 if (req0) 6921 hwrm_req_drop(bp, req0); 6922 } 6923 6924 static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp) 6925 { 6926 struct hwrm_stat_ctx_alloc_output *resp; 6927 struct hwrm_stat_ctx_alloc_input *req; 6928 int rc, i; 6929 6930 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 6931 return 0; 6932 6933 rc = hwrm_req_init(bp, req, HWRM_STAT_CTX_ALLOC); 6934 if (rc) 6935 return rc; 6936 6937 req->stats_dma_length = cpu_to_le16(bp->hw_ring_stats_size); 6938 req->update_period_ms = cpu_to_le32(bp->stats_coal_ticks / 1000); 6939 6940 resp = hwrm_req_hold(bp, req); 6941 for (i = 0; i < bp->cp_nr_rings; i++) { 6942 struct bnxt_napi *bnapi = bp->bnapi[i]; 6943 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 6944 6945 req->stats_dma_addr = cpu_to_le64(cpr->stats.hw_stats_map); 6946 6947 rc = hwrm_req_send(bp, req); 6948 if (rc) 6949 break; 6950 6951 cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id); 6952 6953 bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id; 6954 } 6955 hwrm_req_drop(bp, req); 6956 return rc; 6957 } 6958 6959 static int bnxt_hwrm_func_qcfg(struct bnxt *bp) 6960 { 6961 struct hwrm_func_qcfg_output *resp; 6962 struct hwrm_func_qcfg_input *req; 6963 u32 min_db_offset = 0; 6964 u16 flags; 6965 int rc; 6966 6967 rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG); 6968 if (rc) 6969 return rc; 6970 6971 req->fid = cpu_to_le16(0xffff); 6972 resp = hwrm_req_hold(bp, req); 6973 rc = hwrm_req_send(bp, req); 6974 if (rc) 6975 goto func_qcfg_exit; 6976 6977 #ifdef CONFIG_BNXT_SRIOV 6978 if (BNXT_VF(bp)) { 6979 struct bnxt_vf_info *vf = &bp->vf; 6980 6981 vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK; 6982 } else { 6983 bp->pf.registered_vfs = le16_to_cpu(resp->registered_vfs); 6984 } 6985 #endif 6986 flags = le16_to_cpu(resp->flags); 6987 if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED | 6988 FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED)) { 6989 bp->fw_cap |= BNXT_FW_CAP_LLDP_AGENT; 6990 if (flags & FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED) 6991 bp->fw_cap |= BNXT_FW_CAP_DCBX_AGENT; 6992 } 6993 if (BNXT_PF(bp) && (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST)) 6994 bp->flags |= BNXT_FLAG_MULTI_HOST; 6995 6996 if (flags & FUNC_QCFG_RESP_FLAGS_RING_MONITOR_ENABLED) 6997 bp->fw_cap |= BNXT_FW_CAP_RING_MONITOR; 6998 6999 switch (resp->port_partition_type) { 7000 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0: 7001 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5: 7002 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0: 7003 bp->port_partition_type = resp->port_partition_type; 7004 break; 7005 } 7006 if (bp->hwrm_spec_code < 0x10707 || 7007 resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEB) 7008 bp->br_mode = BRIDGE_MODE_VEB; 7009 else if (resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEPA) 7010 bp->br_mode = BRIDGE_MODE_VEPA; 7011 else 7012 bp->br_mode = BRIDGE_MODE_UNDEF; 7013 7014 bp->max_mtu = le16_to_cpu(resp->max_mtu_configured); 7015 if (!bp->max_mtu) 7016 bp->max_mtu = BNXT_MAX_MTU; 7017 7018 if (bp->db_size) 7019 goto func_qcfg_exit; 7020 7021 if (bp->flags & BNXT_FLAG_CHIP_P5) { 7022 if (BNXT_PF(bp)) 7023 min_db_offset = DB_PF_OFFSET_P5; 7024 else 7025 min_db_offset = DB_VF_OFFSET_P5; 7026 } 7027 bp->db_size = PAGE_ALIGN(le16_to_cpu(resp->l2_doorbell_bar_size_kb) * 7028 1024); 7029 if (!bp->db_size || bp->db_size > pci_resource_len(bp->pdev, 2) || 7030 bp->db_size <= min_db_offset) 7031 bp->db_size = pci_resource_len(bp->pdev, 2); 7032 7033 func_qcfg_exit: 7034 hwrm_req_drop(bp, req); 7035 return rc; 7036 } 7037 7038 static void bnxt_init_ctx_initializer(struct bnxt_ctx_mem_info *ctx, 7039 struct hwrm_func_backing_store_qcaps_output *resp) 7040 { 7041 struct bnxt_mem_init *mem_init; 7042 u16 init_mask; 7043 u8 init_val; 7044 u8 *offset; 7045 int i; 7046 7047 init_val = resp->ctx_kind_initializer; 7048 init_mask = le16_to_cpu(resp->ctx_init_mask); 7049 offset = &resp->qp_init_offset; 7050 mem_init = &ctx->mem_init[BNXT_CTX_MEM_INIT_QP]; 7051 for (i = 0; i < BNXT_CTX_MEM_INIT_MAX; i++, mem_init++, offset++) { 7052 mem_init->init_val = init_val; 7053 mem_init->offset = BNXT_MEM_INVALID_OFFSET; 7054 if (!init_mask) 7055 continue; 7056 if (i == BNXT_CTX_MEM_INIT_STAT) 7057 offset = &resp->stat_init_offset; 7058 if (init_mask & (1 << i)) 7059 mem_init->offset = *offset * 4; 7060 else 7061 mem_init->init_val = 0; 7062 } 7063 ctx->mem_init[BNXT_CTX_MEM_INIT_QP].size = ctx->qp_entry_size; 7064 ctx->mem_init[BNXT_CTX_MEM_INIT_SRQ].size = ctx->srq_entry_size; 7065 ctx->mem_init[BNXT_CTX_MEM_INIT_CQ].size = ctx->cq_entry_size; 7066 ctx->mem_init[BNXT_CTX_MEM_INIT_VNIC].size = ctx->vnic_entry_size; 7067 ctx->mem_init[BNXT_CTX_MEM_INIT_STAT].size = ctx->stat_entry_size; 7068 ctx->mem_init[BNXT_CTX_MEM_INIT_MRAV].size = ctx->mrav_entry_size; 7069 } 7070 7071 static int bnxt_hwrm_func_backing_store_qcaps(struct bnxt *bp) 7072 { 7073 struct hwrm_func_backing_store_qcaps_output *resp; 7074 struct hwrm_func_backing_store_qcaps_input *req; 7075 int rc; 7076 7077 if (bp->hwrm_spec_code < 0x10902 || BNXT_VF(bp) || bp->ctx) 7078 return 0; 7079 7080 rc = hwrm_req_init(bp, req, HWRM_FUNC_BACKING_STORE_QCAPS); 7081 if (rc) 7082 return rc; 7083 7084 resp = hwrm_req_hold(bp, req); 7085 rc = hwrm_req_send_silent(bp, req); 7086 if (!rc) { 7087 struct bnxt_ctx_pg_info *ctx_pg; 7088 struct bnxt_ctx_mem_info *ctx; 7089 int i, tqm_rings; 7090 7091 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 7092 if (!ctx) { 7093 rc = -ENOMEM; 7094 goto ctx_err; 7095 } 7096 ctx->qp_max_entries = le32_to_cpu(resp->qp_max_entries); 7097 ctx->qp_min_qp1_entries = le16_to_cpu(resp->qp_min_qp1_entries); 7098 ctx->qp_max_l2_entries = le16_to_cpu(resp->qp_max_l2_entries); 7099 ctx->qp_entry_size = le16_to_cpu(resp->qp_entry_size); 7100 ctx->srq_max_l2_entries = le16_to_cpu(resp->srq_max_l2_entries); 7101 ctx->srq_max_entries = le32_to_cpu(resp->srq_max_entries); 7102 ctx->srq_entry_size = le16_to_cpu(resp->srq_entry_size); 7103 ctx->cq_max_l2_entries = le16_to_cpu(resp->cq_max_l2_entries); 7104 ctx->cq_max_entries = le32_to_cpu(resp->cq_max_entries); 7105 ctx->cq_entry_size = le16_to_cpu(resp->cq_entry_size); 7106 ctx->vnic_max_vnic_entries = 7107 le16_to_cpu(resp->vnic_max_vnic_entries); 7108 ctx->vnic_max_ring_table_entries = 7109 le16_to_cpu(resp->vnic_max_ring_table_entries); 7110 ctx->vnic_entry_size = le16_to_cpu(resp->vnic_entry_size); 7111 ctx->stat_max_entries = le32_to_cpu(resp->stat_max_entries); 7112 ctx->stat_entry_size = le16_to_cpu(resp->stat_entry_size); 7113 ctx->tqm_entry_size = le16_to_cpu(resp->tqm_entry_size); 7114 ctx->tqm_min_entries_per_ring = 7115 le32_to_cpu(resp->tqm_min_entries_per_ring); 7116 ctx->tqm_max_entries_per_ring = 7117 le32_to_cpu(resp->tqm_max_entries_per_ring); 7118 ctx->tqm_entries_multiple = resp->tqm_entries_multiple; 7119 if (!ctx->tqm_entries_multiple) 7120 ctx->tqm_entries_multiple = 1; 7121 ctx->mrav_max_entries = le32_to_cpu(resp->mrav_max_entries); 7122 ctx->mrav_entry_size = le16_to_cpu(resp->mrav_entry_size); 7123 ctx->mrav_num_entries_units = 7124 le16_to_cpu(resp->mrav_num_entries_units); 7125 ctx->tim_entry_size = le16_to_cpu(resp->tim_entry_size); 7126 ctx->tim_max_entries = le32_to_cpu(resp->tim_max_entries); 7127 7128 bnxt_init_ctx_initializer(ctx, resp); 7129 7130 ctx->tqm_fp_rings_count = resp->tqm_fp_rings_count; 7131 if (!ctx->tqm_fp_rings_count) 7132 ctx->tqm_fp_rings_count = bp->max_q; 7133 else if (ctx->tqm_fp_rings_count > BNXT_MAX_TQM_FP_RINGS) 7134 ctx->tqm_fp_rings_count = BNXT_MAX_TQM_FP_RINGS; 7135 7136 tqm_rings = ctx->tqm_fp_rings_count + BNXT_MAX_TQM_SP_RINGS; 7137 ctx_pg = kcalloc(tqm_rings, sizeof(*ctx_pg), GFP_KERNEL); 7138 if (!ctx_pg) { 7139 kfree(ctx); 7140 rc = -ENOMEM; 7141 goto ctx_err; 7142 } 7143 for (i = 0; i < tqm_rings; i++, ctx_pg++) 7144 ctx->tqm_mem[i] = ctx_pg; 7145 bp->ctx = ctx; 7146 } else { 7147 rc = 0; 7148 } 7149 ctx_err: 7150 hwrm_req_drop(bp, req); 7151 return rc; 7152 } 7153 7154 static void bnxt_hwrm_set_pg_attr(struct bnxt_ring_mem_info *rmem, u8 *pg_attr, 7155 __le64 *pg_dir) 7156 { 7157 if (!rmem->nr_pages) 7158 return; 7159 7160 BNXT_SET_CTX_PAGE_ATTR(*pg_attr); 7161 if (rmem->depth >= 1) { 7162 if (rmem->depth == 2) 7163 *pg_attr |= 2; 7164 else 7165 *pg_attr |= 1; 7166 *pg_dir = cpu_to_le64(rmem->pg_tbl_map); 7167 } else { 7168 *pg_dir = cpu_to_le64(rmem->dma_arr[0]); 7169 } 7170 } 7171 7172 #define FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES \ 7173 (FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP | \ 7174 FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ | \ 7175 FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ | \ 7176 FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC | \ 7177 FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) 7178 7179 static int bnxt_hwrm_func_backing_store_cfg(struct bnxt *bp, u32 enables) 7180 { 7181 struct hwrm_func_backing_store_cfg_input *req; 7182 struct bnxt_ctx_mem_info *ctx = bp->ctx; 7183 struct bnxt_ctx_pg_info *ctx_pg; 7184 void **__req = (void **)&req; 7185 u32 req_len = sizeof(*req); 7186 __le32 *num_entries; 7187 __le64 *pg_dir; 7188 u32 flags = 0; 7189 u8 *pg_attr; 7190 u32 ena; 7191 int rc; 7192 int i; 7193 7194 if (!ctx) 7195 return 0; 7196 7197 if (req_len > bp->hwrm_max_ext_req_len) 7198 req_len = BNXT_BACKING_STORE_CFG_LEGACY_LEN; 7199 rc = __hwrm_req_init(bp, __req, HWRM_FUNC_BACKING_STORE_CFG, req_len); 7200 if (rc) 7201 return rc; 7202 7203 req->enables = cpu_to_le32(enables); 7204 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP) { 7205 ctx_pg = &ctx->qp_mem; 7206 req->qp_num_entries = cpu_to_le32(ctx_pg->entries); 7207 req->qp_num_qp1_entries = cpu_to_le16(ctx->qp_min_qp1_entries); 7208 req->qp_num_l2_entries = cpu_to_le16(ctx->qp_max_l2_entries); 7209 req->qp_entry_size = cpu_to_le16(ctx->qp_entry_size); 7210 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 7211 &req->qpc_pg_size_qpc_lvl, 7212 &req->qpc_page_dir); 7213 } 7214 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ) { 7215 ctx_pg = &ctx->srq_mem; 7216 req->srq_num_entries = cpu_to_le32(ctx_pg->entries); 7217 req->srq_num_l2_entries = cpu_to_le16(ctx->srq_max_l2_entries); 7218 req->srq_entry_size = cpu_to_le16(ctx->srq_entry_size); 7219 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 7220 &req->srq_pg_size_srq_lvl, 7221 &req->srq_page_dir); 7222 } 7223 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ) { 7224 ctx_pg = &ctx->cq_mem; 7225 req->cq_num_entries = cpu_to_le32(ctx_pg->entries); 7226 req->cq_num_l2_entries = cpu_to_le16(ctx->cq_max_l2_entries); 7227 req->cq_entry_size = cpu_to_le16(ctx->cq_entry_size); 7228 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 7229 &req->cq_pg_size_cq_lvl, 7230 &req->cq_page_dir); 7231 } 7232 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC) { 7233 ctx_pg = &ctx->vnic_mem; 7234 req->vnic_num_vnic_entries = 7235 cpu_to_le16(ctx->vnic_max_vnic_entries); 7236 req->vnic_num_ring_table_entries = 7237 cpu_to_le16(ctx->vnic_max_ring_table_entries); 7238 req->vnic_entry_size = cpu_to_le16(ctx->vnic_entry_size); 7239 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 7240 &req->vnic_pg_size_vnic_lvl, 7241 &req->vnic_page_dir); 7242 } 7243 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) { 7244 ctx_pg = &ctx->stat_mem; 7245 req->stat_num_entries = cpu_to_le32(ctx->stat_max_entries); 7246 req->stat_entry_size = cpu_to_le16(ctx->stat_entry_size); 7247 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 7248 &req->stat_pg_size_stat_lvl, 7249 &req->stat_page_dir); 7250 } 7251 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV) { 7252 ctx_pg = &ctx->mrav_mem; 7253 req->mrav_num_entries = cpu_to_le32(ctx_pg->entries); 7254 if (ctx->mrav_num_entries_units) 7255 flags |= 7256 FUNC_BACKING_STORE_CFG_REQ_FLAGS_MRAV_RESERVATION_SPLIT; 7257 req->mrav_entry_size = cpu_to_le16(ctx->mrav_entry_size); 7258 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 7259 &req->mrav_pg_size_mrav_lvl, 7260 &req->mrav_page_dir); 7261 } 7262 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM) { 7263 ctx_pg = &ctx->tim_mem; 7264 req->tim_num_entries = cpu_to_le32(ctx_pg->entries); 7265 req->tim_entry_size = cpu_to_le16(ctx->tim_entry_size); 7266 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 7267 &req->tim_pg_size_tim_lvl, 7268 &req->tim_page_dir); 7269 } 7270 for (i = 0, num_entries = &req->tqm_sp_num_entries, 7271 pg_attr = &req->tqm_sp_pg_size_tqm_sp_lvl, 7272 pg_dir = &req->tqm_sp_page_dir, 7273 ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP; 7274 i < BNXT_MAX_TQM_RINGS; 7275 i++, num_entries++, pg_attr++, pg_dir++, ena <<= 1) { 7276 if (!(enables & ena)) 7277 continue; 7278 7279 req->tqm_entry_size = cpu_to_le16(ctx->tqm_entry_size); 7280 ctx_pg = ctx->tqm_mem[i]; 7281 *num_entries = cpu_to_le32(ctx_pg->entries); 7282 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, pg_attr, pg_dir); 7283 } 7284 req->flags = cpu_to_le32(flags); 7285 return hwrm_req_send(bp, req); 7286 } 7287 7288 static int bnxt_alloc_ctx_mem_blk(struct bnxt *bp, 7289 struct bnxt_ctx_pg_info *ctx_pg) 7290 { 7291 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem; 7292 7293 rmem->page_size = BNXT_PAGE_SIZE; 7294 rmem->pg_arr = ctx_pg->ctx_pg_arr; 7295 rmem->dma_arr = ctx_pg->ctx_dma_arr; 7296 rmem->flags = BNXT_RMEM_VALID_PTE_FLAG; 7297 if (rmem->depth >= 1) 7298 rmem->flags |= BNXT_RMEM_USE_FULL_PAGE_FLAG; 7299 return bnxt_alloc_ring(bp, rmem); 7300 } 7301 7302 static int bnxt_alloc_ctx_pg_tbls(struct bnxt *bp, 7303 struct bnxt_ctx_pg_info *ctx_pg, u32 mem_size, 7304 u8 depth, struct bnxt_mem_init *mem_init) 7305 { 7306 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem; 7307 int rc; 7308 7309 if (!mem_size) 7310 return -EINVAL; 7311 7312 ctx_pg->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE); 7313 if (ctx_pg->nr_pages > MAX_CTX_TOTAL_PAGES) { 7314 ctx_pg->nr_pages = 0; 7315 return -EINVAL; 7316 } 7317 if (ctx_pg->nr_pages > MAX_CTX_PAGES || depth > 1) { 7318 int nr_tbls, i; 7319 7320 rmem->depth = 2; 7321 ctx_pg->ctx_pg_tbl = kcalloc(MAX_CTX_PAGES, sizeof(ctx_pg), 7322 GFP_KERNEL); 7323 if (!ctx_pg->ctx_pg_tbl) 7324 return -ENOMEM; 7325 nr_tbls = DIV_ROUND_UP(ctx_pg->nr_pages, MAX_CTX_PAGES); 7326 rmem->nr_pages = nr_tbls; 7327 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg); 7328 if (rc) 7329 return rc; 7330 for (i = 0; i < nr_tbls; i++) { 7331 struct bnxt_ctx_pg_info *pg_tbl; 7332 7333 pg_tbl = kzalloc(sizeof(*pg_tbl), GFP_KERNEL); 7334 if (!pg_tbl) 7335 return -ENOMEM; 7336 ctx_pg->ctx_pg_tbl[i] = pg_tbl; 7337 rmem = &pg_tbl->ring_mem; 7338 rmem->pg_tbl = ctx_pg->ctx_pg_arr[i]; 7339 rmem->pg_tbl_map = ctx_pg->ctx_dma_arr[i]; 7340 rmem->depth = 1; 7341 rmem->nr_pages = MAX_CTX_PAGES; 7342 rmem->mem_init = mem_init; 7343 if (i == (nr_tbls - 1)) { 7344 int rem = ctx_pg->nr_pages % MAX_CTX_PAGES; 7345 7346 if (rem) 7347 rmem->nr_pages = rem; 7348 } 7349 rc = bnxt_alloc_ctx_mem_blk(bp, pg_tbl); 7350 if (rc) 7351 break; 7352 } 7353 } else { 7354 rmem->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE); 7355 if (rmem->nr_pages > 1 || depth) 7356 rmem->depth = 1; 7357 rmem->mem_init = mem_init; 7358 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg); 7359 } 7360 return rc; 7361 } 7362 7363 static void bnxt_free_ctx_pg_tbls(struct bnxt *bp, 7364 struct bnxt_ctx_pg_info *ctx_pg) 7365 { 7366 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem; 7367 7368 if (rmem->depth > 1 || ctx_pg->nr_pages > MAX_CTX_PAGES || 7369 ctx_pg->ctx_pg_tbl) { 7370 int i, nr_tbls = rmem->nr_pages; 7371 7372 for (i = 0; i < nr_tbls; i++) { 7373 struct bnxt_ctx_pg_info *pg_tbl; 7374 struct bnxt_ring_mem_info *rmem2; 7375 7376 pg_tbl = ctx_pg->ctx_pg_tbl[i]; 7377 if (!pg_tbl) 7378 continue; 7379 rmem2 = &pg_tbl->ring_mem; 7380 bnxt_free_ring(bp, rmem2); 7381 ctx_pg->ctx_pg_arr[i] = NULL; 7382 kfree(pg_tbl); 7383 ctx_pg->ctx_pg_tbl[i] = NULL; 7384 } 7385 kfree(ctx_pg->ctx_pg_tbl); 7386 ctx_pg->ctx_pg_tbl = NULL; 7387 } 7388 bnxt_free_ring(bp, rmem); 7389 ctx_pg->nr_pages = 0; 7390 } 7391 7392 void bnxt_free_ctx_mem(struct bnxt *bp) 7393 { 7394 struct bnxt_ctx_mem_info *ctx = bp->ctx; 7395 int i; 7396 7397 if (!ctx) 7398 return; 7399 7400 if (ctx->tqm_mem[0]) { 7401 for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++) 7402 bnxt_free_ctx_pg_tbls(bp, ctx->tqm_mem[i]); 7403 kfree(ctx->tqm_mem[0]); 7404 ctx->tqm_mem[0] = NULL; 7405 } 7406 7407 bnxt_free_ctx_pg_tbls(bp, &ctx->tim_mem); 7408 bnxt_free_ctx_pg_tbls(bp, &ctx->mrav_mem); 7409 bnxt_free_ctx_pg_tbls(bp, &ctx->stat_mem); 7410 bnxt_free_ctx_pg_tbls(bp, &ctx->vnic_mem); 7411 bnxt_free_ctx_pg_tbls(bp, &ctx->cq_mem); 7412 bnxt_free_ctx_pg_tbls(bp, &ctx->srq_mem); 7413 bnxt_free_ctx_pg_tbls(bp, &ctx->qp_mem); 7414 ctx->flags &= ~BNXT_CTX_FLAG_INITED; 7415 } 7416 7417 static int bnxt_alloc_ctx_mem(struct bnxt *bp) 7418 { 7419 struct bnxt_ctx_pg_info *ctx_pg; 7420 struct bnxt_ctx_mem_info *ctx; 7421 struct bnxt_mem_init *init; 7422 u32 mem_size, ena, entries; 7423 u32 entries_sp, min; 7424 u32 num_mr, num_ah; 7425 u32 extra_srqs = 0; 7426 u32 extra_qps = 0; 7427 u8 pg_lvl = 1; 7428 int i, rc; 7429 7430 rc = bnxt_hwrm_func_backing_store_qcaps(bp); 7431 if (rc) { 7432 netdev_err(bp->dev, "Failed querying context mem capability, rc = %d.\n", 7433 rc); 7434 return rc; 7435 } 7436 ctx = bp->ctx; 7437 if (!ctx || (ctx->flags & BNXT_CTX_FLAG_INITED)) 7438 return 0; 7439 7440 if ((bp->flags & BNXT_FLAG_ROCE_CAP) && !is_kdump_kernel()) { 7441 pg_lvl = 2; 7442 extra_qps = 65536; 7443 extra_srqs = 8192; 7444 } 7445 7446 ctx_pg = &ctx->qp_mem; 7447 ctx_pg->entries = ctx->qp_min_qp1_entries + ctx->qp_max_l2_entries + 7448 extra_qps; 7449 if (ctx->qp_entry_size) { 7450 mem_size = ctx->qp_entry_size * ctx_pg->entries; 7451 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_QP]; 7452 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, init); 7453 if (rc) 7454 return rc; 7455 } 7456 7457 ctx_pg = &ctx->srq_mem; 7458 ctx_pg->entries = ctx->srq_max_l2_entries + extra_srqs; 7459 if (ctx->srq_entry_size) { 7460 mem_size = ctx->srq_entry_size * ctx_pg->entries; 7461 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_SRQ]; 7462 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, init); 7463 if (rc) 7464 return rc; 7465 } 7466 7467 ctx_pg = &ctx->cq_mem; 7468 ctx_pg->entries = ctx->cq_max_l2_entries + extra_qps * 2; 7469 if (ctx->cq_entry_size) { 7470 mem_size = ctx->cq_entry_size * ctx_pg->entries; 7471 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_CQ]; 7472 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, init); 7473 if (rc) 7474 return rc; 7475 } 7476 7477 ctx_pg = &ctx->vnic_mem; 7478 ctx_pg->entries = ctx->vnic_max_vnic_entries + 7479 ctx->vnic_max_ring_table_entries; 7480 if (ctx->vnic_entry_size) { 7481 mem_size = ctx->vnic_entry_size * ctx_pg->entries; 7482 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_VNIC]; 7483 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, init); 7484 if (rc) 7485 return rc; 7486 } 7487 7488 ctx_pg = &ctx->stat_mem; 7489 ctx_pg->entries = ctx->stat_max_entries; 7490 if (ctx->stat_entry_size) { 7491 mem_size = ctx->stat_entry_size * ctx_pg->entries; 7492 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_STAT]; 7493 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, init); 7494 if (rc) 7495 return rc; 7496 } 7497 7498 ena = 0; 7499 if (!(bp->flags & BNXT_FLAG_ROCE_CAP)) 7500 goto skip_rdma; 7501 7502 ctx_pg = &ctx->mrav_mem; 7503 /* 128K extra is needed to accommodate static AH context 7504 * allocation by f/w. 7505 */ 7506 num_mr = 1024 * 256; 7507 num_ah = 1024 * 128; 7508 ctx_pg->entries = num_mr + num_ah; 7509 if (ctx->mrav_entry_size) { 7510 mem_size = ctx->mrav_entry_size * ctx_pg->entries; 7511 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_MRAV]; 7512 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 2, init); 7513 if (rc) 7514 return rc; 7515 } 7516 ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV; 7517 if (ctx->mrav_num_entries_units) 7518 ctx_pg->entries = 7519 ((num_mr / ctx->mrav_num_entries_units) << 16) | 7520 (num_ah / ctx->mrav_num_entries_units); 7521 7522 ctx_pg = &ctx->tim_mem; 7523 ctx_pg->entries = ctx->qp_mem.entries; 7524 if (ctx->tim_entry_size) { 7525 mem_size = ctx->tim_entry_size * ctx_pg->entries; 7526 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, NULL); 7527 if (rc) 7528 return rc; 7529 } 7530 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM; 7531 7532 skip_rdma: 7533 min = ctx->tqm_min_entries_per_ring; 7534 entries_sp = ctx->vnic_max_vnic_entries + ctx->qp_max_l2_entries + 7535 2 * (extra_qps + ctx->qp_min_qp1_entries) + min; 7536 entries_sp = roundup(entries_sp, ctx->tqm_entries_multiple); 7537 entries = ctx->qp_max_l2_entries + 2 * (extra_qps + ctx->qp_min_qp1_entries); 7538 entries = roundup(entries, ctx->tqm_entries_multiple); 7539 entries = clamp_t(u32, entries, min, ctx->tqm_max_entries_per_ring); 7540 for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++) { 7541 ctx_pg = ctx->tqm_mem[i]; 7542 ctx_pg->entries = i ? entries : entries_sp; 7543 if (ctx->tqm_entry_size) { 7544 mem_size = ctx->tqm_entry_size * ctx_pg->entries; 7545 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, 7546 NULL); 7547 if (rc) 7548 return rc; 7549 } 7550 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP << i; 7551 } 7552 ena |= FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES; 7553 rc = bnxt_hwrm_func_backing_store_cfg(bp, ena); 7554 if (rc) { 7555 netdev_err(bp->dev, "Failed configuring context mem, rc = %d.\n", 7556 rc); 7557 return rc; 7558 } 7559 ctx->flags |= BNXT_CTX_FLAG_INITED; 7560 return 0; 7561 } 7562 7563 int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp, bool all) 7564 { 7565 struct hwrm_func_resource_qcaps_output *resp; 7566 struct hwrm_func_resource_qcaps_input *req; 7567 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 7568 int rc; 7569 7570 rc = hwrm_req_init(bp, req, HWRM_FUNC_RESOURCE_QCAPS); 7571 if (rc) 7572 return rc; 7573 7574 req->fid = cpu_to_le16(0xffff); 7575 resp = hwrm_req_hold(bp, req); 7576 rc = hwrm_req_send_silent(bp, req); 7577 if (rc) 7578 goto hwrm_func_resc_qcaps_exit; 7579 7580 hw_resc->max_tx_sch_inputs = le16_to_cpu(resp->max_tx_scheduler_inputs); 7581 if (!all) 7582 goto hwrm_func_resc_qcaps_exit; 7583 7584 hw_resc->min_rsscos_ctxs = le16_to_cpu(resp->min_rsscos_ctx); 7585 hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx); 7586 hw_resc->min_cp_rings = le16_to_cpu(resp->min_cmpl_rings); 7587 hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings); 7588 hw_resc->min_tx_rings = le16_to_cpu(resp->min_tx_rings); 7589 hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings); 7590 hw_resc->min_rx_rings = le16_to_cpu(resp->min_rx_rings); 7591 hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings); 7592 hw_resc->min_hw_ring_grps = le16_to_cpu(resp->min_hw_ring_grps); 7593 hw_resc->max_hw_ring_grps = le16_to_cpu(resp->max_hw_ring_grps); 7594 hw_resc->min_l2_ctxs = le16_to_cpu(resp->min_l2_ctxs); 7595 hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs); 7596 hw_resc->min_vnics = le16_to_cpu(resp->min_vnics); 7597 hw_resc->max_vnics = le16_to_cpu(resp->max_vnics); 7598 hw_resc->min_stat_ctxs = le16_to_cpu(resp->min_stat_ctx); 7599 hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx); 7600 7601 if (bp->flags & BNXT_FLAG_CHIP_P5) { 7602 u16 max_msix = le16_to_cpu(resp->max_msix); 7603 7604 hw_resc->max_nqs = max_msix; 7605 hw_resc->max_hw_ring_grps = hw_resc->max_rx_rings; 7606 } 7607 7608 if (BNXT_PF(bp)) { 7609 struct bnxt_pf_info *pf = &bp->pf; 7610 7611 pf->vf_resv_strategy = 7612 le16_to_cpu(resp->vf_reservation_strategy); 7613 if (pf->vf_resv_strategy > BNXT_VF_RESV_STRATEGY_MINIMAL_STATIC) 7614 pf->vf_resv_strategy = BNXT_VF_RESV_STRATEGY_MAXIMAL; 7615 } 7616 hwrm_func_resc_qcaps_exit: 7617 hwrm_req_drop(bp, req); 7618 return rc; 7619 } 7620 7621 static int __bnxt_hwrm_ptp_qcfg(struct bnxt *bp) 7622 { 7623 struct hwrm_port_mac_ptp_qcfg_output *resp; 7624 struct hwrm_port_mac_ptp_qcfg_input *req; 7625 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; 7626 bool phc_cfg; 7627 u8 flags; 7628 int rc; 7629 7630 if (bp->hwrm_spec_code < 0x10801 || !BNXT_CHIP_P5_THOR(bp)) { 7631 rc = -ENODEV; 7632 goto no_ptp; 7633 } 7634 7635 rc = hwrm_req_init(bp, req, HWRM_PORT_MAC_PTP_QCFG); 7636 if (rc) 7637 goto no_ptp; 7638 7639 req->port_id = cpu_to_le16(bp->pf.port_id); 7640 resp = hwrm_req_hold(bp, req); 7641 rc = hwrm_req_send(bp, req); 7642 if (rc) 7643 goto exit; 7644 7645 flags = resp->flags; 7646 if (!(flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_HWRM_ACCESS)) { 7647 rc = -ENODEV; 7648 goto exit; 7649 } 7650 if (!ptp) { 7651 ptp = kzalloc(sizeof(*ptp), GFP_KERNEL); 7652 if (!ptp) { 7653 rc = -ENOMEM; 7654 goto exit; 7655 } 7656 ptp->bp = bp; 7657 bp->ptp_cfg = ptp; 7658 } 7659 if (flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_PARTIAL_DIRECT_ACCESS_REF_CLOCK) { 7660 ptp->refclk_regs[0] = le32_to_cpu(resp->ts_ref_clock_reg_lower); 7661 ptp->refclk_regs[1] = le32_to_cpu(resp->ts_ref_clock_reg_upper); 7662 } else if (bp->flags & BNXT_FLAG_CHIP_P5) { 7663 ptp->refclk_regs[0] = BNXT_TS_REG_TIMESYNC_TS0_LOWER; 7664 ptp->refclk_regs[1] = BNXT_TS_REG_TIMESYNC_TS0_UPPER; 7665 } else { 7666 rc = -ENODEV; 7667 goto exit; 7668 } 7669 phc_cfg = (flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_RTC_CONFIGURED) != 0; 7670 rc = bnxt_ptp_init(bp, phc_cfg); 7671 if (rc) 7672 netdev_warn(bp->dev, "PTP initialization failed.\n"); 7673 exit: 7674 hwrm_req_drop(bp, req); 7675 if (!rc) 7676 return 0; 7677 7678 no_ptp: 7679 bnxt_ptp_clear(bp); 7680 kfree(ptp); 7681 bp->ptp_cfg = NULL; 7682 return rc; 7683 } 7684 7685 static int __bnxt_hwrm_func_qcaps(struct bnxt *bp) 7686 { 7687 struct hwrm_func_qcaps_output *resp; 7688 struct hwrm_func_qcaps_input *req; 7689 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 7690 u32 flags, flags_ext, flags_ext2; 7691 int rc; 7692 7693 rc = hwrm_req_init(bp, req, HWRM_FUNC_QCAPS); 7694 if (rc) 7695 return rc; 7696 7697 req->fid = cpu_to_le16(0xffff); 7698 resp = hwrm_req_hold(bp, req); 7699 rc = hwrm_req_send(bp, req); 7700 if (rc) 7701 goto hwrm_func_qcaps_exit; 7702 7703 flags = le32_to_cpu(resp->flags); 7704 if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED) 7705 bp->flags |= BNXT_FLAG_ROCEV1_CAP; 7706 if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED) 7707 bp->flags |= BNXT_FLAG_ROCEV2_CAP; 7708 if (flags & FUNC_QCAPS_RESP_FLAGS_PCIE_STATS_SUPPORTED) 7709 bp->fw_cap |= BNXT_FW_CAP_PCIE_STATS_SUPPORTED; 7710 if (flags & FUNC_QCAPS_RESP_FLAGS_HOT_RESET_CAPABLE) 7711 bp->fw_cap |= BNXT_FW_CAP_HOT_RESET; 7712 if (flags & FUNC_QCAPS_RESP_FLAGS_EXT_STATS_SUPPORTED) 7713 bp->fw_cap |= BNXT_FW_CAP_EXT_STATS_SUPPORTED; 7714 if (flags & FUNC_QCAPS_RESP_FLAGS_ERROR_RECOVERY_CAPABLE) 7715 bp->fw_cap |= BNXT_FW_CAP_ERROR_RECOVERY; 7716 if (flags & FUNC_QCAPS_RESP_FLAGS_ERR_RECOVER_RELOAD) 7717 bp->fw_cap |= BNXT_FW_CAP_ERR_RECOVER_RELOAD; 7718 if (!(flags & FUNC_QCAPS_RESP_FLAGS_VLAN_ACCELERATION_TX_DISABLED)) 7719 bp->fw_cap |= BNXT_FW_CAP_VLAN_TX_INSERT; 7720 if (flags & FUNC_QCAPS_RESP_FLAGS_DBG_QCAPS_CMD_SUPPORTED) 7721 bp->fw_cap |= BNXT_FW_CAP_DBG_QCAPS; 7722 7723 flags_ext = le32_to_cpu(resp->flags_ext); 7724 if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_EXT_HW_STATS_SUPPORTED) 7725 bp->fw_cap |= BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED; 7726 if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_PTP_PPS_SUPPORTED)) 7727 bp->fw_cap |= BNXT_FW_CAP_PTP_PPS; 7728 if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_PTP_64BIT_RTC_SUPPORTED) 7729 bp->fw_cap |= BNXT_FW_CAP_PTP_RTC; 7730 if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_HOT_RESET_IF_SUPPORT)) 7731 bp->fw_cap |= BNXT_FW_CAP_HOT_RESET_IF; 7732 if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_FW_LIVEPATCH_SUPPORTED)) 7733 bp->fw_cap |= BNXT_FW_CAP_LIVEPATCH; 7734 7735 flags_ext2 = le32_to_cpu(resp->flags_ext2); 7736 if (flags_ext2 & FUNC_QCAPS_RESP_FLAGS_EXT2_RX_ALL_PKTS_TIMESTAMPS_SUPPORTED) 7737 bp->fw_cap |= BNXT_FW_CAP_RX_ALL_PKT_TS; 7738 7739 bp->tx_push_thresh = 0; 7740 if ((flags & FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED) && 7741 BNXT_FW_MAJ(bp) > 217) 7742 bp->tx_push_thresh = BNXT_TX_PUSH_THRESH; 7743 7744 hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx); 7745 hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings); 7746 hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings); 7747 hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings); 7748 hw_resc->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps); 7749 if (!hw_resc->max_hw_ring_grps) 7750 hw_resc->max_hw_ring_grps = hw_resc->max_tx_rings; 7751 hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs); 7752 hw_resc->max_vnics = le16_to_cpu(resp->max_vnics); 7753 hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx); 7754 7755 if (BNXT_PF(bp)) { 7756 struct bnxt_pf_info *pf = &bp->pf; 7757 7758 pf->fw_fid = le16_to_cpu(resp->fid); 7759 pf->port_id = le16_to_cpu(resp->port_id); 7760 memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN); 7761 pf->first_vf_id = le16_to_cpu(resp->first_vf_id); 7762 pf->max_vfs = le16_to_cpu(resp->max_vfs); 7763 pf->max_encap_records = le32_to_cpu(resp->max_encap_records); 7764 pf->max_decap_records = le32_to_cpu(resp->max_decap_records); 7765 pf->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows); 7766 pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows); 7767 pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows); 7768 pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows); 7769 bp->flags &= ~BNXT_FLAG_WOL_CAP; 7770 if (flags & FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED) 7771 bp->flags |= BNXT_FLAG_WOL_CAP; 7772 if (flags & FUNC_QCAPS_RESP_FLAGS_PTP_SUPPORTED) { 7773 __bnxt_hwrm_ptp_qcfg(bp); 7774 } else { 7775 bnxt_ptp_clear(bp); 7776 kfree(bp->ptp_cfg); 7777 bp->ptp_cfg = NULL; 7778 } 7779 } else { 7780 #ifdef CONFIG_BNXT_SRIOV 7781 struct bnxt_vf_info *vf = &bp->vf; 7782 7783 vf->fw_fid = le16_to_cpu(resp->fid); 7784 memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN); 7785 #endif 7786 } 7787 7788 hwrm_func_qcaps_exit: 7789 hwrm_req_drop(bp, req); 7790 return rc; 7791 } 7792 7793 static void bnxt_hwrm_dbg_qcaps(struct bnxt *bp) 7794 { 7795 struct hwrm_dbg_qcaps_output *resp; 7796 struct hwrm_dbg_qcaps_input *req; 7797 int rc; 7798 7799 bp->fw_dbg_cap = 0; 7800 if (!(bp->fw_cap & BNXT_FW_CAP_DBG_QCAPS)) 7801 return; 7802 7803 rc = hwrm_req_init(bp, req, HWRM_DBG_QCAPS); 7804 if (rc) 7805 return; 7806 7807 req->fid = cpu_to_le16(0xffff); 7808 resp = hwrm_req_hold(bp, req); 7809 rc = hwrm_req_send(bp, req); 7810 if (rc) 7811 goto hwrm_dbg_qcaps_exit; 7812 7813 bp->fw_dbg_cap = le32_to_cpu(resp->flags); 7814 7815 hwrm_dbg_qcaps_exit: 7816 hwrm_req_drop(bp, req); 7817 } 7818 7819 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp); 7820 7821 int bnxt_hwrm_func_qcaps(struct bnxt *bp) 7822 { 7823 int rc; 7824 7825 rc = __bnxt_hwrm_func_qcaps(bp); 7826 if (rc) 7827 return rc; 7828 7829 bnxt_hwrm_dbg_qcaps(bp); 7830 7831 rc = bnxt_hwrm_queue_qportcfg(bp); 7832 if (rc) { 7833 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %d\n", rc); 7834 return rc; 7835 } 7836 if (bp->hwrm_spec_code >= 0x10803) { 7837 rc = bnxt_alloc_ctx_mem(bp); 7838 if (rc) 7839 return rc; 7840 rc = bnxt_hwrm_func_resc_qcaps(bp, true); 7841 if (!rc) 7842 bp->fw_cap |= BNXT_FW_CAP_NEW_RM; 7843 } 7844 return 0; 7845 } 7846 7847 static int bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(struct bnxt *bp) 7848 { 7849 struct hwrm_cfa_adv_flow_mgnt_qcaps_output *resp; 7850 struct hwrm_cfa_adv_flow_mgnt_qcaps_input *req; 7851 u32 flags; 7852 int rc; 7853 7854 if (!(bp->fw_cap & BNXT_FW_CAP_CFA_ADV_FLOW)) 7855 return 0; 7856 7857 rc = hwrm_req_init(bp, req, HWRM_CFA_ADV_FLOW_MGNT_QCAPS); 7858 if (rc) 7859 return rc; 7860 7861 resp = hwrm_req_hold(bp, req); 7862 rc = hwrm_req_send(bp, req); 7863 if (rc) 7864 goto hwrm_cfa_adv_qcaps_exit; 7865 7866 flags = le32_to_cpu(resp->flags); 7867 if (flags & 7868 CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_V2_SUPPORTED) 7869 bp->fw_cap |= BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2; 7870 7871 hwrm_cfa_adv_qcaps_exit: 7872 hwrm_req_drop(bp, req); 7873 return rc; 7874 } 7875 7876 static int __bnxt_alloc_fw_health(struct bnxt *bp) 7877 { 7878 if (bp->fw_health) 7879 return 0; 7880 7881 bp->fw_health = kzalloc(sizeof(*bp->fw_health), GFP_KERNEL); 7882 if (!bp->fw_health) 7883 return -ENOMEM; 7884 7885 mutex_init(&bp->fw_health->lock); 7886 return 0; 7887 } 7888 7889 static int bnxt_alloc_fw_health(struct bnxt *bp) 7890 { 7891 int rc; 7892 7893 if (!(bp->fw_cap & BNXT_FW_CAP_HOT_RESET) && 7894 !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) 7895 return 0; 7896 7897 rc = __bnxt_alloc_fw_health(bp); 7898 if (rc) { 7899 bp->fw_cap &= ~BNXT_FW_CAP_HOT_RESET; 7900 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY; 7901 return rc; 7902 } 7903 7904 return 0; 7905 } 7906 7907 static void __bnxt_map_fw_health_reg(struct bnxt *bp, u32 reg) 7908 { 7909 writel(reg & BNXT_GRC_BASE_MASK, bp->bar0 + 7910 BNXT_GRCPF_REG_WINDOW_BASE_OUT + 7911 BNXT_FW_HEALTH_WIN_MAP_OFF); 7912 } 7913 7914 static void bnxt_inv_fw_health_reg(struct bnxt *bp) 7915 { 7916 struct bnxt_fw_health *fw_health = bp->fw_health; 7917 u32 reg_type; 7918 7919 if (!fw_health) 7920 return; 7921 7922 reg_type = BNXT_FW_HEALTH_REG_TYPE(fw_health->regs[BNXT_FW_HEALTH_REG]); 7923 if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC) 7924 fw_health->status_reliable = false; 7925 7926 reg_type = BNXT_FW_HEALTH_REG_TYPE(fw_health->regs[BNXT_FW_RESET_CNT_REG]); 7927 if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC) 7928 fw_health->resets_reliable = false; 7929 } 7930 7931 static void bnxt_try_map_fw_health_reg(struct bnxt *bp) 7932 { 7933 void __iomem *hs; 7934 u32 status_loc; 7935 u32 reg_type; 7936 u32 sig; 7937 7938 if (bp->fw_health) 7939 bp->fw_health->status_reliable = false; 7940 7941 __bnxt_map_fw_health_reg(bp, HCOMM_STATUS_STRUCT_LOC); 7942 hs = bp->bar0 + BNXT_FW_HEALTH_WIN_OFF(HCOMM_STATUS_STRUCT_LOC); 7943 7944 sig = readl(hs + offsetof(struct hcomm_status, sig_ver)); 7945 if ((sig & HCOMM_STATUS_SIGNATURE_MASK) != HCOMM_STATUS_SIGNATURE_VAL) { 7946 if (!bp->chip_num) { 7947 __bnxt_map_fw_health_reg(bp, BNXT_GRC_REG_BASE); 7948 bp->chip_num = readl(bp->bar0 + 7949 BNXT_FW_HEALTH_WIN_BASE + 7950 BNXT_GRC_REG_CHIP_NUM); 7951 } 7952 if (!BNXT_CHIP_P5(bp)) 7953 return; 7954 7955 status_loc = BNXT_GRC_REG_STATUS_P5 | 7956 BNXT_FW_HEALTH_REG_TYPE_BAR0; 7957 } else { 7958 status_loc = readl(hs + offsetof(struct hcomm_status, 7959 fw_status_loc)); 7960 } 7961 7962 if (__bnxt_alloc_fw_health(bp)) { 7963 netdev_warn(bp->dev, "no memory for firmware status checks\n"); 7964 return; 7965 } 7966 7967 bp->fw_health->regs[BNXT_FW_HEALTH_REG] = status_loc; 7968 reg_type = BNXT_FW_HEALTH_REG_TYPE(status_loc); 7969 if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC) { 7970 __bnxt_map_fw_health_reg(bp, status_loc); 7971 bp->fw_health->mapped_regs[BNXT_FW_HEALTH_REG] = 7972 BNXT_FW_HEALTH_WIN_OFF(status_loc); 7973 } 7974 7975 bp->fw_health->status_reliable = true; 7976 } 7977 7978 static int bnxt_map_fw_health_regs(struct bnxt *bp) 7979 { 7980 struct bnxt_fw_health *fw_health = bp->fw_health; 7981 u32 reg_base = 0xffffffff; 7982 int i; 7983 7984 bp->fw_health->status_reliable = false; 7985 bp->fw_health->resets_reliable = false; 7986 /* Only pre-map the monitoring GRC registers using window 3 */ 7987 for (i = 0; i < 4; i++) { 7988 u32 reg = fw_health->regs[i]; 7989 7990 if (BNXT_FW_HEALTH_REG_TYPE(reg) != BNXT_FW_HEALTH_REG_TYPE_GRC) 7991 continue; 7992 if (reg_base == 0xffffffff) 7993 reg_base = reg & BNXT_GRC_BASE_MASK; 7994 if ((reg & BNXT_GRC_BASE_MASK) != reg_base) 7995 return -ERANGE; 7996 fw_health->mapped_regs[i] = BNXT_FW_HEALTH_WIN_OFF(reg); 7997 } 7998 bp->fw_health->status_reliable = true; 7999 bp->fw_health->resets_reliable = true; 8000 if (reg_base == 0xffffffff) 8001 return 0; 8002 8003 __bnxt_map_fw_health_reg(bp, reg_base); 8004 return 0; 8005 } 8006 8007 static void bnxt_remap_fw_health_regs(struct bnxt *bp) 8008 { 8009 if (!bp->fw_health) 8010 return; 8011 8012 if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) { 8013 bp->fw_health->status_reliable = true; 8014 bp->fw_health->resets_reliable = true; 8015 } else { 8016 bnxt_try_map_fw_health_reg(bp); 8017 } 8018 } 8019 8020 static int bnxt_hwrm_error_recovery_qcfg(struct bnxt *bp) 8021 { 8022 struct bnxt_fw_health *fw_health = bp->fw_health; 8023 struct hwrm_error_recovery_qcfg_output *resp; 8024 struct hwrm_error_recovery_qcfg_input *req; 8025 int rc, i; 8026 8027 if (!(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) 8028 return 0; 8029 8030 rc = hwrm_req_init(bp, req, HWRM_ERROR_RECOVERY_QCFG); 8031 if (rc) 8032 return rc; 8033 8034 resp = hwrm_req_hold(bp, req); 8035 rc = hwrm_req_send(bp, req); 8036 if (rc) 8037 goto err_recovery_out; 8038 fw_health->flags = le32_to_cpu(resp->flags); 8039 if ((fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) && 8040 !(bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL)) { 8041 rc = -EINVAL; 8042 goto err_recovery_out; 8043 } 8044 fw_health->polling_dsecs = le32_to_cpu(resp->driver_polling_freq); 8045 fw_health->master_func_wait_dsecs = 8046 le32_to_cpu(resp->master_func_wait_period); 8047 fw_health->normal_func_wait_dsecs = 8048 le32_to_cpu(resp->normal_func_wait_period); 8049 fw_health->post_reset_wait_dsecs = 8050 le32_to_cpu(resp->master_func_wait_period_after_reset); 8051 fw_health->post_reset_max_wait_dsecs = 8052 le32_to_cpu(resp->max_bailout_time_after_reset); 8053 fw_health->regs[BNXT_FW_HEALTH_REG] = 8054 le32_to_cpu(resp->fw_health_status_reg); 8055 fw_health->regs[BNXT_FW_HEARTBEAT_REG] = 8056 le32_to_cpu(resp->fw_heartbeat_reg); 8057 fw_health->regs[BNXT_FW_RESET_CNT_REG] = 8058 le32_to_cpu(resp->fw_reset_cnt_reg); 8059 fw_health->regs[BNXT_FW_RESET_INPROG_REG] = 8060 le32_to_cpu(resp->reset_inprogress_reg); 8061 fw_health->fw_reset_inprog_reg_mask = 8062 le32_to_cpu(resp->reset_inprogress_reg_mask); 8063 fw_health->fw_reset_seq_cnt = resp->reg_array_cnt; 8064 if (fw_health->fw_reset_seq_cnt >= 16) { 8065 rc = -EINVAL; 8066 goto err_recovery_out; 8067 } 8068 for (i = 0; i < fw_health->fw_reset_seq_cnt; i++) { 8069 fw_health->fw_reset_seq_regs[i] = 8070 le32_to_cpu(resp->reset_reg[i]); 8071 fw_health->fw_reset_seq_vals[i] = 8072 le32_to_cpu(resp->reset_reg_val[i]); 8073 fw_health->fw_reset_seq_delay_msec[i] = 8074 resp->delay_after_reset[i]; 8075 } 8076 err_recovery_out: 8077 hwrm_req_drop(bp, req); 8078 if (!rc) 8079 rc = bnxt_map_fw_health_regs(bp); 8080 if (rc) 8081 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY; 8082 return rc; 8083 } 8084 8085 static int bnxt_hwrm_func_reset(struct bnxt *bp) 8086 { 8087 struct hwrm_func_reset_input *req; 8088 int rc; 8089 8090 rc = hwrm_req_init(bp, req, HWRM_FUNC_RESET); 8091 if (rc) 8092 return rc; 8093 8094 req->enables = 0; 8095 hwrm_req_timeout(bp, req, HWRM_RESET_TIMEOUT); 8096 return hwrm_req_send(bp, req); 8097 } 8098 8099 static void bnxt_nvm_cfg_ver_get(struct bnxt *bp) 8100 { 8101 struct hwrm_nvm_get_dev_info_output nvm_info; 8102 8103 if (!bnxt_hwrm_nvm_get_dev_info(bp, &nvm_info)) 8104 snprintf(bp->nvm_cfg_ver, FW_VER_STR_LEN, "%d.%d.%d", 8105 nvm_info.nvm_cfg_ver_maj, nvm_info.nvm_cfg_ver_min, 8106 nvm_info.nvm_cfg_ver_upd); 8107 } 8108 8109 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp) 8110 { 8111 struct hwrm_queue_qportcfg_output *resp; 8112 struct hwrm_queue_qportcfg_input *req; 8113 u8 i, j, *qptr; 8114 bool no_rdma; 8115 int rc = 0; 8116 8117 rc = hwrm_req_init(bp, req, HWRM_QUEUE_QPORTCFG); 8118 if (rc) 8119 return rc; 8120 8121 resp = hwrm_req_hold(bp, req); 8122 rc = hwrm_req_send(bp, req); 8123 if (rc) 8124 goto qportcfg_exit; 8125 8126 if (!resp->max_configurable_queues) { 8127 rc = -EINVAL; 8128 goto qportcfg_exit; 8129 } 8130 bp->max_tc = resp->max_configurable_queues; 8131 bp->max_lltc = resp->max_configurable_lossless_queues; 8132 if (bp->max_tc > BNXT_MAX_QUEUE) 8133 bp->max_tc = BNXT_MAX_QUEUE; 8134 8135 no_rdma = !(bp->flags & BNXT_FLAG_ROCE_CAP); 8136 qptr = &resp->queue_id0; 8137 for (i = 0, j = 0; i < bp->max_tc; i++) { 8138 bp->q_info[j].queue_id = *qptr; 8139 bp->q_ids[i] = *qptr++; 8140 bp->q_info[j].queue_profile = *qptr++; 8141 bp->tc_to_qidx[j] = j; 8142 if (!BNXT_CNPQ(bp->q_info[j].queue_profile) || 8143 (no_rdma && BNXT_PF(bp))) 8144 j++; 8145 } 8146 bp->max_q = bp->max_tc; 8147 bp->max_tc = max_t(u8, j, 1); 8148 8149 if (resp->queue_cfg_info & QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG) 8150 bp->max_tc = 1; 8151 8152 if (bp->max_lltc > bp->max_tc) 8153 bp->max_lltc = bp->max_tc; 8154 8155 qportcfg_exit: 8156 hwrm_req_drop(bp, req); 8157 return rc; 8158 } 8159 8160 static int bnxt_hwrm_poll(struct bnxt *bp) 8161 { 8162 struct hwrm_ver_get_input *req; 8163 int rc; 8164 8165 rc = hwrm_req_init(bp, req, HWRM_VER_GET); 8166 if (rc) 8167 return rc; 8168 8169 req->hwrm_intf_maj = HWRM_VERSION_MAJOR; 8170 req->hwrm_intf_min = HWRM_VERSION_MINOR; 8171 req->hwrm_intf_upd = HWRM_VERSION_UPDATE; 8172 8173 hwrm_req_flags(bp, req, BNXT_HWRM_CTX_SILENT | BNXT_HWRM_FULL_WAIT); 8174 rc = hwrm_req_send(bp, req); 8175 return rc; 8176 } 8177 8178 static int bnxt_hwrm_ver_get(struct bnxt *bp) 8179 { 8180 struct hwrm_ver_get_output *resp; 8181 struct hwrm_ver_get_input *req; 8182 u16 fw_maj, fw_min, fw_bld, fw_rsv; 8183 u32 dev_caps_cfg, hwrm_ver; 8184 int rc, len; 8185 8186 rc = hwrm_req_init(bp, req, HWRM_VER_GET); 8187 if (rc) 8188 return rc; 8189 8190 hwrm_req_flags(bp, req, BNXT_HWRM_FULL_WAIT); 8191 bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN; 8192 req->hwrm_intf_maj = HWRM_VERSION_MAJOR; 8193 req->hwrm_intf_min = HWRM_VERSION_MINOR; 8194 req->hwrm_intf_upd = HWRM_VERSION_UPDATE; 8195 8196 resp = hwrm_req_hold(bp, req); 8197 rc = hwrm_req_send(bp, req); 8198 if (rc) 8199 goto hwrm_ver_get_exit; 8200 8201 memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output)); 8202 8203 bp->hwrm_spec_code = resp->hwrm_intf_maj_8b << 16 | 8204 resp->hwrm_intf_min_8b << 8 | 8205 resp->hwrm_intf_upd_8b; 8206 if (resp->hwrm_intf_maj_8b < 1) { 8207 netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n", 8208 resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b, 8209 resp->hwrm_intf_upd_8b); 8210 netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n"); 8211 } 8212 8213 hwrm_ver = HWRM_VERSION_MAJOR << 16 | HWRM_VERSION_MINOR << 8 | 8214 HWRM_VERSION_UPDATE; 8215 8216 if (bp->hwrm_spec_code > hwrm_ver) 8217 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d", 8218 HWRM_VERSION_MAJOR, HWRM_VERSION_MINOR, 8219 HWRM_VERSION_UPDATE); 8220 else 8221 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d", 8222 resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b, 8223 resp->hwrm_intf_upd_8b); 8224 8225 fw_maj = le16_to_cpu(resp->hwrm_fw_major); 8226 if (bp->hwrm_spec_code > 0x10803 && fw_maj) { 8227 fw_min = le16_to_cpu(resp->hwrm_fw_minor); 8228 fw_bld = le16_to_cpu(resp->hwrm_fw_build); 8229 fw_rsv = le16_to_cpu(resp->hwrm_fw_patch); 8230 len = FW_VER_STR_LEN; 8231 } else { 8232 fw_maj = resp->hwrm_fw_maj_8b; 8233 fw_min = resp->hwrm_fw_min_8b; 8234 fw_bld = resp->hwrm_fw_bld_8b; 8235 fw_rsv = resp->hwrm_fw_rsvd_8b; 8236 len = BC_HWRM_STR_LEN; 8237 } 8238 bp->fw_ver_code = BNXT_FW_VER_CODE(fw_maj, fw_min, fw_bld, fw_rsv); 8239 snprintf(bp->fw_ver_str, len, "%d.%d.%d.%d", fw_maj, fw_min, fw_bld, 8240 fw_rsv); 8241 8242 if (strlen(resp->active_pkg_name)) { 8243 int fw_ver_len = strlen(bp->fw_ver_str); 8244 8245 snprintf(bp->fw_ver_str + fw_ver_len, 8246 FW_VER_STR_LEN - fw_ver_len - 1, "/pkg %s", 8247 resp->active_pkg_name); 8248 bp->fw_cap |= BNXT_FW_CAP_PKG_VER; 8249 } 8250 8251 bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout); 8252 if (!bp->hwrm_cmd_timeout) 8253 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT; 8254 bp->hwrm_cmd_max_timeout = le16_to_cpu(resp->max_req_timeout) * 1000; 8255 if (!bp->hwrm_cmd_max_timeout) 8256 bp->hwrm_cmd_max_timeout = HWRM_CMD_MAX_TIMEOUT; 8257 else if (bp->hwrm_cmd_max_timeout > HWRM_CMD_MAX_TIMEOUT) 8258 netdev_warn(bp->dev, "Device requests max timeout of %d seconds, may trigger hung task watchdog\n", 8259 bp->hwrm_cmd_max_timeout / 1000); 8260 8261 if (resp->hwrm_intf_maj_8b >= 1) { 8262 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len); 8263 bp->hwrm_max_ext_req_len = le16_to_cpu(resp->max_ext_req_len); 8264 } 8265 if (bp->hwrm_max_ext_req_len < HWRM_MAX_REQ_LEN) 8266 bp->hwrm_max_ext_req_len = HWRM_MAX_REQ_LEN; 8267 8268 bp->chip_num = le16_to_cpu(resp->chip_num); 8269 bp->chip_rev = resp->chip_rev; 8270 if (bp->chip_num == CHIP_NUM_58700 && !resp->chip_rev && 8271 !resp->chip_metal) 8272 bp->flags |= BNXT_FLAG_CHIP_NITRO_A0; 8273 8274 dev_caps_cfg = le32_to_cpu(resp->dev_caps_cfg); 8275 if ((dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) && 8276 (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED)) 8277 bp->fw_cap |= BNXT_FW_CAP_SHORT_CMD; 8278 8279 if (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_KONG_MB_CHNL_SUPPORTED) 8280 bp->fw_cap |= BNXT_FW_CAP_KONG_MB_CHNL; 8281 8282 if (dev_caps_cfg & 8283 VER_GET_RESP_DEV_CAPS_CFG_FLOW_HANDLE_64BIT_SUPPORTED) 8284 bp->fw_cap |= BNXT_FW_CAP_OVS_64BIT_HANDLE; 8285 8286 if (dev_caps_cfg & 8287 VER_GET_RESP_DEV_CAPS_CFG_TRUSTED_VF_SUPPORTED) 8288 bp->fw_cap |= BNXT_FW_CAP_TRUSTED_VF; 8289 8290 if (dev_caps_cfg & 8291 VER_GET_RESP_DEV_CAPS_CFG_CFA_ADV_FLOW_MGNT_SUPPORTED) 8292 bp->fw_cap |= BNXT_FW_CAP_CFA_ADV_FLOW; 8293 8294 hwrm_ver_get_exit: 8295 hwrm_req_drop(bp, req); 8296 return rc; 8297 } 8298 8299 int bnxt_hwrm_fw_set_time(struct bnxt *bp) 8300 { 8301 struct hwrm_fw_set_time_input *req; 8302 struct tm tm; 8303 time64_t now = ktime_get_real_seconds(); 8304 int rc; 8305 8306 if ((BNXT_VF(bp) && bp->hwrm_spec_code < 0x10901) || 8307 bp->hwrm_spec_code < 0x10400) 8308 return -EOPNOTSUPP; 8309 8310 time64_to_tm(now, 0, &tm); 8311 rc = hwrm_req_init(bp, req, HWRM_FW_SET_TIME); 8312 if (rc) 8313 return rc; 8314 8315 req->year = cpu_to_le16(1900 + tm.tm_year); 8316 req->month = 1 + tm.tm_mon; 8317 req->day = tm.tm_mday; 8318 req->hour = tm.tm_hour; 8319 req->minute = tm.tm_min; 8320 req->second = tm.tm_sec; 8321 return hwrm_req_send(bp, req); 8322 } 8323 8324 static void bnxt_add_one_ctr(u64 hw, u64 *sw, u64 mask) 8325 { 8326 u64 sw_tmp; 8327 8328 hw &= mask; 8329 sw_tmp = (*sw & ~mask) | hw; 8330 if (hw < (*sw & mask)) 8331 sw_tmp += mask + 1; 8332 WRITE_ONCE(*sw, sw_tmp); 8333 } 8334 8335 static void __bnxt_accumulate_stats(__le64 *hw_stats, u64 *sw_stats, u64 *masks, 8336 int count, bool ignore_zero) 8337 { 8338 int i; 8339 8340 for (i = 0; i < count; i++) { 8341 u64 hw = le64_to_cpu(READ_ONCE(hw_stats[i])); 8342 8343 if (ignore_zero && !hw) 8344 continue; 8345 8346 if (masks[i] == -1ULL) 8347 sw_stats[i] = hw; 8348 else 8349 bnxt_add_one_ctr(hw, &sw_stats[i], masks[i]); 8350 } 8351 } 8352 8353 static void bnxt_accumulate_stats(struct bnxt_stats_mem *stats) 8354 { 8355 if (!stats->hw_stats) 8356 return; 8357 8358 __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats, 8359 stats->hw_masks, stats->len / 8, false); 8360 } 8361 8362 static void bnxt_accumulate_all_stats(struct bnxt *bp) 8363 { 8364 struct bnxt_stats_mem *ring0_stats; 8365 bool ignore_zero = false; 8366 int i; 8367 8368 /* Chip bug. Counter intermittently becomes 0. */ 8369 if (bp->flags & BNXT_FLAG_CHIP_P5) 8370 ignore_zero = true; 8371 8372 for (i = 0; i < bp->cp_nr_rings; i++) { 8373 struct bnxt_napi *bnapi = bp->bnapi[i]; 8374 struct bnxt_cp_ring_info *cpr; 8375 struct bnxt_stats_mem *stats; 8376 8377 cpr = &bnapi->cp_ring; 8378 stats = &cpr->stats; 8379 if (!i) 8380 ring0_stats = stats; 8381 __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats, 8382 ring0_stats->hw_masks, 8383 ring0_stats->len / 8, ignore_zero); 8384 } 8385 if (bp->flags & BNXT_FLAG_PORT_STATS) { 8386 struct bnxt_stats_mem *stats = &bp->port_stats; 8387 __le64 *hw_stats = stats->hw_stats; 8388 u64 *sw_stats = stats->sw_stats; 8389 u64 *masks = stats->hw_masks; 8390 int cnt; 8391 8392 cnt = sizeof(struct rx_port_stats) / 8; 8393 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false); 8394 8395 hw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 8396 sw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 8397 masks += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 8398 cnt = sizeof(struct tx_port_stats) / 8; 8399 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false); 8400 } 8401 if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) { 8402 bnxt_accumulate_stats(&bp->rx_port_stats_ext); 8403 bnxt_accumulate_stats(&bp->tx_port_stats_ext); 8404 } 8405 } 8406 8407 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags) 8408 { 8409 struct hwrm_port_qstats_input *req; 8410 struct bnxt_pf_info *pf = &bp->pf; 8411 int rc; 8412 8413 if (!(bp->flags & BNXT_FLAG_PORT_STATS)) 8414 return 0; 8415 8416 if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED)) 8417 return -EOPNOTSUPP; 8418 8419 rc = hwrm_req_init(bp, req, HWRM_PORT_QSTATS); 8420 if (rc) 8421 return rc; 8422 8423 req->flags = flags; 8424 req->port_id = cpu_to_le16(pf->port_id); 8425 req->tx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map + 8426 BNXT_TX_PORT_STATS_BYTE_OFFSET); 8427 req->rx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map); 8428 return hwrm_req_send(bp, req); 8429 } 8430 8431 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags) 8432 { 8433 struct hwrm_queue_pri2cos_qcfg_output *resp_qc; 8434 struct hwrm_queue_pri2cos_qcfg_input *req_qc; 8435 struct hwrm_port_qstats_ext_output *resp_qs; 8436 struct hwrm_port_qstats_ext_input *req_qs; 8437 struct bnxt_pf_info *pf = &bp->pf; 8438 u32 tx_stat_size; 8439 int rc; 8440 8441 if (!(bp->flags & BNXT_FLAG_PORT_STATS_EXT)) 8442 return 0; 8443 8444 if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED)) 8445 return -EOPNOTSUPP; 8446 8447 rc = hwrm_req_init(bp, req_qs, HWRM_PORT_QSTATS_EXT); 8448 if (rc) 8449 return rc; 8450 8451 req_qs->flags = flags; 8452 req_qs->port_id = cpu_to_le16(pf->port_id); 8453 req_qs->rx_stat_size = cpu_to_le16(sizeof(struct rx_port_stats_ext)); 8454 req_qs->rx_stat_host_addr = cpu_to_le64(bp->rx_port_stats_ext.hw_stats_map); 8455 tx_stat_size = bp->tx_port_stats_ext.hw_stats ? 8456 sizeof(struct tx_port_stats_ext) : 0; 8457 req_qs->tx_stat_size = cpu_to_le16(tx_stat_size); 8458 req_qs->tx_stat_host_addr = cpu_to_le64(bp->tx_port_stats_ext.hw_stats_map); 8459 resp_qs = hwrm_req_hold(bp, req_qs); 8460 rc = hwrm_req_send(bp, req_qs); 8461 if (!rc) { 8462 bp->fw_rx_stats_ext_size = 8463 le16_to_cpu(resp_qs->rx_stat_size) / 8; 8464 if (BNXT_FW_MAJ(bp) < 220 && 8465 bp->fw_rx_stats_ext_size > BNXT_RX_STATS_EXT_NUM_LEGACY) 8466 bp->fw_rx_stats_ext_size = BNXT_RX_STATS_EXT_NUM_LEGACY; 8467 8468 bp->fw_tx_stats_ext_size = tx_stat_size ? 8469 le16_to_cpu(resp_qs->tx_stat_size) / 8 : 0; 8470 } else { 8471 bp->fw_rx_stats_ext_size = 0; 8472 bp->fw_tx_stats_ext_size = 0; 8473 } 8474 hwrm_req_drop(bp, req_qs); 8475 8476 if (flags) 8477 return rc; 8478 8479 if (bp->fw_tx_stats_ext_size <= 8480 offsetof(struct tx_port_stats_ext, pfc_pri0_tx_duration_us) / 8) { 8481 bp->pri2cos_valid = 0; 8482 return rc; 8483 } 8484 8485 rc = hwrm_req_init(bp, req_qc, HWRM_QUEUE_PRI2COS_QCFG); 8486 if (rc) 8487 return rc; 8488 8489 req_qc->flags = cpu_to_le32(QUEUE_PRI2COS_QCFG_REQ_FLAGS_IVLAN); 8490 8491 resp_qc = hwrm_req_hold(bp, req_qc); 8492 rc = hwrm_req_send(bp, req_qc); 8493 if (!rc) { 8494 u8 *pri2cos; 8495 int i, j; 8496 8497 pri2cos = &resp_qc->pri0_cos_queue_id; 8498 for (i = 0; i < 8; i++) { 8499 u8 queue_id = pri2cos[i]; 8500 u8 queue_idx; 8501 8502 /* Per port queue IDs start from 0, 10, 20, etc */ 8503 queue_idx = queue_id % 10; 8504 if (queue_idx > BNXT_MAX_QUEUE) { 8505 bp->pri2cos_valid = false; 8506 hwrm_req_drop(bp, req_qc); 8507 return rc; 8508 } 8509 for (j = 0; j < bp->max_q; j++) { 8510 if (bp->q_ids[j] == queue_id) 8511 bp->pri2cos_idx[i] = queue_idx; 8512 } 8513 } 8514 bp->pri2cos_valid = true; 8515 } 8516 hwrm_req_drop(bp, req_qc); 8517 8518 return rc; 8519 } 8520 8521 static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp) 8522 { 8523 bnxt_hwrm_tunnel_dst_port_free(bp, 8524 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN); 8525 bnxt_hwrm_tunnel_dst_port_free(bp, 8526 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE); 8527 } 8528 8529 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa) 8530 { 8531 int rc, i; 8532 u32 tpa_flags = 0; 8533 8534 if (set_tpa) 8535 tpa_flags = bp->flags & BNXT_FLAG_TPA; 8536 else if (BNXT_NO_FW_ACCESS(bp)) 8537 return 0; 8538 for (i = 0; i < bp->nr_vnics; i++) { 8539 rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags); 8540 if (rc) { 8541 netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n", 8542 i, rc); 8543 return rc; 8544 } 8545 } 8546 return 0; 8547 } 8548 8549 static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp) 8550 { 8551 int i; 8552 8553 for (i = 0; i < bp->nr_vnics; i++) 8554 bnxt_hwrm_vnic_set_rss(bp, i, false); 8555 } 8556 8557 static void bnxt_clear_vnic(struct bnxt *bp) 8558 { 8559 if (!bp->vnic_info) 8560 return; 8561 8562 bnxt_hwrm_clear_vnic_filter(bp); 8563 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) { 8564 /* clear all RSS setting before free vnic ctx */ 8565 bnxt_hwrm_clear_vnic_rss(bp); 8566 bnxt_hwrm_vnic_ctx_free(bp); 8567 } 8568 /* before free the vnic, undo the vnic tpa settings */ 8569 if (bp->flags & BNXT_FLAG_TPA) 8570 bnxt_set_tpa(bp, false); 8571 bnxt_hwrm_vnic_free(bp); 8572 if (bp->flags & BNXT_FLAG_CHIP_P5) 8573 bnxt_hwrm_vnic_ctx_free(bp); 8574 } 8575 8576 static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path, 8577 bool irq_re_init) 8578 { 8579 bnxt_clear_vnic(bp); 8580 bnxt_hwrm_ring_free(bp, close_path); 8581 bnxt_hwrm_ring_grp_free(bp); 8582 if (irq_re_init) { 8583 bnxt_hwrm_stat_ctx_free(bp); 8584 bnxt_hwrm_free_tunnel_ports(bp); 8585 } 8586 } 8587 8588 static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode) 8589 { 8590 struct hwrm_func_cfg_input *req; 8591 u8 evb_mode; 8592 int rc; 8593 8594 if (br_mode == BRIDGE_MODE_VEB) 8595 evb_mode = FUNC_CFG_REQ_EVB_MODE_VEB; 8596 else if (br_mode == BRIDGE_MODE_VEPA) 8597 evb_mode = FUNC_CFG_REQ_EVB_MODE_VEPA; 8598 else 8599 return -EINVAL; 8600 8601 rc = hwrm_req_init(bp, req, HWRM_FUNC_CFG); 8602 if (rc) 8603 return rc; 8604 8605 req->fid = cpu_to_le16(0xffff); 8606 req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_EVB_MODE); 8607 req->evb_mode = evb_mode; 8608 return hwrm_req_send(bp, req); 8609 } 8610 8611 static int bnxt_hwrm_set_cache_line_size(struct bnxt *bp, int size) 8612 { 8613 struct hwrm_func_cfg_input *req; 8614 int rc; 8615 8616 if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10803) 8617 return 0; 8618 8619 rc = hwrm_req_init(bp, req, HWRM_FUNC_CFG); 8620 if (rc) 8621 return rc; 8622 8623 req->fid = cpu_to_le16(0xffff); 8624 req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_CACHE_LINESIZE); 8625 req->options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_64; 8626 if (size == 128) 8627 req->options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_128; 8628 8629 return hwrm_req_send(bp, req); 8630 } 8631 8632 static int __bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id) 8633 { 8634 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 8635 int rc; 8636 8637 if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) 8638 goto skip_rss_ctx; 8639 8640 /* allocate context for vnic */ 8641 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 0); 8642 if (rc) { 8643 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n", 8644 vnic_id, rc); 8645 goto vnic_setup_err; 8646 } 8647 bp->rsscos_nr_ctxs++; 8648 8649 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 8650 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 1); 8651 if (rc) { 8652 netdev_err(bp->dev, "hwrm vnic %d cos ctx alloc failure rc: %x\n", 8653 vnic_id, rc); 8654 goto vnic_setup_err; 8655 } 8656 bp->rsscos_nr_ctxs++; 8657 } 8658 8659 skip_rss_ctx: 8660 /* configure default vnic, ring grp */ 8661 rc = bnxt_hwrm_vnic_cfg(bp, vnic_id); 8662 if (rc) { 8663 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n", 8664 vnic_id, rc); 8665 goto vnic_setup_err; 8666 } 8667 8668 /* Enable RSS hashing on vnic */ 8669 rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true); 8670 if (rc) { 8671 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n", 8672 vnic_id, rc); 8673 goto vnic_setup_err; 8674 } 8675 8676 if (bp->flags & BNXT_FLAG_AGG_RINGS) { 8677 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id); 8678 if (rc) { 8679 netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n", 8680 vnic_id, rc); 8681 } 8682 } 8683 8684 vnic_setup_err: 8685 return rc; 8686 } 8687 8688 static int __bnxt_setup_vnic_p5(struct bnxt *bp, u16 vnic_id) 8689 { 8690 int rc, i, nr_ctxs; 8691 8692 nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings); 8693 for (i = 0; i < nr_ctxs; i++) { 8694 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, i); 8695 if (rc) { 8696 netdev_err(bp->dev, "hwrm vnic %d ctx %d alloc failure rc: %x\n", 8697 vnic_id, i, rc); 8698 break; 8699 } 8700 bp->rsscos_nr_ctxs++; 8701 } 8702 if (i < nr_ctxs) 8703 return -ENOMEM; 8704 8705 rc = bnxt_hwrm_vnic_set_rss_p5(bp, vnic_id, true); 8706 if (rc) { 8707 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %d\n", 8708 vnic_id, rc); 8709 return rc; 8710 } 8711 rc = bnxt_hwrm_vnic_cfg(bp, vnic_id); 8712 if (rc) { 8713 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n", 8714 vnic_id, rc); 8715 return rc; 8716 } 8717 if (bp->flags & BNXT_FLAG_AGG_RINGS) { 8718 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id); 8719 if (rc) { 8720 netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n", 8721 vnic_id, rc); 8722 } 8723 } 8724 return rc; 8725 } 8726 8727 static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id) 8728 { 8729 if (bp->flags & BNXT_FLAG_CHIP_P5) 8730 return __bnxt_setup_vnic_p5(bp, vnic_id); 8731 else 8732 return __bnxt_setup_vnic(bp, vnic_id); 8733 } 8734 8735 static int bnxt_alloc_rfs_vnics(struct bnxt *bp) 8736 { 8737 #ifdef CONFIG_RFS_ACCEL 8738 int i, rc = 0; 8739 8740 if (bp->flags & BNXT_FLAG_CHIP_P5) 8741 return 0; 8742 8743 for (i = 0; i < bp->rx_nr_rings; i++) { 8744 struct bnxt_vnic_info *vnic; 8745 u16 vnic_id = i + 1; 8746 u16 ring_id = i; 8747 8748 if (vnic_id >= bp->nr_vnics) 8749 break; 8750 8751 vnic = &bp->vnic_info[vnic_id]; 8752 vnic->flags |= BNXT_VNIC_RFS_FLAG; 8753 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP) 8754 vnic->flags |= BNXT_VNIC_RFS_NEW_RSS_FLAG; 8755 rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, ring_id, 1); 8756 if (rc) { 8757 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n", 8758 vnic_id, rc); 8759 break; 8760 } 8761 rc = bnxt_setup_vnic(bp, vnic_id); 8762 if (rc) 8763 break; 8764 } 8765 return rc; 8766 #else 8767 return 0; 8768 #endif 8769 } 8770 8771 /* Allow PF, trusted VFs and VFs with default VLAN to be in promiscuous mode */ 8772 static bool bnxt_promisc_ok(struct bnxt *bp) 8773 { 8774 #ifdef CONFIG_BNXT_SRIOV 8775 if (BNXT_VF(bp) && !bp->vf.vlan && !bnxt_is_trusted_vf(bp, &bp->vf)) 8776 return false; 8777 #endif 8778 return true; 8779 } 8780 8781 static int bnxt_setup_nitroa0_vnic(struct bnxt *bp) 8782 { 8783 unsigned int rc = 0; 8784 8785 rc = bnxt_hwrm_vnic_alloc(bp, 1, bp->rx_nr_rings - 1, 1); 8786 if (rc) { 8787 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n", 8788 rc); 8789 return rc; 8790 } 8791 8792 rc = bnxt_hwrm_vnic_cfg(bp, 1); 8793 if (rc) { 8794 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n", 8795 rc); 8796 return rc; 8797 } 8798 return rc; 8799 } 8800 8801 static int bnxt_cfg_rx_mode(struct bnxt *); 8802 static bool bnxt_mc_list_updated(struct bnxt *, u32 *); 8803 8804 static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init) 8805 { 8806 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 8807 int rc = 0; 8808 unsigned int rx_nr_rings = bp->rx_nr_rings; 8809 8810 if (irq_re_init) { 8811 rc = bnxt_hwrm_stat_ctx_alloc(bp); 8812 if (rc) { 8813 netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n", 8814 rc); 8815 goto err_out; 8816 } 8817 } 8818 8819 rc = bnxt_hwrm_ring_alloc(bp); 8820 if (rc) { 8821 netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc); 8822 goto err_out; 8823 } 8824 8825 rc = bnxt_hwrm_ring_grp_alloc(bp); 8826 if (rc) { 8827 netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc); 8828 goto err_out; 8829 } 8830 8831 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 8832 rx_nr_rings--; 8833 8834 /* default vnic 0 */ 8835 rc = bnxt_hwrm_vnic_alloc(bp, 0, 0, rx_nr_rings); 8836 if (rc) { 8837 netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc); 8838 goto err_out; 8839 } 8840 8841 rc = bnxt_setup_vnic(bp, 0); 8842 if (rc) 8843 goto err_out; 8844 if (bp->fw_cap & BNXT_FW_CAP_RSS_HASH_TYPE_DELTA) 8845 bnxt_hwrm_update_rss_hash_cfg(bp); 8846 8847 if (bp->flags & BNXT_FLAG_RFS) { 8848 rc = bnxt_alloc_rfs_vnics(bp); 8849 if (rc) 8850 goto err_out; 8851 } 8852 8853 if (bp->flags & BNXT_FLAG_TPA) { 8854 rc = bnxt_set_tpa(bp, true); 8855 if (rc) 8856 goto err_out; 8857 } 8858 8859 if (BNXT_VF(bp)) 8860 bnxt_update_vf_mac(bp); 8861 8862 /* Filter for default vnic 0 */ 8863 rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr); 8864 if (rc) { 8865 if (BNXT_VF(bp) && rc == -ENODEV) 8866 netdev_err(bp->dev, "Cannot configure L2 filter while PF is unavailable\n"); 8867 else 8868 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc); 8869 goto err_out; 8870 } 8871 vnic->uc_filter_count = 1; 8872 8873 vnic->rx_mask = 0; 8874 if (test_bit(BNXT_STATE_HALF_OPEN, &bp->state)) 8875 goto skip_rx_mask; 8876 8877 if (bp->dev->flags & IFF_BROADCAST) 8878 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST; 8879 8880 if (bp->dev->flags & IFF_PROMISC) 8881 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 8882 8883 if (bp->dev->flags & IFF_ALLMULTI) { 8884 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 8885 vnic->mc_list_count = 0; 8886 } else if (bp->dev->flags & IFF_MULTICAST) { 8887 u32 mask = 0; 8888 8889 bnxt_mc_list_updated(bp, &mask); 8890 vnic->rx_mask |= mask; 8891 } 8892 8893 rc = bnxt_cfg_rx_mode(bp); 8894 if (rc) 8895 goto err_out; 8896 8897 skip_rx_mask: 8898 rc = bnxt_hwrm_set_coal(bp); 8899 if (rc) 8900 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n", 8901 rc); 8902 8903 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 8904 rc = bnxt_setup_nitroa0_vnic(bp); 8905 if (rc) 8906 netdev_err(bp->dev, "Special vnic setup failure for NS2 A0 rc: %x\n", 8907 rc); 8908 } 8909 8910 if (BNXT_VF(bp)) { 8911 bnxt_hwrm_func_qcfg(bp); 8912 netdev_update_features(bp->dev); 8913 } 8914 8915 return 0; 8916 8917 err_out: 8918 bnxt_hwrm_resource_free(bp, 0, true); 8919 8920 return rc; 8921 } 8922 8923 static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init) 8924 { 8925 bnxt_hwrm_resource_free(bp, 1, irq_re_init); 8926 return 0; 8927 } 8928 8929 static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init) 8930 { 8931 bnxt_init_cp_rings(bp); 8932 bnxt_init_rx_rings(bp); 8933 bnxt_init_tx_rings(bp); 8934 bnxt_init_ring_grps(bp, irq_re_init); 8935 bnxt_init_vnics(bp); 8936 8937 return bnxt_init_chip(bp, irq_re_init); 8938 } 8939 8940 static int bnxt_set_real_num_queues(struct bnxt *bp) 8941 { 8942 int rc; 8943 struct net_device *dev = bp->dev; 8944 8945 rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings - 8946 bp->tx_nr_rings_xdp); 8947 if (rc) 8948 return rc; 8949 8950 rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings); 8951 if (rc) 8952 return rc; 8953 8954 #ifdef CONFIG_RFS_ACCEL 8955 if (bp->flags & BNXT_FLAG_RFS) 8956 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings); 8957 #endif 8958 8959 return rc; 8960 } 8961 8962 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max, 8963 bool shared) 8964 { 8965 int _rx = *rx, _tx = *tx; 8966 8967 if (shared) { 8968 *rx = min_t(int, _rx, max); 8969 *tx = min_t(int, _tx, max); 8970 } else { 8971 if (max < 2) 8972 return -ENOMEM; 8973 8974 while (_rx + _tx > max) { 8975 if (_rx > _tx && _rx > 1) 8976 _rx--; 8977 else if (_tx > 1) 8978 _tx--; 8979 } 8980 *rx = _rx; 8981 *tx = _tx; 8982 } 8983 return 0; 8984 } 8985 8986 static void bnxt_setup_msix(struct bnxt *bp) 8987 { 8988 const int len = sizeof(bp->irq_tbl[0].name); 8989 struct net_device *dev = bp->dev; 8990 int tcs, i; 8991 8992 tcs = netdev_get_num_tc(dev); 8993 if (tcs) { 8994 int i, off, count; 8995 8996 for (i = 0; i < tcs; i++) { 8997 count = bp->tx_nr_rings_per_tc; 8998 off = i * count; 8999 netdev_set_tc_queue(dev, i, count, off); 9000 } 9001 } 9002 9003 for (i = 0; i < bp->cp_nr_rings; i++) { 9004 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 9005 char *attr; 9006 9007 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 9008 attr = "TxRx"; 9009 else if (i < bp->rx_nr_rings) 9010 attr = "rx"; 9011 else 9012 attr = "tx"; 9013 9014 snprintf(bp->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name, 9015 attr, i); 9016 bp->irq_tbl[map_idx].handler = bnxt_msix; 9017 } 9018 } 9019 9020 static void bnxt_setup_inta(struct bnxt *bp) 9021 { 9022 const int len = sizeof(bp->irq_tbl[0].name); 9023 9024 if (netdev_get_num_tc(bp->dev)) 9025 netdev_reset_tc(bp->dev); 9026 9027 snprintf(bp->irq_tbl[0].name, len, "%s-%s-%d", bp->dev->name, "TxRx", 9028 0); 9029 bp->irq_tbl[0].handler = bnxt_inta; 9030 } 9031 9032 static int bnxt_init_int_mode(struct bnxt *bp); 9033 9034 static int bnxt_setup_int_mode(struct bnxt *bp) 9035 { 9036 int rc; 9037 9038 if (!bp->irq_tbl) { 9039 rc = bnxt_init_int_mode(bp); 9040 if (rc || !bp->irq_tbl) 9041 return rc ?: -ENODEV; 9042 } 9043 9044 if (bp->flags & BNXT_FLAG_USING_MSIX) 9045 bnxt_setup_msix(bp); 9046 else 9047 bnxt_setup_inta(bp); 9048 9049 rc = bnxt_set_real_num_queues(bp); 9050 return rc; 9051 } 9052 9053 #ifdef CONFIG_RFS_ACCEL 9054 static unsigned int bnxt_get_max_func_rss_ctxs(struct bnxt *bp) 9055 { 9056 return bp->hw_resc.max_rsscos_ctxs; 9057 } 9058 9059 static unsigned int bnxt_get_max_func_vnics(struct bnxt *bp) 9060 { 9061 return bp->hw_resc.max_vnics; 9062 } 9063 #endif 9064 9065 unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp) 9066 { 9067 return bp->hw_resc.max_stat_ctxs; 9068 } 9069 9070 unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp) 9071 { 9072 return bp->hw_resc.max_cp_rings; 9073 } 9074 9075 static unsigned int bnxt_get_max_func_cp_rings_for_en(struct bnxt *bp) 9076 { 9077 unsigned int cp = bp->hw_resc.max_cp_rings; 9078 9079 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 9080 cp -= bnxt_get_ulp_msix_num(bp); 9081 9082 return cp; 9083 } 9084 9085 static unsigned int bnxt_get_max_func_irqs(struct bnxt *bp) 9086 { 9087 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 9088 9089 if (bp->flags & BNXT_FLAG_CHIP_P5) 9090 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_nqs); 9091 9092 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_cp_rings); 9093 } 9094 9095 static void bnxt_set_max_func_irqs(struct bnxt *bp, unsigned int max_irqs) 9096 { 9097 bp->hw_resc.max_irqs = max_irqs; 9098 } 9099 9100 unsigned int bnxt_get_avail_cp_rings_for_en(struct bnxt *bp) 9101 { 9102 unsigned int cp; 9103 9104 cp = bnxt_get_max_func_cp_rings_for_en(bp); 9105 if (bp->flags & BNXT_FLAG_CHIP_P5) 9106 return cp - bp->rx_nr_rings - bp->tx_nr_rings; 9107 else 9108 return cp - bp->cp_nr_rings; 9109 } 9110 9111 unsigned int bnxt_get_avail_stat_ctxs_for_en(struct bnxt *bp) 9112 { 9113 return bnxt_get_max_func_stat_ctxs(bp) - bnxt_get_func_stat_ctxs(bp); 9114 } 9115 9116 int bnxt_get_avail_msix(struct bnxt *bp, int num) 9117 { 9118 int max_cp = bnxt_get_max_func_cp_rings(bp); 9119 int max_irq = bnxt_get_max_func_irqs(bp); 9120 int total_req = bp->cp_nr_rings + num; 9121 int max_idx, avail_msix; 9122 9123 max_idx = bp->total_irqs; 9124 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 9125 max_idx = min_t(int, bp->total_irqs, max_cp); 9126 avail_msix = max_idx - bp->cp_nr_rings; 9127 if (!BNXT_NEW_RM(bp) || avail_msix >= num) 9128 return avail_msix; 9129 9130 if (max_irq < total_req) { 9131 num = max_irq - bp->cp_nr_rings; 9132 if (num <= 0) 9133 return 0; 9134 } 9135 return num; 9136 } 9137 9138 static int bnxt_get_num_msix(struct bnxt *bp) 9139 { 9140 if (!BNXT_NEW_RM(bp)) 9141 return bnxt_get_max_func_irqs(bp); 9142 9143 return bnxt_nq_rings_in_use(bp); 9144 } 9145 9146 static int bnxt_init_msix(struct bnxt *bp) 9147 { 9148 int i, total_vecs, max, rc = 0, min = 1, ulp_msix; 9149 struct msix_entry *msix_ent; 9150 9151 total_vecs = bnxt_get_num_msix(bp); 9152 max = bnxt_get_max_func_irqs(bp); 9153 if (total_vecs > max) 9154 total_vecs = max; 9155 9156 if (!total_vecs) 9157 return 0; 9158 9159 msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL); 9160 if (!msix_ent) 9161 return -ENOMEM; 9162 9163 for (i = 0; i < total_vecs; i++) { 9164 msix_ent[i].entry = i; 9165 msix_ent[i].vector = 0; 9166 } 9167 9168 if (!(bp->flags & BNXT_FLAG_SHARED_RINGS)) 9169 min = 2; 9170 9171 total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs); 9172 ulp_msix = bnxt_get_ulp_msix_num(bp); 9173 if (total_vecs < 0 || total_vecs < ulp_msix) { 9174 rc = -ENODEV; 9175 goto msix_setup_exit; 9176 } 9177 9178 bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL); 9179 if (bp->irq_tbl) { 9180 for (i = 0; i < total_vecs; i++) 9181 bp->irq_tbl[i].vector = msix_ent[i].vector; 9182 9183 bp->total_irqs = total_vecs; 9184 /* Trim rings based upon num of vectors allocated */ 9185 rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings, 9186 total_vecs - ulp_msix, min == 1); 9187 if (rc) 9188 goto msix_setup_exit; 9189 9190 bp->cp_nr_rings = (min == 1) ? 9191 max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) : 9192 bp->tx_nr_rings + bp->rx_nr_rings; 9193 9194 } else { 9195 rc = -ENOMEM; 9196 goto msix_setup_exit; 9197 } 9198 bp->flags |= BNXT_FLAG_USING_MSIX; 9199 kfree(msix_ent); 9200 return 0; 9201 9202 msix_setup_exit: 9203 netdev_err(bp->dev, "bnxt_init_msix err: %x\n", rc); 9204 kfree(bp->irq_tbl); 9205 bp->irq_tbl = NULL; 9206 pci_disable_msix(bp->pdev); 9207 kfree(msix_ent); 9208 return rc; 9209 } 9210 9211 static int bnxt_init_inta(struct bnxt *bp) 9212 { 9213 bp->irq_tbl = kzalloc(sizeof(struct bnxt_irq), GFP_KERNEL); 9214 if (!bp->irq_tbl) 9215 return -ENOMEM; 9216 9217 bp->total_irqs = 1; 9218 bp->rx_nr_rings = 1; 9219 bp->tx_nr_rings = 1; 9220 bp->cp_nr_rings = 1; 9221 bp->flags |= BNXT_FLAG_SHARED_RINGS; 9222 bp->irq_tbl[0].vector = bp->pdev->irq; 9223 return 0; 9224 } 9225 9226 static int bnxt_init_int_mode(struct bnxt *bp) 9227 { 9228 int rc = -ENODEV; 9229 9230 if (bp->flags & BNXT_FLAG_MSIX_CAP) 9231 rc = bnxt_init_msix(bp); 9232 9233 if (!(bp->flags & BNXT_FLAG_USING_MSIX) && BNXT_PF(bp)) { 9234 /* fallback to INTA */ 9235 rc = bnxt_init_inta(bp); 9236 } 9237 return rc; 9238 } 9239 9240 static void bnxt_clear_int_mode(struct bnxt *bp) 9241 { 9242 if (bp->flags & BNXT_FLAG_USING_MSIX) 9243 pci_disable_msix(bp->pdev); 9244 9245 kfree(bp->irq_tbl); 9246 bp->irq_tbl = NULL; 9247 bp->flags &= ~BNXT_FLAG_USING_MSIX; 9248 } 9249 9250 int bnxt_reserve_rings(struct bnxt *bp, bool irq_re_init) 9251 { 9252 int tcs = netdev_get_num_tc(bp->dev); 9253 bool irq_cleared = false; 9254 int rc; 9255 9256 if (!bnxt_need_reserve_rings(bp)) 9257 return 0; 9258 9259 if (irq_re_init && BNXT_NEW_RM(bp) && 9260 bnxt_get_num_msix(bp) != bp->total_irqs) { 9261 bnxt_ulp_irq_stop(bp); 9262 bnxt_clear_int_mode(bp); 9263 irq_cleared = true; 9264 } 9265 rc = __bnxt_reserve_rings(bp); 9266 if (irq_cleared) { 9267 if (!rc) 9268 rc = bnxt_init_int_mode(bp); 9269 bnxt_ulp_irq_restart(bp, rc); 9270 } 9271 if (rc) { 9272 netdev_err(bp->dev, "ring reservation/IRQ init failure rc: %d\n", rc); 9273 return rc; 9274 } 9275 if (tcs && (bp->tx_nr_rings_per_tc * tcs != 9276 bp->tx_nr_rings - bp->tx_nr_rings_xdp)) { 9277 netdev_err(bp->dev, "tx ring reservation failure\n"); 9278 netdev_reset_tc(bp->dev); 9279 if (bp->tx_nr_rings_xdp) 9280 bp->tx_nr_rings_per_tc = bp->tx_nr_rings_xdp; 9281 else 9282 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 9283 return -ENOMEM; 9284 } 9285 return 0; 9286 } 9287 9288 static void bnxt_free_irq(struct bnxt *bp) 9289 { 9290 struct bnxt_irq *irq; 9291 int i; 9292 9293 #ifdef CONFIG_RFS_ACCEL 9294 free_irq_cpu_rmap(bp->dev->rx_cpu_rmap); 9295 bp->dev->rx_cpu_rmap = NULL; 9296 #endif 9297 if (!bp->irq_tbl || !bp->bnapi) 9298 return; 9299 9300 for (i = 0; i < bp->cp_nr_rings; i++) { 9301 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 9302 9303 irq = &bp->irq_tbl[map_idx]; 9304 if (irq->requested) { 9305 if (irq->have_cpumask) { 9306 irq_set_affinity_hint(irq->vector, NULL); 9307 free_cpumask_var(irq->cpu_mask); 9308 irq->have_cpumask = 0; 9309 } 9310 free_irq(irq->vector, bp->bnapi[i]); 9311 } 9312 9313 irq->requested = 0; 9314 } 9315 } 9316 9317 static int bnxt_request_irq(struct bnxt *bp) 9318 { 9319 int i, j, rc = 0; 9320 unsigned long flags = 0; 9321 #ifdef CONFIG_RFS_ACCEL 9322 struct cpu_rmap *rmap; 9323 #endif 9324 9325 rc = bnxt_setup_int_mode(bp); 9326 if (rc) { 9327 netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n", 9328 rc); 9329 return rc; 9330 } 9331 #ifdef CONFIG_RFS_ACCEL 9332 rmap = bp->dev->rx_cpu_rmap; 9333 #endif 9334 if (!(bp->flags & BNXT_FLAG_USING_MSIX)) 9335 flags = IRQF_SHARED; 9336 9337 for (i = 0, j = 0; i < bp->cp_nr_rings; i++) { 9338 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 9339 struct bnxt_irq *irq = &bp->irq_tbl[map_idx]; 9340 9341 #ifdef CONFIG_RFS_ACCEL 9342 if (rmap && bp->bnapi[i]->rx_ring) { 9343 rc = irq_cpu_rmap_add(rmap, irq->vector); 9344 if (rc) 9345 netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n", 9346 j); 9347 j++; 9348 } 9349 #endif 9350 rc = request_irq(irq->vector, irq->handler, flags, irq->name, 9351 bp->bnapi[i]); 9352 if (rc) 9353 break; 9354 9355 irq->requested = 1; 9356 9357 if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) { 9358 int numa_node = dev_to_node(&bp->pdev->dev); 9359 9360 irq->have_cpumask = 1; 9361 cpumask_set_cpu(cpumask_local_spread(i, numa_node), 9362 irq->cpu_mask); 9363 rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask); 9364 if (rc) { 9365 netdev_warn(bp->dev, 9366 "Set affinity failed, IRQ = %d\n", 9367 irq->vector); 9368 break; 9369 } 9370 } 9371 } 9372 return rc; 9373 } 9374 9375 static void bnxt_del_napi(struct bnxt *bp) 9376 { 9377 int i; 9378 9379 if (!bp->bnapi) 9380 return; 9381 9382 for (i = 0; i < bp->cp_nr_rings; i++) { 9383 struct bnxt_napi *bnapi = bp->bnapi[i]; 9384 9385 __netif_napi_del(&bnapi->napi); 9386 } 9387 /* We called __netif_napi_del(), we need 9388 * to respect an RCU grace period before freeing napi structures. 9389 */ 9390 synchronize_net(); 9391 } 9392 9393 static void bnxt_init_napi(struct bnxt *bp) 9394 { 9395 int i; 9396 unsigned int cp_nr_rings = bp->cp_nr_rings; 9397 struct bnxt_napi *bnapi; 9398 9399 if (bp->flags & BNXT_FLAG_USING_MSIX) { 9400 int (*poll_fn)(struct napi_struct *, int) = bnxt_poll; 9401 9402 if (bp->flags & BNXT_FLAG_CHIP_P5) 9403 poll_fn = bnxt_poll_p5; 9404 else if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 9405 cp_nr_rings--; 9406 for (i = 0; i < cp_nr_rings; i++) { 9407 bnapi = bp->bnapi[i]; 9408 netif_napi_add(bp->dev, &bnapi->napi, poll_fn); 9409 } 9410 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 9411 bnapi = bp->bnapi[cp_nr_rings]; 9412 netif_napi_add(bp->dev, &bnapi->napi, 9413 bnxt_poll_nitroa0); 9414 } 9415 } else { 9416 bnapi = bp->bnapi[0]; 9417 netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll); 9418 } 9419 } 9420 9421 static void bnxt_disable_napi(struct bnxt *bp) 9422 { 9423 int i; 9424 9425 if (!bp->bnapi || 9426 test_and_set_bit(BNXT_STATE_NAPI_DISABLED, &bp->state)) 9427 return; 9428 9429 for (i = 0; i < bp->cp_nr_rings; i++) { 9430 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring; 9431 9432 napi_disable(&bp->bnapi[i]->napi); 9433 if (bp->bnapi[i]->rx_ring) 9434 cancel_work_sync(&cpr->dim.work); 9435 } 9436 } 9437 9438 static void bnxt_enable_napi(struct bnxt *bp) 9439 { 9440 int i; 9441 9442 clear_bit(BNXT_STATE_NAPI_DISABLED, &bp->state); 9443 for (i = 0; i < bp->cp_nr_rings; i++) { 9444 struct bnxt_napi *bnapi = bp->bnapi[i]; 9445 struct bnxt_cp_ring_info *cpr; 9446 9447 cpr = &bnapi->cp_ring; 9448 if (bnapi->in_reset) 9449 cpr->sw_stats.rx.rx_resets++; 9450 bnapi->in_reset = false; 9451 9452 if (bnapi->rx_ring) { 9453 INIT_WORK(&cpr->dim.work, bnxt_dim_work); 9454 cpr->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; 9455 } 9456 napi_enable(&bnapi->napi); 9457 } 9458 } 9459 9460 void bnxt_tx_disable(struct bnxt *bp) 9461 { 9462 int i; 9463 struct bnxt_tx_ring_info *txr; 9464 9465 if (bp->tx_ring) { 9466 for (i = 0; i < bp->tx_nr_rings; i++) { 9467 txr = &bp->tx_ring[i]; 9468 WRITE_ONCE(txr->dev_state, BNXT_DEV_STATE_CLOSING); 9469 } 9470 } 9471 /* Make sure napi polls see @dev_state change */ 9472 synchronize_net(); 9473 /* Drop carrier first to prevent TX timeout */ 9474 netif_carrier_off(bp->dev); 9475 /* Stop all TX queues */ 9476 netif_tx_disable(bp->dev); 9477 } 9478 9479 void bnxt_tx_enable(struct bnxt *bp) 9480 { 9481 int i; 9482 struct bnxt_tx_ring_info *txr; 9483 9484 for (i = 0; i < bp->tx_nr_rings; i++) { 9485 txr = &bp->tx_ring[i]; 9486 WRITE_ONCE(txr->dev_state, 0); 9487 } 9488 /* Make sure napi polls see @dev_state change */ 9489 synchronize_net(); 9490 netif_tx_wake_all_queues(bp->dev); 9491 if (BNXT_LINK_IS_UP(bp)) 9492 netif_carrier_on(bp->dev); 9493 } 9494 9495 static char *bnxt_report_fec(struct bnxt_link_info *link_info) 9496 { 9497 u8 active_fec = link_info->active_fec_sig_mode & 9498 PORT_PHY_QCFG_RESP_ACTIVE_FEC_MASK; 9499 9500 switch (active_fec) { 9501 default: 9502 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_NONE_ACTIVE: 9503 return "None"; 9504 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE74_ACTIVE: 9505 return "Clause 74 BaseR"; 9506 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE91_ACTIVE: 9507 return "Clause 91 RS(528,514)"; 9508 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_1XN_ACTIVE: 9509 return "Clause 91 RS544_1XN"; 9510 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_IEEE_ACTIVE: 9511 return "Clause 91 RS(544,514)"; 9512 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_1XN_ACTIVE: 9513 return "Clause 91 RS272_1XN"; 9514 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_IEEE_ACTIVE: 9515 return "Clause 91 RS(272,257)"; 9516 } 9517 } 9518 9519 void bnxt_report_link(struct bnxt *bp) 9520 { 9521 if (BNXT_LINK_IS_UP(bp)) { 9522 const char *signal = ""; 9523 const char *flow_ctrl; 9524 const char *duplex; 9525 u32 speed; 9526 u16 fec; 9527 9528 netif_carrier_on(bp->dev); 9529 speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed); 9530 if (speed == SPEED_UNKNOWN) { 9531 netdev_info(bp->dev, "NIC Link is Up, speed unknown\n"); 9532 return; 9533 } 9534 if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL) 9535 duplex = "full"; 9536 else 9537 duplex = "half"; 9538 if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH) 9539 flow_ctrl = "ON - receive & transmit"; 9540 else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX) 9541 flow_ctrl = "ON - transmit"; 9542 else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX) 9543 flow_ctrl = "ON - receive"; 9544 else 9545 flow_ctrl = "none"; 9546 if (bp->link_info.phy_qcfg_resp.option_flags & 9547 PORT_PHY_QCFG_RESP_OPTION_FLAGS_SIGNAL_MODE_KNOWN) { 9548 u8 sig_mode = bp->link_info.active_fec_sig_mode & 9549 PORT_PHY_QCFG_RESP_SIGNAL_MODE_MASK; 9550 switch (sig_mode) { 9551 case PORT_PHY_QCFG_RESP_SIGNAL_MODE_NRZ: 9552 signal = "(NRZ) "; 9553 break; 9554 case PORT_PHY_QCFG_RESP_SIGNAL_MODE_PAM4: 9555 signal = "(PAM4) "; 9556 break; 9557 default: 9558 break; 9559 } 9560 } 9561 netdev_info(bp->dev, "NIC Link is Up, %u Mbps %s%s duplex, Flow control: %s\n", 9562 speed, signal, duplex, flow_ctrl); 9563 if (bp->phy_flags & BNXT_PHY_FL_EEE_CAP) 9564 netdev_info(bp->dev, "EEE is %s\n", 9565 bp->eee.eee_active ? "active" : 9566 "not active"); 9567 fec = bp->link_info.fec_cfg; 9568 if (!(fec & PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED)) 9569 netdev_info(bp->dev, "FEC autoneg %s encoding: %s\n", 9570 (fec & BNXT_FEC_AUTONEG) ? "on" : "off", 9571 bnxt_report_fec(&bp->link_info)); 9572 } else { 9573 netif_carrier_off(bp->dev); 9574 netdev_err(bp->dev, "NIC Link is Down\n"); 9575 } 9576 } 9577 9578 static bool bnxt_phy_qcaps_no_speed(struct hwrm_port_phy_qcaps_output *resp) 9579 { 9580 if (!resp->supported_speeds_auto_mode && 9581 !resp->supported_speeds_force_mode && 9582 !resp->supported_pam4_speeds_auto_mode && 9583 !resp->supported_pam4_speeds_force_mode) 9584 return true; 9585 return false; 9586 } 9587 9588 static int bnxt_hwrm_phy_qcaps(struct bnxt *bp) 9589 { 9590 struct bnxt_link_info *link_info = &bp->link_info; 9591 struct hwrm_port_phy_qcaps_output *resp; 9592 struct hwrm_port_phy_qcaps_input *req; 9593 int rc = 0; 9594 9595 if (bp->hwrm_spec_code < 0x10201) 9596 return 0; 9597 9598 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_QCAPS); 9599 if (rc) 9600 return rc; 9601 9602 resp = hwrm_req_hold(bp, req); 9603 rc = hwrm_req_send(bp, req); 9604 if (rc) 9605 goto hwrm_phy_qcaps_exit; 9606 9607 bp->phy_flags = resp->flags | (le16_to_cpu(resp->flags2) << 8); 9608 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) { 9609 struct ethtool_eee *eee = &bp->eee; 9610 u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode); 9611 9612 eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0); 9613 bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) & 9614 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK; 9615 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) & 9616 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK; 9617 } 9618 9619 if (bp->hwrm_spec_code >= 0x10a01) { 9620 if (bnxt_phy_qcaps_no_speed(resp)) { 9621 link_info->phy_state = BNXT_PHY_STATE_DISABLED; 9622 netdev_warn(bp->dev, "Ethernet link disabled\n"); 9623 } else if (link_info->phy_state == BNXT_PHY_STATE_DISABLED) { 9624 link_info->phy_state = BNXT_PHY_STATE_ENABLED; 9625 netdev_info(bp->dev, "Ethernet link enabled\n"); 9626 /* Phy re-enabled, reprobe the speeds */ 9627 link_info->support_auto_speeds = 0; 9628 link_info->support_pam4_auto_speeds = 0; 9629 } 9630 } 9631 if (resp->supported_speeds_auto_mode) 9632 link_info->support_auto_speeds = 9633 le16_to_cpu(resp->supported_speeds_auto_mode); 9634 if (resp->supported_pam4_speeds_auto_mode) 9635 link_info->support_pam4_auto_speeds = 9636 le16_to_cpu(resp->supported_pam4_speeds_auto_mode); 9637 9638 bp->port_count = resp->port_cnt; 9639 9640 hwrm_phy_qcaps_exit: 9641 hwrm_req_drop(bp, req); 9642 return rc; 9643 } 9644 9645 static bool bnxt_support_dropped(u16 advertising, u16 supported) 9646 { 9647 u16 diff = advertising ^ supported; 9648 9649 return ((supported | diff) != supported); 9650 } 9651 9652 int bnxt_update_link(struct bnxt *bp, bool chng_link_state) 9653 { 9654 struct bnxt_link_info *link_info = &bp->link_info; 9655 struct hwrm_port_phy_qcfg_output *resp; 9656 struct hwrm_port_phy_qcfg_input *req; 9657 u8 link_state = link_info->link_state; 9658 bool support_changed = false; 9659 int rc; 9660 9661 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_QCFG); 9662 if (rc) 9663 return rc; 9664 9665 resp = hwrm_req_hold(bp, req); 9666 rc = hwrm_req_send(bp, req); 9667 if (rc) { 9668 hwrm_req_drop(bp, req); 9669 if (BNXT_VF(bp) && rc == -ENODEV) { 9670 netdev_warn(bp->dev, "Cannot obtain link state while PF unavailable.\n"); 9671 rc = 0; 9672 } 9673 return rc; 9674 } 9675 9676 memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp)); 9677 link_info->phy_link_status = resp->link; 9678 link_info->duplex = resp->duplex_cfg; 9679 if (bp->hwrm_spec_code >= 0x10800) 9680 link_info->duplex = resp->duplex_state; 9681 link_info->pause = resp->pause; 9682 link_info->auto_mode = resp->auto_mode; 9683 link_info->auto_pause_setting = resp->auto_pause; 9684 link_info->lp_pause = resp->link_partner_adv_pause; 9685 link_info->force_pause_setting = resp->force_pause; 9686 link_info->duplex_setting = resp->duplex_cfg; 9687 if (link_info->phy_link_status == BNXT_LINK_LINK) 9688 link_info->link_speed = le16_to_cpu(resp->link_speed); 9689 else 9690 link_info->link_speed = 0; 9691 link_info->force_link_speed = le16_to_cpu(resp->force_link_speed); 9692 link_info->force_pam4_link_speed = 9693 le16_to_cpu(resp->force_pam4_link_speed); 9694 link_info->support_speeds = le16_to_cpu(resp->support_speeds); 9695 link_info->support_pam4_speeds = le16_to_cpu(resp->support_pam4_speeds); 9696 link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask); 9697 link_info->auto_pam4_link_speeds = 9698 le16_to_cpu(resp->auto_pam4_link_speed_mask); 9699 link_info->lp_auto_link_speeds = 9700 le16_to_cpu(resp->link_partner_adv_speeds); 9701 link_info->lp_auto_pam4_link_speeds = 9702 resp->link_partner_pam4_adv_speeds; 9703 link_info->preemphasis = le32_to_cpu(resp->preemphasis); 9704 link_info->phy_ver[0] = resp->phy_maj; 9705 link_info->phy_ver[1] = resp->phy_min; 9706 link_info->phy_ver[2] = resp->phy_bld; 9707 link_info->media_type = resp->media_type; 9708 link_info->phy_type = resp->phy_type; 9709 link_info->transceiver = resp->xcvr_pkg_type; 9710 link_info->phy_addr = resp->eee_config_phy_addr & 9711 PORT_PHY_QCFG_RESP_PHY_ADDR_MASK; 9712 link_info->module_status = resp->module_status; 9713 9714 if (bp->phy_flags & BNXT_PHY_FL_EEE_CAP) { 9715 struct ethtool_eee *eee = &bp->eee; 9716 u16 fw_speeds; 9717 9718 eee->eee_active = 0; 9719 if (resp->eee_config_phy_addr & 9720 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) { 9721 eee->eee_active = 1; 9722 fw_speeds = le16_to_cpu( 9723 resp->link_partner_adv_eee_link_speed_mask); 9724 eee->lp_advertised = 9725 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0); 9726 } 9727 9728 /* Pull initial EEE config */ 9729 if (!chng_link_state) { 9730 if (resp->eee_config_phy_addr & 9731 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED) 9732 eee->eee_enabled = 1; 9733 9734 fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask); 9735 eee->advertised = 9736 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0); 9737 9738 if (resp->eee_config_phy_addr & 9739 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) { 9740 __le32 tmr; 9741 9742 eee->tx_lpi_enabled = 1; 9743 tmr = resp->xcvr_identifier_type_tx_lpi_timer; 9744 eee->tx_lpi_timer = le32_to_cpu(tmr) & 9745 PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK; 9746 } 9747 } 9748 } 9749 9750 link_info->fec_cfg = PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED; 9751 if (bp->hwrm_spec_code >= 0x10504) { 9752 link_info->fec_cfg = le16_to_cpu(resp->fec_cfg); 9753 link_info->active_fec_sig_mode = resp->active_fec_signal_mode; 9754 } 9755 /* TODO: need to add more logic to report VF link */ 9756 if (chng_link_state) { 9757 if (link_info->phy_link_status == BNXT_LINK_LINK) 9758 link_info->link_state = BNXT_LINK_STATE_UP; 9759 else 9760 link_info->link_state = BNXT_LINK_STATE_DOWN; 9761 if (link_state != link_info->link_state) 9762 bnxt_report_link(bp); 9763 } else { 9764 /* always link down if not require to update link state */ 9765 link_info->link_state = BNXT_LINK_STATE_DOWN; 9766 } 9767 hwrm_req_drop(bp, req); 9768 9769 if (!BNXT_PHY_CFG_ABLE(bp)) 9770 return 0; 9771 9772 /* Check if any advertised speeds are no longer supported. The caller 9773 * holds the link_lock mutex, so we can modify link_info settings. 9774 */ 9775 if (bnxt_support_dropped(link_info->advertising, 9776 link_info->support_auto_speeds)) { 9777 link_info->advertising = link_info->support_auto_speeds; 9778 support_changed = true; 9779 } 9780 if (bnxt_support_dropped(link_info->advertising_pam4, 9781 link_info->support_pam4_auto_speeds)) { 9782 link_info->advertising_pam4 = link_info->support_pam4_auto_speeds; 9783 support_changed = true; 9784 } 9785 if (support_changed && (link_info->autoneg & BNXT_AUTONEG_SPEED)) 9786 bnxt_hwrm_set_link_setting(bp, true, false); 9787 return 0; 9788 } 9789 9790 static void bnxt_get_port_module_status(struct bnxt *bp) 9791 { 9792 struct bnxt_link_info *link_info = &bp->link_info; 9793 struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp; 9794 u8 module_status; 9795 9796 if (bnxt_update_link(bp, true)) 9797 return; 9798 9799 module_status = link_info->module_status; 9800 switch (module_status) { 9801 case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX: 9802 case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN: 9803 case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG: 9804 netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n", 9805 bp->pf.port_id); 9806 if (bp->hwrm_spec_code >= 0x10201) { 9807 netdev_warn(bp->dev, "Module part number %s\n", 9808 resp->phy_vendor_partnumber); 9809 } 9810 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX) 9811 netdev_warn(bp->dev, "TX is disabled\n"); 9812 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN) 9813 netdev_warn(bp->dev, "SFP+ module is shutdown\n"); 9814 } 9815 } 9816 9817 static void 9818 bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req) 9819 { 9820 if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) { 9821 if (bp->hwrm_spec_code >= 0x10201) 9822 req->auto_pause = 9823 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE; 9824 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX) 9825 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX; 9826 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX) 9827 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX; 9828 req->enables |= 9829 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE); 9830 } else { 9831 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX) 9832 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX; 9833 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX) 9834 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX; 9835 req->enables |= 9836 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE); 9837 if (bp->hwrm_spec_code >= 0x10201) { 9838 req->auto_pause = req->force_pause; 9839 req->enables |= cpu_to_le32( 9840 PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE); 9841 } 9842 } 9843 } 9844 9845 static void bnxt_hwrm_set_link_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req) 9846 { 9847 if (bp->link_info.autoneg & BNXT_AUTONEG_SPEED) { 9848 req->auto_mode |= PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK; 9849 if (bp->link_info.advertising) { 9850 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK); 9851 req->auto_link_speed_mask = cpu_to_le16(bp->link_info.advertising); 9852 } 9853 if (bp->link_info.advertising_pam4) { 9854 req->enables |= 9855 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAM4_LINK_SPEED_MASK); 9856 req->auto_link_pam4_speed_mask = 9857 cpu_to_le16(bp->link_info.advertising_pam4); 9858 } 9859 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE); 9860 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG); 9861 } else { 9862 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE); 9863 if (bp->link_info.req_signal_mode == BNXT_SIG_MODE_PAM4) { 9864 req->force_pam4_link_speed = cpu_to_le16(bp->link_info.req_link_speed); 9865 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAM4_LINK_SPEED); 9866 } else { 9867 req->force_link_speed = cpu_to_le16(bp->link_info.req_link_speed); 9868 } 9869 } 9870 9871 /* tell chimp that the setting takes effect immediately */ 9872 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY); 9873 } 9874 9875 int bnxt_hwrm_set_pause(struct bnxt *bp) 9876 { 9877 struct hwrm_port_phy_cfg_input *req; 9878 int rc; 9879 9880 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG); 9881 if (rc) 9882 return rc; 9883 9884 bnxt_hwrm_set_pause_common(bp, req); 9885 9886 if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) || 9887 bp->link_info.force_link_chng) 9888 bnxt_hwrm_set_link_common(bp, req); 9889 9890 rc = hwrm_req_send(bp, req); 9891 if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) { 9892 /* since changing of pause setting doesn't trigger any link 9893 * change event, the driver needs to update the current pause 9894 * result upon successfully return of the phy_cfg command 9895 */ 9896 bp->link_info.pause = 9897 bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl; 9898 bp->link_info.auto_pause_setting = 0; 9899 if (!bp->link_info.force_link_chng) 9900 bnxt_report_link(bp); 9901 } 9902 bp->link_info.force_link_chng = false; 9903 return rc; 9904 } 9905 9906 static void bnxt_hwrm_set_eee(struct bnxt *bp, 9907 struct hwrm_port_phy_cfg_input *req) 9908 { 9909 struct ethtool_eee *eee = &bp->eee; 9910 9911 if (eee->eee_enabled) { 9912 u16 eee_speeds; 9913 u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE; 9914 9915 if (eee->tx_lpi_enabled) 9916 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE; 9917 else 9918 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE; 9919 9920 req->flags |= cpu_to_le32(flags); 9921 eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised); 9922 req->eee_link_speed_mask = cpu_to_le16(eee_speeds); 9923 req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer); 9924 } else { 9925 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE); 9926 } 9927 } 9928 9929 int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee) 9930 { 9931 struct hwrm_port_phy_cfg_input *req; 9932 int rc; 9933 9934 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG); 9935 if (rc) 9936 return rc; 9937 9938 if (set_pause) 9939 bnxt_hwrm_set_pause_common(bp, req); 9940 9941 bnxt_hwrm_set_link_common(bp, req); 9942 9943 if (set_eee) 9944 bnxt_hwrm_set_eee(bp, req); 9945 return hwrm_req_send(bp, req); 9946 } 9947 9948 static int bnxt_hwrm_shutdown_link(struct bnxt *bp) 9949 { 9950 struct hwrm_port_phy_cfg_input *req; 9951 int rc; 9952 9953 if (!BNXT_SINGLE_PF(bp)) 9954 return 0; 9955 9956 if (pci_num_vf(bp->pdev) && 9957 !(bp->phy_flags & BNXT_PHY_FL_FW_MANAGED_LKDN)) 9958 return 0; 9959 9960 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG); 9961 if (rc) 9962 return rc; 9963 9964 req->flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN); 9965 rc = hwrm_req_send(bp, req); 9966 if (!rc) { 9967 mutex_lock(&bp->link_lock); 9968 /* Device is not obliged link down in certain scenarios, even 9969 * when forced. Setting the state unknown is consistent with 9970 * driver startup and will force link state to be reported 9971 * during subsequent open based on PORT_PHY_QCFG. 9972 */ 9973 bp->link_info.link_state = BNXT_LINK_STATE_UNKNOWN; 9974 mutex_unlock(&bp->link_lock); 9975 } 9976 return rc; 9977 } 9978 9979 static int bnxt_fw_reset_via_optee(struct bnxt *bp) 9980 { 9981 #ifdef CONFIG_TEE_BNXT_FW 9982 int rc = tee_bnxt_fw_load(); 9983 9984 if (rc) 9985 netdev_err(bp->dev, "Failed FW reset via OP-TEE, rc=%d\n", rc); 9986 9987 return rc; 9988 #else 9989 netdev_err(bp->dev, "OP-TEE not supported\n"); 9990 return -ENODEV; 9991 #endif 9992 } 9993 9994 static int bnxt_try_recover_fw(struct bnxt *bp) 9995 { 9996 if (bp->fw_health && bp->fw_health->status_reliable) { 9997 int retry = 0, rc; 9998 u32 sts; 9999 10000 do { 10001 sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG); 10002 rc = bnxt_hwrm_poll(bp); 10003 if (!BNXT_FW_IS_BOOTING(sts) && 10004 !BNXT_FW_IS_RECOVERING(sts)) 10005 break; 10006 retry++; 10007 } while (rc == -EBUSY && retry < BNXT_FW_RETRY); 10008 10009 if (!BNXT_FW_IS_HEALTHY(sts)) { 10010 netdev_err(bp->dev, 10011 "Firmware not responding, status: 0x%x\n", 10012 sts); 10013 rc = -ENODEV; 10014 } 10015 if (sts & FW_STATUS_REG_CRASHED_NO_MASTER) { 10016 netdev_warn(bp->dev, "Firmware recover via OP-TEE requested\n"); 10017 return bnxt_fw_reset_via_optee(bp); 10018 } 10019 return rc; 10020 } 10021 10022 return -ENODEV; 10023 } 10024 10025 static void bnxt_clear_reservations(struct bnxt *bp, bool fw_reset) 10026 { 10027 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 10028 10029 if (!BNXT_NEW_RM(bp)) 10030 return; /* no resource reservations required */ 10031 10032 hw_resc->resv_cp_rings = 0; 10033 hw_resc->resv_stat_ctxs = 0; 10034 hw_resc->resv_irqs = 0; 10035 hw_resc->resv_tx_rings = 0; 10036 hw_resc->resv_rx_rings = 0; 10037 hw_resc->resv_hw_ring_grps = 0; 10038 hw_resc->resv_vnics = 0; 10039 if (!fw_reset) { 10040 bp->tx_nr_rings = 0; 10041 bp->rx_nr_rings = 0; 10042 } 10043 } 10044 10045 int bnxt_cancel_reservations(struct bnxt *bp, bool fw_reset) 10046 { 10047 int rc; 10048 10049 if (!BNXT_NEW_RM(bp)) 10050 return 0; /* no resource reservations required */ 10051 10052 rc = bnxt_hwrm_func_resc_qcaps(bp, true); 10053 if (rc) 10054 netdev_err(bp->dev, "resc_qcaps failed\n"); 10055 10056 bnxt_clear_reservations(bp, fw_reset); 10057 10058 return rc; 10059 } 10060 10061 static int bnxt_hwrm_if_change(struct bnxt *bp, bool up) 10062 { 10063 struct hwrm_func_drv_if_change_output *resp; 10064 struct hwrm_func_drv_if_change_input *req; 10065 bool fw_reset = !bp->irq_tbl; 10066 bool resc_reinit = false; 10067 int rc, retry = 0; 10068 u32 flags = 0; 10069 10070 if (!(bp->fw_cap & BNXT_FW_CAP_IF_CHANGE)) 10071 return 0; 10072 10073 rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_IF_CHANGE); 10074 if (rc) 10075 return rc; 10076 10077 if (up) 10078 req->flags = cpu_to_le32(FUNC_DRV_IF_CHANGE_REQ_FLAGS_UP); 10079 resp = hwrm_req_hold(bp, req); 10080 10081 hwrm_req_flags(bp, req, BNXT_HWRM_FULL_WAIT); 10082 while (retry < BNXT_FW_IF_RETRY) { 10083 rc = hwrm_req_send(bp, req); 10084 if (rc != -EAGAIN) 10085 break; 10086 10087 msleep(50); 10088 retry++; 10089 } 10090 10091 if (rc == -EAGAIN) { 10092 hwrm_req_drop(bp, req); 10093 return rc; 10094 } else if (!rc) { 10095 flags = le32_to_cpu(resp->flags); 10096 } else if (up) { 10097 rc = bnxt_try_recover_fw(bp); 10098 fw_reset = true; 10099 } 10100 hwrm_req_drop(bp, req); 10101 if (rc) 10102 return rc; 10103 10104 if (!up) { 10105 bnxt_inv_fw_health_reg(bp); 10106 return 0; 10107 } 10108 10109 if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_RESC_CHANGE) 10110 resc_reinit = true; 10111 if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_HOT_FW_RESET_DONE || 10112 test_bit(BNXT_STATE_FW_RESET_DET, &bp->state)) 10113 fw_reset = true; 10114 else 10115 bnxt_remap_fw_health_regs(bp); 10116 10117 if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state) && !fw_reset) { 10118 netdev_err(bp->dev, "RESET_DONE not set during FW reset.\n"); 10119 set_bit(BNXT_STATE_ABORT_ERR, &bp->state); 10120 return -ENODEV; 10121 } 10122 if (resc_reinit || fw_reset) { 10123 if (fw_reset) { 10124 set_bit(BNXT_STATE_FW_RESET_DET, &bp->state); 10125 if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 10126 bnxt_ulp_stop(bp); 10127 bnxt_free_ctx_mem(bp); 10128 kfree(bp->ctx); 10129 bp->ctx = NULL; 10130 bnxt_dcb_free(bp); 10131 rc = bnxt_fw_init_one(bp); 10132 if (rc) { 10133 clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state); 10134 set_bit(BNXT_STATE_ABORT_ERR, &bp->state); 10135 return rc; 10136 } 10137 bnxt_clear_int_mode(bp); 10138 rc = bnxt_init_int_mode(bp); 10139 if (rc) { 10140 clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state); 10141 netdev_err(bp->dev, "init int mode failed\n"); 10142 return rc; 10143 } 10144 } 10145 rc = bnxt_cancel_reservations(bp, fw_reset); 10146 } 10147 return rc; 10148 } 10149 10150 static int bnxt_hwrm_port_led_qcaps(struct bnxt *bp) 10151 { 10152 struct hwrm_port_led_qcaps_output *resp; 10153 struct hwrm_port_led_qcaps_input *req; 10154 struct bnxt_pf_info *pf = &bp->pf; 10155 int rc; 10156 10157 bp->num_leds = 0; 10158 if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10601) 10159 return 0; 10160 10161 rc = hwrm_req_init(bp, req, HWRM_PORT_LED_QCAPS); 10162 if (rc) 10163 return rc; 10164 10165 req->port_id = cpu_to_le16(pf->port_id); 10166 resp = hwrm_req_hold(bp, req); 10167 rc = hwrm_req_send(bp, req); 10168 if (rc) { 10169 hwrm_req_drop(bp, req); 10170 return rc; 10171 } 10172 if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) { 10173 int i; 10174 10175 bp->num_leds = resp->num_leds; 10176 memcpy(bp->leds, &resp->led0_id, sizeof(bp->leds[0]) * 10177 bp->num_leds); 10178 for (i = 0; i < bp->num_leds; i++) { 10179 struct bnxt_led_info *led = &bp->leds[i]; 10180 __le16 caps = led->led_state_caps; 10181 10182 if (!led->led_group_id || 10183 !BNXT_LED_ALT_BLINK_CAP(caps)) { 10184 bp->num_leds = 0; 10185 break; 10186 } 10187 } 10188 } 10189 hwrm_req_drop(bp, req); 10190 return 0; 10191 } 10192 10193 int bnxt_hwrm_alloc_wol_fltr(struct bnxt *bp) 10194 { 10195 struct hwrm_wol_filter_alloc_output *resp; 10196 struct hwrm_wol_filter_alloc_input *req; 10197 int rc; 10198 10199 rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_ALLOC); 10200 if (rc) 10201 return rc; 10202 10203 req->port_id = cpu_to_le16(bp->pf.port_id); 10204 req->wol_type = WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT; 10205 req->enables = cpu_to_le32(WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS); 10206 memcpy(req->mac_address, bp->dev->dev_addr, ETH_ALEN); 10207 10208 resp = hwrm_req_hold(bp, req); 10209 rc = hwrm_req_send(bp, req); 10210 if (!rc) 10211 bp->wol_filter_id = resp->wol_filter_id; 10212 hwrm_req_drop(bp, req); 10213 return rc; 10214 } 10215 10216 int bnxt_hwrm_free_wol_fltr(struct bnxt *bp) 10217 { 10218 struct hwrm_wol_filter_free_input *req; 10219 int rc; 10220 10221 rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_FREE); 10222 if (rc) 10223 return rc; 10224 10225 req->port_id = cpu_to_le16(bp->pf.port_id); 10226 req->enables = cpu_to_le32(WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID); 10227 req->wol_filter_id = bp->wol_filter_id; 10228 10229 return hwrm_req_send(bp, req); 10230 } 10231 10232 static u16 bnxt_hwrm_get_wol_fltrs(struct bnxt *bp, u16 handle) 10233 { 10234 struct hwrm_wol_filter_qcfg_output *resp; 10235 struct hwrm_wol_filter_qcfg_input *req; 10236 u16 next_handle = 0; 10237 int rc; 10238 10239 rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_QCFG); 10240 if (rc) 10241 return rc; 10242 10243 req->port_id = cpu_to_le16(bp->pf.port_id); 10244 req->handle = cpu_to_le16(handle); 10245 resp = hwrm_req_hold(bp, req); 10246 rc = hwrm_req_send(bp, req); 10247 if (!rc) { 10248 next_handle = le16_to_cpu(resp->next_handle); 10249 if (next_handle != 0) { 10250 if (resp->wol_type == 10251 WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT) { 10252 bp->wol = 1; 10253 bp->wol_filter_id = resp->wol_filter_id; 10254 } 10255 } 10256 } 10257 hwrm_req_drop(bp, req); 10258 return next_handle; 10259 } 10260 10261 static void bnxt_get_wol_settings(struct bnxt *bp) 10262 { 10263 u16 handle = 0; 10264 10265 bp->wol = 0; 10266 if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_WOL_CAP)) 10267 return; 10268 10269 do { 10270 handle = bnxt_hwrm_get_wol_fltrs(bp, handle); 10271 } while (handle && handle != 0xffff); 10272 } 10273 10274 #ifdef CONFIG_BNXT_HWMON 10275 static ssize_t bnxt_show_temp(struct device *dev, 10276 struct device_attribute *devattr, char *buf) 10277 { 10278 struct hwrm_temp_monitor_query_output *resp; 10279 struct hwrm_temp_monitor_query_input *req; 10280 struct bnxt *bp = dev_get_drvdata(dev); 10281 u32 len = 0; 10282 int rc; 10283 10284 rc = hwrm_req_init(bp, req, HWRM_TEMP_MONITOR_QUERY); 10285 if (rc) 10286 return rc; 10287 resp = hwrm_req_hold(bp, req); 10288 rc = hwrm_req_send(bp, req); 10289 if (!rc) 10290 len = sprintf(buf, "%u\n", resp->temp * 1000); /* display millidegree */ 10291 hwrm_req_drop(bp, req); 10292 if (rc) 10293 return rc; 10294 return len; 10295 } 10296 static SENSOR_DEVICE_ATTR(temp1_input, 0444, bnxt_show_temp, NULL, 0); 10297 10298 static struct attribute *bnxt_attrs[] = { 10299 &sensor_dev_attr_temp1_input.dev_attr.attr, 10300 NULL 10301 }; 10302 ATTRIBUTE_GROUPS(bnxt); 10303 10304 static void bnxt_hwmon_close(struct bnxt *bp) 10305 { 10306 if (bp->hwmon_dev) { 10307 hwmon_device_unregister(bp->hwmon_dev); 10308 bp->hwmon_dev = NULL; 10309 } 10310 } 10311 10312 static void bnxt_hwmon_open(struct bnxt *bp) 10313 { 10314 struct hwrm_temp_monitor_query_input *req; 10315 struct pci_dev *pdev = bp->pdev; 10316 int rc; 10317 10318 rc = hwrm_req_init(bp, req, HWRM_TEMP_MONITOR_QUERY); 10319 if (!rc) 10320 rc = hwrm_req_send_silent(bp, req); 10321 if (rc == -EACCES || rc == -EOPNOTSUPP) { 10322 bnxt_hwmon_close(bp); 10323 return; 10324 } 10325 10326 if (bp->hwmon_dev) 10327 return; 10328 10329 bp->hwmon_dev = hwmon_device_register_with_groups(&pdev->dev, 10330 DRV_MODULE_NAME, bp, 10331 bnxt_groups); 10332 if (IS_ERR(bp->hwmon_dev)) { 10333 bp->hwmon_dev = NULL; 10334 dev_warn(&pdev->dev, "Cannot register hwmon device\n"); 10335 } 10336 } 10337 #else 10338 static void bnxt_hwmon_close(struct bnxt *bp) 10339 { 10340 } 10341 10342 static void bnxt_hwmon_open(struct bnxt *bp) 10343 { 10344 } 10345 #endif 10346 10347 static bool bnxt_eee_config_ok(struct bnxt *bp) 10348 { 10349 struct ethtool_eee *eee = &bp->eee; 10350 struct bnxt_link_info *link_info = &bp->link_info; 10351 10352 if (!(bp->phy_flags & BNXT_PHY_FL_EEE_CAP)) 10353 return true; 10354 10355 if (eee->eee_enabled) { 10356 u32 advertising = 10357 _bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0); 10358 10359 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) { 10360 eee->eee_enabled = 0; 10361 return false; 10362 } 10363 if (eee->advertised & ~advertising) { 10364 eee->advertised = advertising & eee->supported; 10365 return false; 10366 } 10367 } 10368 return true; 10369 } 10370 10371 static int bnxt_update_phy_setting(struct bnxt *bp) 10372 { 10373 int rc; 10374 bool update_link = false; 10375 bool update_pause = false; 10376 bool update_eee = false; 10377 struct bnxt_link_info *link_info = &bp->link_info; 10378 10379 rc = bnxt_update_link(bp, true); 10380 if (rc) { 10381 netdev_err(bp->dev, "failed to update link (rc: %x)\n", 10382 rc); 10383 return rc; 10384 } 10385 if (!BNXT_SINGLE_PF(bp)) 10386 return 0; 10387 10388 if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) && 10389 (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) != 10390 link_info->req_flow_ctrl) 10391 update_pause = true; 10392 if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) && 10393 link_info->force_pause_setting != link_info->req_flow_ctrl) 10394 update_pause = true; 10395 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) { 10396 if (BNXT_AUTO_MODE(link_info->auto_mode)) 10397 update_link = true; 10398 if (link_info->req_signal_mode == BNXT_SIG_MODE_NRZ && 10399 link_info->req_link_speed != link_info->force_link_speed) 10400 update_link = true; 10401 else if (link_info->req_signal_mode == BNXT_SIG_MODE_PAM4 && 10402 link_info->req_link_speed != link_info->force_pam4_link_speed) 10403 update_link = true; 10404 if (link_info->req_duplex != link_info->duplex_setting) 10405 update_link = true; 10406 } else { 10407 if (link_info->auto_mode == BNXT_LINK_AUTO_NONE) 10408 update_link = true; 10409 if (link_info->advertising != link_info->auto_link_speeds || 10410 link_info->advertising_pam4 != link_info->auto_pam4_link_speeds) 10411 update_link = true; 10412 } 10413 10414 /* The last close may have shutdown the link, so need to call 10415 * PHY_CFG to bring it back up. 10416 */ 10417 if (!BNXT_LINK_IS_UP(bp)) 10418 update_link = true; 10419 10420 if (!bnxt_eee_config_ok(bp)) 10421 update_eee = true; 10422 10423 if (update_link) 10424 rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee); 10425 else if (update_pause) 10426 rc = bnxt_hwrm_set_pause(bp); 10427 if (rc) { 10428 netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n", 10429 rc); 10430 return rc; 10431 } 10432 10433 return rc; 10434 } 10435 10436 /* Common routine to pre-map certain register block to different GRC window. 10437 * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows 10438 * in PF and 3 windows in VF that can be customized to map in different 10439 * register blocks. 10440 */ 10441 static void bnxt_preset_reg_win(struct bnxt *bp) 10442 { 10443 if (BNXT_PF(bp)) { 10444 /* CAG registers map to GRC window #4 */ 10445 writel(BNXT_CAG_REG_BASE, 10446 bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12); 10447 } 10448 } 10449 10450 static int bnxt_init_dflt_ring_mode(struct bnxt *bp); 10451 10452 static int bnxt_reinit_after_abort(struct bnxt *bp) 10453 { 10454 int rc; 10455 10456 if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 10457 return -EBUSY; 10458 10459 if (bp->dev->reg_state == NETREG_UNREGISTERED) 10460 return -ENODEV; 10461 10462 rc = bnxt_fw_init_one(bp); 10463 if (!rc) { 10464 bnxt_clear_int_mode(bp); 10465 rc = bnxt_init_int_mode(bp); 10466 if (!rc) { 10467 clear_bit(BNXT_STATE_ABORT_ERR, &bp->state); 10468 set_bit(BNXT_STATE_FW_RESET_DET, &bp->state); 10469 } 10470 } 10471 return rc; 10472 } 10473 10474 static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) 10475 { 10476 int rc = 0; 10477 10478 bnxt_preset_reg_win(bp); 10479 netif_carrier_off(bp->dev); 10480 if (irq_re_init) { 10481 /* Reserve rings now if none were reserved at driver probe. */ 10482 rc = bnxt_init_dflt_ring_mode(bp); 10483 if (rc) { 10484 netdev_err(bp->dev, "Failed to reserve default rings at open\n"); 10485 return rc; 10486 } 10487 } 10488 rc = bnxt_reserve_rings(bp, irq_re_init); 10489 if (rc) 10490 return rc; 10491 if ((bp->flags & BNXT_FLAG_RFS) && 10492 !(bp->flags & BNXT_FLAG_USING_MSIX)) { 10493 /* disable RFS if falling back to INTA */ 10494 bp->dev->hw_features &= ~NETIF_F_NTUPLE; 10495 bp->flags &= ~BNXT_FLAG_RFS; 10496 } 10497 10498 rc = bnxt_alloc_mem(bp, irq_re_init); 10499 if (rc) { 10500 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc); 10501 goto open_err_free_mem; 10502 } 10503 10504 if (irq_re_init) { 10505 bnxt_init_napi(bp); 10506 rc = bnxt_request_irq(bp); 10507 if (rc) { 10508 netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc); 10509 goto open_err_irq; 10510 } 10511 } 10512 10513 rc = bnxt_init_nic(bp, irq_re_init); 10514 if (rc) { 10515 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc); 10516 goto open_err_irq; 10517 } 10518 10519 bnxt_enable_napi(bp); 10520 bnxt_debug_dev_init(bp); 10521 10522 if (link_re_init) { 10523 mutex_lock(&bp->link_lock); 10524 rc = bnxt_update_phy_setting(bp); 10525 mutex_unlock(&bp->link_lock); 10526 if (rc) { 10527 netdev_warn(bp->dev, "failed to update phy settings\n"); 10528 if (BNXT_SINGLE_PF(bp)) { 10529 bp->link_info.phy_retry = true; 10530 bp->link_info.phy_retry_expires = 10531 jiffies + 5 * HZ; 10532 } 10533 } 10534 } 10535 10536 if (irq_re_init) 10537 udp_tunnel_nic_reset_ntf(bp->dev); 10538 10539 if (bp->tx_nr_rings_xdp < num_possible_cpus()) { 10540 if (!static_key_enabled(&bnxt_xdp_locking_key)) 10541 static_branch_enable(&bnxt_xdp_locking_key); 10542 } else if (static_key_enabled(&bnxt_xdp_locking_key)) { 10543 static_branch_disable(&bnxt_xdp_locking_key); 10544 } 10545 set_bit(BNXT_STATE_OPEN, &bp->state); 10546 bnxt_enable_int(bp); 10547 /* Enable TX queues */ 10548 bnxt_tx_enable(bp); 10549 mod_timer(&bp->timer, jiffies + bp->current_interval); 10550 /* Poll link status and check for SFP+ module status */ 10551 mutex_lock(&bp->link_lock); 10552 bnxt_get_port_module_status(bp); 10553 mutex_unlock(&bp->link_lock); 10554 10555 /* VF-reps may need to be re-opened after the PF is re-opened */ 10556 if (BNXT_PF(bp)) 10557 bnxt_vf_reps_open(bp); 10558 bnxt_ptp_init_rtc(bp, true); 10559 bnxt_ptp_cfg_tstamp_filters(bp); 10560 return 0; 10561 10562 open_err_irq: 10563 bnxt_del_napi(bp); 10564 10565 open_err_free_mem: 10566 bnxt_free_skbs(bp); 10567 bnxt_free_irq(bp); 10568 bnxt_free_mem(bp, true); 10569 return rc; 10570 } 10571 10572 /* rtnl_lock held */ 10573 int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) 10574 { 10575 int rc = 0; 10576 10577 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) 10578 rc = -EIO; 10579 if (!rc) 10580 rc = __bnxt_open_nic(bp, irq_re_init, link_re_init); 10581 if (rc) { 10582 netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc); 10583 dev_close(bp->dev); 10584 } 10585 return rc; 10586 } 10587 10588 /* rtnl_lock held, open the NIC half way by allocating all resources, but 10589 * NAPI, IRQ, and TX are not enabled. This is mainly used for offline 10590 * self tests. 10591 */ 10592 int bnxt_half_open_nic(struct bnxt *bp) 10593 { 10594 int rc = 0; 10595 10596 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) { 10597 netdev_err(bp->dev, "A previous firmware reset has not completed, aborting half open\n"); 10598 rc = -ENODEV; 10599 goto half_open_err; 10600 } 10601 10602 rc = bnxt_alloc_mem(bp, true); 10603 if (rc) { 10604 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc); 10605 goto half_open_err; 10606 } 10607 set_bit(BNXT_STATE_HALF_OPEN, &bp->state); 10608 rc = bnxt_init_nic(bp, true); 10609 if (rc) { 10610 clear_bit(BNXT_STATE_HALF_OPEN, &bp->state); 10611 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc); 10612 goto half_open_err; 10613 } 10614 return 0; 10615 10616 half_open_err: 10617 bnxt_free_skbs(bp); 10618 bnxt_free_mem(bp, true); 10619 dev_close(bp->dev); 10620 return rc; 10621 } 10622 10623 /* rtnl_lock held, this call can only be made after a previous successful 10624 * call to bnxt_half_open_nic(). 10625 */ 10626 void bnxt_half_close_nic(struct bnxt *bp) 10627 { 10628 bnxt_hwrm_resource_free(bp, false, true); 10629 bnxt_free_skbs(bp); 10630 bnxt_free_mem(bp, true); 10631 clear_bit(BNXT_STATE_HALF_OPEN, &bp->state); 10632 } 10633 10634 void bnxt_reenable_sriov(struct bnxt *bp) 10635 { 10636 if (BNXT_PF(bp)) { 10637 struct bnxt_pf_info *pf = &bp->pf; 10638 int n = pf->active_vfs; 10639 10640 if (n) 10641 bnxt_cfg_hw_sriov(bp, &n, true); 10642 } 10643 } 10644 10645 static int bnxt_open(struct net_device *dev) 10646 { 10647 struct bnxt *bp = netdev_priv(dev); 10648 int rc; 10649 10650 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) { 10651 rc = bnxt_reinit_after_abort(bp); 10652 if (rc) { 10653 if (rc == -EBUSY) 10654 netdev_err(bp->dev, "A previous firmware reset has not completed, aborting\n"); 10655 else 10656 netdev_err(bp->dev, "Failed to reinitialize after aborted firmware reset\n"); 10657 return -ENODEV; 10658 } 10659 } 10660 10661 rc = bnxt_hwrm_if_change(bp, true); 10662 if (rc) 10663 return rc; 10664 10665 rc = __bnxt_open_nic(bp, true, true); 10666 if (rc) { 10667 bnxt_hwrm_if_change(bp, false); 10668 } else { 10669 if (test_and_clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state)) { 10670 if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) { 10671 bnxt_ulp_start(bp, 0); 10672 bnxt_reenable_sriov(bp); 10673 } 10674 } 10675 bnxt_hwmon_open(bp); 10676 } 10677 10678 return rc; 10679 } 10680 10681 static bool bnxt_drv_busy(struct bnxt *bp) 10682 { 10683 return (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state) || 10684 test_bit(BNXT_STATE_READ_STATS, &bp->state)); 10685 } 10686 10687 static void bnxt_get_ring_stats(struct bnxt *bp, 10688 struct rtnl_link_stats64 *stats); 10689 10690 static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init, 10691 bool link_re_init) 10692 { 10693 /* Close the VF-reps before closing PF */ 10694 if (BNXT_PF(bp)) 10695 bnxt_vf_reps_close(bp); 10696 10697 /* Change device state to avoid TX queue wake up's */ 10698 bnxt_tx_disable(bp); 10699 10700 clear_bit(BNXT_STATE_OPEN, &bp->state); 10701 smp_mb__after_atomic(); 10702 while (bnxt_drv_busy(bp)) 10703 msleep(20); 10704 10705 /* Flush rings and disable interrupts */ 10706 bnxt_shutdown_nic(bp, irq_re_init); 10707 10708 /* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */ 10709 10710 bnxt_debug_dev_exit(bp); 10711 bnxt_disable_napi(bp); 10712 del_timer_sync(&bp->timer); 10713 bnxt_free_skbs(bp); 10714 10715 /* Save ring stats before shutdown */ 10716 if (bp->bnapi && irq_re_init) 10717 bnxt_get_ring_stats(bp, &bp->net_stats_prev); 10718 if (irq_re_init) { 10719 bnxt_free_irq(bp); 10720 bnxt_del_napi(bp); 10721 } 10722 bnxt_free_mem(bp, irq_re_init); 10723 } 10724 10725 int bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) 10726 { 10727 int rc = 0; 10728 10729 if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) { 10730 /* If we get here, it means firmware reset is in progress 10731 * while we are trying to close. We can safely proceed with 10732 * the close because we are holding rtnl_lock(). Some firmware 10733 * messages may fail as we proceed to close. We set the 10734 * ABORT_ERR flag here so that the FW reset thread will later 10735 * abort when it gets the rtnl_lock() and sees the flag. 10736 */ 10737 netdev_warn(bp->dev, "FW reset in progress during close, FW reset will be aborted\n"); 10738 set_bit(BNXT_STATE_ABORT_ERR, &bp->state); 10739 } 10740 10741 #ifdef CONFIG_BNXT_SRIOV 10742 if (bp->sriov_cfg) { 10743 rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait, 10744 !bp->sriov_cfg, 10745 BNXT_SRIOV_CFG_WAIT_TMO); 10746 if (rc) 10747 netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete!\n"); 10748 } 10749 #endif 10750 __bnxt_close_nic(bp, irq_re_init, link_re_init); 10751 return rc; 10752 } 10753 10754 static int bnxt_close(struct net_device *dev) 10755 { 10756 struct bnxt *bp = netdev_priv(dev); 10757 10758 bnxt_hwmon_close(bp); 10759 bnxt_close_nic(bp, true, true); 10760 bnxt_hwrm_shutdown_link(bp); 10761 bnxt_hwrm_if_change(bp, false); 10762 return 0; 10763 } 10764 10765 static int bnxt_hwrm_port_phy_read(struct bnxt *bp, u16 phy_addr, u16 reg, 10766 u16 *val) 10767 { 10768 struct hwrm_port_phy_mdio_read_output *resp; 10769 struct hwrm_port_phy_mdio_read_input *req; 10770 int rc; 10771 10772 if (bp->hwrm_spec_code < 0x10a00) 10773 return -EOPNOTSUPP; 10774 10775 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_MDIO_READ); 10776 if (rc) 10777 return rc; 10778 10779 req->port_id = cpu_to_le16(bp->pf.port_id); 10780 req->phy_addr = phy_addr; 10781 req->reg_addr = cpu_to_le16(reg & 0x1f); 10782 if (mdio_phy_id_is_c45(phy_addr)) { 10783 req->cl45_mdio = 1; 10784 req->phy_addr = mdio_phy_id_prtad(phy_addr); 10785 req->dev_addr = mdio_phy_id_devad(phy_addr); 10786 req->reg_addr = cpu_to_le16(reg); 10787 } 10788 10789 resp = hwrm_req_hold(bp, req); 10790 rc = hwrm_req_send(bp, req); 10791 if (!rc) 10792 *val = le16_to_cpu(resp->reg_data); 10793 hwrm_req_drop(bp, req); 10794 return rc; 10795 } 10796 10797 static int bnxt_hwrm_port_phy_write(struct bnxt *bp, u16 phy_addr, u16 reg, 10798 u16 val) 10799 { 10800 struct hwrm_port_phy_mdio_write_input *req; 10801 int rc; 10802 10803 if (bp->hwrm_spec_code < 0x10a00) 10804 return -EOPNOTSUPP; 10805 10806 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_MDIO_WRITE); 10807 if (rc) 10808 return rc; 10809 10810 req->port_id = cpu_to_le16(bp->pf.port_id); 10811 req->phy_addr = phy_addr; 10812 req->reg_addr = cpu_to_le16(reg & 0x1f); 10813 if (mdio_phy_id_is_c45(phy_addr)) { 10814 req->cl45_mdio = 1; 10815 req->phy_addr = mdio_phy_id_prtad(phy_addr); 10816 req->dev_addr = mdio_phy_id_devad(phy_addr); 10817 req->reg_addr = cpu_to_le16(reg); 10818 } 10819 req->reg_data = cpu_to_le16(val); 10820 10821 return hwrm_req_send(bp, req); 10822 } 10823 10824 /* rtnl_lock held */ 10825 static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) 10826 { 10827 struct mii_ioctl_data *mdio = if_mii(ifr); 10828 struct bnxt *bp = netdev_priv(dev); 10829 int rc; 10830 10831 switch (cmd) { 10832 case SIOCGMIIPHY: 10833 mdio->phy_id = bp->link_info.phy_addr; 10834 10835 fallthrough; 10836 case SIOCGMIIREG: { 10837 u16 mii_regval = 0; 10838 10839 if (!netif_running(dev)) 10840 return -EAGAIN; 10841 10842 rc = bnxt_hwrm_port_phy_read(bp, mdio->phy_id, mdio->reg_num, 10843 &mii_regval); 10844 mdio->val_out = mii_regval; 10845 return rc; 10846 } 10847 10848 case SIOCSMIIREG: 10849 if (!netif_running(dev)) 10850 return -EAGAIN; 10851 10852 return bnxt_hwrm_port_phy_write(bp, mdio->phy_id, mdio->reg_num, 10853 mdio->val_in); 10854 10855 case SIOCSHWTSTAMP: 10856 return bnxt_hwtstamp_set(dev, ifr); 10857 10858 case SIOCGHWTSTAMP: 10859 return bnxt_hwtstamp_get(dev, ifr); 10860 10861 default: 10862 /* do nothing */ 10863 break; 10864 } 10865 return -EOPNOTSUPP; 10866 } 10867 10868 static void bnxt_get_ring_stats(struct bnxt *bp, 10869 struct rtnl_link_stats64 *stats) 10870 { 10871 int i; 10872 10873 for (i = 0; i < bp->cp_nr_rings; i++) { 10874 struct bnxt_napi *bnapi = bp->bnapi[i]; 10875 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 10876 u64 *sw = cpr->stats.sw_stats; 10877 10878 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_ucast_pkts); 10879 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts); 10880 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_bcast_pkts); 10881 10882 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_ucast_pkts); 10883 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_mcast_pkts); 10884 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_bcast_pkts); 10885 10886 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_ucast_bytes); 10887 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_mcast_bytes); 10888 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_bcast_bytes); 10889 10890 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_ucast_bytes); 10891 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_mcast_bytes); 10892 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_bcast_bytes); 10893 10894 stats->rx_missed_errors += 10895 BNXT_GET_RING_STATS64(sw, rx_discard_pkts); 10896 10897 stats->multicast += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts); 10898 10899 stats->tx_dropped += BNXT_GET_RING_STATS64(sw, tx_error_pkts); 10900 10901 stats->rx_dropped += 10902 cpr->sw_stats.rx.rx_netpoll_discards + 10903 cpr->sw_stats.rx.rx_oom_discards; 10904 } 10905 } 10906 10907 static void bnxt_add_prev_stats(struct bnxt *bp, 10908 struct rtnl_link_stats64 *stats) 10909 { 10910 struct rtnl_link_stats64 *prev_stats = &bp->net_stats_prev; 10911 10912 stats->rx_packets += prev_stats->rx_packets; 10913 stats->tx_packets += prev_stats->tx_packets; 10914 stats->rx_bytes += prev_stats->rx_bytes; 10915 stats->tx_bytes += prev_stats->tx_bytes; 10916 stats->rx_missed_errors += prev_stats->rx_missed_errors; 10917 stats->multicast += prev_stats->multicast; 10918 stats->rx_dropped += prev_stats->rx_dropped; 10919 stats->tx_dropped += prev_stats->tx_dropped; 10920 } 10921 10922 static void 10923 bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) 10924 { 10925 struct bnxt *bp = netdev_priv(dev); 10926 10927 set_bit(BNXT_STATE_READ_STATS, &bp->state); 10928 /* Make sure bnxt_close_nic() sees that we are reading stats before 10929 * we check the BNXT_STATE_OPEN flag. 10930 */ 10931 smp_mb__after_atomic(); 10932 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 10933 clear_bit(BNXT_STATE_READ_STATS, &bp->state); 10934 *stats = bp->net_stats_prev; 10935 return; 10936 } 10937 10938 bnxt_get_ring_stats(bp, stats); 10939 bnxt_add_prev_stats(bp, stats); 10940 10941 if (bp->flags & BNXT_FLAG_PORT_STATS) { 10942 u64 *rx = bp->port_stats.sw_stats; 10943 u64 *tx = bp->port_stats.sw_stats + 10944 BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 10945 10946 stats->rx_crc_errors = 10947 BNXT_GET_RX_PORT_STATS64(rx, rx_fcs_err_frames); 10948 stats->rx_frame_errors = 10949 BNXT_GET_RX_PORT_STATS64(rx, rx_align_err_frames); 10950 stats->rx_length_errors = 10951 BNXT_GET_RX_PORT_STATS64(rx, rx_undrsz_frames) + 10952 BNXT_GET_RX_PORT_STATS64(rx, rx_ovrsz_frames) + 10953 BNXT_GET_RX_PORT_STATS64(rx, rx_runt_frames); 10954 stats->rx_errors = 10955 BNXT_GET_RX_PORT_STATS64(rx, rx_false_carrier_frames) + 10956 BNXT_GET_RX_PORT_STATS64(rx, rx_jbr_frames); 10957 stats->collisions = 10958 BNXT_GET_TX_PORT_STATS64(tx, tx_total_collisions); 10959 stats->tx_fifo_errors = 10960 BNXT_GET_TX_PORT_STATS64(tx, tx_fifo_underruns); 10961 stats->tx_errors = BNXT_GET_TX_PORT_STATS64(tx, tx_err); 10962 } 10963 clear_bit(BNXT_STATE_READ_STATS, &bp->state); 10964 } 10965 10966 static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask) 10967 { 10968 struct net_device *dev = bp->dev; 10969 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 10970 struct netdev_hw_addr *ha; 10971 u8 *haddr; 10972 int mc_count = 0; 10973 bool update = false; 10974 int off = 0; 10975 10976 netdev_for_each_mc_addr(ha, dev) { 10977 if (mc_count >= BNXT_MAX_MC_ADDRS) { 10978 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 10979 vnic->mc_list_count = 0; 10980 return false; 10981 } 10982 haddr = ha->addr; 10983 if (!ether_addr_equal(haddr, vnic->mc_list + off)) { 10984 memcpy(vnic->mc_list + off, haddr, ETH_ALEN); 10985 update = true; 10986 } 10987 off += ETH_ALEN; 10988 mc_count++; 10989 } 10990 if (mc_count) 10991 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST; 10992 10993 if (mc_count != vnic->mc_list_count) { 10994 vnic->mc_list_count = mc_count; 10995 update = true; 10996 } 10997 return update; 10998 } 10999 11000 static bool bnxt_uc_list_updated(struct bnxt *bp) 11001 { 11002 struct net_device *dev = bp->dev; 11003 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 11004 struct netdev_hw_addr *ha; 11005 int off = 0; 11006 11007 if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1)) 11008 return true; 11009 11010 netdev_for_each_uc_addr(ha, dev) { 11011 if (!ether_addr_equal(ha->addr, vnic->uc_list + off)) 11012 return true; 11013 11014 off += ETH_ALEN; 11015 } 11016 return false; 11017 } 11018 11019 static void bnxt_set_rx_mode(struct net_device *dev) 11020 { 11021 struct bnxt *bp = netdev_priv(dev); 11022 struct bnxt_vnic_info *vnic; 11023 bool mc_update = false; 11024 bool uc_update; 11025 u32 mask; 11026 11027 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) 11028 return; 11029 11030 vnic = &bp->vnic_info[0]; 11031 mask = vnic->rx_mask; 11032 mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS | 11033 CFA_L2_SET_RX_MASK_REQ_MASK_MCAST | 11034 CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST | 11035 CFA_L2_SET_RX_MASK_REQ_MASK_BCAST); 11036 11037 if (dev->flags & IFF_PROMISC) 11038 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 11039 11040 uc_update = bnxt_uc_list_updated(bp); 11041 11042 if (dev->flags & IFF_BROADCAST) 11043 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST; 11044 if (dev->flags & IFF_ALLMULTI) { 11045 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 11046 vnic->mc_list_count = 0; 11047 } else if (dev->flags & IFF_MULTICAST) { 11048 mc_update = bnxt_mc_list_updated(bp, &mask); 11049 } 11050 11051 if (mask != vnic->rx_mask || uc_update || mc_update) { 11052 vnic->rx_mask = mask; 11053 11054 set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event); 11055 bnxt_queue_sp_work(bp); 11056 } 11057 } 11058 11059 static int bnxt_cfg_rx_mode(struct bnxt *bp) 11060 { 11061 struct net_device *dev = bp->dev; 11062 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 11063 struct hwrm_cfa_l2_filter_free_input *req; 11064 struct netdev_hw_addr *ha; 11065 int i, off = 0, rc; 11066 bool uc_update; 11067 11068 netif_addr_lock_bh(dev); 11069 uc_update = bnxt_uc_list_updated(bp); 11070 netif_addr_unlock_bh(dev); 11071 11072 if (!uc_update) 11073 goto skip_uc; 11074 11075 rc = hwrm_req_init(bp, req, HWRM_CFA_L2_FILTER_FREE); 11076 if (rc) 11077 return rc; 11078 hwrm_req_hold(bp, req); 11079 for (i = 1; i < vnic->uc_filter_count; i++) { 11080 req->l2_filter_id = vnic->fw_l2_filter_id[i]; 11081 11082 rc = hwrm_req_send(bp, req); 11083 } 11084 hwrm_req_drop(bp, req); 11085 11086 vnic->uc_filter_count = 1; 11087 11088 netif_addr_lock_bh(dev); 11089 if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) { 11090 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 11091 } else { 11092 netdev_for_each_uc_addr(ha, dev) { 11093 memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN); 11094 off += ETH_ALEN; 11095 vnic->uc_filter_count++; 11096 } 11097 } 11098 netif_addr_unlock_bh(dev); 11099 11100 for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) { 11101 rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off); 11102 if (rc) { 11103 if (BNXT_VF(bp) && rc == -ENODEV) { 11104 if (!test_and_set_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state)) 11105 netdev_warn(bp->dev, "Cannot configure L2 filters while PF is unavailable, will retry\n"); 11106 else 11107 netdev_dbg(bp->dev, "PF still unavailable while configuring L2 filters.\n"); 11108 rc = 0; 11109 } else { 11110 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc); 11111 } 11112 vnic->uc_filter_count = i; 11113 return rc; 11114 } 11115 } 11116 if (test_and_clear_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state)) 11117 netdev_notice(bp->dev, "Retry of L2 filter configuration successful.\n"); 11118 11119 skip_uc: 11120 if ((vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS) && 11121 !bnxt_promisc_ok(bp)) 11122 vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 11123 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0); 11124 if (rc && (vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_MCAST)) { 11125 netdev_info(bp->dev, "Failed setting MC filters rc: %d, turning on ALL_MCAST mode\n", 11126 rc); 11127 vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_MCAST; 11128 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 11129 vnic->mc_list_count = 0; 11130 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0); 11131 } 11132 if (rc) 11133 netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %d\n", 11134 rc); 11135 11136 return rc; 11137 } 11138 11139 static bool bnxt_can_reserve_rings(struct bnxt *bp) 11140 { 11141 #ifdef CONFIG_BNXT_SRIOV 11142 if (BNXT_NEW_RM(bp) && BNXT_VF(bp)) { 11143 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 11144 11145 /* No minimum rings were provisioned by the PF. Don't 11146 * reserve rings by default when device is down. 11147 */ 11148 if (hw_resc->min_tx_rings || hw_resc->resv_tx_rings) 11149 return true; 11150 11151 if (!netif_running(bp->dev)) 11152 return false; 11153 } 11154 #endif 11155 return true; 11156 } 11157 11158 /* If the chip and firmware supports RFS */ 11159 static bool bnxt_rfs_supported(struct bnxt *bp) 11160 { 11161 if (bp->flags & BNXT_FLAG_CHIP_P5) { 11162 if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2) 11163 return true; 11164 return false; 11165 } 11166 /* 212 firmware is broken for aRFS */ 11167 if (BNXT_FW_MAJ(bp) == 212) 11168 return false; 11169 if (BNXT_PF(bp) && !BNXT_CHIP_TYPE_NITRO_A0(bp)) 11170 return true; 11171 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP) 11172 return true; 11173 return false; 11174 } 11175 11176 /* If runtime conditions support RFS */ 11177 static bool bnxt_rfs_capable(struct bnxt *bp) 11178 { 11179 #ifdef CONFIG_RFS_ACCEL 11180 int vnics, max_vnics, max_rss_ctxs; 11181 11182 if (bp->flags & BNXT_FLAG_CHIP_P5) 11183 return bnxt_rfs_supported(bp); 11184 if (!(bp->flags & BNXT_FLAG_MSIX_CAP) || !bnxt_can_reserve_rings(bp) || !bp->rx_nr_rings) 11185 return false; 11186 11187 vnics = 1 + bp->rx_nr_rings; 11188 max_vnics = bnxt_get_max_func_vnics(bp); 11189 max_rss_ctxs = bnxt_get_max_func_rss_ctxs(bp); 11190 11191 /* RSS contexts not a limiting factor */ 11192 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP) 11193 max_rss_ctxs = max_vnics; 11194 if (vnics > max_vnics || vnics > max_rss_ctxs) { 11195 if (bp->rx_nr_rings > 1) 11196 netdev_warn(bp->dev, 11197 "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n", 11198 min(max_rss_ctxs - 1, max_vnics - 1)); 11199 return false; 11200 } 11201 11202 if (!BNXT_NEW_RM(bp)) 11203 return true; 11204 11205 if (vnics == bp->hw_resc.resv_vnics) 11206 return true; 11207 11208 bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, vnics); 11209 if (vnics <= bp->hw_resc.resv_vnics) 11210 return true; 11211 11212 netdev_warn(bp->dev, "Unable to reserve resources to support NTUPLE filters.\n"); 11213 bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, 1); 11214 return false; 11215 #else 11216 return false; 11217 #endif 11218 } 11219 11220 static netdev_features_t bnxt_fix_features(struct net_device *dev, 11221 netdev_features_t features) 11222 { 11223 struct bnxt *bp = netdev_priv(dev); 11224 netdev_features_t vlan_features; 11225 11226 if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp)) 11227 features &= ~NETIF_F_NTUPLE; 11228 11229 if ((bp->flags & BNXT_FLAG_NO_AGG_RINGS) || bp->xdp_prog) 11230 features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); 11231 11232 if (!(features & NETIF_F_GRO)) 11233 features &= ~NETIF_F_GRO_HW; 11234 11235 if (features & NETIF_F_GRO_HW) 11236 features &= ~NETIF_F_LRO; 11237 11238 /* Both CTAG and STAG VLAN accelaration on the RX side have to be 11239 * turned on or off together. 11240 */ 11241 vlan_features = features & BNXT_HW_FEATURE_VLAN_ALL_RX; 11242 if (vlan_features != BNXT_HW_FEATURE_VLAN_ALL_RX) { 11243 if (dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX) 11244 features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX; 11245 else if (vlan_features) 11246 features |= BNXT_HW_FEATURE_VLAN_ALL_RX; 11247 } 11248 #ifdef CONFIG_BNXT_SRIOV 11249 if (BNXT_VF(bp) && bp->vf.vlan) 11250 features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX; 11251 #endif 11252 return features; 11253 } 11254 11255 static int bnxt_set_features(struct net_device *dev, netdev_features_t features) 11256 { 11257 struct bnxt *bp = netdev_priv(dev); 11258 u32 flags = bp->flags; 11259 u32 changes; 11260 int rc = 0; 11261 bool re_init = false; 11262 bool update_tpa = false; 11263 11264 flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS; 11265 if (features & NETIF_F_GRO_HW) 11266 flags |= BNXT_FLAG_GRO; 11267 else if (features & NETIF_F_LRO) 11268 flags |= BNXT_FLAG_LRO; 11269 11270 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS) 11271 flags &= ~BNXT_FLAG_TPA; 11272 11273 if (features & BNXT_HW_FEATURE_VLAN_ALL_RX) 11274 flags |= BNXT_FLAG_STRIP_VLAN; 11275 11276 if (features & NETIF_F_NTUPLE) 11277 flags |= BNXT_FLAG_RFS; 11278 11279 changes = flags ^ bp->flags; 11280 if (changes & BNXT_FLAG_TPA) { 11281 update_tpa = true; 11282 if ((bp->flags & BNXT_FLAG_TPA) == 0 || 11283 (flags & BNXT_FLAG_TPA) == 0 || 11284 (bp->flags & BNXT_FLAG_CHIP_P5)) 11285 re_init = true; 11286 } 11287 11288 if (changes & ~BNXT_FLAG_TPA) 11289 re_init = true; 11290 11291 if (flags != bp->flags) { 11292 u32 old_flags = bp->flags; 11293 11294 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 11295 bp->flags = flags; 11296 if (update_tpa) 11297 bnxt_set_ring_params(bp); 11298 return rc; 11299 } 11300 11301 if (re_init) { 11302 bnxt_close_nic(bp, false, false); 11303 bp->flags = flags; 11304 if (update_tpa) 11305 bnxt_set_ring_params(bp); 11306 11307 return bnxt_open_nic(bp, false, false); 11308 } 11309 if (update_tpa) { 11310 bp->flags = flags; 11311 rc = bnxt_set_tpa(bp, 11312 (flags & BNXT_FLAG_TPA) ? 11313 true : false); 11314 if (rc) 11315 bp->flags = old_flags; 11316 } 11317 } 11318 return rc; 11319 } 11320 11321 static bool bnxt_exthdr_check(struct bnxt *bp, struct sk_buff *skb, int nw_off, 11322 u8 **nextp) 11323 { 11324 struct ipv6hdr *ip6h = (struct ipv6hdr *)(skb->data + nw_off); 11325 struct hop_jumbo_hdr *jhdr; 11326 int hdr_count = 0; 11327 u8 *nexthdr; 11328 int start; 11329 11330 /* Check that there are at most 2 IPv6 extension headers, no 11331 * fragment header, and each is <= 64 bytes. 11332 */ 11333 start = nw_off + sizeof(*ip6h); 11334 nexthdr = &ip6h->nexthdr; 11335 while (ipv6_ext_hdr(*nexthdr)) { 11336 struct ipv6_opt_hdr *hp; 11337 int hdrlen; 11338 11339 if (hdr_count >= 3 || *nexthdr == NEXTHDR_NONE || 11340 *nexthdr == NEXTHDR_FRAGMENT) 11341 return false; 11342 hp = __skb_header_pointer(NULL, start, sizeof(*hp), skb->data, 11343 skb_headlen(skb), NULL); 11344 if (!hp) 11345 return false; 11346 if (*nexthdr == NEXTHDR_AUTH) 11347 hdrlen = ipv6_authlen(hp); 11348 else 11349 hdrlen = ipv6_optlen(hp); 11350 11351 if (hdrlen > 64) 11352 return false; 11353 11354 /* The ext header may be a hop-by-hop header inserted for 11355 * big TCP purposes. This will be removed before sending 11356 * from NIC, so do not count it. 11357 */ 11358 if (*nexthdr == NEXTHDR_HOP) { 11359 if (likely(skb->len <= GRO_LEGACY_MAX_SIZE)) 11360 goto increment_hdr; 11361 11362 jhdr = (struct hop_jumbo_hdr *)hp; 11363 if (jhdr->tlv_type != IPV6_TLV_JUMBO || jhdr->hdrlen != 0 || 11364 jhdr->nexthdr != IPPROTO_TCP) 11365 goto increment_hdr; 11366 11367 goto next_hdr; 11368 } 11369 increment_hdr: 11370 hdr_count++; 11371 next_hdr: 11372 nexthdr = &hp->nexthdr; 11373 start += hdrlen; 11374 } 11375 if (nextp) { 11376 /* Caller will check inner protocol */ 11377 if (skb->encapsulation) { 11378 *nextp = nexthdr; 11379 return true; 11380 } 11381 *nextp = NULL; 11382 } 11383 /* Only support TCP/UDP for non-tunneled ipv6 and inner ipv6 */ 11384 return *nexthdr == IPPROTO_TCP || *nexthdr == IPPROTO_UDP; 11385 } 11386 11387 /* For UDP, we can only handle 1 Vxlan port and 1 Geneve port. */ 11388 static bool bnxt_udp_tunl_check(struct bnxt *bp, struct sk_buff *skb) 11389 { 11390 struct udphdr *uh = udp_hdr(skb); 11391 __be16 udp_port = uh->dest; 11392 11393 if (udp_port != bp->vxlan_port && udp_port != bp->nge_port) 11394 return false; 11395 if (skb->inner_protocol_type == ENCAP_TYPE_ETHER) { 11396 struct ethhdr *eh = inner_eth_hdr(skb); 11397 11398 switch (eh->h_proto) { 11399 case htons(ETH_P_IP): 11400 return true; 11401 case htons(ETH_P_IPV6): 11402 return bnxt_exthdr_check(bp, skb, 11403 skb_inner_network_offset(skb), 11404 NULL); 11405 } 11406 } 11407 return false; 11408 } 11409 11410 static bool bnxt_tunl_check(struct bnxt *bp, struct sk_buff *skb, u8 l4_proto) 11411 { 11412 switch (l4_proto) { 11413 case IPPROTO_UDP: 11414 return bnxt_udp_tunl_check(bp, skb); 11415 case IPPROTO_IPIP: 11416 return true; 11417 case IPPROTO_GRE: { 11418 switch (skb->inner_protocol) { 11419 default: 11420 return false; 11421 case htons(ETH_P_IP): 11422 return true; 11423 case htons(ETH_P_IPV6): 11424 fallthrough; 11425 } 11426 } 11427 case IPPROTO_IPV6: 11428 /* Check ext headers of inner ipv6 */ 11429 return bnxt_exthdr_check(bp, skb, skb_inner_network_offset(skb), 11430 NULL); 11431 } 11432 return false; 11433 } 11434 11435 static netdev_features_t bnxt_features_check(struct sk_buff *skb, 11436 struct net_device *dev, 11437 netdev_features_t features) 11438 { 11439 struct bnxt *bp = netdev_priv(dev); 11440 u8 *l4_proto; 11441 11442 features = vlan_features_check(skb, features); 11443 switch (vlan_get_protocol(skb)) { 11444 case htons(ETH_P_IP): 11445 if (!skb->encapsulation) 11446 return features; 11447 l4_proto = &ip_hdr(skb)->protocol; 11448 if (bnxt_tunl_check(bp, skb, *l4_proto)) 11449 return features; 11450 break; 11451 case htons(ETH_P_IPV6): 11452 if (!bnxt_exthdr_check(bp, skb, skb_network_offset(skb), 11453 &l4_proto)) 11454 break; 11455 if (!l4_proto || bnxt_tunl_check(bp, skb, *l4_proto)) 11456 return features; 11457 break; 11458 } 11459 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 11460 } 11461 11462 int bnxt_dbg_hwrm_rd_reg(struct bnxt *bp, u32 reg_off, u16 num_words, 11463 u32 *reg_buf) 11464 { 11465 struct hwrm_dbg_read_direct_output *resp; 11466 struct hwrm_dbg_read_direct_input *req; 11467 __le32 *dbg_reg_buf; 11468 dma_addr_t mapping; 11469 int rc, i; 11470 11471 rc = hwrm_req_init(bp, req, HWRM_DBG_READ_DIRECT); 11472 if (rc) 11473 return rc; 11474 11475 dbg_reg_buf = hwrm_req_dma_slice(bp, req, num_words * 4, 11476 &mapping); 11477 if (!dbg_reg_buf) { 11478 rc = -ENOMEM; 11479 goto dbg_rd_reg_exit; 11480 } 11481 11482 req->host_dest_addr = cpu_to_le64(mapping); 11483 11484 resp = hwrm_req_hold(bp, req); 11485 req->read_addr = cpu_to_le32(reg_off + CHIMP_REG_VIEW_ADDR); 11486 req->read_len32 = cpu_to_le32(num_words); 11487 11488 rc = hwrm_req_send(bp, req); 11489 if (rc || resp->error_code) { 11490 rc = -EIO; 11491 goto dbg_rd_reg_exit; 11492 } 11493 for (i = 0; i < num_words; i++) 11494 reg_buf[i] = le32_to_cpu(dbg_reg_buf[i]); 11495 11496 dbg_rd_reg_exit: 11497 hwrm_req_drop(bp, req); 11498 return rc; 11499 } 11500 11501 static int bnxt_dbg_hwrm_ring_info_get(struct bnxt *bp, u8 ring_type, 11502 u32 ring_id, u32 *prod, u32 *cons) 11503 { 11504 struct hwrm_dbg_ring_info_get_output *resp; 11505 struct hwrm_dbg_ring_info_get_input *req; 11506 int rc; 11507 11508 rc = hwrm_req_init(bp, req, HWRM_DBG_RING_INFO_GET); 11509 if (rc) 11510 return rc; 11511 11512 req->ring_type = ring_type; 11513 req->fw_ring_id = cpu_to_le32(ring_id); 11514 resp = hwrm_req_hold(bp, req); 11515 rc = hwrm_req_send(bp, req); 11516 if (!rc) { 11517 *prod = le32_to_cpu(resp->producer_index); 11518 *cons = le32_to_cpu(resp->consumer_index); 11519 } 11520 hwrm_req_drop(bp, req); 11521 return rc; 11522 } 11523 11524 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi) 11525 { 11526 struct bnxt_tx_ring_info *txr = bnapi->tx_ring; 11527 int i = bnapi->index; 11528 11529 if (!txr) 11530 return; 11531 11532 netdev_info(bnapi->bp->dev, "[%d]: tx{fw_ring: %d prod: %x cons: %x}\n", 11533 i, txr->tx_ring_struct.fw_ring_id, txr->tx_prod, 11534 txr->tx_cons); 11535 } 11536 11537 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi) 11538 { 11539 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 11540 int i = bnapi->index; 11541 11542 if (!rxr) 11543 return; 11544 11545 netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n", 11546 i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod, 11547 rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod, 11548 rxr->rx_sw_agg_prod); 11549 } 11550 11551 static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi) 11552 { 11553 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 11554 int i = bnapi->index; 11555 11556 netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n", 11557 i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons); 11558 } 11559 11560 static void bnxt_dbg_dump_states(struct bnxt *bp) 11561 { 11562 int i; 11563 struct bnxt_napi *bnapi; 11564 11565 for (i = 0; i < bp->cp_nr_rings; i++) { 11566 bnapi = bp->bnapi[i]; 11567 if (netif_msg_drv(bp)) { 11568 bnxt_dump_tx_sw_state(bnapi); 11569 bnxt_dump_rx_sw_state(bnapi); 11570 bnxt_dump_cp_sw_state(bnapi); 11571 } 11572 } 11573 } 11574 11575 static int bnxt_hwrm_rx_ring_reset(struct bnxt *bp, int ring_nr) 11576 { 11577 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr]; 11578 struct hwrm_ring_reset_input *req; 11579 struct bnxt_napi *bnapi = rxr->bnapi; 11580 struct bnxt_cp_ring_info *cpr; 11581 u16 cp_ring_id; 11582 int rc; 11583 11584 rc = hwrm_req_init(bp, req, HWRM_RING_RESET); 11585 if (rc) 11586 return rc; 11587 11588 cpr = &bnapi->cp_ring; 11589 cp_ring_id = cpr->cp_ring_struct.fw_ring_id; 11590 req->cmpl_ring = cpu_to_le16(cp_ring_id); 11591 req->ring_type = RING_RESET_REQ_RING_TYPE_RX_RING_GRP; 11592 req->ring_id = cpu_to_le16(bp->grp_info[bnapi->index].fw_grp_id); 11593 return hwrm_req_send_silent(bp, req); 11594 } 11595 11596 static void bnxt_reset_task(struct bnxt *bp, bool silent) 11597 { 11598 if (!silent) 11599 bnxt_dbg_dump_states(bp); 11600 if (netif_running(bp->dev)) { 11601 int rc; 11602 11603 if (silent) { 11604 bnxt_close_nic(bp, false, false); 11605 bnxt_open_nic(bp, false, false); 11606 } else { 11607 bnxt_ulp_stop(bp); 11608 bnxt_close_nic(bp, true, false); 11609 rc = bnxt_open_nic(bp, true, false); 11610 bnxt_ulp_start(bp, rc); 11611 } 11612 } 11613 } 11614 11615 static void bnxt_tx_timeout(struct net_device *dev, unsigned int txqueue) 11616 { 11617 struct bnxt *bp = netdev_priv(dev); 11618 11619 netdev_err(bp->dev, "TX timeout detected, starting reset task!\n"); 11620 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event); 11621 bnxt_queue_sp_work(bp); 11622 } 11623 11624 static void bnxt_fw_health_check(struct bnxt *bp) 11625 { 11626 struct bnxt_fw_health *fw_health = bp->fw_health; 11627 u32 val; 11628 11629 if (!fw_health->enabled || test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 11630 return; 11631 11632 /* Make sure it is enabled before checking the tmr_counter. */ 11633 smp_rmb(); 11634 if (fw_health->tmr_counter) { 11635 fw_health->tmr_counter--; 11636 return; 11637 } 11638 11639 val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG); 11640 if (val == fw_health->last_fw_heartbeat) { 11641 fw_health->arrests++; 11642 goto fw_reset; 11643 } 11644 11645 fw_health->last_fw_heartbeat = val; 11646 11647 val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG); 11648 if (val != fw_health->last_fw_reset_cnt) { 11649 fw_health->discoveries++; 11650 goto fw_reset; 11651 } 11652 11653 fw_health->tmr_counter = fw_health->tmr_multiplier; 11654 return; 11655 11656 fw_reset: 11657 set_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event); 11658 bnxt_queue_sp_work(bp); 11659 } 11660 11661 static void bnxt_timer(struct timer_list *t) 11662 { 11663 struct bnxt *bp = from_timer(bp, t, timer); 11664 struct net_device *dev = bp->dev; 11665 11666 if (!netif_running(dev) || !test_bit(BNXT_STATE_OPEN, &bp->state)) 11667 return; 11668 11669 if (atomic_read(&bp->intr_sem) != 0) 11670 goto bnxt_restart_timer; 11671 11672 if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) 11673 bnxt_fw_health_check(bp); 11674 11675 if (BNXT_LINK_IS_UP(bp) && bp->stats_coal_ticks) { 11676 set_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event); 11677 bnxt_queue_sp_work(bp); 11678 } 11679 11680 if (bnxt_tc_flower_enabled(bp)) { 11681 set_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event); 11682 bnxt_queue_sp_work(bp); 11683 } 11684 11685 #ifdef CONFIG_RFS_ACCEL 11686 if ((bp->flags & BNXT_FLAG_RFS) && bp->ntp_fltr_count) { 11687 set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event); 11688 bnxt_queue_sp_work(bp); 11689 } 11690 #endif /*CONFIG_RFS_ACCEL*/ 11691 11692 if (bp->link_info.phy_retry) { 11693 if (time_after(jiffies, bp->link_info.phy_retry_expires)) { 11694 bp->link_info.phy_retry = false; 11695 netdev_warn(bp->dev, "failed to update phy settings after maximum retries.\n"); 11696 } else { 11697 set_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event); 11698 bnxt_queue_sp_work(bp); 11699 } 11700 } 11701 11702 if (test_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state)) { 11703 set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event); 11704 bnxt_queue_sp_work(bp); 11705 } 11706 11707 if ((bp->flags & BNXT_FLAG_CHIP_P5) && !bp->chip_rev && 11708 netif_carrier_ok(dev)) { 11709 set_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event); 11710 bnxt_queue_sp_work(bp); 11711 } 11712 bnxt_restart_timer: 11713 mod_timer(&bp->timer, jiffies + bp->current_interval); 11714 } 11715 11716 static void bnxt_rtnl_lock_sp(struct bnxt *bp) 11717 { 11718 /* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK 11719 * set. If the device is being closed, bnxt_close() may be holding 11720 * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear. So we 11721 * must clear BNXT_STATE_IN_SP_TASK before holding rtnl(). 11722 */ 11723 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 11724 rtnl_lock(); 11725 } 11726 11727 static void bnxt_rtnl_unlock_sp(struct bnxt *bp) 11728 { 11729 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 11730 rtnl_unlock(); 11731 } 11732 11733 /* Only called from bnxt_sp_task() */ 11734 static void bnxt_reset(struct bnxt *bp, bool silent) 11735 { 11736 bnxt_rtnl_lock_sp(bp); 11737 if (test_bit(BNXT_STATE_OPEN, &bp->state)) 11738 bnxt_reset_task(bp, silent); 11739 bnxt_rtnl_unlock_sp(bp); 11740 } 11741 11742 /* Only called from bnxt_sp_task() */ 11743 static void bnxt_rx_ring_reset(struct bnxt *bp) 11744 { 11745 int i; 11746 11747 bnxt_rtnl_lock_sp(bp); 11748 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 11749 bnxt_rtnl_unlock_sp(bp); 11750 return; 11751 } 11752 /* Disable and flush TPA before resetting the RX ring */ 11753 if (bp->flags & BNXT_FLAG_TPA) 11754 bnxt_set_tpa(bp, false); 11755 for (i = 0; i < bp->rx_nr_rings; i++) { 11756 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 11757 struct bnxt_cp_ring_info *cpr; 11758 int rc; 11759 11760 if (!rxr->bnapi->in_reset) 11761 continue; 11762 11763 rc = bnxt_hwrm_rx_ring_reset(bp, i); 11764 if (rc) { 11765 if (rc == -EINVAL || rc == -EOPNOTSUPP) 11766 netdev_info_once(bp->dev, "RX ring reset not supported by firmware, falling back to global reset\n"); 11767 else 11768 netdev_warn(bp->dev, "RX ring reset failed, rc = %d, falling back to global reset\n", 11769 rc); 11770 bnxt_reset_task(bp, true); 11771 break; 11772 } 11773 bnxt_free_one_rx_ring_skbs(bp, i); 11774 rxr->rx_prod = 0; 11775 rxr->rx_agg_prod = 0; 11776 rxr->rx_sw_agg_prod = 0; 11777 rxr->rx_next_cons = 0; 11778 rxr->bnapi->in_reset = false; 11779 bnxt_alloc_one_rx_ring(bp, i); 11780 cpr = &rxr->bnapi->cp_ring; 11781 cpr->sw_stats.rx.rx_resets++; 11782 if (bp->flags & BNXT_FLAG_AGG_RINGS) 11783 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); 11784 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 11785 } 11786 if (bp->flags & BNXT_FLAG_TPA) 11787 bnxt_set_tpa(bp, true); 11788 bnxt_rtnl_unlock_sp(bp); 11789 } 11790 11791 static void bnxt_fw_reset_close(struct bnxt *bp) 11792 { 11793 bnxt_ulp_stop(bp); 11794 /* When firmware is in fatal state, quiesce device and disable 11795 * bus master to prevent any potential bad DMAs before freeing 11796 * kernel memory. 11797 */ 11798 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) { 11799 u16 val = 0; 11800 11801 pci_read_config_word(bp->pdev, PCI_SUBSYSTEM_ID, &val); 11802 if (val == 0xffff) 11803 bp->fw_reset_min_dsecs = 0; 11804 bnxt_tx_disable(bp); 11805 bnxt_disable_napi(bp); 11806 bnxt_disable_int_sync(bp); 11807 bnxt_free_irq(bp); 11808 bnxt_clear_int_mode(bp); 11809 pci_disable_device(bp->pdev); 11810 } 11811 __bnxt_close_nic(bp, true, false); 11812 bnxt_vf_reps_free(bp); 11813 bnxt_clear_int_mode(bp); 11814 bnxt_hwrm_func_drv_unrgtr(bp); 11815 if (pci_is_enabled(bp->pdev)) 11816 pci_disable_device(bp->pdev); 11817 bnxt_free_ctx_mem(bp); 11818 kfree(bp->ctx); 11819 bp->ctx = NULL; 11820 } 11821 11822 static bool is_bnxt_fw_ok(struct bnxt *bp) 11823 { 11824 struct bnxt_fw_health *fw_health = bp->fw_health; 11825 bool no_heartbeat = false, has_reset = false; 11826 u32 val; 11827 11828 val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG); 11829 if (val == fw_health->last_fw_heartbeat) 11830 no_heartbeat = true; 11831 11832 val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG); 11833 if (val != fw_health->last_fw_reset_cnt) 11834 has_reset = true; 11835 11836 if (!no_heartbeat && has_reset) 11837 return true; 11838 11839 return false; 11840 } 11841 11842 /* rtnl_lock is acquired before calling this function */ 11843 static void bnxt_force_fw_reset(struct bnxt *bp) 11844 { 11845 struct bnxt_fw_health *fw_health = bp->fw_health; 11846 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; 11847 u32 wait_dsecs; 11848 11849 if (!test_bit(BNXT_STATE_OPEN, &bp->state) || 11850 test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 11851 return; 11852 11853 if (ptp) { 11854 spin_lock_bh(&ptp->ptp_lock); 11855 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 11856 spin_unlock_bh(&ptp->ptp_lock); 11857 } else { 11858 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 11859 } 11860 bnxt_fw_reset_close(bp); 11861 wait_dsecs = fw_health->master_func_wait_dsecs; 11862 if (fw_health->primary) { 11863 if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) 11864 wait_dsecs = 0; 11865 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW; 11866 } else { 11867 bp->fw_reset_timestamp = jiffies + wait_dsecs * HZ / 10; 11868 wait_dsecs = fw_health->normal_func_wait_dsecs; 11869 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 11870 } 11871 11872 bp->fw_reset_min_dsecs = fw_health->post_reset_wait_dsecs; 11873 bp->fw_reset_max_dsecs = fw_health->post_reset_max_wait_dsecs; 11874 bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10); 11875 } 11876 11877 void bnxt_fw_exception(struct bnxt *bp) 11878 { 11879 netdev_warn(bp->dev, "Detected firmware fatal condition, initiating reset\n"); 11880 set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state); 11881 bnxt_rtnl_lock_sp(bp); 11882 bnxt_force_fw_reset(bp); 11883 bnxt_rtnl_unlock_sp(bp); 11884 } 11885 11886 /* Returns the number of registered VFs, or 1 if VF configuration is pending, or 11887 * < 0 on error. 11888 */ 11889 static int bnxt_get_registered_vfs(struct bnxt *bp) 11890 { 11891 #ifdef CONFIG_BNXT_SRIOV 11892 int rc; 11893 11894 if (!BNXT_PF(bp)) 11895 return 0; 11896 11897 rc = bnxt_hwrm_func_qcfg(bp); 11898 if (rc) { 11899 netdev_err(bp->dev, "func_qcfg cmd failed, rc = %d\n", rc); 11900 return rc; 11901 } 11902 if (bp->pf.registered_vfs) 11903 return bp->pf.registered_vfs; 11904 if (bp->sriov_cfg) 11905 return 1; 11906 #endif 11907 return 0; 11908 } 11909 11910 void bnxt_fw_reset(struct bnxt *bp) 11911 { 11912 bnxt_rtnl_lock_sp(bp); 11913 if (test_bit(BNXT_STATE_OPEN, &bp->state) && 11914 !test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) { 11915 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; 11916 int n = 0, tmo; 11917 11918 if (ptp) { 11919 spin_lock_bh(&ptp->ptp_lock); 11920 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 11921 spin_unlock_bh(&ptp->ptp_lock); 11922 } else { 11923 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 11924 } 11925 if (bp->pf.active_vfs && 11926 !test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) 11927 n = bnxt_get_registered_vfs(bp); 11928 if (n < 0) { 11929 netdev_err(bp->dev, "Firmware reset aborted, rc = %d\n", 11930 n); 11931 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 11932 dev_close(bp->dev); 11933 goto fw_reset_exit; 11934 } else if (n > 0) { 11935 u16 vf_tmo_dsecs = n * 10; 11936 11937 if (bp->fw_reset_max_dsecs < vf_tmo_dsecs) 11938 bp->fw_reset_max_dsecs = vf_tmo_dsecs; 11939 bp->fw_reset_state = 11940 BNXT_FW_RESET_STATE_POLL_VF; 11941 bnxt_queue_fw_reset_work(bp, HZ / 10); 11942 goto fw_reset_exit; 11943 } 11944 bnxt_fw_reset_close(bp); 11945 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) { 11946 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN; 11947 tmo = HZ / 10; 11948 } else { 11949 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 11950 tmo = bp->fw_reset_min_dsecs * HZ / 10; 11951 } 11952 bnxt_queue_fw_reset_work(bp, tmo); 11953 } 11954 fw_reset_exit: 11955 bnxt_rtnl_unlock_sp(bp); 11956 } 11957 11958 static void bnxt_chk_missed_irq(struct bnxt *bp) 11959 { 11960 int i; 11961 11962 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 11963 return; 11964 11965 for (i = 0; i < bp->cp_nr_rings; i++) { 11966 struct bnxt_napi *bnapi = bp->bnapi[i]; 11967 struct bnxt_cp_ring_info *cpr; 11968 u32 fw_ring_id; 11969 int j; 11970 11971 if (!bnapi) 11972 continue; 11973 11974 cpr = &bnapi->cp_ring; 11975 for (j = 0; j < 2; j++) { 11976 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j]; 11977 u32 val[2]; 11978 11979 if (!cpr2 || cpr2->has_more_work || 11980 !bnxt_has_work(bp, cpr2)) 11981 continue; 11982 11983 if (cpr2->cp_raw_cons != cpr2->last_cp_raw_cons) { 11984 cpr2->last_cp_raw_cons = cpr2->cp_raw_cons; 11985 continue; 11986 } 11987 fw_ring_id = cpr2->cp_ring_struct.fw_ring_id; 11988 bnxt_dbg_hwrm_ring_info_get(bp, 11989 DBG_RING_INFO_GET_REQ_RING_TYPE_L2_CMPL, 11990 fw_ring_id, &val[0], &val[1]); 11991 cpr->sw_stats.cmn.missed_irqs++; 11992 } 11993 } 11994 } 11995 11996 static void bnxt_cfg_ntp_filters(struct bnxt *); 11997 11998 static void bnxt_init_ethtool_link_settings(struct bnxt *bp) 11999 { 12000 struct bnxt_link_info *link_info = &bp->link_info; 12001 12002 if (BNXT_AUTO_MODE(link_info->auto_mode)) { 12003 link_info->autoneg = BNXT_AUTONEG_SPEED; 12004 if (bp->hwrm_spec_code >= 0x10201) { 12005 if (link_info->auto_pause_setting & 12006 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE) 12007 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL; 12008 } else { 12009 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL; 12010 } 12011 link_info->advertising = link_info->auto_link_speeds; 12012 link_info->advertising_pam4 = link_info->auto_pam4_link_speeds; 12013 } else { 12014 link_info->req_link_speed = link_info->force_link_speed; 12015 link_info->req_signal_mode = BNXT_SIG_MODE_NRZ; 12016 if (link_info->force_pam4_link_speed) { 12017 link_info->req_link_speed = 12018 link_info->force_pam4_link_speed; 12019 link_info->req_signal_mode = BNXT_SIG_MODE_PAM4; 12020 } 12021 link_info->req_duplex = link_info->duplex_setting; 12022 } 12023 if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) 12024 link_info->req_flow_ctrl = 12025 link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH; 12026 else 12027 link_info->req_flow_ctrl = link_info->force_pause_setting; 12028 } 12029 12030 static void bnxt_fw_echo_reply(struct bnxt *bp) 12031 { 12032 struct bnxt_fw_health *fw_health = bp->fw_health; 12033 struct hwrm_func_echo_response_input *req; 12034 int rc; 12035 12036 rc = hwrm_req_init(bp, req, HWRM_FUNC_ECHO_RESPONSE); 12037 if (rc) 12038 return; 12039 req->event_data1 = cpu_to_le32(fw_health->echo_req_data1); 12040 req->event_data2 = cpu_to_le32(fw_health->echo_req_data2); 12041 hwrm_req_send(bp, req); 12042 } 12043 12044 static void bnxt_sp_task(struct work_struct *work) 12045 { 12046 struct bnxt *bp = container_of(work, struct bnxt, sp_task); 12047 12048 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 12049 smp_mb__after_atomic(); 12050 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 12051 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 12052 return; 12053 } 12054 12055 if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event)) 12056 bnxt_cfg_rx_mode(bp); 12057 12058 if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event)) 12059 bnxt_cfg_ntp_filters(bp); 12060 if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event)) 12061 bnxt_hwrm_exec_fwd_req(bp); 12062 if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event)) { 12063 bnxt_hwrm_port_qstats(bp, 0); 12064 bnxt_hwrm_port_qstats_ext(bp, 0); 12065 bnxt_accumulate_all_stats(bp); 12066 } 12067 12068 if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) { 12069 int rc; 12070 12071 mutex_lock(&bp->link_lock); 12072 if (test_and_clear_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, 12073 &bp->sp_event)) 12074 bnxt_hwrm_phy_qcaps(bp); 12075 12076 rc = bnxt_update_link(bp, true); 12077 if (rc) 12078 netdev_err(bp->dev, "SP task can't update link (rc: %x)\n", 12079 rc); 12080 12081 if (test_and_clear_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT, 12082 &bp->sp_event)) 12083 bnxt_init_ethtool_link_settings(bp); 12084 mutex_unlock(&bp->link_lock); 12085 } 12086 if (test_and_clear_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event)) { 12087 int rc; 12088 12089 mutex_lock(&bp->link_lock); 12090 rc = bnxt_update_phy_setting(bp); 12091 mutex_unlock(&bp->link_lock); 12092 if (rc) { 12093 netdev_warn(bp->dev, "update phy settings retry failed\n"); 12094 } else { 12095 bp->link_info.phy_retry = false; 12096 netdev_info(bp->dev, "update phy settings retry succeeded\n"); 12097 } 12098 } 12099 if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event)) { 12100 mutex_lock(&bp->link_lock); 12101 bnxt_get_port_module_status(bp); 12102 mutex_unlock(&bp->link_lock); 12103 } 12104 12105 if (test_and_clear_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event)) 12106 bnxt_tc_flow_stats_work(bp); 12107 12108 if (test_and_clear_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event)) 12109 bnxt_chk_missed_irq(bp); 12110 12111 if (test_and_clear_bit(BNXT_FW_ECHO_REQUEST_SP_EVENT, &bp->sp_event)) 12112 bnxt_fw_echo_reply(bp); 12113 12114 /* These functions below will clear BNXT_STATE_IN_SP_TASK. They 12115 * must be the last functions to be called before exiting. 12116 */ 12117 if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event)) 12118 bnxt_reset(bp, false); 12119 12120 if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event)) 12121 bnxt_reset(bp, true); 12122 12123 if (test_and_clear_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event)) 12124 bnxt_rx_ring_reset(bp); 12125 12126 if (test_and_clear_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event)) { 12127 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state) || 12128 test_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state)) 12129 bnxt_devlink_health_fw_report(bp); 12130 else 12131 bnxt_fw_reset(bp); 12132 } 12133 12134 if (test_and_clear_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event)) { 12135 if (!is_bnxt_fw_ok(bp)) 12136 bnxt_devlink_health_fw_report(bp); 12137 } 12138 12139 smp_mb__before_atomic(); 12140 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 12141 } 12142 12143 /* Under rtnl_lock */ 12144 int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs, 12145 int tx_xdp) 12146 { 12147 int max_rx, max_tx, tx_sets = 1; 12148 int tx_rings_needed, stats; 12149 int rx_rings = rx; 12150 int cp, vnics, rc; 12151 12152 if (tcs) 12153 tx_sets = tcs; 12154 12155 rc = bnxt_get_max_rings(bp, &max_rx, &max_tx, sh); 12156 if (rc) 12157 return rc; 12158 12159 if (max_rx < rx) 12160 return -ENOMEM; 12161 12162 tx_rings_needed = tx * tx_sets + tx_xdp; 12163 if (max_tx < tx_rings_needed) 12164 return -ENOMEM; 12165 12166 vnics = 1; 12167 if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS) 12168 vnics += rx_rings; 12169 12170 if (bp->flags & BNXT_FLAG_AGG_RINGS) 12171 rx_rings <<= 1; 12172 cp = sh ? max_t(int, tx_rings_needed, rx) : tx_rings_needed + rx; 12173 stats = cp; 12174 if (BNXT_NEW_RM(bp)) { 12175 cp += bnxt_get_ulp_msix_num(bp); 12176 stats += bnxt_get_ulp_stat_ctxs(bp); 12177 } 12178 return bnxt_hwrm_check_rings(bp, tx_rings_needed, rx_rings, rx, cp, 12179 stats, vnics); 12180 } 12181 12182 static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev) 12183 { 12184 if (bp->bar2) { 12185 pci_iounmap(pdev, bp->bar2); 12186 bp->bar2 = NULL; 12187 } 12188 12189 if (bp->bar1) { 12190 pci_iounmap(pdev, bp->bar1); 12191 bp->bar1 = NULL; 12192 } 12193 12194 if (bp->bar0) { 12195 pci_iounmap(pdev, bp->bar0); 12196 bp->bar0 = NULL; 12197 } 12198 } 12199 12200 static void bnxt_cleanup_pci(struct bnxt *bp) 12201 { 12202 bnxt_unmap_bars(bp, bp->pdev); 12203 pci_release_regions(bp->pdev); 12204 if (pci_is_enabled(bp->pdev)) 12205 pci_disable_device(bp->pdev); 12206 } 12207 12208 static void bnxt_init_dflt_coal(struct bnxt *bp) 12209 { 12210 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 12211 struct bnxt_coal *coal; 12212 u16 flags = 0; 12213 12214 if (coal_cap->cmpl_params & 12215 RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_TIMER_RESET) 12216 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET; 12217 12218 /* Tick values in micro seconds. 12219 * 1 coal_buf x bufs_per_record = 1 completion record. 12220 */ 12221 coal = &bp->rx_coal; 12222 coal->coal_ticks = 10; 12223 coal->coal_bufs = 30; 12224 coal->coal_ticks_irq = 1; 12225 coal->coal_bufs_irq = 2; 12226 coal->idle_thresh = 50; 12227 coal->bufs_per_record = 2; 12228 coal->budget = 64; /* NAPI budget */ 12229 coal->flags = flags; 12230 12231 coal = &bp->tx_coal; 12232 coal->coal_ticks = 28; 12233 coal->coal_bufs = 30; 12234 coal->coal_ticks_irq = 2; 12235 coal->coal_bufs_irq = 2; 12236 coal->bufs_per_record = 1; 12237 coal->flags = flags; 12238 12239 bp->stats_coal_ticks = BNXT_DEF_STATS_COAL_TICKS; 12240 } 12241 12242 static int bnxt_fw_init_one_p1(struct bnxt *bp) 12243 { 12244 int rc; 12245 12246 bp->fw_cap = 0; 12247 rc = bnxt_hwrm_ver_get(bp); 12248 bnxt_try_map_fw_health_reg(bp); 12249 if (rc) { 12250 rc = bnxt_try_recover_fw(bp); 12251 if (rc) 12252 return rc; 12253 rc = bnxt_hwrm_ver_get(bp); 12254 if (rc) 12255 return rc; 12256 } 12257 12258 bnxt_nvm_cfg_ver_get(bp); 12259 12260 rc = bnxt_hwrm_func_reset(bp); 12261 if (rc) 12262 return -ENODEV; 12263 12264 bnxt_hwrm_fw_set_time(bp); 12265 return 0; 12266 } 12267 12268 static int bnxt_fw_init_one_p2(struct bnxt *bp) 12269 { 12270 int rc; 12271 12272 /* Get the MAX capabilities for this function */ 12273 rc = bnxt_hwrm_func_qcaps(bp); 12274 if (rc) { 12275 netdev_err(bp->dev, "hwrm query capability failure rc: %x\n", 12276 rc); 12277 return -ENODEV; 12278 } 12279 12280 rc = bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(bp); 12281 if (rc) 12282 netdev_warn(bp->dev, "hwrm query adv flow mgnt failure rc: %d\n", 12283 rc); 12284 12285 if (bnxt_alloc_fw_health(bp)) { 12286 netdev_warn(bp->dev, "no memory for firmware error recovery\n"); 12287 } else { 12288 rc = bnxt_hwrm_error_recovery_qcfg(bp); 12289 if (rc) 12290 netdev_warn(bp->dev, "hwrm query error recovery failure rc: %d\n", 12291 rc); 12292 } 12293 12294 rc = bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false); 12295 if (rc) 12296 return -ENODEV; 12297 12298 bnxt_hwrm_func_qcfg(bp); 12299 bnxt_hwrm_vnic_qcaps(bp); 12300 bnxt_hwrm_port_led_qcaps(bp); 12301 bnxt_ethtool_init(bp); 12302 bnxt_dcb_init(bp); 12303 return 0; 12304 } 12305 12306 static void bnxt_set_dflt_rss_hash_type(struct bnxt *bp) 12307 { 12308 bp->flags &= ~BNXT_FLAG_UDP_RSS_CAP; 12309 bp->rss_hash_cfg = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 | 12310 VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 | 12311 VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 | 12312 VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6; 12313 if (bp->fw_cap & BNXT_FW_CAP_RSS_HASH_TYPE_DELTA) 12314 bp->rss_hash_delta = bp->rss_hash_cfg; 12315 if (BNXT_CHIP_P4_PLUS(bp) && bp->hwrm_spec_code >= 0x10501) { 12316 bp->flags |= BNXT_FLAG_UDP_RSS_CAP; 12317 bp->rss_hash_cfg |= VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 | 12318 VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6; 12319 } 12320 } 12321 12322 static void bnxt_set_dflt_rfs(struct bnxt *bp) 12323 { 12324 struct net_device *dev = bp->dev; 12325 12326 dev->hw_features &= ~NETIF_F_NTUPLE; 12327 dev->features &= ~NETIF_F_NTUPLE; 12328 bp->flags &= ~BNXT_FLAG_RFS; 12329 if (bnxt_rfs_supported(bp)) { 12330 dev->hw_features |= NETIF_F_NTUPLE; 12331 if (bnxt_rfs_capable(bp)) { 12332 bp->flags |= BNXT_FLAG_RFS; 12333 dev->features |= NETIF_F_NTUPLE; 12334 } 12335 } 12336 } 12337 12338 static void bnxt_fw_init_one_p3(struct bnxt *bp) 12339 { 12340 struct pci_dev *pdev = bp->pdev; 12341 12342 bnxt_set_dflt_rss_hash_type(bp); 12343 bnxt_set_dflt_rfs(bp); 12344 12345 bnxt_get_wol_settings(bp); 12346 if (bp->flags & BNXT_FLAG_WOL_CAP) 12347 device_set_wakeup_enable(&pdev->dev, bp->wol); 12348 else 12349 device_set_wakeup_capable(&pdev->dev, false); 12350 12351 bnxt_hwrm_set_cache_line_size(bp, cache_line_size()); 12352 bnxt_hwrm_coal_params_qcaps(bp); 12353 } 12354 12355 static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt); 12356 12357 int bnxt_fw_init_one(struct bnxt *bp) 12358 { 12359 int rc; 12360 12361 rc = bnxt_fw_init_one_p1(bp); 12362 if (rc) { 12363 netdev_err(bp->dev, "Firmware init phase 1 failed\n"); 12364 return rc; 12365 } 12366 rc = bnxt_fw_init_one_p2(bp); 12367 if (rc) { 12368 netdev_err(bp->dev, "Firmware init phase 2 failed\n"); 12369 return rc; 12370 } 12371 rc = bnxt_probe_phy(bp, false); 12372 if (rc) 12373 return rc; 12374 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, false); 12375 if (rc) 12376 return rc; 12377 12378 bnxt_fw_init_one_p3(bp); 12379 return 0; 12380 } 12381 12382 static void bnxt_fw_reset_writel(struct bnxt *bp, int reg_idx) 12383 { 12384 struct bnxt_fw_health *fw_health = bp->fw_health; 12385 u32 reg = fw_health->fw_reset_seq_regs[reg_idx]; 12386 u32 val = fw_health->fw_reset_seq_vals[reg_idx]; 12387 u32 reg_type, reg_off, delay_msecs; 12388 12389 delay_msecs = fw_health->fw_reset_seq_delay_msec[reg_idx]; 12390 reg_type = BNXT_FW_HEALTH_REG_TYPE(reg); 12391 reg_off = BNXT_FW_HEALTH_REG_OFF(reg); 12392 switch (reg_type) { 12393 case BNXT_FW_HEALTH_REG_TYPE_CFG: 12394 pci_write_config_dword(bp->pdev, reg_off, val); 12395 break; 12396 case BNXT_FW_HEALTH_REG_TYPE_GRC: 12397 writel(reg_off & BNXT_GRC_BASE_MASK, 12398 bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4); 12399 reg_off = (reg_off & BNXT_GRC_OFFSET_MASK) + 0x2000; 12400 fallthrough; 12401 case BNXT_FW_HEALTH_REG_TYPE_BAR0: 12402 writel(val, bp->bar0 + reg_off); 12403 break; 12404 case BNXT_FW_HEALTH_REG_TYPE_BAR1: 12405 writel(val, bp->bar1 + reg_off); 12406 break; 12407 } 12408 if (delay_msecs) { 12409 pci_read_config_dword(bp->pdev, 0, &val); 12410 msleep(delay_msecs); 12411 } 12412 } 12413 12414 bool bnxt_hwrm_reset_permitted(struct bnxt *bp) 12415 { 12416 struct hwrm_func_qcfg_output *resp; 12417 struct hwrm_func_qcfg_input *req; 12418 bool result = true; /* firmware will enforce if unknown */ 12419 12420 if (~bp->fw_cap & BNXT_FW_CAP_HOT_RESET_IF) 12421 return result; 12422 12423 if (hwrm_req_init(bp, req, HWRM_FUNC_QCFG)) 12424 return result; 12425 12426 req->fid = cpu_to_le16(0xffff); 12427 resp = hwrm_req_hold(bp, req); 12428 if (!hwrm_req_send(bp, req)) 12429 result = !!(le16_to_cpu(resp->flags) & 12430 FUNC_QCFG_RESP_FLAGS_HOT_RESET_ALLOWED); 12431 hwrm_req_drop(bp, req); 12432 return result; 12433 } 12434 12435 static void bnxt_reset_all(struct bnxt *bp) 12436 { 12437 struct bnxt_fw_health *fw_health = bp->fw_health; 12438 int i, rc; 12439 12440 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) { 12441 bnxt_fw_reset_via_optee(bp); 12442 bp->fw_reset_timestamp = jiffies; 12443 return; 12444 } 12445 12446 if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_HOST) { 12447 for (i = 0; i < fw_health->fw_reset_seq_cnt; i++) 12448 bnxt_fw_reset_writel(bp, i); 12449 } else if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) { 12450 struct hwrm_fw_reset_input *req; 12451 12452 rc = hwrm_req_init(bp, req, HWRM_FW_RESET); 12453 if (!rc) { 12454 req->target_id = cpu_to_le16(HWRM_TARGET_ID_KONG); 12455 req->embedded_proc_type = FW_RESET_REQ_EMBEDDED_PROC_TYPE_CHIP; 12456 req->selfrst_status = FW_RESET_REQ_SELFRST_STATUS_SELFRSTASAP; 12457 req->flags = FW_RESET_REQ_FLAGS_RESET_GRACEFUL; 12458 rc = hwrm_req_send(bp, req); 12459 } 12460 if (rc != -ENODEV) 12461 netdev_warn(bp->dev, "Unable to reset FW rc=%d\n", rc); 12462 } 12463 bp->fw_reset_timestamp = jiffies; 12464 } 12465 12466 static bool bnxt_fw_reset_timeout(struct bnxt *bp) 12467 { 12468 return time_after(jiffies, bp->fw_reset_timestamp + 12469 (bp->fw_reset_max_dsecs * HZ / 10)); 12470 } 12471 12472 static void bnxt_fw_reset_abort(struct bnxt *bp, int rc) 12473 { 12474 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 12475 if (bp->fw_reset_state != BNXT_FW_RESET_STATE_POLL_VF) { 12476 bnxt_ulp_start(bp, rc); 12477 bnxt_dl_health_fw_status_update(bp, false); 12478 } 12479 bp->fw_reset_state = 0; 12480 dev_close(bp->dev); 12481 } 12482 12483 static void bnxt_fw_reset_task(struct work_struct *work) 12484 { 12485 struct bnxt *bp = container_of(work, struct bnxt, fw_reset_task.work); 12486 int rc = 0; 12487 12488 if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) { 12489 netdev_err(bp->dev, "bnxt_fw_reset_task() called when not in fw reset mode!\n"); 12490 return; 12491 } 12492 12493 switch (bp->fw_reset_state) { 12494 case BNXT_FW_RESET_STATE_POLL_VF: { 12495 int n = bnxt_get_registered_vfs(bp); 12496 int tmo; 12497 12498 if (n < 0) { 12499 netdev_err(bp->dev, "Firmware reset aborted, subsequent func_qcfg cmd failed, rc = %d, %d msecs since reset timestamp\n", 12500 n, jiffies_to_msecs(jiffies - 12501 bp->fw_reset_timestamp)); 12502 goto fw_reset_abort; 12503 } else if (n > 0) { 12504 if (bnxt_fw_reset_timeout(bp)) { 12505 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 12506 bp->fw_reset_state = 0; 12507 netdev_err(bp->dev, "Firmware reset aborted, bnxt_get_registered_vfs() returns %d\n", 12508 n); 12509 return; 12510 } 12511 bnxt_queue_fw_reset_work(bp, HZ / 10); 12512 return; 12513 } 12514 bp->fw_reset_timestamp = jiffies; 12515 rtnl_lock(); 12516 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) { 12517 bnxt_fw_reset_abort(bp, rc); 12518 rtnl_unlock(); 12519 return; 12520 } 12521 bnxt_fw_reset_close(bp); 12522 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) { 12523 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN; 12524 tmo = HZ / 10; 12525 } else { 12526 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 12527 tmo = bp->fw_reset_min_dsecs * HZ / 10; 12528 } 12529 rtnl_unlock(); 12530 bnxt_queue_fw_reset_work(bp, tmo); 12531 return; 12532 } 12533 case BNXT_FW_RESET_STATE_POLL_FW_DOWN: { 12534 u32 val; 12535 12536 val = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG); 12537 if (!(val & BNXT_FW_STATUS_SHUTDOWN) && 12538 !bnxt_fw_reset_timeout(bp)) { 12539 bnxt_queue_fw_reset_work(bp, HZ / 5); 12540 return; 12541 } 12542 12543 if (!bp->fw_health->primary) { 12544 u32 wait_dsecs = bp->fw_health->normal_func_wait_dsecs; 12545 12546 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 12547 bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10); 12548 return; 12549 } 12550 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW; 12551 } 12552 fallthrough; 12553 case BNXT_FW_RESET_STATE_RESET_FW: 12554 bnxt_reset_all(bp); 12555 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 12556 bnxt_queue_fw_reset_work(bp, bp->fw_reset_min_dsecs * HZ / 10); 12557 return; 12558 case BNXT_FW_RESET_STATE_ENABLE_DEV: 12559 bnxt_inv_fw_health_reg(bp); 12560 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state) && 12561 !bp->fw_reset_min_dsecs) { 12562 u16 val; 12563 12564 pci_read_config_word(bp->pdev, PCI_SUBSYSTEM_ID, &val); 12565 if (val == 0xffff) { 12566 if (bnxt_fw_reset_timeout(bp)) { 12567 netdev_err(bp->dev, "Firmware reset aborted, PCI config space invalid\n"); 12568 rc = -ETIMEDOUT; 12569 goto fw_reset_abort; 12570 } 12571 bnxt_queue_fw_reset_work(bp, HZ / 1000); 12572 return; 12573 } 12574 } 12575 clear_bit(BNXT_STATE_FW_FATAL_COND, &bp->state); 12576 clear_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state); 12577 if (test_and_clear_bit(BNXT_STATE_FW_ACTIVATE_RESET, &bp->state) && 12578 !test_bit(BNXT_STATE_FW_ACTIVATE, &bp->state)) 12579 bnxt_dl_remote_reload(bp); 12580 if (pci_enable_device(bp->pdev)) { 12581 netdev_err(bp->dev, "Cannot re-enable PCI device\n"); 12582 rc = -ENODEV; 12583 goto fw_reset_abort; 12584 } 12585 pci_set_master(bp->pdev); 12586 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW; 12587 fallthrough; 12588 case BNXT_FW_RESET_STATE_POLL_FW: 12589 bp->hwrm_cmd_timeout = SHORT_HWRM_CMD_TIMEOUT; 12590 rc = bnxt_hwrm_poll(bp); 12591 if (rc) { 12592 if (bnxt_fw_reset_timeout(bp)) { 12593 netdev_err(bp->dev, "Firmware reset aborted\n"); 12594 goto fw_reset_abort_status; 12595 } 12596 bnxt_queue_fw_reset_work(bp, HZ / 5); 12597 return; 12598 } 12599 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT; 12600 bp->fw_reset_state = BNXT_FW_RESET_STATE_OPENING; 12601 fallthrough; 12602 case BNXT_FW_RESET_STATE_OPENING: 12603 while (!rtnl_trylock()) { 12604 bnxt_queue_fw_reset_work(bp, HZ / 10); 12605 return; 12606 } 12607 rc = bnxt_open(bp->dev); 12608 if (rc) { 12609 netdev_err(bp->dev, "bnxt_open() failed during FW reset\n"); 12610 bnxt_fw_reset_abort(bp, rc); 12611 rtnl_unlock(); 12612 return; 12613 } 12614 12615 if ((bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) && 12616 bp->fw_health->enabled) { 12617 bp->fw_health->last_fw_reset_cnt = 12618 bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG); 12619 } 12620 bp->fw_reset_state = 0; 12621 /* Make sure fw_reset_state is 0 before clearing the flag */ 12622 smp_mb__before_atomic(); 12623 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 12624 bnxt_ulp_start(bp, 0); 12625 bnxt_reenable_sriov(bp); 12626 bnxt_vf_reps_alloc(bp); 12627 bnxt_vf_reps_open(bp); 12628 bnxt_ptp_reapply_pps(bp); 12629 clear_bit(BNXT_STATE_FW_ACTIVATE, &bp->state); 12630 if (test_and_clear_bit(BNXT_STATE_RECOVER, &bp->state)) { 12631 bnxt_dl_health_fw_recovery_done(bp); 12632 bnxt_dl_health_fw_status_update(bp, true); 12633 } 12634 rtnl_unlock(); 12635 break; 12636 } 12637 return; 12638 12639 fw_reset_abort_status: 12640 if (bp->fw_health->status_reliable || 12641 (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) { 12642 u32 sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG); 12643 12644 netdev_err(bp->dev, "fw_health_status 0x%x\n", sts); 12645 } 12646 fw_reset_abort: 12647 rtnl_lock(); 12648 bnxt_fw_reset_abort(bp, rc); 12649 rtnl_unlock(); 12650 } 12651 12652 static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev) 12653 { 12654 int rc; 12655 struct bnxt *bp = netdev_priv(dev); 12656 12657 SET_NETDEV_DEV(dev, &pdev->dev); 12658 12659 /* enable device (incl. PCI PM wakeup), and bus-mastering */ 12660 rc = pci_enable_device(pdev); 12661 if (rc) { 12662 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n"); 12663 goto init_err; 12664 } 12665 12666 if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) { 12667 dev_err(&pdev->dev, 12668 "Cannot find PCI device base address, aborting\n"); 12669 rc = -ENODEV; 12670 goto init_err_disable; 12671 } 12672 12673 rc = pci_request_regions(pdev, DRV_MODULE_NAME); 12674 if (rc) { 12675 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n"); 12676 goto init_err_disable; 12677 } 12678 12679 if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 && 12680 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) { 12681 dev_err(&pdev->dev, "System does not support DMA, aborting\n"); 12682 rc = -EIO; 12683 goto init_err_release; 12684 } 12685 12686 pci_set_master(pdev); 12687 12688 bp->dev = dev; 12689 bp->pdev = pdev; 12690 12691 /* Doorbell BAR bp->bar1 is mapped after bnxt_fw_init_one_p2() 12692 * determines the BAR size. 12693 */ 12694 bp->bar0 = pci_ioremap_bar(pdev, 0); 12695 if (!bp->bar0) { 12696 dev_err(&pdev->dev, "Cannot map device registers, aborting\n"); 12697 rc = -ENOMEM; 12698 goto init_err_release; 12699 } 12700 12701 bp->bar2 = pci_ioremap_bar(pdev, 4); 12702 if (!bp->bar2) { 12703 dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n"); 12704 rc = -ENOMEM; 12705 goto init_err_release; 12706 } 12707 12708 pci_enable_pcie_error_reporting(pdev); 12709 12710 INIT_WORK(&bp->sp_task, bnxt_sp_task); 12711 INIT_DELAYED_WORK(&bp->fw_reset_task, bnxt_fw_reset_task); 12712 12713 spin_lock_init(&bp->ntp_fltr_lock); 12714 #if BITS_PER_LONG == 32 12715 spin_lock_init(&bp->db_lock); 12716 #endif 12717 12718 bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE; 12719 bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE; 12720 12721 timer_setup(&bp->timer, bnxt_timer, 0); 12722 bp->current_interval = BNXT_TIMER_INTERVAL; 12723 12724 bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID; 12725 bp->nge_fw_dst_port_id = INVALID_HW_RING_ID; 12726 12727 clear_bit(BNXT_STATE_OPEN, &bp->state); 12728 return 0; 12729 12730 init_err_release: 12731 bnxt_unmap_bars(bp, pdev); 12732 pci_release_regions(pdev); 12733 12734 init_err_disable: 12735 pci_disable_device(pdev); 12736 12737 init_err: 12738 return rc; 12739 } 12740 12741 /* rtnl_lock held */ 12742 static int bnxt_change_mac_addr(struct net_device *dev, void *p) 12743 { 12744 struct sockaddr *addr = p; 12745 struct bnxt *bp = netdev_priv(dev); 12746 int rc = 0; 12747 12748 if (!is_valid_ether_addr(addr->sa_data)) 12749 return -EADDRNOTAVAIL; 12750 12751 if (ether_addr_equal(addr->sa_data, dev->dev_addr)) 12752 return 0; 12753 12754 rc = bnxt_approve_mac(bp, addr->sa_data, true); 12755 if (rc) 12756 return rc; 12757 12758 eth_hw_addr_set(dev, addr->sa_data); 12759 if (netif_running(dev)) { 12760 bnxt_close_nic(bp, false, false); 12761 rc = bnxt_open_nic(bp, false, false); 12762 } 12763 12764 return rc; 12765 } 12766 12767 /* rtnl_lock held */ 12768 static int bnxt_change_mtu(struct net_device *dev, int new_mtu) 12769 { 12770 struct bnxt *bp = netdev_priv(dev); 12771 12772 if (netif_running(dev)) 12773 bnxt_close_nic(bp, true, false); 12774 12775 dev->mtu = new_mtu; 12776 bnxt_set_ring_params(bp); 12777 12778 if (netif_running(dev)) 12779 return bnxt_open_nic(bp, true, false); 12780 12781 return 0; 12782 } 12783 12784 int bnxt_setup_mq_tc(struct net_device *dev, u8 tc) 12785 { 12786 struct bnxt *bp = netdev_priv(dev); 12787 bool sh = false; 12788 int rc; 12789 12790 if (tc > bp->max_tc) { 12791 netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n", 12792 tc, bp->max_tc); 12793 return -EINVAL; 12794 } 12795 12796 if (netdev_get_num_tc(dev) == tc) 12797 return 0; 12798 12799 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 12800 sh = true; 12801 12802 rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings, 12803 sh, tc, bp->tx_nr_rings_xdp); 12804 if (rc) 12805 return rc; 12806 12807 /* Needs to close the device and do hw resource re-allocations */ 12808 if (netif_running(bp->dev)) 12809 bnxt_close_nic(bp, true, false); 12810 12811 if (tc) { 12812 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc; 12813 netdev_set_num_tc(dev, tc); 12814 } else { 12815 bp->tx_nr_rings = bp->tx_nr_rings_per_tc; 12816 netdev_reset_tc(dev); 12817 } 12818 bp->tx_nr_rings += bp->tx_nr_rings_xdp; 12819 bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) : 12820 bp->tx_nr_rings + bp->rx_nr_rings; 12821 12822 if (netif_running(bp->dev)) 12823 return bnxt_open_nic(bp, true, false); 12824 12825 return 0; 12826 } 12827 12828 static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 12829 void *cb_priv) 12830 { 12831 struct bnxt *bp = cb_priv; 12832 12833 if (!bnxt_tc_flower_enabled(bp) || 12834 !tc_cls_can_offload_and_chain0(bp->dev, type_data)) 12835 return -EOPNOTSUPP; 12836 12837 switch (type) { 12838 case TC_SETUP_CLSFLOWER: 12839 return bnxt_tc_setup_flower(bp, bp->pf.fw_fid, type_data); 12840 default: 12841 return -EOPNOTSUPP; 12842 } 12843 } 12844 12845 LIST_HEAD(bnxt_block_cb_list); 12846 12847 static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type, 12848 void *type_data) 12849 { 12850 struct bnxt *bp = netdev_priv(dev); 12851 12852 switch (type) { 12853 case TC_SETUP_BLOCK: 12854 return flow_block_cb_setup_simple(type_data, 12855 &bnxt_block_cb_list, 12856 bnxt_setup_tc_block_cb, 12857 bp, bp, true); 12858 case TC_SETUP_QDISC_MQPRIO: { 12859 struct tc_mqprio_qopt *mqprio = type_data; 12860 12861 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 12862 12863 return bnxt_setup_mq_tc(dev, mqprio->num_tc); 12864 } 12865 default: 12866 return -EOPNOTSUPP; 12867 } 12868 } 12869 12870 #ifdef CONFIG_RFS_ACCEL 12871 static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1, 12872 struct bnxt_ntuple_filter *f2) 12873 { 12874 struct flow_keys *keys1 = &f1->fkeys; 12875 struct flow_keys *keys2 = &f2->fkeys; 12876 12877 if (keys1->basic.n_proto != keys2->basic.n_proto || 12878 keys1->basic.ip_proto != keys2->basic.ip_proto) 12879 return false; 12880 12881 if (keys1->basic.n_proto == htons(ETH_P_IP)) { 12882 if (keys1->addrs.v4addrs.src != keys2->addrs.v4addrs.src || 12883 keys1->addrs.v4addrs.dst != keys2->addrs.v4addrs.dst) 12884 return false; 12885 } else { 12886 if (memcmp(&keys1->addrs.v6addrs.src, &keys2->addrs.v6addrs.src, 12887 sizeof(keys1->addrs.v6addrs.src)) || 12888 memcmp(&keys1->addrs.v6addrs.dst, &keys2->addrs.v6addrs.dst, 12889 sizeof(keys1->addrs.v6addrs.dst))) 12890 return false; 12891 } 12892 12893 if (keys1->ports.ports == keys2->ports.ports && 12894 keys1->control.flags == keys2->control.flags && 12895 ether_addr_equal(f1->src_mac_addr, f2->src_mac_addr) && 12896 ether_addr_equal(f1->dst_mac_addr, f2->dst_mac_addr)) 12897 return true; 12898 12899 return false; 12900 } 12901 12902 static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb, 12903 u16 rxq_index, u32 flow_id) 12904 { 12905 struct bnxt *bp = netdev_priv(dev); 12906 struct bnxt_ntuple_filter *fltr, *new_fltr; 12907 struct flow_keys *fkeys; 12908 struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb); 12909 int rc = 0, idx, bit_id, l2_idx = 0; 12910 struct hlist_head *head; 12911 u32 flags; 12912 12913 if (!ether_addr_equal(dev->dev_addr, eth->h_dest)) { 12914 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 12915 int off = 0, j; 12916 12917 netif_addr_lock_bh(dev); 12918 for (j = 0; j < vnic->uc_filter_count; j++, off += ETH_ALEN) { 12919 if (ether_addr_equal(eth->h_dest, 12920 vnic->uc_list + off)) { 12921 l2_idx = j + 1; 12922 break; 12923 } 12924 } 12925 netif_addr_unlock_bh(dev); 12926 if (!l2_idx) 12927 return -EINVAL; 12928 } 12929 new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC); 12930 if (!new_fltr) 12931 return -ENOMEM; 12932 12933 fkeys = &new_fltr->fkeys; 12934 if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) { 12935 rc = -EPROTONOSUPPORT; 12936 goto err_free; 12937 } 12938 12939 if ((fkeys->basic.n_proto != htons(ETH_P_IP) && 12940 fkeys->basic.n_proto != htons(ETH_P_IPV6)) || 12941 ((fkeys->basic.ip_proto != IPPROTO_TCP) && 12942 (fkeys->basic.ip_proto != IPPROTO_UDP))) { 12943 rc = -EPROTONOSUPPORT; 12944 goto err_free; 12945 } 12946 if (fkeys->basic.n_proto == htons(ETH_P_IPV6) && 12947 bp->hwrm_spec_code < 0x10601) { 12948 rc = -EPROTONOSUPPORT; 12949 goto err_free; 12950 } 12951 flags = fkeys->control.flags; 12952 if (((flags & FLOW_DIS_ENCAPSULATION) && 12953 bp->hwrm_spec_code < 0x10601) || (flags & FLOW_DIS_IS_FRAGMENT)) { 12954 rc = -EPROTONOSUPPORT; 12955 goto err_free; 12956 } 12957 12958 memcpy(new_fltr->dst_mac_addr, eth->h_dest, ETH_ALEN); 12959 memcpy(new_fltr->src_mac_addr, eth->h_source, ETH_ALEN); 12960 12961 idx = skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK; 12962 head = &bp->ntp_fltr_hash_tbl[idx]; 12963 rcu_read_lock(); 12964 hlist_for_each_entry_rcu(fltr, head, hash) { 12965 if (bnxt_fltr_match(fltr, new_fltr)) { 12966 rc = fltr->sw_id; 12967 rcu_read_unlock(); 12968 goto err_free; 12969 } 12970 } 12971 rcu_read_unlock(); 12972 12973 spin_lock_bh(&bp->ntp_fltr_lock); 12974 bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap, 12975 BNXT_NTP_FLTR_MAX_FLTR, 0); 12976 if (bit_id < 0) { 12977 spin_unlock_bh(&bp->ntp_fltr_lock); 12978 rc = -ENOMEM; 12979 goto err_free; 12980 } 12981 12982 new_fltr->sw_id = (u16)bit_id; 12983 new_fltr->flow_id = flow_id; 12984 new_fltr->l2_fltr_idx = l2_idx; 12985 new_fltr->rxq = rxq_index; 12986 hlist_add_head_rcu(&new_fltr->hash, head); 12987 bp->ntp_fltr_count++; 12988 spin_unlock_bh(&bp->ntp_fltr_lock); 12989 12990 set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event); 12991 bnxt_queue_sp_work(bp); 12992 12993 return new_fltr->sw_id; 12994 12995 err_free: 12996 kfree(new_fltr); 12997 return rc; 12998 } 12999 13000 static void bnxt_cfg_ntp_filters(struct bnxt *bp) 13001 { 13002 int i; 13003 13004 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) { 13005 struct hlist_head *head; 13006 struct hlist_node *tmp; 13007 struct bnxt_ntuple_filter *fltr; 13008 int rc; 13009 13010 head = &bp->ntp_fltr_hash_tbl[i]; 13011 hlist_for_each_entry_safe(fltr, tmp, head, hash) { 13012 bool del = false; 13013 13014 if (test_bit(BNXT_FLTR_VALID, &fltr->state)) { 13015 if (rps_may_expire_flow(bp->dev, fltr->rxq, 13016 fltr->flow_id, 13017 fltr->sw_id)) { 13018 bnxt_hwrm_cfa_ntuple_filter_free(bp, 13019 fltr); 13020 del = true; 13021 } 13022 } else { 13023 rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp, 13024 fltr); 13025 if (rc) 13026 del = true; 13027 else 13028 set_bit(BNXT_FLTR_VALID, &fltr->state); 13029 } 13030 13031 if (del) { 13032 spin_lock_bh(&bp->ntp_fltr_lock); 13033 hlist_del_rcu(&fltr->hash); 13034 bp->ntp_fltr_count--; 13035 spin_unlock_bh(&bp->ntp_fltr_lock); 13036 synchronize_rcu(); 13037 clear_bit(fltr->sw_id, bp->ntp_fltr_bmap); 13038 kfree(fltr); 13039 } 13040 } 13041 } 13042 if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event)) 13043 netdev_info(bp->dev, "Receive PF driver unload event!\n"); 13044 } 13045 13046 #else 13047 13048 static void bnxt_cfg_ntp_filters(struct bnxt *bp) 13049 { 13050 } 13051 13052 #endif /* CONFIG_RFS_ACCEL */ 13053 13054 static int bnxt_udp_tunnel_sync(struct net_device *netdev, unsigned int table) 13055 { 13056 struct bnxt *bp = netdev_priv(netdev); 13057 struct udp_tunnel_info ti; 13058 unsigned int cmd; 13059 13060 udp_tunnel_nic_get_port(netdev, table, 0, &ti); 13061 if (ti.type == UDP_TUNNEL_TYPE_VXLAN) 13062 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN; 13063 else 13064 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE; 13065 13066 if (ti.port) 13067 return bnxt_hwrm_tunnel_dst_port_alloc(bp, ti.port, cmd); 13068 13069 return bnxt_hwrm_tunnel_dst_port_free(bp, cmd); 13070 } 13071 13072 static const struct udp_tunnel_nic_info bnxt_udp_tunnels = { 13073 .sync_table = bnxt_udp_tunnel_sync, 13074 .flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP | 13075 UDP_TUNNEL_NIC_INFO_OPEN_ONLY, 13076 .tables = { 13077 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN, }, 13078 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_GENEVE, }, 13079 }, 13080 }; 13081 13082 static int bnxt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 13083 struct net_device *dev, u32 filter_mask, 13084 int nlflags) 13085 { 13086 struct bnxt *bp = netdev_priv(dev); 13087 13088 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bp->br_mode, 0, 0, 13089 nlflags, filter_mask, NULL); 13090 } 13091 13092 static int bnxt_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh, 13093 u16 flags, struct netlink_ext_ack *extack) 13094 { 13095 struct bnxt *bp = netdev_priv(dev); 13096 struct nlattr *attr, *br_spec; 13097 int rem, rc = 0; 13098 13099 if (bp->hwrm_spec_code < 0x10708 || !BNXT_SINGLE_PF(bp)) 13100 return -EOPNOTSUPP; 13101 13102 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 13103 if (!br_spec) 13104 return -EINVAL; 13105 13106 nla_for_each_nested(attr, br_spec, rem) { 13107 u16 mode; 13108 13109 if (nla_type(attr) != IFLA_BRIDGE_MODE) 13110 continue; 13111 13112 if (nla_len(attr) < sizeof(mode)) 13113 return -EINVAL; 13114 13115 mode = nla_get_u16(attr); 13116 if (mode == bp->br_mode) 13117 break; 13118 13119 rc = bnxt_hwrm_set_br_mode(bp, mode); 13120 if (!rc) 13121 bp->br_mode = mode; 13122 break; 13123 } 13124 return rc; 13125 } 13126 13127 int bnxt_get_port_parent_id(struct net_device *dev, 13128 struct netdev_phys_item_id *ppid) 13129 { 13130 struct bnxt *bp = netdev_priv(dev); 13131 13132 if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV) 13133 return -EOPNOTSUPP; 13134 13135 /* The PF and it's VF-reps only support the switchdev framework */ 13136 if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_DSN_VALID)) 13137 return -EOPNOTSUPP; 13138 13139 ppid->id_len = sizeof(bp->dsn); 13140 memcpy(ppid->id, bp->dsn, ppid->id_len); 13141 13142 return 0; 13143 } 13144 13145 static const struct net_device_ops bnxt_netdev_ops = { 13146 .ndo_open = bnxt_open, 13147 .ndo_start_xmit = bnxt_start_xmit, 13148 .ndo_stop = bnxt_close, 13149 .ndo_get_stats64 = bnxt_get_stats64, 13150 .ndo_set_rx_mode = bnxt_set_rx_mode, 13151 .ndo_eth_ioctl = bnxt_ioctl, 13152 .ndo_validate_addr = eth_validate_addr, 13153 .ndo_set_mac_address = bnxt_change_mac_addr, 13154 .ndo_change_mtu = bnxt_change_mtu, 13155 .ndo_fix_features = bnxt_fix_features, 13156 .ndo_set_features = bnxt_set_features, 13157 .ndo_features_check = bnxt_features_check, 13158 .ndo_tx_timeout = bnxt_tx_timeout, 13159 #ifdef CONFIG_BNXT_SRIOV 13160 .ndo_get_vf_config = bnxt_get_vf_config, 13161 .ndo_set_vf_mac = bnxt_set_vf_mac, 13162 .ndo_set_vf_vlan = bnxt_set_vf_vlan, 13163 .ndo_set_vf_rate = bnxt_set_vf_bw, 13164 .ndo_set_vf_link_state = bnxt_set_vf_link_state, 13165 .ndo_set_vf_spoofchk = bnxt_set_vf_spoofchk, 13166 .ndo_set_vf_trust = bnxt_set_vf_trust, 13167 #endif 13168 .ndo_setup_tc = bnxt_setup_tc, 13169 #ifdef CONFIG_RFS_ACCEL 13170 .ndo_rx_flow_steer = bnxt_rx_flow_steer, 13171 #endif 13172 .ndo_bpf = bnxt_xdp, 13173 .ndo_xdp_xmit = bnxt_xdp_xmit, 13174 .ndo_bridge_getlink = bnxt_bridge_getlink, 13175 .ndo_bridge_setlink = bnxt_bridge_setlink, 13176 }; 13177 13178 static void bnxt_remove_one(struct pci_dev *pdev) 13179 { 13180 struct net_device *dev = pci_get_drvdata(pdev); 13181 struct bnxt *bp = netdev_priv(dev); 13182 13183 if (BNXT_PF(bp)) 13184 bnxt_sriov_disable(bp); 13185 13186 bnxt_rdma_aux_device_uninit(bp); 13187 13188 bnxt_ptp_clear(bp); 13189 pci_disable_pcie_error_reporting(pdev); 13190 unregister_netdev(dev); 13191 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 13192 /* Flush any pending tasks */ 13193 cancel_work_sync(&bp->sp_task); 13194 cancel_delayed_work_sync(&bp->fw_reset_task); 13195 bp->sp_event = 0; 13196 13197 bnxt_dl_fw_reporters_destroy(bp); 13198 bnxt_dl_unregister(bp); 13199 bnxt_shutdown_tc(bp); 13200 13201 bnxt_clear_int_mode(bp); 13202 bnxt_hwrm_func_drv_unrgtr(bp); 13203 bnxt_free_hwrm_resources(bp); 13204 bnxt_ethtool_free(bp); 13205 bnxt_dcb_free(bp); 13206 kfree(bp->ptp_cfg); 13207 bp->ptp_cfg = NULL; 13208 kfree(bp->fw_health); 13209 bp->fw_health = NULL; 13210 bnxt_cleanup_pci(bp); 13211 bnxt_free_ctx_mem(bp); 13212 kfree(bp->ctx); 13213 bp->ctx = NULL; 13214 kfree(bp->rss_indir_tbl); 13215 bp->rss_indir_tbl = NULL; 13216 bnxt_free_port_stats(bp); 13217 free_netdev(dev); 13218 } 13219 13220 static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt) 13221 { 13222 int rc = 0; 13223 struct bnxt_link_info *link_info = &bp->link_info; 13224 13225 bp->phy_flags = 0; 13226 rc = bnxt_hwrm_phy_qcaps(bp); 13227 if (rc) { 13228 netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n", 13229 rc); 13230 return rc; 13231 } 13232 if (bp->phy_flags & BNXT_PHY_FL_NO_FCS) 13233 bp->dev->priv_flags |= IFF_SUPP_NOFCS; 13234 else 13235 bp->dev->priv_flags &= ~IFF_SUPP_NOFCS; 13236 if (!fw_dflt) 13237 return 0; 13238 13239 mutex_lock(&bp->link_lock); 13240 rc = bnxt_update_link(bp, false); 13241 if (rc) { 13242 mutex_unlock(&bp->link_lock); 13243 netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n", 13244 rc); 13245 return rc; 13246 } 13247 13248 /* Older firmware does not have supported_auto_speeds, so assume 13249 * that all supported speeds can be autonegotiated. 13250 */ 13251 if (link_info->auto_link_speeds && !link_info->support_auto_speeds) 13252 link_info->support_auto_speeds = link_info->support_speeds; 13253 13254 bnxt_init_ethtool_link_settings(bp); 13255 mutex_unlock(&bp->link_lock); 13256 return 0; 13257 } 13258 13259 static int bnxt_get_max_irq(struct pci_dev *pdev) 13260 { 13261 u16 ctrl; 13262 13263 if (!pdev->msix_cap) 13264 return 1; 13265 13266 pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl); 13267 return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1; 13268 } 13269 13270 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, 13271 int *max_cp) 13272 { 13273 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 13274 int max_ring_grps = 0, max_irq; 13275 13276 *max_tx = hw_resc->max_tx_rings; 13277 *max_rx = hw_resc->max_rx_rings; 13278 *max_cp = bnxt_get_max_func_cp_rings_for_en(bp); 13279 max_irq = min_t(int, bnxt_get_max_func_irqs(bp) - 13280 bnxt_get_ulp_msix_num(bp), 13281 hw_resc->max_stat_ctxs - bnxt_get_ulp_stat_ctxs(bp)); 13282 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 13283 *max_cp = min_t(int, *max_cp, max_irq); 13284 max_ring_grps = hw_resc->max_hw_ring_grps; 13285 if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) { 13286 *max_cp -= 1; 13287 *max_rx -= 2; 13288 } 13289 if (bp->flags & BNXT_FLAG_AGG_RINGS) 13290 *max_rx >>= 1; 13291 if (bp->flags & BNXT_FLAG_CHIP_P5) { 13292 bnxt_trim_rings(bp, max_rx, max_tx, *max_cp, false); 13293 /* On P5 chips, max_cp output param should be available NQs */ 13294 *max_cp = max_irq; 13295 } 13296 *max_rx = min_t(int, *max_rx, max_ring_grps); 13297 } 13298 13299 int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared) 13300 { 13301 int rx, tx, cp; 13302 13303 _bnxt_get_max_rings(bp, &rx, &tx, &cp); 13304 *max_rx = rx; 13305 *max_tx = tx; 13306 if (!rx || !tx || !cp) 13307 return -ENOMEM; 13308 13309 return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared); 13310 } 13311 13312 static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx, 13313 bool shared) 13314 { 13315 int rc; 13316 13317 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared); 13318 if (rc && (bp->flags & BNXT_FLAG_AGG_RINGS)) { 13319 /* Not enough rings, try disabling agg rings. */ 13320 bp->flags &= ~BNXT_FLAG_AGG_RINGS; 13321 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared); 13322 if (rc) { 13323 /* set BNXT_FLAG_AGG_RINGS back for consistency */ 13324 bp->flags |= BNXT_FLAG_AGG_RINGS; 13325 return rc; 13326 } 13327 bp->flags |= BNXT_FLAG_NO_AGG_RINGS; 13328 bp->dev->hw_features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); 13329 bp->dev->features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); 13330 bnxt_set_ring_params(bp); 13331 } 13332 13333 if (bp->flags & BNXT_FLAG_ROCE_CAP) { 13334 int max_cp, max_stat, max_irq; 13335 13336 /* Reserve minimum resources for RoCE */ 13337 max_cp = bnxt_get_max_func_cp_rings(bp); 13338 max_stat = bnxt_get_max_func_stat_ctxs(bp); 13339 max_irq = bnxt_get_max_func_irqs(bp); 13340 if (max_cp <= BNXT_MIN_ROCE_CP_RINGS || 13341 max_irq <= BNXT_MIN_ROCE_CP_RINGS || 13342 max_stat <= BNXT_MIN_ROCE_STAT_CTXS) 13343 return 0; 13344 13345 max_cp -= BNXT_MIN_ROCE_CP_RINGS; 13346 max_irq -= BNXT_MIN_ROCE_CP_RINGS; 13347 max_stat -= BNXT_MIN_ROCE_STAT_CTXS; 13348 max_cp = min_t(int, max_cp, max_irq); 13349 max_cp = min_t(int, max_cp, max_stat); 13350 rc = bnxt_trim_rings(bp, max_rx, max_tx, max_cp, shared); 13351 if (rc) 13352 rc = 0; 13353 } 13354 return rc; 13355 } 13356 13357 /* In initial default shared ring setting, each shared ring must have a 13358 * RX/TX ring pair. 13359 */ 13360 static void bnxt_trim_dflt_sh_rings(struct bnxt *bp) 13361 { 13362 bp->cp_nr_rings = min_t(int, bp->tx_nr_rings_per_tc, bp->rx_nr_rings); 13363 bp->rx_nr_rings = bp->cp_nr_rings; 13364 bp->tx_nr_rings_per_tc = bp->cp_nr_rings; 13365 bp->tx_nr_rings = bp->tx_nr_rings_per_tc; 13366 } 13367 13368 static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh) 13369 { 13370 int dflt_rings, max_rx_rings, max_tx_rings, rc; 13371 13372 if (!bnxt_can_reserve_rings(bp)) 13373 return 0; 13374 13375 if (sh) 13376 bp->flags |= BNXT_FLAG_SHARED_RINGS; 13377 dflt_rings = is_kdump_kernel() ? 1 : netif_get_num_default_rss_queues(); 13378 /* Reduce default rings on multi-port cards so that total default 13379 * rings do not exceed CPU count. 13380 */ 13381 if (bp->port_count > 1) { 13382 int max_rings = 13383 max_t(int, num_online_cpus() / bp->port_count, 1); 13384 13385 dflt_rings = min_t(int, dflt_rings, max_rings); 13386 } 13387 rc = bnxt_get_dflt_rings(bp, &max_rx_rings, &max_tx_rings, sh); 13388 if (rc) 13389 return rc; 13390 bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings); 13391 bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings); 13392 if (sh) 13393 bnxt_trim_dflt_sh_rings(bp); 13394 else 13395 bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings; 13396 bp->tx_nr_rings = bp->tx_nr_rings_per_tc; 13397 13398 rc = __bnxt_reserve_rings(bp); 13399 if (rc && rc != -ENODEV) 13400 netdev_warn(bp->dev, "Unable to reserve tx rings\n"); 13401 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 13402 if (sh) 13403 bnxt_trim_dflt_sh_rings(bp); 13404 13405 /* Rings may have been trimmed, re-reserve the trimmed rings. */ 13406 if (bnxt_need_reserve_rings(bp)) { 13407 rc = __bnxt_reserve_rings(bp); 13408 if (rc && rc != -ENODEV) 13409 netdev_warn(bp->dev, "2nd rings reservation failed.\n"); 13410 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 13411 } 13412 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 13413 bp->rx_nr_rings++; 13414 bp->cp_nr_rings++; 13415 } 13416 if (rc) { 13417 bp->tx_nr_rings = 0; 13418 bp->rx_nr_rings = 0; 13419 } 13420 return rc; 13421 } 13422 13423 static int bnxt_init_dflt_ring_mode(struct bnxt *bp) 13424 { 13425 int rc; 13426 13427 if (bp->tx_nr_rings) 13428 return 0; 13429 13430 bnxt_ulp_irq_stop(bp); 13431 bnxt_clear_int_mode(bp); 13432 rc = bnxt_set_dflt_rings(bp, true); 13433 if (rc) { 13434 if (BNXT_VF(bp) && rc == -ENODEV) 13435 netdev_err(bp->dev, "Cannot configure VF rings while PF is unavailable.\n"); 13436 else 13437 netdev_err(bp->dev, "Not enough rings available.\n"); 13438 goto init_dflt_ring_err; 13439 } 13440 rc = bnxt_init_int_mode(bp); 13441 if (rc) 13442 goto init_dflt_ring_err; 13443 13444 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 13445 13446 bnxt_set_dflt_rfs(bp); 13447 13448 init_dflt_ring_err: 13449 bnxt_ulp_irq_restart(bp, rc); 13450 return rc; 13451 } 13452 13453 int bnxt_restore_pf_fw_resources(struct bnxt *bp) 13454 { 13455 int rc; 13456 13457 ASSERT_RTNL(); 13458 bnxt_hwrm_func_qcaps(bp); 13459 13460 if (netif_running(bp->dev)) 13461 __bnxt_close_nic(bp, true, false); 13462 13463 bnxt_ulp_irq_stop(bp); 13464 bnxt_clear_int_mode(bp); 13465 rc = bnxt_init_int_mode(bp); 13466 bnxt_ulp_irq_restart(bp, rc); 13467 13468 if (netif_running(bp->dev)) { 13469 if (rc) 13470 dev_close(bp->dev); 13471 else 13472 rc = bnxt_open_nic(bp, true, false); 13473 } 13474 13475 return rc; 13476 } 13477 13478 static int bnxt_init_mac_addr(struct bnxt *bp) 13479 { 13480 int rc = 0; 13481 13482 if (BNXT_PF(bp)) { 13483 eth_hw_addr_set(bp->dev, bp->pf.mac_addr); 13484 } else { 13485 #ifdef CONFIG_BNXT_SRIOV 13486 struct bnxt_vf_info *vf = &bp->vf; 13487 bool strict_approval = true; 13488 13489 if (is_valid_ether_addr(vf->mac_addr)) { 13490 /* overwrite netdev dev_addr with admin VF MAC */ 13491 eth_hw_addr_set(bp->dev, vf->mac_addr); 13492 /* Older PF driver or firmware may not approve this 13493 * correctly. 13494 */ 13495 strict_approval = false; 13496 } else { 13497 eth_hw_addr_random(bp->dev); 13498 } 13499 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, strict_approval); 13500 #endif 13501 } 13502 return rc; 13503 } 13504 13505 static void bnxt_vpd_read_info(struct bnxt *bp) 13506 { 13507 struct pci_dev *pdev = bp->pdev; 13508 unsigned int vpd_size, kw_len; 13509 int pos, size; 13510 u8 *vpd_data; 13511 13512 vpd_data = pci_vpd_alloc(pdev, &vpd_size); 13513 if (IS_ERR(vpd_data)) { 13514 pci_warn(pdev, "Unable to read VPD\n"); 13515 return; 13516 } 13517 13518 pos = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size, 13519 PCI_VPD_RO_KEYWORD_PARTNO, &kw_len); 13520 if (pos < 0) 13521 goto read_sn; 13522 13523 size = min_t(int, kw_len, BNXT_VPD_FLD_LEN - 1); 13524 memcpy(bp->board_partno, &vpd_data[pos], size); 13525 13526 read_sn: 13527 pos = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size, 13528 PCI_VPD_RO_KEYWORD_SERIALNO, 13529 &kw_len); 13530 if (pos < 0) 13531 goto exit; 13532 13533 size = min_t(int, kw_len, BNXT_VPD_FLD_LEN - 1); 13534 memcpy(bp->board_serialno, &vpd_data[pos], size); 13535 exit: 13536 kfree(vpd_data); 13537 } 13538 13539 static int bnxt_pcie_dsn_get(struct bnxt *bp, u8 dsn[]) 13540 { 13541 struct pci_dev *pdev = bp->pdev; 13542 u64 qword; 13543 13544 qword = pci_get_dsn(pdev); 13545 if (!qword) { 13546 netdev_info(bp->dev, "Unable to read adapter's DSN\n"); 13547 return -EOPNOTSUPP; 13548 } 13549 13550 put_unaligned_le64(qword, dsn); 13551 13552 bp->flags |= BNXT_FLAG_DSN_VALID; 13553 return 0; 13554 } 13555 13556 static int bnxt_map_db_bar(struct bnxt *bp) 13557 { 13558 if (!bp->db_size) 13559 return -ENODEV; 13560 bp->bar1 = pci_iomap(bp->pdev, 2, bp->db_size); 13561 if (!bp->bar1) 13562 return -ENOMEM; 13563 return 0; 13564 } 13565 13566 void bnxt_print_device_info(struct bnxt *bp) 13567 { 13568 netdev_info(bp->dev, "%s found at mem %lx, node addr %pM\n", 13569 board_info[bp->board_idx].name, 13570 (long)pci_resource_start(bp->pdev, 0), bp->dev->dev_addr); 13571 13572 pcie_print_link_status(bp->pdev); 13573 } 13574 13575 static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) 13576 { 13577 struct net_device *dev; 13578 struct bnxt *bp; 13579 int rc, max_irqs; 13580 13581 if (pci_is_bridge(pdev)) 13582 return -ENODEV; 13583 13584 /* Clear any pending DMA transactions from crash kernel 13585 * while loading driver in capture kernel. 13586 */ 13587 if (is_kdump_kernel()) { 13588 pci_clear_master(pdev); 13589 pcie_flr(pdev); 13590 } 13591 13592 max_irqs = bnxt_get_max_irq(pdev); 13593 dev = alloc_etherdev_mq(sizeof(*bp), max_irqs); 13594 if (!dev) 13595 return -ENOMEM; 13596 13597 bp = netdev_priv(dev); 13598 bp->board_idx = ent->driver_data; 13599 bp->msg_enable = BNXT_DEF_MSG_ENABLE; 13600 bnxt_set_max_func_irqs(bp, max_irqs); 13601 13602 if (bnxt_vf_pciid(bp->board_idx)) 13603 bp->flags |= BNXT_FLAG_VF; 13604 13605 /* No devlink port registration in case of a VF */ 13606 if (BNXT_PF(bp)) 13607 SET_NETDEV_DEVLINK_PORT(dev, &bp->dl_port); 13608 13609 if (pdev->msix_cap) 13610 bp->flags |= BNXT_FLAG_MSIX_CAP; 13611 13612 rc = bnxt_init_board(pdev, dev); 13613 if (rc < 0) 13614 goto init_err_free; 13615 13616 dev->netdev_ops = &bnxt_netdev_ops; 13617 dev->watchdog_timeo = BNXT_TX_TIMEOUT; 13618 dev->ethtool_ops = &bnxt_ethtool_ops; 13619 pci_set_drvdata(pdev, dev); 13620 13621 rc = bnxt_alloc_hwrm_resources(bp); 13622 if (rc) 13623 goto init_err_pci_clean; 13624 13625 mutex_init(&bp->hwrm_cmd_lock); 13626 mutex_init(&bp->link_lock); 13627 13628 rc = bnxt_fw_init_one_p1(bp); 13629 if (rc) 13630 goto init_err_pci_clean; 13631 13632 if (BNXT_PF(bp)) 13633 bnxt_vpd_read_info(bp); 13634 13635 if (BNXT_CHIP_P5(bp)) { 13636 bp->flags |= BNXT_FLAG_CHIP_P5; 13637 if (BNXT_CHIP_SR2(bp)) 13638 bp->flags |= BNXT_FLAG_CHIP_SR2; 13639 } 13640 13641 rc = bnxt_alloc_rss_indir_tbl(bp); 13642 if (rc) 13643 goto init_err_pci_clean; 13644 13645 rc = bnxt_fw_init_one_p2(bp); 13646 if (rc) 13647 goto init_err_pci_clean; 13648 13649 rc = bnxt_map_db_bar(bp); 13650 if (rc) { 13651 dev_err(&pdev->dev, "Cannot map doorbell BAR rc = %d, aborting\n", 13652 rc); 13653 goto init_err_pci_clean; 13654 } 13655 13656 dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG | 13657 NETIF_F_TSO | NETIF_F_TSO6 | 13658 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE | 13659 NETIF_F_GSO_IPXIP4 | 13660 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM | 13661 NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH | 13662 NETIF_F_RXCSUM | NETIF_F_GRO; 13663 13664 if (BNXT_SUPPORTS_TPA(bp)) 13665 dev->hw_features |= NETIF_F_LRO; 13666 13667 dev->hw_enc_features = 13668 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG | 13669 NETIF_F_TSO | NETIF_F_TSO6 | 13670 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE | 13671 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM | 13672 NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL; 13673 dev->udp_tunnel_nic_info = &bnxt_udp_tunnels; 13674 13675 dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM | 13676 NETIF_F_GSO_GRE_CSUM; 13677 dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA; 13678 if (bp->fw_cap & BNXT_FW_CAP_VLAN_RX_STRIP) 13679 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_RX; 13680 if (bp->fw_cap & BNXT_FW_CAP_VLAN_TX_INSERT) 13681 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_TX; 13682 if (BNXT_SUPPORTS_TPA(bp)) 13683 dev->hw_features |= NETIF_F_GRO_HW; 13684 dev->features |= dev->hw_features | NETIF_F_HIGHDMA; 13685 if (dev->features & NETIF_F_GRO_HW) 13686 dev->features &= ~NETIF_F_LRO; 13687 dev->priv_flags |= IFF_UNICAST_FLT; 13688 13689 netif_set_tso_max_size(dev, GSO_MAX_SIZE); 13690 13691 dev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | 13692 NETDEV_XDP_ACT_RX_SG; 13693 13694 #ifdef CONFIG_BNXT_SRIOV 13695 init_waitqueue_head(&bp->sriov_cfg_wait); 13696 #endif 13697 if (BNXT_SUPPORTS_TPA(bp)) { 13698 bp->gro_func = bnxt_gro_func_5730x; 13699 if (BNXT_CHIP_P4(bp)) 13700 bp->gro_func = bnxt_gro_func_5731x; 13701 else if (BNXT_CHIP_P5(bp)) 13702 bp->gro_func = bnxt_gro_func_5750x; 13703 } 13704 if (!BNXT_CHIP_P4_PLUS(bp)) 13705 bp->flags |= BNXT_FLAG_DOUBLE_DB; 13706 13707 rc = bnxt_init_mac_addr(bp); 13708 if (rc) { 13709 dev_err(&pdev->dev, "Unable to initialize mac address.\n"); 13710 rc = -EADDRNOTAVAIL; 13711 goto init_err_pci_clean; 13712 } 13713 13714 if (BNXT_PF(bp)) { 13715 /* Read the adapter's DSN to use as the eswitch switch_id */ 13716 rc = bnxt_pcie_dsn_get(bp, bp->dsn); 13717 } 13718 13719 /* MTU range: 60 - FW defined max */ 13720 dev->min_mtu = ETH_ZLEN; 13721 dev->max_mtu = bp->max_mtu; 13722 13723 rc = bnxt_probe_phy(bp, true); 13724 if (rc) 13725 goto init_err_pci_clean; 13726 13727 bnxt_set_rx_skb_mode(bp, false); 13728 bnxt_set_tpa_flags(bp); 13729 bnxt_set_ring_params(bp); 13730 rc = bnxt_set_dflt_rings(bp, true); 13731 if (rc) { 13732 if (BNXT_VF(bp) && rc == -ENODEV) { 13733 netdev_err(bp->dev, "Cannot configure VF rings while PF is unavailable.\n"); 13734 } else { 13735 netdev_err(bp->dev, "Not enough rings available.\n"); 13736 rc = -ENOMEM; 13737 } 13738 goto init_err_pci_clean; 13739 } 13740 13741 bnxt_fw_init_one_p3(bp); 13742 13743 bnxt_init_dflt_coal(bp); 13744 13745 if (dev->hw_features & BNXT_HW_FEATURE_VLAN_ALL_RX) 13746 bp->flags |= BNXT_FLAG_STRIP_VLAN; 13747 13748 rc = bnxt_init_int_mode(bp); 13749 if (rc) 13750 goto init_err_pci_clean; 13751 13752 /* No TC has been set yet and rings may have been trimmed due to 13753 * limited MSIX, so we re-initialize the TX rings per TC. 13754 */ 13755 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 13756 13757 if (BNXT_PF(bp)) { 13758 if (!bnxt_pf_wq) { 13759 bnxt_pf_wq = 13760 create_singlethread_workqueue("bnxt_pf_wq"); 13761 if (!bnxt_pf_wq) { 13762 dev_err(&pdev->dev, "Unable to create workqueue.\n"); 13763 rc = -ENOMEM; 13764 goto init_err_pci_clean; 13765 } 13766 } 13767 rc = bnxt_init_tc(bp); 13768 if (rc) 13769 netdev_err(dev, "Failed to initialize TC flower offload, err = %d.\n", 13770 rc); 13771 } 13772 13773 bnxt_inv_fw_health_reg(bp); 13774 rc = bnxt_dl_register(bp); 13775 if (rc) 13776 goto init_err_dl; 13777 13778 rc = register_netdev(dev); 13779 if (rc) 13780 goto init_err_cleanup; 13781 13782 bnxt_dl_fw_reporters_create(bp); 13783 13784 bnxt_rdma_aux_device_init(bp); 13785 13786 bnxt_print_device_info(bp); 13787 13788 pci_save_state(pdev); 13789 13790 return 0; 13791 init_err_cleanup: 13792 bnxt_dl_unregister(bp); 13793 init_err_dl: 13794 bnxt_shutdown_tc(bp); 13795 bnxt_clear_int_mode(bp); 13796 13797 init_err_pci_clean: 13798 bnxt_hwrm_func_drv_unrgtr(bp); 13799 bnxt_free_hwrm_resources(bp); 13800 bnxt_ethtool_free(bp); 13801 bnxt_ptp_clear(bp); 13802 kfree(bp->ptp_cfg); 13803 bp->ptp_cfg = NULL; 13804 kfree(bp->fw_health); 13805 bp->fw_health = NULL; 13806 bnxt_cleanup_pci(bp); 13807 bnxt_free_ctx_mem(bp); 13808 kfree(bp->ctx); 13809 bp->ctx = NULL; 13810 kfree(bp->rss_indir_tbl); 13811 bp->rss_indir_tbl = NULL; 13812 13813 init_err_free: 13814 free_netdev(dev); 13815 return rc; 13816 } 13817 13818 static void bnxt_shutdown(struct pci_dev *pdev) 13819 { 13820 struct net_device *dev = pci_get_drvdata(pdev); 13821 struct bnxt *bp; 13822 13823 if (!dev) 13824 return; 13825 13826 rtnl_lock(); 13827 bp = netdev_priv(dev); 13828 if (!bp) 13829 goto shutdown_exit; 13830 13831 if (netif_running(dev)) 13832 dev_close(dev); 13833 13834 bnxt_clear_int_mode(bp); 13835 pci_disable_device(pdev); 13836 13837 if (system_state == SYSTEM_POWER_OFF) { 13838 pci_wake_from_d3(pdev, bp->wol); 13839 pci_set_power_state(pdev, PCI_D3hot); 13840 } 13841 13842 shutdown_exit: 13843 rtnl_unlock(); 13844 } 13845 13846 #ifdef CONFIG_PM_SLEEP 13847 static int bnxt_suspend(struct device *device) 13848 { 13849 struct net_device *dev = dev_get_drvdata(device); 13850 struct bnxt *bp = netdev_priv(dev); 13851 int rc = 0; 13852 13853 rtnl_lock(); 13854 bnxt_ulp_stop(bp); 13855 if (netif_running(dev)) { 13856 netif_device_detach(dev); 13857 rc = bnxt_close(dev); 13858 } 13859 bnxt_hwrm_func_drv_unrgtr(bp); 13860 pci_disable_device(bp->pdev); 13861 bnxt_free_ctx_mem(bp); 13862 kfree(bp->ctx); 13863 bp->ctx = NULL; 13864 rtnl_unlock(); 13865 return rc; 13866 } 13867 13868 static int bnxt_resume(struct device *device) 13869 { 13870 struct net_device *dev = dev_get_drvdata(device); 13871 struct bnxt *bp = netdev_priv(dev); 13872 int rc = 0; 13873 13874 rtnl_lock(); 13875 rc = pci_enable_device(bp->pdev); 13876 if (rc) { 13877 netdev_err(dev, "Cannot re-enable PCI device during resume, err = %d\n", 13878 rc); 13879 goto resume_exit; 13880 } 13881 pci_set_master(bp->pdev); 13882 if (bnxt_hwrm_ver_get(bp)) { 13883 rc = -ENODEV; 13884 goto resume_exit; 13885 } 13886 rc = bnxt_hwrm_func_reset(bp); 13887 if (rc) { 13888 rc = -EBUSY; 13889 goto resume_exit; 13890 } 13891 13892 rc = bnxt_hwrm_func_qcaps(bp); 13893 if (rc) 13894 goto resume_exit; 13895 13896 if (bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false)) { 13897 rc = -ENODEV; 13898 goto resume_exit; 13899 } 13900 13901 bnxt_get_wol_settings(bp); 13902 if (netif_running(dev)) { 13903 rc = bnxt_open(dev); 13904 if (!rc) 13905 netif_device_attach(dev); 13906 } 13907 13908 resume_exit: 13909 bnxt_ulp_start(bp, rc); 13910 if (!rc) 13911 bnxt_reenable_sriov(bp); 13912 rtnl_unlock(); 13913 return rc; 13914 } 13915 13916 static SIMPLE_DEV_PM_OPS(bnxt_pm_ops, bnxt_suspend, bnxt_resume); 13917 #define BNXT_PM_OPS (&bnxt_pm_ops) 13918 13919 #else 13920 13921 #define BNXT_PM_OPS NULL 13922 13923 #endif /* CONFIG_PM_SLEEP */ 13924 13925 /** 13926 * bnxt_io_error_detected - called when PCI error is detected 13927 * @pdev: Pointer to PCI device 13928 * @state: The current pci connection state 13929 * 13930 * This function is called after a PCI bus error affecting 13931 * this device has been detected. 13932 */ 13933 static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev, 13934 pci_channel_state_t state) 13935 { 13936 struct net_device *netdev = pci_get_drvdata(pdev); 13937 struct bnxt *bp = netdev_priv(netdev); 13938 13939 netdev_info(netdev, "PCI I/O error detected\n"); 13940 13941 rtnl_lock(); 13942 netif_device_detach(netdev); 13943 13944 bnxt_ulp_stop(bp); 13945 13946 if (state == pci_channel_io_perm_failure) { 13947 rtnl_unlock(); 13948 return PCI_ERS_RESULT_DISCONNECT; 13949 } 13950 13951 if (state == pci_channel_io_frozen) 13952 set_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, &bp->state); 13953 13954 if (netif_running(netdev)) 13955 bnxt_close(netdev); 13956 13957 if (pci_is_enabled(pdev)) 13958 pci_disable_device(pdev); 13959 bnxt_free_ctx_mem(bp); 13960 kfree(bp->ctx); 13961 bp->ctx = NULL; 13962 rtnl_unlock(); 13963 13964 /* Request a slot slot reset. */ 13965 return PCI_ERS_RESULT_NEED_RESET; 13966 } 13967 13968 /** 13969 * bnxt_io_slot_reset - called after the pci bus has been reset. 13970 * @pdev: Pointer to PCI device 13971 * 13972 * Restart the card from scratch, as if from a cold-boot. 13973 * At this point, the card has exprienced a hard reset, 13974 * followed by fixups by BIOS, and has its config space 13975 * set up identically to what it was at cold boot. 13976 */ 13977 static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev) 13978 { 13979 pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT; 13980 struct net_device *netdev = pci_get_drvdata(pdev); 13981 struct bnxt *bp = netdev_priv(netdev); 13982 int retry = 0; 13983 int err = 0; 13984 int off; 13985 13986 netdev_info(bp->dev, "PCI Slot Reset\n"); 13987 13988 rtnl_lock(); 13989 13990 if (pci_enable_device(pdev)) { 13991 dev_err(&pdev->dev, 13992 "Cannot re-enable PCI device after reset.\n"); 13993 } else { 13994 pci_set_master(pdev); 13995 /* Upon fatal error, our device internal logic that latches to 13996 * BAR value is getting reset and will restore only upon 13997 * rewritting the BARs. 13998 * 13999 * As pci_restore_state() does not re-write the BARs if the 14000 * value is same as saved value earlier, driver needs to 14001 * write the BARs to 0 to force restore, in case of fatal error. 14002 */ 14003 if (test_and_clear_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, 14004 &bp->state)) { 14005 for (off = PCI_BASE_ADDRESS_0; 14006 off <= PCI_BASE_ADDRESS_5; off += 4) 14007 pci_write_config_dword(bp->pdev, off, 0); 14008 } 14009 pci_restore_state(pdev); 14010 pci_save_state(pdev); 14011 14012 bnxt_inv_fw_health_reg(bp); 14013 bnxt_try_map_fw_health_reg(bp); 14014 14015 /* In some PCIe AER scenarios, firmware may take up to 14016 * 10 seconds to become ready in the worst case. 14017 */ 14018 do { 14019 err = bnxt_try_recover_fw(bp); 14020 if (!err) 14021 break; 14022 retry++; 14023 } while (retry < BNXT_FW_SLOT_RESET_RETRY); 14024 14025 if (err) { 14026 dev_err(&pdev->dev, "Firmware not ready\n"); 14027 goto reset_exit; 14028 } 14029 14030 err = bnxt_hwrm_func_reset(bp); 14031 if (!err) 14032 result = PCI_ERS_RESULT_RECOVERED; 14033 14034 bnxt_ulp_irq_stop(bp); 14035 bnxt_clear_int_mode(bp); 14036 err = bnxt_init_int_mode(bp); 14037 bnxt_ulp_irq_restart(bp, err); 14038 } 14039 14040 reset_exit: 14041 bnxt_clear_reservations(bp, true); 14042 rtnl_unlock(); 14043 14044 return result; 14045 } 14046 14047 /** 14048 * bnxt_io_resume - called when traffic can start flowing again. 14049 * @pdev: Pointer to PCI device 14050 * 14051 * This callback is called when the error recovery driver tells 14052 * us that its OK to resume normal operation. 14053 */ 14054 static void bnxt_io_resume(struct pci_dev *pdev) 14055 { 14056 struct net_device *netdev = pci_get_drvdata(pdev); 14057 struct bnxt *bp = netdev_priv(netdev); 14058 int err; 14059 14060 netdev_info(bp->dev, "PCI Slot Resume\n"); 14061 rtnl_lock(); 14062 14063 err = bnxt_hwrm_func_qcaps(bp); 14064 if (!err && netif_running(netdev)) 14065 err = bnxt_open(netdev); 14066 14067 bnxt_ulp_start(bp, err); 14068 if (!err) { 14069 bnxt_reenable_sriov(bp); 14070 netif_device_attach(netdev); 14071 } 14072 14073 rtnl_unlock(); 14074 } 14075 14076 static const struct pci_error_handlers bnxt_err_handler = { 14077 .error_detected = bnxt_io_error_detected, 14078 .slot_reset = bnxt_io_slot_reset, 14079 .resume = bnxt_io_resume 14080 }; 14081 14082 static struct pci_driver bnxt_pci_driver = { 14083 .name = DRV_MODULE_NAME, 14084 .id_table = bnxt_pci_tbl, 14085 .probe = bnxt_init_one, 14086 .remove = bnxt_remove_one, 14087 .shutdown = bnxt_shutdown, 14088 .driver.pm = BNXT_PM_OPS, 14089 .err_handler = &bnxt_err_handler, 14090 #if defined(CONFIG_BNXT_SRIOV) 14091 .sriov_configure = bnxt_sriov_configure, 14092 #endif 14093 }; 14094 14095 static int __init bnxt_init(void) 14096 { 14097 int err; 14098 14099 bnxt_debug_init(); 14100 err = pci_register_driver(&bnxt_pci_driver); 14101 if (err) { 14102 bnxt_debug_exit(); 14103 return err; 14104 } 14105 14106 return 0; 14107 } 14108 14109 static void __exit bnxt_exit(void) 14110 { 14111 pci_unregister_driver(&bnxt_pci_driver); 14112 if (bnxt_pf_wq) 14113 destroy_workqueue(bnxt_pf_wq); 14114 bnxt_debug_exit(); 14115 } 14116 14117 module_init(bnxt_init); 14118 module_exit(bnxt_exit); 14119