1 /* Broadcom NetXtreme-C/E network driver. 2 * 3 * Copyright (c) 2014-2016 Broadcom Corporation 4 * Copyright (c) 2016-2019 Broadcom Limited 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License as published by 8 * the Free Software Foundation. 9 */ 10 11 #include <linux/module.h> 12 13 #include <linux/stringify.h> 14 #include <linux/kernel.h> 15 #include <linux/timer.h> 16 #include <linux/errno.h> 17 #include <linux/ioport.h> 18 #include <linux/slab.h> 19 #include <linux/vmalloc.h> 20 #include <linux/interrupt.h> 21 #include <linux/pci.h> 22 #include <linux/netdevice.h> 23 #include <linux/etherdevice.h> 24 #include <linux/skbuff.h> 25 #include <linux/dma-mapping.h> 26 #include <linux/bitops.h> 27 #include <linux/io.h> 28 #include <linux/irq.h> 29 #include <linux/delay.h> 30 #include <asm/byteorder.h> 31 #include <asm/page.h> 32 #include <linux/time.h> 33 #include <linux/mii.h> 34 #include <linux/mdio.h> 35 #include <linux/if.h> 36 #include <linux/if_vlan.h> 37 #include <linux/if_bridge.h> 38 #include <linux/rtc.h> 39 #include <linux/bpf.h> 40 #include <net/gro.h> 41 #include <net/ip.h> 42 #include <net/tcp.h> 43 #include <net/udp.h> 44 #include <net/checksum.h> 45 #include <net/ip6_checksum.h> 46 #include <net/udp_tunnel.h> 47 #include <linux/workqueue.h> 48 #include <linux/prefetch.h> 49 #include <linux/cache.h> 50 #include <linux/log2.h> 51 #include <linux/bitmap.h> 52 #include <linux/cpu_rmap.h> 53 #include <linux/cpumask.h> 54 #include <net/pkt_cls.h> 55 #include <linux/hwmon.h> 56 #include <linux/hwmon-sysfs.h> 57 #include <net/page_pool/helpers.h> 58 #include <linux/align.h> 59 #include <net/netdev_queues.h> 60 61 #include "bnxt_hsi.h" 62 #include "bnxt.h" 63 #include "bnxt_hwrm.h" 64 #include "bnxt_ulp.h" 65 #include "bnxt_sriov.h" 66 #include "bnxt_ethtool.h" 67 #include "bnxt_dcb.h" 68 #include "bnxt_xdp.h" 69 #include "bnxt_ptp.h" 70 #include "bnxt_vfr.h" 71 #include "bnxt_tc.h" 72 #include "bnxt_devlink.h" 73 #include "bnxt_debugfs.h" 74 75 #define BNXT_TX_TIMEOUT (5 * HZ) 76 #define BNXT_DEF_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_HW | \ 77 NETIF_MSG_TX_ERR) 78 79 MODULE_LICENSE("GPL"); 80 MODULE_DESCRIPTION("Broadcom BCM573xx network driver"); 81 82 #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN) 83 #define BNXT_RX_DMA_OFFSET NET_SKB_PAD 84 #define BNXT_RX_COPY_THRESH 256 85 86 #define BNXT_TX_PUSH_THRESH 164 87 88 /* indexed by enum board_idx */ 89 static const struct { 90 char *name; 91 } board_info[] = { 92 [BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" }, 93 [BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" }, 94 [BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" }, 95 [BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" }, 96 [BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" }, 97 [BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" }, 98 [BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" }, 99 [BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" }, 100 [BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" }, 101 [BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" }, 102 [BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" }, 103 [BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" }, 104 [BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" }, 105 [BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" }, 106 [BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" }, 107 [BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" }, 108 [BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" }, 109 [BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" }, 110 [BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" }, 111 [BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" }, 112 [BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" }, 113 [BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" }, 114 [BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" }, 115 [BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" }, 116 [BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" }, 117 [BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" }, 118 [BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" }, 119 [BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" }, 120 [BCM5745x_NPAR] = { "Broadcom BCM5745x NetXtreme-E Ethernet Partition" }, 121 [BCM57508] = { "Broadcom BCM57508 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" }, 122 [BCM57504] = { "Broadcom BCM57504 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" }, 123 [BCM57502] = { "Broadcom BCM57502 NetXtreme-E 10Gb/25Gb/50Gb Ethernet" }, 124 [BCM57508_NPAR] = { "Broadcom BCM57508 NetXtreme-E Ethernet Partition" }, 125 [BCM57504_NPAR] = { "Broadcom BCM57504 NetXtreme-E Ethernet Partition" }, 126 [BCM57502_NPAR] = { "Broadcom BCM57502 NetXtreme-E Ethernet Partition" }, 127 [BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" }, 128 [BCM58804] = { "Broadcom BCM58804 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" }, 129 [BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" }, 130 [NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" }, 131 [NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" }, 132 [NETXTREME_S_VF] = { "Broadcom NetXtreme-S Ethernet Virtual Function" }, 133 [NETXTREME_C_VF_HV] = { "Broadcom NetXtreme-C Virtual Function for Hyper-V" }, 134 [NETXTREME_E_VF_HV] = { "Broadcom NetXtreme-E Virtual Function for Hyper-V" }, 135 [NETXTREME_E_P5_VF] = { "Broadcom BCM5750X NetXtreme-E Ethernet Virtual Function" }, 136 [NETXTREME_E_P5_VF_HV] = { "Broadcom BCM5750X NetXtreme-E Virtual Function for Hyper-V" }, 137 }; 138 139 static const struct pci_device_id bnxt_pci_tbl[] = { 140 { PCI_VDEVICE(BROADCOM, 0x1604), .driver_data = BCM5745x_NPAR }, 141 { PCI_VDEVICE(BROADCOM, 0x1605), .driver_data = BCM5745x_NPAR }, 142 { PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 }, 143 { PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR }, 144 { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 }, 145 { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 }, 146 { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 }, 147 { PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR }, 148 { PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 }, 149 { PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 }, 150 { PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 }, 151 { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 }, 152 { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 }, 153 { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 }, 154 { PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR }, 155 { PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 }, 156 { PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 }, 157 { PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 }, 158 { PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 }, 159 { PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 }, 160 { PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR }, 161 { PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 }, 162 { PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP }, 163 { PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP }, 164 { PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR }, 165 { PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR }, 166 { PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP }, 167 { PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR }, 168 { PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR }, 169 { PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR }, 170 { PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR }, 171 { PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR }, 172 { PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR }, 173 { PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 }, 174 { PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 }, 175 { PCI_VDEVICE(BROADCOM, 0x1750), .driver_data = BCM57508 }, 176 { PCI_VDEVICE(BROADCOM, 0x1751), .driver_data = BCM57504 }, 177 { PCI_VDEVICE(BROADCOM, 0x1752), .driver_data = BCM57502 }, 178 { PCI_VDEVICE(BROADCOM, 0x1800), .driver_data = BCM57502_NPAR }, 179 { PCI_VDEVICE(BROADCOM, 0x1801), .driver_data = BCM57504_NPAR }, 180 { PCI_VDEVICE(BROADCOM, 0x1802), .driver_data = BCM57508_NPAR }, 181 { PCI_VDEVICE(BROADCOM, 0x1803), .driver_data = BCM57502_NPAR }, 182 { PCI_VDEVICE(BROADCOM, 0x1804), .driver_data = BCM57504_NPAR }, 183 { PCI_VDEVICE(BROADCOM, 0x1805), .driver_data = BCM57508_NPAR }, 184 { PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 }, 185 { PCI_VDEVICE(BROADCOM, 0xd804), .driver_data = BCM58804 }, 186 #ifdef CONFIG_BNXT_SRIOV 187 { PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF }, 188 { PCI_VDEVICE(BROADCOM, 0x1607), .driver_data = NETXTREME_E_VF_HV }, 189 { PCI_VDEVICE(BROADCOM, 0x1608), .driver_data = NETXTREME_E_VF_HV }, 190 { PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF }, 191 { PCI_VDEVICE(BROADCOM, 0x16bd), .driver_data = NETXTREME_E_VF_HV }, 192 { PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF }, 193 { PCI_VDEVICE(BROADCOM, 0x16c2), .driver_data = NETXTREME_C_VF_HV }, 194 { PCI_VDEVICE(BROADCOM, 0x16c3), .driver_data = NETXTREME_C_VF_HV }, 195 { PCI_VDEVICE(BROADCOM, 0x16c4), .driver_data = NETXTREME_E_VF_HV }, 196 { PCI_VDEVICE(BROADCOM, 0x16c5), .driver_data = NETXTREME_E_VF_HV }, 197 { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF }, 198 { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF }, 199 { PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF }, 200 { PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF }, 201 { PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF }, 202 { PCI_VDEVICE(BROADCOM, 0x16e6), .driver_data = NETXTREME_C_VF_HV }, 203 { PCI_VDEVICE(BROADCOM, 0x1806), .driver_data = NETXTREME_E_P5_VF }, 204 { PCI_VDEVICE(BROADCOM, 0x1807), .driver_data = NETXTREME_E_P5_VF }, 205 { PCI_VDEVICE(BROADCOM, 0x1808), .driver_data = NETXTREME_E_P5_VF_HV }, 206 { PCI_VDEVICE(BROADCOM, 0x1809), .driver_data = NETXTREME_E_P5_VF_HV }, 207 { PCI_VDEVICE(BROADCOM, 0xd800), .driver_data = NETXTREME_S_VF }, 208 #endif 209 { 0 } 210 }; 211 212 MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl); 213 214 static const u16 bnxt_vf_req_snif[] = { 215 HWRM_FUNC_CFG, 216 HWRM_FUNC_VF_CFG, 217 HWRM_PORT_PHY_QCFG, 218 HWRM_CFA_L2_FILTER_ALLOC, 219 }; 220 221 static const u16 bnxt_async_events_arr[] = { 222 ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE, 223 ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE, 224 ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD, 225 ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED, 226 ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE, 227 ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE, 228 ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE, 229 ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY, 230 ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY, 231 ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION, 232 ASYNC_EVENT_CMPL_EVENT_ID_DEFERRED_RESPONSE, 233 ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG, 234 ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST, 235 ASYNC_EVENT_CMPL_EVENT_ID_PPS_TIMESTAMP, 236 ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT, 237 ASYNC_EVENT_CMPL_EVENT_ID_PHC_UPDATE, 238 }; 239 240 static struct workqueue_struct *bnxt_pf_wq; 241 242 static bool bnxt_vf_pciid(enum board_idx idx) 243 { 244 return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF || 245 idx == NETXTREME_S_VF || idx == NETXTREME_C_VF_HV || 246 idx == NETXTREME_E_VF_HV || idx == NETXTREME_E_P5_VF || 247 idx == NETXTREME_E_P5_VF_HV); 248 } 249 250 #define DB_CP_REARM_FLAGS (DB_KEY_CP | DB_IDX_VALID) 251 #define DB_CP_FLAGS (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS) 252 #define DB_CP_IRQ_DIS_FLAGS (DB_KEY_CP | DB_IRQ_DIS) 253 254 #define BNXT_CP_DB_IRQ_DIS(db) \ 255 writel(DB_CP_IRQ_DIS_FLAGS, db) 256 257 #define BNXT_DB_CQ(db, idx) \ 258 writel(DB_CP_FLAGS | RING_CMP(idx), (db)->doorbell) 259 260 #define BNXT_DB_NQ_P5(db, idx) \ 261 bnxt_writeq(bp, (db)->db_key64 | DBR_TYPE_NQ | RING_CMP(idx), \ 262 (db)->doorbell) 263 264 #define BNXT_DB_CQ_ARM(db, idx) \ 265 writel(DB_CP_REARM_FLAGS | RING_CMP(idx), (db)->doorbell) 266 267 #define BNXT_DB_NQ_ARM_P5(db, idx) \ 268 bnxt_writeq(bp, (db)->db_key64 | DBR_TYPE_NQ_ARM | RING_CMP(idx),\ 269 (db)->doorbell) 270 271 static void bnxt_db_nq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx) 272 { 273 if (bp->flags & BNXT_FLAG_CHIP_P5) 274 BNXT_DB_NQ_P5(db, idx); 275 else 276 BNXT_DB_CQ(db, idx); 277 } 278 279 static void bnxt_db_nq_arm(struct bnxt *bp, struct bnxt_db_info *db, u32 idx) 280 { 281 if (bp->flags & BNXT_FLAG_CHIP_P5) 282 BNXT_DB_NQ_ARM_P5(db, idx); 283 else 284 BNXT_DB_CQ_ARM(db, idx); 285 } 286 287 static void bnxt_db_cq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx) 288 { 289 if (bp->flags & BNXT_FLAG_CHIP_P5) 290 bnxt_writeq(bp, db->db_key64 | DBR_TYPE_CQ_ARMALL | 291 RING_CMP(idx), db->doorbell); 292 else 293 BNXT_DB_CQ(db, idx); 294 } 295 296 static void bnxt_queue_fw_reset_work(struct bnxt *bp, unsigned long delay) 297 { 298 if (!(test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))) 299 return; 300 301 if (BNXT_PF(bp)) 302 queue_delayed_work(bnxt_pf_wq, &bp->fw_reset_task, delay); 303 else 304 schedule_delayed_work(&bp->fw_reset_task, delay); 305 } 306 307 static void __bnxt_queue_sp_work(struct bnxt *bp) 308 { 309 if (BNXT_PF(bp)) 310 queue_work(bnxt_pf_wq, &bp->sp_task); 311 else 312 schedule_work(&bp->sp_task); 313 } 314 315 static void bnxt_queue_sp_work(struct bnxt *bp, unsigned int event) 316 { 317 set_bit(event, &bp->sp_event); 318 __bnxt_queue_sp_work(bp); 319 } 320 321 static void bnxt_sched_reset_rxr(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) 322 { 323 if (!rxr->bnapi->in_reset) { 324 rxr->bnapi->in_reset = true; 325 if (bp->flags & BNXT_FLAG_CHIP_P5) 326 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event); 327 else 328 set_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event); 329 __bnxt_queue_sp_work(bp); 330 } 331 rxr->rx_next_cons = 0xffff; 332 } 333 334 void bnxt_sched_reset_txr(struct bnxt *bp, struct bnxt_tx_ring_info *txr, 335 int idx) 336 { 337 struct bnxt_napi *bnapi = txr->bnapi; 338 339 if (bnapi->tx_fault) 340 return; 341 342 netdev_err(bp->dev, "Invalid Tx completion (ring:%d tx_pkts:%d cons:%u prod:%u i:%d)", 343 txr->txq_index, bnapi->tx_pkts, 344 txr->tx_cons, txr->tx_prod, idx); 345 WARN_ON_ONCE(1); 346 bnapi->tx_fault = 1; 347 bnxt_queue_sp_work(bp, BNXT_RESET_TASK_SP_EVENT); 348 } 349 350 const u16 bnxt_lhint_arr[] = { 351 TX_BD_FLAGS_LHINT_512_AND_SMALLER, 352 TX_BD_FLAGS_LHINT_512_TO_1023, 353 TX_BD_FLAGS_LHINT_1024_TO_2047, 354 TX_BD_FLAGS_LHINT_1024_TO_2047, 355 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 356 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 357 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 358 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 359 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 360 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 361 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 362 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 363 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 364 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 365 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 366 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 367 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 368 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 369 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 370 }; 371 372 static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb) 373 { 374 struct metadata_dst *md_dst = skb_metadata_dst(skb); 375 376 if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX) 377 return 0; 378 379 return md_dst->u.port_info.port_id; 380 } 381 382 static void bnxt_txr_db_kick(struct bnxt *bp, struct bnxt_tx_ring_info *txr, 383 u16 prod) 384 { 385 bnxt_db_write(bp, &txr->tx_db, prod); 386 txr->kick_pending = 0; 387 } 388 389 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev) 390 { 391 struct bnxt *bp = netdev_priv(dev); 392 struct tx_bd *txbd; 393 struct tx_bd_ext *txbd1; 394 struct netdev_queue *txq; 395 int i; 396 dma_addr_t mapping; 397 unsigned int length, pad = 0; 398 u32 len, free_size, vlan_tag_flags, cfa_action, flags; 399 u16 prod, last_frag; 400 struct pci_dev *pdev = bp->pdev; 401 struct bnxt_tx_ring_info *txr; 402 struct bnxt_sw_tx_bd *tx_buf; 403 __le32 lflags = 0; 404 405 i = skb_get_queue_mapping(skb); 406 if (unlikely(i >= bp->tx_nr_rings)) { 407 dev_kfree_skb_any(skb); 408 dev_core_stats_tx_dropped_inc(dev); 409 return NETDEV_TX_OK; 410 } 411 412 txq = netdev_get_tx_queue(dev, i); 413 txr = &bp->tx_ring[bp->tx_ring_map[i]]; 414 prod = txr->tx_prod; 415 416 free_size = bnxt_tx_avail(bp, txr); 417 if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) { 418 /* We must have raced with NAPI cleanup */ 419 if (net_ratelimit() && txr->kick_pending) 420 netif_warn(bp, tx_err, dev, 421 "bnxt: ring busy w/ flush pending!\n"); 422 if (!netif_txq_try_stop(txq, bnxt_tx_avail(bp, txr), 423 bp->tx_wake_thresh)) 424 return NETDEV_TX_BUSY; 425 } 426 427 if (unlikely(ipv6_hopopt_jumbo_remove(skb))) 428 goto tx_free; 429 430 length = skb->len; 431 len = skb_headlen(skb); 432 last_frag = skb_shinfo(skb)->nr_frags; 433 434 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 435 436 txbd->tx_bd_opaque = prod; 437 438 tx_buf = &txr->tx_buf_ring[prod]; 439 tx_buf->skb = skb; 440 tx_buf->nr_frags = last_frag; 441 442 vlan_tag_flags = 0; 443 cfa_action = bnxt_xmit_get_cfa_action(skb); 444 if (skb_vlan_tag_present(skb)) { 445 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN | 446 skb_vlan_tag_get(skb); 447 /* Currently supports 8021Q, 8021AD vlan offloads 448 * QINQ1, QINQ2, QINQ3 vlan headers are deprecated 449 */ 450 if (skb->vlan_proto == htons(ETH_P_8021Q)) 451 vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT; 452 } 453 454 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) { 455 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; 456 457 if (ptp && ptp->tx_tstamp_en && !skb_is_gso(skb) && 458 atomic_dec_if_positive(&ptp->tx_avail) >= 0) { 459 if (!bnxt_ptp_parse(skb, &ptp->tx_seqid, 460 &ptp->tx_hdr_off)) { 461 if (vlan_tag_flags) 462 ptp->tx_hdr_off += VLAN_HLEN; 463 lflags |= cpu_to_le32(TX_BD_FLAGS_STAMP); 464 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 465 } else { 466 atomic_inc(&bp->ptp_cfg->tx_avail); 467 } 468 } 469 } 470 471 if (unlikely(skb->no_fcs)) 472 lflags |= cpu_to_le32(TX_BD_FLAGS_NO_CRC); 473 474 if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh && 475 !lflags) { 476 struct tx_push_buffer *tx_push_buf = txr->tx_push; 477 struct tx_push_bd *tx_push = &tx_push_buf->push_bd; 478 struct tx_bd_ext *tx_push1 = &tx_push->txbd2; 479 void __iomem *db = txr->tx_db.doorbell; 480 void *pdata = tx_push_buf->data; 481 u64 *end; 482 int j, push_len; 483 484 /* Set COAL_NOW to be ready quickly for the next push */ 485 tx_push->tx_bd_len_flags_type = 486 cpu_to_le32((length << TX_BD_LEN_SHIFT) | 487 TX_BD_TYPE_LONG_TX_BD | 488 TX_BD_FLAGS_LHINT_512_AND_SMALLER | 489 TX_BD_FLAGS_COAL_NOW | 490 TX_BD_FLAGS_PACKET_END | 491 (2 << TX_BD_FLAGS_BD_CNT_SHIFT)); 492 493 if (skb->ip_summed == CHECKSUM_PARTIAL) 494 tx_push1->tx_bd_hsize_lflags = 495 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM); 496 else 497 tx_push1->tx_bd_hsize_lflags = 0; 498 499 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags); 500 tx_push1->tx_bd_cfa_action = 501 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT); 502 503 end = pdata + length; 504 end = PTR_ALIGN(end, 8) - 1; 505 *end = 0; 506 507 skb_copy_from_linear_data(skb, pdata, len); 508 pdata += len; 509 for (j = 0; j < last_frag; j++) { 510 skb_frag_t *frag = &skb_shinfo(skb)->frags[j]; 511 void *fptr; 512 513 fptr = skb_frag_address_safe(frag); 514 if (!fptr) 515 goto normal_tx; 516 517 memcpy(pdata, fptr, skb_frag_size(frag)); 518 pdata += skb_frag_size(frag); 519 } 520 521 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type; 522 txbd->tx_bd_haddr = txr->data_mapping; 523 prod = NEXT_TX(prod); 524 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 525 memcpy(txbd, tx_push1, sizeof(*txbd)); 526 prod = NEXT_TX(prod); 527 tx_push->doorbell = 528 cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | prod); 529 WRITE_ONCE(txr->tx_prod, prod); 530 531 tx_buf->is_push = 1; 532 netdev_tx_sent_queue(txq, skb->len); 533 wmb(); /* Sync is_push and byte queue before pushing data */ 534 535 push_len = (length + sizeof(*tx_push) + 7) / 8; 536 if (push_len > 16) { 537 __iowrite64_copy(db, tx_push_buf, 16); 538 __iowrite32_copy(db + 4, tx_push_buf + 1, 539 (push_len - 16) << 1); 540 } else { 541 __iowrite64_copy(db, tx_push_buf, push_len); 542 } 543 544 goto tx_done; 545 } 546 547 normal_tx: 548 if (length < BNXT_MIN_PKT_SIZE) { 549 pad = BNXT_MIN_PKT_SIZE - length; 550 if (skb_pad(skb, pad)) 551 /* SKB already freed. */ 552 goto tx_kick_pending; 553 length = BNXT_MIN_PKT_SIZE; 554 } 555 556 mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE); 557 558 if (unlikely(dma_mapping_error(&pdev->dev, mapping))) 559 goto tx_free; 560 561 dma_unmap_addr_set(tx_buf, mapping, mapping); 562 flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD | 563 ((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT); 564 565 txbd->tx_bd_haddr = cpu_to_le64(mapping); 566 567 prod = NEXT_TX(prod); 568 txbd1 = (struct tx_bd_ext *) 569 &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 570 571 txbd1->tx_bd_hsize_lflags = lflags; 572 if (skb_is_gso(skb)) { 573 u32 hdr_len; 574 575 if (skb->encapsulation) 576 hdr_len = skb_inner_tcp_all_headers(skb); 577 else 578 hdr_len = skb_tcp_all_headers(skb); 579 580 txbd1->tx_bd_hsize_lflags |= cpu_to_le32(TX_BD_FLAGS_LSO | 581 TX_BD_FLAGS_T_IPID | 582 (hdr_len << (TX_BD_HSIZE_SHIFT - 1))); 583 length = skb_shinfo(skb)->gso_size; 584 txbd1->tx_bd_mss = cpu_to_le32(length); 585 length += hdr_len; 586 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { 587 txbd1->tx_bd_hsize_lflags |= 588 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM); 589 txbd1->tx_bd_mss = 0; 590 } 591 592 length >>= 9; 593 if (unlikely(length >= ARRAY_SIZE(bnxt_lhint_arr))) { 594 dev_warn_ratelimited(&pdev->dev, "Dropped oversize %d bytes TX packet.\n", 595 skb->len); 596 i = 0; 597 goto tx_dma_error; 598 } 599 flags |= bnxt_lhint_arr[length]; 600 txbd->tx_bd_len_flags_type = cpu_to_le32(flags); 601 602 txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags); 603 txbd1->tx_bd_cfa_action = 604 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT); 605 for (i = 0; i < last_frag; i++) { 606 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 607 608 prod = NEXT_TX(prod); 609 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 610 611 len = skb_frag_size(frag); 612 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len, 613 DMA_TO_DEVICE); 614 615 if (unlikely(dma_mapping_error(&pdev->dev, mapping))) 616 goto tx_dma_error; 617 618 tx_buf = &txr->tx_buf_ring[prod]; 619 dma_unmap_addr_set(tx_buf, mapping, mapping); 620 621 txbd->tx_bd_haddr = cpu_to_le64(mapping); 622 623 flags = len << TX_BD_LEN_SHIFT; 624 txbd->tx_bd_len_flags_type = cpu_to_le32(flags); 625 } 626 627 flags &= ~TX_BD_LEN; 628 txbd->tx_bd_len_flags_type = 629 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags | 630 TX_BD_FLAGS_PACKET_END); 631 632 netdev_tx_sent_queue(txq, skb->len); 633 634 skb_tx_timestamp(skb); 635 636 /* Sync BD data before updating doorbell */ 637 wmb(); 638 639 prod = NEXT_TX(prod); 640 WRITE_ONCE(txr->tx_prod, prod); 641 642 if (!netdev_xmit_more() || netif_xmit_stopped(txq)) 643 bnxt_txr_db_kick(bp, txr, prod); 644 else 645 txr->kick_pending = 1; 646 647 tx_done: 648 649 if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) { 650 if (netdev_xmit_more() && !tx_buf->is_push) 651 bnxt_txr_db_kick(bp, txr, prod); 652 653 netif_txq_try_stop(txq, bnxt_tx_avail(bp, txr), 654 bp->tx_wake_thresh); 655 } 656 return NETDEV_TX_OK; 657 658 tx_dma_error: 659 if (BNXT_TX_PTP_IS_SET(lflags)) 660 atomic_inc(&bp->ptp_cfg->tx_avail); 661 662 last_frag = i; 663 664 /* start back at beginning and unmap skb */ 665 prod = txr->tx_prod; 666 tx_buf = &txr->tx_buf_ring[prod]; 667 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping), 668 skb_headlen(skb), DMA_TO_DEVICE); 669 prod = NEXT_TX(prod); 670 671 /* unmap remaining mapped pages */ 672 for (i = 0; i < last_frag; i++) { 673 prod = NEXT_TX(prod); 674 tx_buf = &txr->tx_buf_ring[prod]; 675 dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping), 676 skb_frag_size(&skb_shinfo(skb)->frags[i]), 677 DMA_TO_DEVICE); 678 } 679 680 tx_free: 681 dev_kfree_skb_any(skb); 682 tx_kick_pending: 683 if (txr->kick_pending) 684 bnxt_txr_db_kick(bp, txr, txr->tx_prod); 685 txr->tx_buf_ring[txr->tx_prod].skb = NULL; 686 dev_core_stats_tx_dropped_inc(dev); 687 return NETDEV_TX_OK; 688 } 689 690 static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int budget) 691 { 692 struct bnxt_tx_ring_info *txr = bnapi->tx_ring; 693 struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index); 694 u16 cons = txr->tx_cons; 695 struct pci_dev *pdev = bp->pdev; 696 int nr_pkts = bnapi->tx_pkts; 697 int i; 698 unsigned int tx_bytes = 0; 699 700 for (i = 0; i < nr_pkts; i++) { 701 struct bnxt_sw_tx_bd *tx_buf; 702 struct sk_buff *skb; 703 int j, last; 704 705 tx_buf = &txr->tx_buf_ring[cons]; 706 cons = NEXT_TX(cons); 707 skb = tx_buf->skb; 708 tx_buf->skb = NULL; 709 710 if (unlikely(!skb)) { 711 bnxt_sched_reset_txr(bp, txr, i); 712 return; 713 } 714 715 tx_bytes += skb->len; 716 717 if (tx_buf->is_push) { 718 tx_buf->is_push = 0; 719 goto next_tx_int; 720 } 721 722 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping), 723 skb_headlen(skb), DMA_TO_DEVICE); 724 last = tx_buf->nr_frags; 725 726 for (j = 0; j < last; j++) { 727 cons = NEXT_TX(cons); 728 tx_buf = &txr->tx_buf_ring[cons]; 729 dma_unmap_page( 730 &pdev->dev, 731 dma_unmap_addr(tx_buf, mapping), 732 skb_frag_size(&skb_shinfo(skb)->frags[j]), 733 DMA_TO_DEVICE); 734 } 735 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)) { 736 if (bp->flags & BNXT_FLAG_CHIP_P5) { 737 /* PTP worker takes ownership of the skb */ 738 if (!bnxt_get_tx_ts_p5(bp, skb)) 739 skb = NULL; 740 else 741 atomic_inc(&bp->ptp_cfg->tx_avail); 742 } 743 } 744 745 next_tx_int: 746 cons = NEXT_TX(cons); 747 748 dev_consume_skb_any(skb); 749 } 750 751 bnapi->tx_pkts = 0; 752 WRITE_ONCE(txr->tx_cons, cons); 753 754 __netif_txq_completed_wake(txq, nr_pkts, tx_bytes, 755 bnxt_tx_avail(bp, txr), bp->tx_wake_thresh, 756 READ_ONCE(txr->dev_state) == BNXT_DEV_STATE_CLOSING); 757 } 758 759 static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping, 760 struct bnxt_rx_ring_info *rxr, 761 unsigned int *offset, 762 gfp_t gfp) 763 { 764 struct page *page; 765 766 if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) { 767 page = page_pool_dev_alloc_frag(rxr->page_pool, offset, 768 BNXT_RX_PAGE_SIZE); 769 } else { 770 page = page_pool_dev_alloc_pages(rxr->page_pool); 771 *offset = 0; 772 } 773 if (!page) 774 return NULL; 775 776 *mapping = page_pool_get_dma_addr(page) + *offset; 777 return page; 778 } 779 780 static inline u8 *__bnxt_alloc_rx_frag(struct bnxt *bp, dma_addr_t *mapping, 781 gfp_t gfp) 782 { 783 u8 *data; 784 struct pci_dev *pdev = bp->pdev; 785 786 if (gfp == GFP_ATOMIC) 787 data = napi_alloc_frag(bp->rx_buf_size); 788 else 789 data = netdev_alloc_frag(bp->rx_buf_size); 790 if (!data) 791 return NULL; 792 793 *mapping = dma_map_single_attrs(&pdev->dev, data + bp->rx_dma_offset, 794 bp->rx_buf_use_size, bp->rx_dir, 795 DMA_ATTR_WEAK_ORDERING); 796 797 if (dma_mapping_error(&pdev->dev, *mapping)) { 798 skb_free_frag(data); 799 data = NULL; 800 } 801 return data; 802 } 803 804 int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, 805 u16 prod, gfp_t gfp) 806 { 807 struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 808 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod]; 809 dma_addr_t mapping; 810 811 if (BNXT_RX_PAGE_MODE(bp)) { 812 unsigned int offset; 813 struct page *page = 814 __bnxt_alloc_rx_page(bp, &mapping, rxr, &offset, gfp); 815 816 if (!page) 817 return -ENOMEM; 818 819 mapping += bp->rx_dma_offset; 820 rx_buf->data = page; 821 rx_buf->data_ptr = page_address(page) + offset + bp->rx_offset; 822 } else { 823 u8 *data = __bnxt_alloc_rx_frag(bp, &mapping, gfp); 824 825 if (!data) 826 return -ENOMEM; 827 828 rx_buf->data = data; 829 rx_buf->data_ptr = data + bp->rx_offset; 830 } 831 rx_buf->mapping = mapping; 832 833 rxbd->rx_bd_haddr = cpu_to_le64(mapping); 834 return 0; 835 } 836 837 void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data) 838 { 839 u16 prod = rxr->rx_prod; 840 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf; 841 struct rx_bd *cons_bd, *prod_bd; 842 843 prod_rx_buf = &rxr->rx_buf_ring[prod]; 844 cons_rx_buf = &rxr->rx_buf_ring[cons]; 845 846 prod_rx_buf->data = data; 847 prod_rx_buf->data_ptr = cons_rx_buf->data_ptr; 848 849 prod_rx_buf->mapping = cons_rx_buf->mapping; 850 851 prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 852 cons_bd = &rxr->rx_desc_ring[RX_RING(cons)][RX_IDX(cons)]; 853 854 prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr; 855 } 856 857 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx) 858 { 859 u16 next, max = rxr->rx_agg_bmap_size; 860 861 next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx); 862 if (next >= max) 863 next = find_first_zero_bit(rxr->rx_agg_bmap, max); 864 return next; 865 } 866 867 static inline int bnxt_alloc_rx_page(struct bnxt *bp, 868 struct bnxt_rx_ring_info *rxr, 869 u16 prod, gfp_t gfp) 870 { 871 struct rx_bd *rxbd = 872 &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 873 struct bnxt_sw_rx_agg_bd *rx_agg_buf; 874 struct page *page; 875 dma_addr_t mapping; 876 u16 sw_prod = rxr->rx_sw_agg_prod; 877 unsigned int offset = 0; 878 879 page = __bnxt_alloc_rx_page(bp, &mapping, rxr, &offset, gfp); 880 881 if (!page) 882 return -ENOMEM; 883 884 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap))) 885 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod); 886 887 __set_bit(sw_prod, rxr->rx_agg_bmap); 888 rx_agg_buf = &rxr->rx_agg_ring[sw_prod]; 889 rxr->rx_sw_agg_prod = NEXT_RX_AGG(sw_prod); 890 891 rx_agg_buf->page = page; 892 rx_agg_buf->offset = offset; 893 rx_agg_buf->mapping = mapping; 894 rxbd->rx_bd_haddr = cpu_to_le64(mapping); 895 rxbd->rx_bd_opaque = sw_prod; 896 return 0; 897 } 898 899 static struct rx_agg_cmp *bnxt_get_agg(struct bnxt *bp, 900 struct bnxt_cp_ring_info *cpr, 901 u16 cp_cons, u16 curr) 902 { 903 struct rx_agg_cmp *agg; 904 905 cp_cons = RING_CMP(ADV_RAW_CMP(cp_cons, curr)); 906 agg = (struct rx_agg_cmp *) 907 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 908 return agg; 909 } 910 911 static struct rx_agg_cmp *bnxt_get_tpa_agg_p5(struct bnxt *bp, 912 struct bnxt_rx_ring_info *rxr, 913 u16 agg_id, u16 curr) 914 { 915 struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[agg_id]; 916 917 return &tpa_info->agg_arr[curr]; 918 } 919 920 static void bnxt_reuse_rx_agg_bufs(struct bnxt_cp_ring_info *cpr, u16 idx, 921 u16 start, u32 agg_bufs, bool tpa) 922 { 923 struct bnxt_napi *bnapi = cpr->bnapi; 924 struct bnxt *bp = bnapi->bp; 925 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 926 u16 prod = rxr->rx_agg_prod; 927 u16 sw_prod = rxr->rx_sw_agg_prod; 928 bool p5_tpa = false; 929 u32 i; 930 931 if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa) 932 p5_tpa = true; 933 934 for (i = 0; i < agg_bufs; i++) { 935 u16 cons; 936 struct rx_agg_cmp *agg; 937 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf; 938 struct rx_bd *prod_bd; 939 struct page *page; 940 941 if (p5_tpa) 942 agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, start + i); 943 else 944 agg = bnxt_get_agg(bp, cpr, idx, start + i); 945 cons = agg->rx_agg_cmp_opaque; 946 __clear_bit(cons, rxr->rx_agg_bmap); 947 948 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap))) 949 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod); 950 951 __set_bit(sw_prod, rxr->rx_agg_bmap); 952 prod_rx_buf = &rxr->rx_agg_ring[sw_prod]; 953 cons_rx_buf = &rxr->rx_agg_ring[cons]; 954 955 /* It is possible for sw_prod to be equal to cons, so 956 * set cons_rx_buf->page to NULL first. 957 */ 958 page = cons_rx_buf->page; 959 cons_rx_buf->page = NULL; 960 prod_rx_buf->page = page; 961 prod_rx_buf->offset = cons_rx_buf->offset; 962 963 prod_rx_buf->mapping = cons_rx_buf->mapping; 964 965 prod_bd = &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 966 967 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping); 968 prod_bd->rx_bd_opaque = sw_prod; 969 970 prod = NEXT_RX_AGG(prod); 971 sw_prod = NEXT_RX_AGG(sw_prod); 972 } 973 rxr->rx_agg_prod = prod; 974 rxr->rx_sw_agg_prod = sw_prod; 975 } 976 977 static struct sk_buff *bnxt_rx_multi_page_skb(struct bnxt *bp, 978 struct bnxt_rx_ring_info *rxr, 979 u16 cons, void *data, u8 *data_ptr, 980 dma_addr_t dma_addr, 981 unsigned int offset_and_len) 982 { 983 unsigned int len = offset_and_len & 0xffff; 984 struct page *page = data; 985 u16 prod = rxr->rx_prod; 986 struct sk_buff *skb; 987 int err; 988 989 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC); 990 if (unlikely(err)) { 991 bnxt_reuse_rx_data(rxr, cons, data); 992 return NULL; 993 } 994 dma_addr -= bp->rx_dma_offset; 995 dma_sync_single_for_cpu(&bp->pdev->dev, dma_addr, BNXT_RX_PAGE_SIZE, 996 bp->rx_dir); 997 skb = napi_build_skb(data_ptr - bp->rx_offset, BNXT_RX_PAGE_SIZE); 998 if (!skb) { 999 page_pool_recycle_direct(rxr->page_pool, page); 1000 return NULL; 1001 } 1002 skb_mark_for_recycle(skb); 1003 skb_reserve(skb, bp->rx_offset); 1004 __skb_put(skb, len); 1005 1006 return skb; 1007 } 1008 1009 static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp, 1010 struct bnxt_rx_ring_info *rxr, 1011 u16 cons, void *data, u8 *data_ptr, 1012 dma_addr_t dma_addr, 1013 unsigned int offset_and_len) 1014 { 1015 unsigned int payload = offset_and_len >> 16; 1016 unsigned int len = offset_and_len & 0xffff; 1017 skb_frag_t *frag; 1018 struct page *page = data; 1019 u16 prod = rxr->rx_prod; 1020 struct sk_buff *skb; 1021 int off, err; 1022 1023 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC); 1024 if (unlikely(err)) { 1025 bnxt_reuse_rx_data(rxr, cons, data); 1026 return NULL; 1027 } 1028 dma_addr -= bp->rx_dma_offset; 1029 dma_sync_single_for_cpu(&bp->pdev->dev, dma_addr, BNXT_RX_PAGE_SIZE, 1030 bp->rx_dir); 1031 1032 if (unlikely(!payload)) 1033 payload = eth_get_headlen(bp->dev, data_ptr, len); 1034 1035 skb = napi_alloc_skb(&rxr->bnapi->napi, payload); 1036 if (!skb) { 1037 page_pool_recycle_direct(rxr->page_pool, page); 1038 return NULL; 1039 } 1040 1041 skb_mark_for_recycle(skb); 1042 off = (void *)data_ptr - page_address(page); 1043 skb_add_rx_frag(skb, 0, page, off, len, BNXT_RX_PAGE_SIZE); 1044 memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN, 1045 payload + NET_IP_ALIGN); 1046 1047 frag = &skb_shinfo(skb)->frags[0]; 1048 skb_frag_size_sub(frag, payload); 1049 skb_frag_off_add(frag, payload); 1050 skb->data_len -= payload; 1051 skb->tail += payload; 1052 1053 return skb; 1054 } 1055 1056 static struct sk_buff *bnxt_rx_skb(struct bnxt *bp, 1057 struct bnxt_rx_ring_info *rxr, u16 cons, 1058 void *data, u8 *data_ptr, 1059 dma_addr_t dma_addr, 1060 unsigned int offset_and_len) 1061 { 1062 u16 prod = rxr->rx_prod; 1063 struct sk_buff *skb; 1064 int err; 1065 1066 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC); 1067 if (unlikely(err)) { 1068 bnxt_reuse_rx_data(rxr, cons, data); 1069 return NULL; 1070 } 1071 1072 skb = napi_build_skb(data, bp->rx_buf_size); 1073 dma_unmap_single_attrs(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size, 1074 bp->rx_dir, DMA_ATTR_WEAK_ORDERING); 1075 if (!skb) { 1076 skb_free_frag(data); 1077 return NULL; 1078 } 1079 1080 skb_reserve(skb, bp->rx_offset); 1081 skb_put(skb, offset_and_len & 0xffff); 1082 return skb; 1083 } 1084 1085 static u32 __bnxt_rx_agg_pages(struct bnxt *bp, 1086 struct bnxt_cp_ring_info *cpr, 1087 struct skb_shared_info *shinfo, 1088 u16 idx, u32 agg_bufs, bool tpa, 1089 struct xdp_buff *xdp) 1090 { 1091 struct bnxt_napi *bnapi = cpr->bnapi; 1092 struct pci_dev *pdev = bp->pdev; 1093 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1094 u16 prod = rxr->rx_agg_prod; 1095 u32 i, total_frag_len = 0; 1096 bool p5_tpa = false; 1097 1098 if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa) 1099 p5_tpa = true; 1100 1101 for (i = 0; i < agg_bufs; i++) { 1102 skb_frag_t *frag = &shinfo->frags[i]; 1103 u16 cons, frag_len; 1104 struct rx_agg_cmp *agg; 1105 struct bnxt_sw_rx_agg_bd *cons_rx_buf; 1106 struct page *page; 1107 dma_addr_t mapping; 1108 1109 if (p5_tpa) 1110 agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, i); 1111 else 1112 agg = bnxt_get_agg(bp, cpr, idx, i); 1113 cons = agg->rx_agg_cmp_opaque; 1114 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) & 1115 RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT; 1116 1117 cons_rx_buf = &rxr->rx_agg_ring[cons]; 1118 skb_frag_fill_page_desc(frag, cons_rx_buf->page, 1119 cons_rx_buf->offset, frag_len); 1120 shinfo->nr_frags = i + 1; 1121 __clear_bit(cons, rxr->rx_agg_bmap); 1122 1123 /* It is possible for bnxt_alloc_rx_page() to allocate 1124 * a sw_prod index that equals the cons index, so we 1125 * need to clear the cons entry now. 1126 */ 1127 mapping = cons_rx_buf->mapping; 1128 page = cons_rx_buf->page; 1129 cons_rx_buf->page = NULL; 1130 1131 if (xdp && page_is_pfmemalloc(page)) 1132 xdp_buff_set_frag_pfmemalloc(xdp); 1133 1134 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) { 1135 --shinfo->nr_frags; 1136 cons_rx_buf->page = page; 1137 1138 /* Update prod since possibly some pages have been 1139 * allocated already. 1140 */ 1141 rxr->rx_agg_prod = prod; 1142 bnxt_reuse_rx_agg_bufs(cpr, idx, i, agg_bufs - i, tpa); 1143 return 0; 1144 } 1145 1146 dma_sync_single_for_cpu(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE, 1147 bp->rx_dir); 1148 1149 total_frag_len += frag_len; 1150 prod = NEXT_RX_AGG(prod); 1151 } 1152 rxr->rx_agg_prod = prod; 1153 return total_frag_len; 1154 } 1155 1156 static struct sk_buff *bnxt_rx_agg_pages_skb(struct bnxt *bp, 1157 struct bnxt_cp_ring_info *cpr, 1158 struct sk_buff *skb, u16 idx, 1159 u32 agg_bufs, bool tpa) 1160 { 1161 struct skb_shared_info *shinfo = skb_shinfo(skb); 1162 u32 total_frag_len = 0; 1163 1164 total_frag_len = __bnxt_rx_agg_pages(bp, cpr, shinfo, idx, 1165 agg_bufs, tpa, NULL); 1166 if (!total_frag_len) { 1167 skb_mark_for_recycle(skb); 1168 dev_kfree_skb(skb); 1169 return NULL; 1170 } 1171 1172 skb->data_len += total_frag_len; 1173 skb->len += total_frag_len; 1174 skb->truesize += BNXT_RX_PAGE_SIZE * agg_bufs; 1175 return skb; 1176 } 1177 1178 static u32 bnxt_rx_agg_pages_xdp(struct bnxt *bp, 1179 struct bnxt_cp_ring_info *cpr, 1180 struct xdp_buff *xdp, u16 idx, 1181 u32 agg_bufs, bool tpa) 1182 { 1183 struct skb_shared_info *shinfo = xdp_get_shared_info_from_buff(xdp); 1184 u32 total_frag_len = 0; 1185 1186 if (!xdp_buff_has_frags(xdp)) 1187 shinfo->nr_frags = 0; 1188 1189 total_frag_len = __bnxt_rx_agg_pages(bp, cpr, shinfo, 1190 idx, agg_bufs, tpa, xdp); 1191 if (total_frag_len) { 1192 xdp_buff_set_frags_flag(xdp); 1193 shinfo->nr_frags = agg_bufs; 1194 shinfo->xdp_frags_size = total_frag_len; 1195 } 1196 return total_frag_len; 1197 } 1198 1199 static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 1200 u8 agg_bufs, u32 *raw_cons) 1201 { 1202 u16 last; 1203 struct rx_agg_cmp *agg; 1204 1205 *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs); 1206 last = RING_CMP(*raw_cons); 1207 agg = (struct rx_agg_cmp *) 1208 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)]; 1209 return RX_AGG_CMP_VALID(agg, *raw_cons); 1210 } 1211 1212 static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data, 1213 unsigned int len, 1214 dma_addr_t mapping) 1215 { 1216 struct bnxt *bp = bnapi->bp; 1217 struct pci_dev *pdev = bp->pdev; 1218 struct sk_buff *skb; 1219 1220 skb = napi_alloc_skb(&bnapi->napi, len); 1221 if (!skb) 1222 return NULL; 1223 1224 dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copy_thresh, 1225 bp->rx_dir); 1226 1227 memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN, 1228 len + NET_IP_ALIGN); 1229 1230 dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copy_thresh, 1231 bp->rx_dir); 1232 1233 skb_put(skb, len); 1234 return skb; 1235 } 1236 1237 static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 1238 u32 *raw_cons, void *cmp) 1239 { 1240 struct rx_cmp *rxcmp = cmp; 1241 u32 tmp_raw_cons = *raw_cons; 1242 u8 cmp_type, agg_bufs = 0; 1243 1244 cmp_type = RX_CMP_TYPE(rxcmp); 1245 1246 if (cmp_type == CMP_TYPE_RX_L2_CMP) { 1247 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) & 1248 RX_CMP_AGG_BUFS) >> 1249 RX_CMP_AGG_BUFS_SHIFT; 1250 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) { 1251 struct rx_tpa_end_cmp *tpa_end = cmp; 1252 1253 if (bp->flags & BNXT_FLAG_CHIP_P5) 1254 return 0; 1255 1256 agg_bufs = TPA_END_AGG_BUFS(tpa_end); 1257 } 1258 1259 if (agg_bufs) { 1260 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons)) 1261 return -EBUSY; 1262 } 1263 *raw_cons = tmp_raw_cons; 1264 return 0; 1265 } 1266 1267 static u16 bnxt_alloc_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id) 1268 { 1269 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map; 1270 u16 idx = agg_id & MAX_TPA_P5_MASK; 1271 1272 if (test_bit(idx, map->agg_idx_bmap)) 1273 idx = find_first_zero_bit(map->agg_idx_bmap, 1274 BNXT_AGG_IDX_BMAP_SIZE); 1275 __set_bit(idx, map->agg_idx_bmap); 1276 map->agg_id_tbl[agg_id] = idx; 1277 return idx; 1278 } 1279 1280 static void bnxt_free_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx) 1281 { 1282 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map; 1283 1284 __clear_bit(idx, map->agg_idx_bmap); 1285 } 1286 1287 static u16 bnxt_lookup_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id) 1288 { 1289 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map; 1290 1291 return map->agg_id_tbl[agg_id]; 1292 } 1293 1294 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, 1295 struct rx_tpa_start_cmp *tpa_start, 1296 struct rx_tpa_start_cmp_ext *tpa_start1) 1297 { 1298 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf; 1299 struct bnxt_tpa_info *tpa_info; 1300 u16 cons, prod, agg_id; 1301 struct rx_bd *prod_bd; 1302 dma_addr_t mapping; 1303 1304 if (bp->flags & BNXT_FLAG_CHIP_P5) { 1305 agg_id = TPA_START_AGG_ID_P5(tpa_start); 1306 agg_id = bnxt_alloc_agg_idx(rxr, agg_id); 1307 } else { 1308 agg_id = TPA_START_AGG_ID(tpa_start); 1309 } 1310 cons = tpa_start->rx_tpa_start_cmp_opaque; 1311 prod = rxr->rx_prod; 1312 cons_rx_buf = &rxr->rx_buf_ring[cons]; 1313 prod_rx_buf = &rxr->rx_buf_ring[prod]; 1314 tpa_info = &rxr->rx_tpa[agg_id]; 1315 1316 if (unlikely(cons != rxr->rx_next_cons || 1317 TPA_START_ERROR(tpa_start))) { 1318 netdev_warn(bp->dev, "TPA cons %x, expected cons %x, error code %x\n", 1319 cons, rxr->rx_next_cons, 1320 TPA_START_ERROR_CODE(tpa_start1)); 1321 bnxt_sched_reset_rxr(bp, rxr); 1322 return; 1323 } 1324 /* Store cfa_code in tpa_info to use in tpa_end 1325 * completion processing. 1326 */ 1327 tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1); 1328 prod_rx_buf->data = tpa_info->data; 1329 prod_rx_buf->data_ptr = tpa_info->data_ptr; 1330 1331 mapping = tpa_info->mapping; 1332 prod_rx_buf->mapping = mapping; 1333 1334 prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 1335 1336 prod_bd->rx_bd_haddr = cpu_to_le64(mapping); 1337 1338 tpa_info->data = cons_rx_buf->data; 1339 tpa_info->data_ptr = cons_rx_buf->data_ptr; 1340 cons_rx_buf->data = NULL; 1341 tpa_info->mapping = cons_rx_buf->mapping; 1342 1343 tpa_info->len = 1344 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >> 1345 RX_TPA_START_CMP_LEN_SHIFT; 1346 if (likely(TPA_START_HASH_VALID(tpa_start))) { 1347 u32 hash_type = TPA_START_HASH_TYPE(tpa_start); 1348 1349 tpa_info->hash_type = PKT_HASH_TYPE_L4; 1350 tpa_info->gso_type = SKB_GSO_TCPV4; 1351 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */ 1352 if (hash_type == 3 || TPA_START_IS_IPV6(tpa_start1)) 1353 tpa_info->gso_type = SKB_GSO_TCPV6; 1354 tpa_info->rss_hash = 1355 le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash); 1356 } else { 1357 tpa_info->hash_type = PKT_HASH_TYPE_NONE; 1358 tpa_info->gso_type = 0; 1359 netif_warn(bp, rx_err, bp->dev, "TPA packet without valid hash\n"); 1360 } 1361 tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2); 1362 tpa_info->metadata = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata); 1363 tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info); 1364 tpa_info->agg_count = 0; 1365 1366 rxr->rx_prod = NEXT_RX(prod); 1367 cons = NEXT_RX(cons); 1368 rxr->rx_next_cons = NEXT_RX(cons); 1369 cons_rx_buf = &rxr->rx_buf_ring[cons]; 1370 1371 bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data); 1372 rxr->rx_prod = NEXT_RX(rxr->rx_prod); 1373 cons_rx_buf->data = NULL; 1374 } 1375 1376 static void bnxt_abort_tpa(struct bnxt_cp_ring_info *cpr, u16 idx, u32 agg_bufs) 1377 { 1378 if (agg_bufs) 1379 bnxt_reuse_rx_agg_bufs(cpr, idx, 0, agg_bufs, true); 1380 } 1381 1382 #ifdef CONFIG_INET 1383 static void bnxt_gro_tunnel(struct sk_buff *skb, __be16 ip_proto) 1384 { 1385 struct udphdr *uh = NULL; 1386 1387 if (ip_proto == htons(ETH_P_IP)) { 1388 struct iphdr *iph = (struct iphdr *)skb->data; 1389 1390 if (iph->protocol == IPPROTO_UDP) 1391 uh = (struct udphdr *)(iph + 1); 1392 } else { 1393 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data; 1394 1395 if (iph->nexthdr == IPPROTO_UDP) 1396 uh = (struct udphdr *)(iph + 1); 1397 } 1398 if (uh) { 1399 if (uh->check) 1400 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM; 1401 else 1402 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL; 1403 } 1404 } 1405 #endif 1406 1407 static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info, 1408 int payload_off, int tcp_ts, 1409 struct sk_buff *skb) 1410 { 1411 #ifdef CONFIG_INET 1412 struct tcphdr *th; 1413 int len, nw_off; 1414 u16 outer_ip_off, inner_ip_off, inner_mac_off; 1415 u32 hdr_info = tpa_info->hdr_info; 1416 bool loopback = false; 1417 1418 inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info); 1419 inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info); 1420 outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info); 1421 1422 /* If the packet is an internal loopback packet, the offsets will 1423 * have an extra 4 bytes. 1424 */ 1425 if (inner_mac_off == 4) { 1426 loopback = true; 1427 } else if (inner_mac_off > 4) { 1428 __be16 proto = *((__be16 *)(skb->data + inner_ip_off - 1429 ETH_HLEN - 2)); 1430 1431 /* We only support inner iPv4/ipv6. If we don't see the 1432 * correct protocol ID, it must be a loopback packet where 1433 * the offsets are off by 4. 1434 */ 1435 if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6)) 1436 loopback = true; 1437 } 1438 if (loopback) { 1439 /* internal loopback packet, subtract all offsets by 4 */ 1440 inner_ip_off -= 4; 1441 inner_mac_off -= 4; 1442 outer_ip_off -= 4; 1443 } 1444 1445 nw_off = inner_ip_off - ETH_HLEN; 1446 skb_set_network_header(skb, nw_off); 1447 if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) { 1448 struct ipv6hdr *iph = ipv6_hdr(skb); 1449 1450 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr)); 1451 len = skb->len - skb_transport_offset(skb); 1452 th = tcp_hdr(skb); 1453 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0); 1454 } else { 1455 struct iphdr *iph = ip_hdr(skb); 1456 1457 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr)); 1458 len = skb->len - skb_transport_offset(skb); 1459 th = tcp_hdr(skb); 1460 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0); 1461 } 1462 1463 if (inner_mac_off) { /* tunnel */ 1464 __be16 proto = *((__be16 *)(skb->data + outer_ip_off - 1465 ETH_HLEN - 2)); 1466 1467 bnxt_gro_tunnel(skb, proto); 1468 } 1469 #endif 1470 return skb; 1471 } 1472 1473 static struct sk_buff *bnxt_gro_func_5750x(struct bnxt_tpa_info *tpa_info, 1474 int payload_off, int tcp_ts, 1475 struct sk_buff *skb) 1476 { 1477 #ifdef CONFIG_INET 1478 u16 outer_ip_off, inner_ip_off, inner_mac_off; 1479 u32 hdr_info = tpa_info->hdr_info; 1480 int iphdr_len, nw_off; 1481 1482 inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info); 1483 inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info); 1484 outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info); 1485 1486 nw_off = inner_ip_off - ETH_HLEN; 1487 skb_set_network_header(skb, nw_off); 1488 iphdr_len = (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) ? 1489 sizeof(struct ipv6hdr) : sizeof(struct iphdr); 1490 skb_set_transport_header(skb, nw_off + iphdr_len); 1491 1492 if (inner_mac_off) { /* tunnel */ 1493 __be16 proto = *((__be16 *)(skb->data + outer_ip_off - 1494 ETH_HLEN - 2)); 1495 1496 bnxt_gro_tunnel(skb, proto); 1497 } 1498 #endif 1499 return skb; 1500 } 1501 1502 #define BNXT_IPV4_HDR_SIZE (sizeof(struct iphdr) + sizeof(struct tcphdr)) 1503 #define BNXT_IPV6_HDR_SIZE (sizeof(struct ipv6hdr) + sizeof(struct tcphdr)) 1504 1505 static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info, 1506 int payload_off, int tcp_ts, 1507 struct sk_buff *skb) 1508 { 1509 #ifdef CONFIG_INET 1510 struct tcphdr *th; 1511 int len, nw_off, tcp_opt_len = 0; 1512 1513 if (tcp_ts) 1514 tcp_opt_len = 12; 1515 1516 if (tpa_info->gso_type == SKB_GSO_TCPV4) { 1517 struct iphdr *iph; 1518 1519 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len - 1520 ETH_HLEN; 1521 skb_set_network_header(skb, nw_off); 1522 iph = ip_hdr(skb); 1523 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr)); 1524 len = skb->len - skb_transport_offset(skb); 1525 th = tcp_hdr(skb); 1526 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0); 1527 } else if (tpa_info->gso_type == SKB_GSO_TCPV6) { 1528 struct ipv6hdr *iph; 1529 1530 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len - 1531 ETH_HLEN; 1532 skb_set_network_header(skb, nw_off); 1533 iph = ipv6_hdr(skb); 1534 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr)); 1535 len = skb->len - skb_transport_offset(skb); 1536 th = tcp_hdr(skb); 1537 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0); 1538 } else { 1539 dev_kfree_skb_any(skb); 1540 return NULL; 1541 } 1542 1543 if (nw_off) /* tunnel */ 1544 bnxt_gro_tunnel(skb, skb->protocol); 1545 #endif 1546 return skb; 1547 } 1548 1549 static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp, 1550 struct bnxt_tpa_info *tpa_info, 1551 struct rx_tpa_end_cmp *tpa_end, 1552 struct rx_tpa_end_cmp_ext *tpa_end1, 1553 struct sk_buff *skb) 1554 { 1555 #ifdef CONFIG_INET 1556 int payload_off; 1557 u16 segs; 1558 1559 segs = TPA_END_TPA_SEGS(tpa_end); 1560 if (segs == 1) 1561 return skb; 1562 1563 NAPI_GRO_CB(skb)->count = segs; 1564 skb_shinfo(skb)->gso_size = 1565 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len); 1566 skb_shinfo(skb)->gso_type = tpa_info->gso_type; 1567 if (bp->flags & BNXT_FLAG_CHIP_P5) 1568 payload_off = TPA_END_PAYLOAD_OFF_P5(tpa_end1); 1569 else 1570 payload_off = TPA_END_PAYLOAD_OFF(tpa_end); 1571 skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb); 1572 if (likely(skb)) 1573 tcp_gro_complete(skb); 1574 #endif 1575 return skb; 1576 } 1577 1578 /* Given the cfa_code of a received packet determine which 1579 * netdev (vf-rep or PF) the packet is destined to. 1580 */ 1581 static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code) 1582 { 1583 struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code); 1584 1585 /* if vf-rep dev is NULL, the must belongs to the PF */ 1586 return dev ? dev : bp->dev; 1587 } 1588 1589 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp, 1590 struct bnxt_cp_ring_info *cpr, 1591 u32 *raw_cons, 1592 struct rx_tpa_end_cmp *tpa_end, 1593 struct rx_tpa_end_cmp_ext *tpa_end1, 1594 u8 *event) 1595 { 1596 struct bnxt_napi *bnapi = cpr->bnapi; 1597 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1598 u8 *data_ptr, agg_bufs; 1599 unsigned int len; 1600 struct bnxt_tpa_info *tpa_info; 1601 dma_addr_t mapping; 1602 struct sk_buff *skb; 1603 u16 idx = 0, agg_id; 1604 void *data; 1605 bool gro; 1606 1607 if (unlikely(bnapi->in_reset)) { 1608 int rc = bnxt_discard_rx(bp, cpr, raw_cons, tpa_end); 1609 1610 if (rc < 0) 1611 return ERR_PTR(-EBUSY); 1612 return NULL; 1613 } 1614 1615 if (bp->flags & BNXT_FLAG_CHIP_P5) { 1616 agg_id = TPA_END_AGG_ID_P5(tpa_end); 1617 agg_id = bnxt_lookup_agg_idx(rxr, agg_id); 1618 agg_bufs = TPA_END_AGG_BUFS_P5(tpa_end1); 1619 tpa_info = &rxr->rx_tpa[agg_id]; 1620 if (unlikely(agg_bufs != tpa_info->agg_count)) { 1621 netdev_warn(bp->dev, "TPA end agg_buf %d != expected agg_bufs %d\n", 1622 agg_bufs, tpa_info->agg_count); 1623 agg_bufs = tpa_info->agg_count; 1624 } 1625 tpa_info->agg_count = 0; 1626 *event |= BNXT_AGG_EVENT; 1627 bnxt_free_agg_idx(rxr, agg_id); 1628 idx = agg_id; 1629 gro = !!(bp->flags & BNXT_FLAG_GRO); 1630 } else { 1631 agg_id = TPA_END_AGG_ID(tpa_end); 1632 agg_bufs = TPA_END_AGG_BUFS(tpa_end); 1633 tpa_info = &rxr->rx_tpa[agg_id]; 1634 idx = RING_CMP(*raw_cons); 1635 if (agg_bufs) { 1636 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons)) 1637 return ERR_PTR(-EBUSY); 1638 1639 *event |= BNXT_AGG_EVENT; 1640 idx = NEXT_CMP(idx); 1641 } 1642 gro = !!TPA_END_GRO(tpa_end); 1643 } 1644 data = tpa_info->data; 1645 data_ptr = tpa_info->data_ptr; 1646 prefetch(data_ptr); 1647 len = tpa_info->len; 1648 mapping = tpa_info->mapping; 1649 1650 if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) { 1651 bnxt_abort_tpa(cpr, idx, agg_bufs); 1652 if (agg_bufs > MAX_SKB_FRAGS) 1653 netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n", 1654 agg_bufs, (int)MAX_SKB_FRAGS); 1655 return NULL; 1656 } 1657 1658 if (len <= bp->rx_copy_thresh) { 1659 skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping); 1660 if (!skb) { 1661 bnxt_abort_tpa(cpr, idx, agg_bufs); 1662 cpr->sw_stats.rx.rx_oom_discards += 1; 1663 return NULL; 1664 } 1665 } else { 1666 u8 *new_data; 1667 dma_addr_t new_mapping; 1668 1669 new_data = __bnxt_alloc_rx_frag(bp, &new_mapping, GFP_ATOMIC); 1670 if (!new_data) { 1671 bnxt_abort_tpa(cpr, idx, agg_bufs); 1672 cpr->sw_stats.rx.rx_oom_discards += 1; 1673 return NULL; 1674 } 1675 1676 tpa_info->data = new_data; 1677 tpa_info->data_ptr = new_data + bp->rx_offset; 1678 tpa_info->mapping = new_mapping; 1679 1680 skb = napi_build_skb(data, bp->rx_buf_size); 1681 dma_unmap_single_attrs(&bp->pdev->dev, mapping, 1682 bp->rx_buf_use_size, bp->rx_dir, 1683 DMA_ATTR_WEAK_ORDERING); 1684 1685 if (!skb) { 1686 skb_free_frag(data); 1687 bnxt_abort_tpa(cpr, idx, agg_bufs); 1688 cpr->sw_stats.rx.rx_oom_discards += 1; 1689 return NULL; 1690 } 1691 skb_reserve(skb, bp->rx_offset); 1692 skb_put(skb, len); 1693 } 1694 1695 if (agg_bufs) { 1696 skb = bnxt_rx_agg_pages_skb(bp, cpr, skb, idx, agg_bufs, true); 1697 if (!skb) { 1698 /* Page reuse already handled by bnxt_rx_pages(). */ 1699 cpr->sw_stats.rx.rx_oom_discards += 1; 1700 return NULL; 1701 } 1702 } 1703 1704 skb->protocol = 1705 eth_type_trans(skb, bnxt_get_pkt_dev(bp, tpa_info->cfa_code)); 1706 1707 if (tpa_info->hash_type != PKT_HASH_TYPE_NONE) 1708 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type); 1709 1710 if ((tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) && 1711 (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) { 1712 __be16 vlan_proto = htons(tpa_info->metadata >> 1713 RX_CMP_FLAGS2_METADATA_TPID_SFT); 1714 u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_TCI_MASK; 1715 1716 if (eth_type_vlan(vlan_proto)) { 1717 __vlan_hwaccel_put_tag(skb, vlan_proto, vtag); 1718 } else { 1719 dev_kfree_skb(skb); 1720 return NULL; 1721 } 1722 } 1723 1724 skb_checksum_none_assert(skb); 1725 if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) { 1726 skb->ip_summed = CHECKSUM_UNNECESSARY; 1727 skb->csum_level = 1728 (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3; 1729 } 1730 1731 if (gro) 1732 skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb); 1733 1734 return skb; 1735 } 1736 1737 static void bnxt_tpa_agg(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, 1738 struct rx_agg_cmp *rx_agg) 1739 { 1740 u16 agg_id = TPA_AGG_AGG_ID(rx_agg); 1741 struct bnxt_tpa_info *tpa_info; 1742 1743 agg_id = bnxt_lookup_agg_idx(rxr, agg_id); 1744 tpa_info = &rxr->rx_tpa[agg_id]; 1745 BUG_ON(tpa_info->agg_count >= MAX_SKB_FRAGS); 1746 tpa_info->agg_arr[tpa_info->agg_count++] = *rx_agg; 1747 } 1748 1749 static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi, 1750 struct sk_buff *skb) 1751 { 1752 if (skb->dev != bp->dev) { 1753 /* this packet belongs to a vf-rep */ 1754 bnxt_vf_rep_rx(bp, skb); 1755 return; 1756 } 1757 skb_record_rx_queue(skb, bnapi->index); 1758 skb_mark_for_recycle(skb); 1759 napi_gro_receive(&bnapi->napi, skb); 1760 } 1761 1762 /* returns the following: 1763 * 1 - 1 packet successfully received 1764 * 0 - successful TPA_START, packet not completed yet 1765 * -EBUSY - completion ring does not have all the agg buffers yet 1766 * -ENOMEM - packet aborted due to out of memory 1767 * -EIO - packet aborted due to hw error indicated in BD 1768 */ 1769 static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 1770 u32 *raw_cons, u8 *event) 1771 { 1772 struct bnxt_napi *bnapi = cpr->bnapi; 1773 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1774 struct net_device *dev = bp->dev; 1775 struct rx_cmp *rxcmp; 1776 struct rx_cmp_ext *rxcmp1; 1777 u32 tmp_raw_cons = *raw_cons; 1778 u16 cfa_code, cons, prod, cp_cons = RING_CMP(tmp_raw_cons); 1779 struct bnxt_sw_rx_bd *rx_buf; 1780 unsigned int len; 1781 u8 *data_ptr, agg_bufs, cmp_type; 1782 bool xdp_active = false; 1783 dma_addr_t dma_addr; 1784 struct sk_buff *skb; 1785 struct xdp_buff xdp; 1786 u32 flags, misc; 1787 void *data; 1788 int rc = 0; 1789 1790 rxcmp = (struct rx_cmp *) 1791 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1792 1793 cmp_type = RX_CMP_TYPE(rxcmp); 1794 1795 if (cmp_type == CMP_TYPE_RX_TPA_AGG_CMP) { 1796 bnxt_tpa_agg(bp, rxr, (struct rx_agg_cmp *)rxcmp); 1797 goto next_rx_no_prod_no_len; 1798 } 1799 1800 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons); 1801 cp_cons = RING_CMP(tmp_raw_cons); 1802 rxcmp1 = (struct rx_cmp_ext *) 1803 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1804 1805 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons)) 1806 return -EBUSY; 1807 1808 /* The valid test of the entry must be done first before 1809 * reading any further. 1810 */ 1811 dma_rmb(); 1812 prod = rxr->rx_prod; 1813 1814 if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) { 1815 bnxt_tpa_start(bp, rxr, (struct rx_tpa_start_cmp *)rxcmp, 1816 (struct rx_tpa_start_cmp_ext *)rxcmp1); 1817 1818 *event |= BNXT_RX_EVENT; 1819 goto next_rx_no_prod_no_len; 1820 1821 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) { 1822 skb = bnxt_tpa_end(bp, cpr, &tmp_raw_cons, 1823 (struct rx_tpa_end_cmp *)rxcmp, 1824 (struct rx_tpa_end_cmp_ext *)rxcmp1, event); 1825 1826 if (IS_ERR(skb)) 1827 return -EBUSY; 1828 1829 rc = -ENOMEM; 1830 if (likely(skb)) { 1831 bnxt_deliver_skb(bp, bnapi, skb); 1832 rc = 1; 1833 } 1834 *event |= BNXT_RX_EVENT; 1835 goto next_rx_no_prod_no_len; 1836 } 1837 1838 cons = rxcmp->rx_cmp_opaque; 1839 if (unlikely(cons != rxr->rx_next_cons)) { 1840 int rc1 = bnxt_discard_rx(bp, cpr, &tmp_raw_cons, rxcmp); 1841 1842 /* 0xffff is forced error, don't print it */ 1843 if (rxr->rx_next_cons != 0xffff) 1844 netdev_warn(bp->dev, "RX cons %x != expected cons %x\n", 1845 cons, rxr->rx_next_cons); 1846 bnxt_sched_reset_rxr(bp, rxr); 1847 if (rc1) 1848 return rc1; 1849 goto next_rx_no_prod_no_len; 1850 } 1851 rx_buf = &rxr->rx_buf_ring[cons]; 1852 data = rx_buf->data; 1853 data_ptr = rx_buf->data_ptr; 1854 prefetch(data_ptr); 1855 1856 misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1); 1857 agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT; 1858 1859 if (agg_bufs) { 1860 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons)) 1861 return -EBUSY; 1862 1863 cp_cons = NEXT_CMP(cp_cons); 1864 *event |= BNXT_AGG_EVENT; 1865 } 1866 *event |= BNXT_RX_EVENT; 1867 1868 rx_buf->data = NULL; 1869 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) { 1870 u32 rx_err = le32_to_cpu(rxcmp1->rx_cmp_cfa_code_errors_v2); 1871 1872 bnxt_reuse_rx_data(rxr, cons, data); 1873 if (agg_bufs) 1874 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, agg_bufs, 1875 false); 1876 1877 rc = -EIO; 1878 if (rx_err & RX_CMPL_ERRORS_BUFFER_ERROR_MASK) { 1879 bnapi->cp_ring.sw_stats.rx.rx_buf_errors++; 1880 if (!(bp->flags & BNXT_FLAG_CHIP_P5) && 1881 !(bp->fw_cap & BNXT_FW_CAP_RING_MONITOR)) { 1882 netdev_warn_once(bp->dev, "RX buffer error %x\n", 1883 rx_err); 1884 bnxt_sched_reset_rxr(bp, rxr); 1885 } 1886 } 1887 goto next_rx_no_len; 1888 } 1889 1890 flags = le32_to_cpu(rxcmp->rx_cmp_len_flags_type); 1891 len = flags >> RX_CMP_LEN_SHIFT; 1892 dma_addr = rx_buf->mapping; 1893 1894 if (bnxt_xdp_attached(bp, rxr)) { 1895 bnxt_xdp_buff_init(bp, rxr, cons, data_ptr, len, &xdp); 1896 if (agg_bufs) { 1897 u32 frag_len = bnxt_rx_agg_pages_xdp(bp, cpr, &xdp, 1898 cp_cons, agg_bufs, 1899 false); 1900 if (!frag_len) { 1901 cpr->sw_stats.rx.rx_oom_discards += 1; 1902 rc = -ENOMEM; 1903 goto next_rx; 1904 } 1905 } 1906 xdp_active = true; 1907 } 1908 1909 if (xdp_active) { 1910 if (bnxt_rx_xdp(bp, rxr, cons, xdp, data, &data_ptr, &len, event)) { 1911 rc = 1; 1912 goto next_rx; 1913 } 1914 } 1915 1916 if (len <= bp->rx_copy_thresh) { 1917 skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr); 1918 bnxt_reuse_rx_data(rxr, cons, data); 1919 if (!skb) { 1920 if (agg_bufs) { 1921 if (!xdp_active) 1922 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, 1923 agg_bufs, false); 1924 else 1925 bnxt_xdp_buff_frags_free(rxr, &xdp); 1926 } 1927 cpr->sw_stats.rx.rx_oom_discards += 1; 1928 rc = -ENOMEM; 1929 goto next_rx; 1930 } 1931 } else { 1932 u32 payload; 1933 1934 if (rx_buf->data_ptr == data_ptr) 1935 payload = misc & RX_CMP_PAYLOAD_OFFSET; 1936 else 1937 payload = 0; 1938 skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr, 1939 payload | len); 1940 if (!skb) { 1941 cpr->sw_stats.rx.rx_oom_discards += 1; 1942 rc = -ENOMEM; 1943 goto next_rx; 1944 } 1945 } 1946 1947 if (agg_bufs) { 1948 if (!xdp_active) { 1949 skb = bnxt_rx_agg_pages_skb(bp, cpr, skb, cp_cons, agg_bufs, false); 1950 if (!skb) { 1951 cpr->sw_stats.rx.rx_oom_discards += 1; 1952 rc = -ENOMEM; 1953 goto next_rx; 1954 } 1955 } else { 1956 skb = bnxt_xdp_build_skb(bp, skb, agg_bufs, rxr->page_pool, &xdp, rxcmp1); 1957 if (!skb) { 1958 /* we should be able to free the old skb here */ 1959 bnxt_xdp_buff_frags_free(rxr, &xdp); 1960 cpr->sw_stats.rx.rx_oom_discards += 1; 1961 rc = -ENOMEM; 1962 goto next_rx; 1963 } 1964 } 1965 } 1966 1967 if (RX_CMP_HASH_VALID(rxcmp)) { 1968 u32 hash_type = RX_CMP_HASH_TYPE(rxcmp); 1969 enum pkt_hash_types type = PKT_HASH_TYPE_L4; 1970 1971 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */ 1972 if (hash_type != 1 && hash_type != 3) 1973 type = PKT_HASH_TYPE_L3; 1974 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type); 1975 } 1976 1977 cfa_code = RX_CMP_CFA_CODE(rxcmp1); 1978 skb->protocol = eth_type_trans(skb, bnxt_get_pkt_dev(bp, cfa_code)); 1979 1980 if ((rxcmp1->rx_cmp_flags2 & 1981 cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) && 1982 (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) { 1983 u32 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data); 1984 u16 vtag = meta_data & RX_CMP_FLAGS2_METADATA_TCI_MASK; 1985 __be16 vlan_proto = htons(meta_data >> 1986 RX_CMP_FLAGS2_METADATA_TPID_SFT); 1987 1988 if (eth_type_vlan(vlan_proto)) { 1989 __vlan_hwaccel_put_tag(skb, vlan_proto, vtag); 1990 } else { 1991 dev_kfree_skb(skb); 1992 goto next_rx; 1993 } 1994 } 1995 1996 skb_checksum_none_assert(skb); 1997 if (RX_CMP_L4_CS_OK(rxcmp1)) { 1998 if (dev->features & NETIF_F_RXCSUM) { 1999 skb->ip_summed = CHECKSUM_UNNECESSARY; 2000 skb->csum_level = RX_CMP_ENCAP(rxcmp1); 2001 } 2002 } else { 2003 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) { 2004 if (dev->features & NETIF_F_RXCSUM) 2005 bnapi->cp_ring.sw_stats.rx.rx_l4_csum_errors++; 2006 } 2007 } 2008 2009 if (unlikely((flags & RX_CMP_FLAGS_ITYPES_MASK) == 2010 RX_CMP_FLAGS_ITYPE_PTP_W_TS) || bp->ptp_all_rx_tstamp) { 2011 if (bp->flags & BNXT_FLAG_CHIP_P5) { 2012 u32 cmpl_ts = le32_to_cpu(rxcmp1->rx_cmp_timestamp); 2013 u64 ns, ts; 2014 2015 if (!bnxt_get_rx_ts_p5(bp, &ts, cmpl_ts)) { 2016 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; 2017 2018 spin_lock_bh(&ptp->ptp_lock); 2019 ns = timecounter_cyc2time(&ptp->tc, ts); 2020 spin_unlock_bh(&ptp->ptp_lock); 2021 memset(skb_hwtstamps(skb), 0, 2022 sizeof(*skb_hwtstamps(skb))); 2023 skb_hwtstamps(skb)->hwtstamp = ns_to_ktime(ns); 2024 } 2025 } 2026 } 2027 bnxt_deliver_skb(bp, bnapi, skb); 2028 rc = 1; 2029 2030 next_rx: 2031 cpr->rx_packets += 1; 2032 cpr->rx_bytes += len; 2033 2034 next_rx_no_len: 2035 rxr->rx_prod = NEXT_RX(prod); 2036 rxr->rx_next_cons = NEXT_RX(cons); 2037 2038 next_rx_no_prod_no_len: 2039 *raw_cons = tmp_raw_cons; 2040 2041 return rc; 2042 } 2043 2044 /* In netpoll mode, if we are using a combined completion ring, we need to 2045 * discard the rx packets and recycle the buffers. 2046 */ 2047 static int bnxt_force_rx_discard(struct bnxt *bp, 2048 struct bnxt_cp_ring_info *cpr, 2049 u32 *raw_cons, u8 *event) 2050 { 2051 u32 tmp_raw_cons = *raw_cons; 2052 struct rx_cmp_ext *rxcmp1; 2053 struct rx_cmp *rxcmp; 2054 u16 cp_cons; 2055 u8 cmp_type; 2056 int rc; 2057 2058 cp_cons = RING_CMP(tmp_raw_cons); 2059 rxcmp = (struct rx_cmp *) 2060 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 2061 2062 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons); 2063 cp_cons = RING_CMP(tmp_raw_cons); 2064 rxcmp1 = (struct rx_cmp_ext *) 2065 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 2066 2067 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons)) 2068 return -EBUSY; 2069 2070 /* The valid test of the entry must be done first before 2071 * reading any further. 2072 */ 2073 dma_rmb(); 2074 cmp_type = RX_CMP_TYPE(rxcmp); 2075 if (cmp_type == CMP_TYPE_RX_L2_CMP) { 2076 rxcmp1->rx_cmp_cfa_code_errors_v2 |= 2077 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR); 2078 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) { 2079 struct rx_tpa_end_cmp_ext *tpa_end1; 2080 2081 tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1; 2082 tpa_end1->rx_tpa_end_cmp_errors_v2 |= 2083 cpu_to_le32(RX_TPA_END_CMP_ERRORS); 2084 } 2085 rc = bnxt_rx_pkt(bp, cpr, raw_cons, event); 2086 if (rc && rc != -EBUSY) 2087 cpr->sw_stats.rx.rx_netpoll_discards += 1; 2088 return rc; 2089 } 2090 2091 u32 bnxt_fw_health_readl(struct bnxt *bp, int reg_idx) 2092 { 2093 struct bnxt_fw_health *fw_health = bp->fw_health; 2094 u32 reg = fw_health->regs[reg_idx]; 2095 u32 reg_type, reg_off, val = 0; 2096 2097 reg_type = BNXT_FW_HEALTH_REG_TYPE(reg); 2098 reg_off = BNXT_FW_HEALTH_REG_OFF(reg); 2099 switch (reg_type) { 2100 case BNXT_FW_HEALTH_REG_TYPE_CFG: 2101 pci_read_config_dword(bp->pdev, reg_off, &val); 2102 break; 2103 case BNXT_FW_HEALTH_REG_TYPE_GRC: 2104 reg_off = fw_health->mapped_regs[reg_idx]; 2105 fallthrough; 2106 case BNXT_FW_HEALTH_REG_TYPE_BAR0: 2107 val = readl(bp->bar0 + reg_off); 2108 break; 2109 case BNXT_FW_HEALTH_REG_TYPE_BAR1: 2110 val = readl(bp->bar1 + reg_off); 2111 break; 2112 } 2113 if (reg_idx == BNXT_FW_RESET_INPROG_REG) 2114 val &= fw_health->fw_reset_inprog_reg_mask; 2115 return val; 2116 } 2117 2118 static u16 bnxt_agg_ring_id_to_grp_idx(struct bnxt *bp, u16 ring_id) 2119 { 2120 int i; 2121 2122 for (i = 0; i < bp->rx_nr_rings; i++) { 2123 u16 grp_idx = bp->rx_ring[i].bnapi->index; 2124 struct bnxt_ring_grp_info *grp_info; 2125 2126 grp_info = &bp->grp_info[grp_idx]; 2127 if (grp_info->agg_fw_ring_id == ring_id) 2128 return grp_idx; 2129 } 2130 return INVALID_HW_RING_ID; 2131 } 2132 2133 static void bnxt_event_error_report(struct bnxt *bp, u32 data1, u32 data2) 2134 { 2135 u32 err_type = BNXT_EVENT_ERROR_REPORT_TYPE(data1); 2136 2137 switch (err_type) { 2138 case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_INVALID_SIGNAL: 2139 netdev_err(bp->dev, "1PPS: Received invalid signal on pin%lu from the external source. Please fix the signal and reconfigure the pin\n", 2140 BNXT_EVENT_INVALID_SIGNAL_DATA(data2)); 2141 break; 2142 case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_PAUSE_STORM: 2143 netdev_warn(bp->dev, "Pause Storm detected!\n"); 2144 break; 2145 case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_DOORBELL_DROP_THRESHOLD: 2146 netdev_warn(bp->dev, "One or more MMIO doorbells dropped by the device!\n"); 2147 break; 2148 default: 2149 netdev_err(bp->dev, "FW reported unknown error type %u\n", 2150 err_type); 2151 break; 2152 } 2153 } 2154 2155 #define BNXT_GET_EVENT_PORT(data) \ 2156 ((data) & \ 2157 ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK) 2158 2159 #define BNXT_EVENT_RING_TYPE(data2) \ 2160 ((data2) & \ 2161 ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_MASK) 2162 2163 #define BNXT_EVENT_RING_TYPE_RX(data2) \ 2164 (BNXT_EVENT_RING_TYPE(data2) == \ 2165 ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_RX) 2166 2167 #define BNXT_EVENT_PHC_EVENT_TYPE(data1) \ 2168 (((data1) & ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_MASK) >>\ 2169 ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_SFT) 2170 2171 #define BNXT_EVENT_PHC_RTC_UPDATE(data1) \ 2172 (((data1) & ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_PHC_TIME_MSB_MASK) >>\ 2173 ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_PHC_TIME_MSB_SFT) 2174 2175 #define BNXT_PHC_BITS 48 2176 2177 static int bnxt_async_event_process(struct bnxt *bp, 2178 struct hwrm_async_event_cmpl *cmpl) 2179 { 2180 u16 event_id = le16_to_cpu(cmpl->event_id); 2181 u32 data1 = le32_to_cpu(cmpl->event_data1); 2182 u32 data2 = le32_to_cpu(cmpl->event_data2); 2183 2184 netdev_dbg(bp->dev, "hwrm event 0x%x {0x%x, 0x%x}\n", 2185 event_id, data1, data2); 2186 2187 /* TODO CHIMP_FW: Define event id's for link change, error etc */ 2188 switch (event_id) { 2189 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: { 2190 struct bnxt_link_info *link_info = &bp->link_info; 2191 2192 if (BNXT_VF(bp)) 2193 goto async_event_process_exit; 2194 2195 /* print unsupported speed warning in forced speed mode only */ 2196 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED) && 2197 (data1 & 0x20000)) { 2198 u16 fw_speed = link_info->force_link_speed; 2199 u32 speed = bnxt_fw_to_ethtool_speed(fw_speed); 2200 2201 if (speed != SPEED_UNKNOWN) 2202 netdev_warn(bp->dev, "Link speed %d no longer supported\n", 2203 speed); 2204 } 2205 set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event); 2206 } 2207 fallthrough; 2208 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE: 2209 case ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE: 2210 set_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT, &bp->sp_event); 2211 fallthrough; 2212 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE: 2213 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event); 2214 break; 2215 case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD: 2216 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event); 2217 break; 2218 case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: { 2219 u16 port_id = BNXT_GET_EVENT_PORT(data1); 2220 2221 if (BNXT_VF(bp)) 2222 break; 2223 2224 if (bp->pf.port_id != port_id) 2225 break; 2226 2227 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event); 2228 break; 2229 } 2230 case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE: 2231 if (BNXT_PF(bp)) 2232 goto async_event_process_exit; 2233 set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event); 2234 break; 2235 case ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY: { 2236 char *type_str = "Solicited"; 2237 2238 if (!bp->fw_health) 2239 goto async_event_process_exit; 2240 2241 bp->fw_reset_timestamp = jiffies; 2242 bp->fw_reset_min_dsecs = cmpl->timestamp_lo; 2243 if (!bp->fw_reset_min_dsecs) 2244 bp->fw_reset_min_dsecs = BNXT_DFLT_FW_RST_MIN_DSECS; 2245 bp->fw_reset_max_dsecs = le16_to_cpu(cmpl->timestamp_hi); 2246 if (!bp->fw_reset_max_dsecs) 2247 bp->fw_reset_max_dsecs = BNXT_DFLT_FW_RST_MAX_DSECS; 2248 if (EVENT_DATA1_RESET_NOTIFY_FW_ACTIVATION(data1)) { 2249 set_bit(BNXT_STATE_FW_ACTIVATE_RESET, &bp->state); 2250 } else if (EVENT_DATA1_RESET_NOTIFY_FATAL(data1)) { 2251 type_str = "Fatal"; 2252 bp->fw_health->fatalities++; 2253 set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state); 2254 } else if (data2 && BNXT_FW_STATUS_HEALTHY != 2255 EVENT_DATA2_RESET_NOTIFY_FW_STATUS_CODE(data2)) { 2256 type_str = "Non-fatal"; 2257 bp->fw_health->survivals++; 2258 set_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state); 2259 } 2260 netif_warn(bp, hw, bp->dev, 2261 "%s firmware reset event, data1: 0x%x, data2: 0x%x, min wait %u ms, max wait %u ms\n", 2262 type_str, data1, data2, 2263 bp->fw_reset_min_dsecs * 100, 2264 bp->fw_reset_max_dsecs * 100); 2265 set_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event); 2266 break; 2267 } 2268 case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY: { 2269 struct bnxt_fw_health *fw_health = bp->fw_health; 2270 char *status_desc = "healthy"; 2271 u32 status; 2272 2273 if (!fw_health) 2274 goto async_event_process_exit; 2275 2276 if (!EVENT_DATA1_RECOVERY_ENABLED(data1)) { 2277 fw_health->enabled = false; 2278 netif_info(bp, drv, bp->dev, "Driver recovery watchdog is disabled\n"); 2279 break; 2280 } 2281 fw_health->primary = EVENT_DATA1_RECOVERY_MASTER_FUNC(data1); 2282 fw_health->tmr_multiplier = 2283 DIV_ROUND_UP(fw_health->polling_dsecs * HZ, 2284 bp->current_interval * 10); 2285 fw_health->tmr_counter = fw_health->tmr_multiplier; 2286 if (!fw_health->enabled) 2287 fw_health->last_fw_heartbeat = 2288 bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG); 2289 fw_health->last_fw_reset_cnt = 2290 bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG); 2291 status = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG); 2292 if (status != BNXT_FW_STATUS_HEALTHY) 2293 status_desc = "unhealthy"; 2294 netif_info(bp, drv, bp->dev, 2295 "Driver recovery watchdog, role: %s, firmware status: 0x%x (%s), resets: %u\n", 2296 fw_health->primary ? "primary" : "backup", status, 2297 status_desc, fw_health->last_fw_reset_cnt); 2298 if (!fw_health->enabled) { 2299 /* Make sure tmr_counter is set and visible to 2300 * bnxt_health_check() before setting enabled to true. 2301 */ 2302 smp_wmb(); 2303 fw_health->enabled = true; 2304 } 2305 goto async_event_process_exit; 2306 } 2307 case ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION: 2308 netif_notice(bp, hw, bp->dev, 2309 "Received firmware debug notification, data1: 0x%x, data2: 0x%x\n", 2310 data1, data2); 2311 goto async_event_process_exit; 2312 case ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG: { 2313 struct bnxt_rx_ring_info *rxr; 2314 u16 grp_idx; 2315 2316 if (bp->flags & BNXT_FLAG_CHIP_P5) 2317 goto async_event_process_exit; 2318 2319 netdev_warn(bp->dev, "Ring monitor event, ring type %lu id 0x%x\n", 2320 BNXT_EVENT_RING_TYPE(data2), data1); 2321 if (!BNXT_EVENT_RING_TYPE_RX(data2)) 2322 goto async_event_process_exit; 2323 2324 grp_idx = bnxt_agg_ring_id_to_grp_idx(bp, data1); 2325 if (grp_idx == INVALID_HW_RING_ID) { 2326 netdev_warn(bp->dev, "Unknown RX agg ring id 0x%x\n", 2327 data1); 2328 goto async_event_process_exit; 2329 } 2330 rxr = bp->bnapi[grp_idx]->rx_ring; 2331 bnxt_sched_reset_rxr(bp, rxr); 2332 goto async_event_process_exit; 2333 } 2334 case ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST: { 2335 struct bnxt_fw_health *fw_health = bp->fw_health; 2336 2337 netif_notice(bp, hw, bp->dev, 2338 "Received firmware echo request, data1: 0x%x, data2: 0x%x\n", 2339 data1, data2); 2340 if (fw_health) { 2341 fw_health->echo_req_data1 = data1; 2342 fw_health->echo_req_data2 = data2; 2343 set_bit(BNXT_FW_ECHO_REQUEST_SP_EVENT, &bp->sp_event); 2344 break; 2345 } 2346 goto async_event_process_exit; 2347 } 2348 case ASYNC_EVENT_CMPL_EVENT_ID_PPS_TIMESTAMP: { 2349 bnxt_ptp_pps_event(bp, data1, data2); 2350 goto async_event_process_exit; 2351 } 2352 case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT: { 2353 bnxt_event_error_report(bp, data1, data2); 2354 goto async_event_process_exit; 2355 } 2356 case ASYNC_EVENT_CMPL_EVENT_ID_PHC_UPDATE: { 2357 switch (BNXT_EVENT_PHC_EVENT_TYPE(data1)) { 2358 case ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_PHC_RTC_UPDATE: 2359 if (BNXT_PTP_USE_RTC(bp)) { 2360 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; 2361 u64 ns; 2362 2363 if (!ptp) 2364 goto async_event_process_exit; 2365 2366 spin_lock_bh(&ptp->ptp_lock); 2367 bnxt_ptp_update_current_time(bp); 2368 ns = (((u64)BNXT_EVENT_PHC_RTC_UPDATE(data1) << 2369 BNXT_PHC_BITS) | ptp->current_time); 2370 bnxt_ptp_rtc_timecounter_init(ptp, ns); 2371 spin_unlock_bh(&ptp->ptp_lock); 2372 } 2373 break; 2374 } 2375 goto async_event_process_exit; 2376 } 2377 case ASYNC_EVENT_CMPL_EVENT_ID_DEFERRED_RESPONSE: { 2378 u16 seq_id = le32_to_cpu(cmpl->event_data2) & 0xffff; 2379 2380 hwrm_update_token(bp, seq_id, BNXT_HWRM_DEFERRED); 2381 goto async_event_process_exit; 2382 } 2383 default: 2384 goto async_event_process_exit; 2385 } 2386 __bnxt_queue_sp_work(bp); 2387 async_event_process_exit: 2388 return 0; 2389 } 2390 2391 static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp) 2392 { 2393 u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id; 2394 struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp; 2395 struct hwrm_fwd_req_cmpl *fwd_req_cmpl = 2396 (struct hwrm_fwd_req_cmpl *)txcmp; 2397 2398 switch (cmpl_type) { 2399 case CMPL_BASE_TYPE_HWRM_DONE: 2400 seq_id = le16_to_cpu(h_cmpl->sequence_id); 2401 hwrm_update_token(bp, seq_id, BNXT_HWRM_COMPLETE); 2402 break; 2403 2404 case CMPL_BASE_TYPE_HWRM_FWD_REQ: 2405 vf_id = le16_to_cpu(fwd_req_cmpl->source_id); 2406 2407 if ((vf_id < bp->pf.first_vf_id) || 2408 (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) { 2409 netdev_err(bp->dev, "Msg contains invalid VF id %x\n", 2410 vf_id); 2411 return -EINVAL; 2412 } 2413 2414 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap); 2415 bnxt_queue_sp_work(bp, BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT); 2416 break; 2417 2418 case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT: 2419 bnxt_async_event_process(bp, 2420 (struct hwrm_async_event_cmpl *)txcmp); 2421 break; 2422 2423 default: 2424 break; 2425 } 2426 2427 return 0; 2428 } 2429 2430 static irqreturn_t bnxt_msix(int irq, void *dev_instance) 2431 { 2432 struct bnxt_napi *bnapi = dev_instance; 2433 struct bnxt *bp = bnapi->bp; 2434 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2435 u32 cons = RING_CMP(cpr->cp_raw_cons); 2436 2437 cpr->event_ctr++; 2438 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]); 2439 napi_schedule(&bnapi->napi); 2440 return IRQ_HANDLED; 2441 } 2442 2443 static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr) 2444 { 2445 u32 raw_cons = cpr->cp_raw_cons; 2446 u16 cons = RING_CMP(raw_cons); 2447 struct tx_cmp *txcmp; 2448 2449 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]; 2450 2451 return TX_CMP_VALID(txcmp, raw_cons); 2452 } 2453 2454 static irqreturn_t bnxt_inta(int irq, void *dev_instance) 2455 { 2456 struct bnxt_napi *bnapi = dev_instance; 2457 struct bnxt *bp = bnapi->bp; 2458 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2459 u32 cons = RING_CMP(cpr->cp_raw_cons); 2460 u32 int_status; 2461 2462 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]); 2463 2464 if (!bnxt_has_work(bp, cpr)) { 2465 int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS); 2466 /* return if erroneous interrupt */ 2467 if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id))) 2468 return IRQ_NONE; 2469 } 2470 2471 /* disable ring IRQ */ 2472 BNXT_CP_DB_IRQ_DIS(cpr->cp_db.doorbell); 2473 2474 /* Return here if interrupt is shared and is disabled. */ 2475 if (unlikely(atomic_read(&bp->intr_sem) != 0)) 2476 return IRQ_HANDLED; 2477 2478 napi_schedule(&bnapi->napi); 2479 return IRQ_HANDLED; 2480 } 2481 2482 static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 2483 int budget) 2484 { 2485 struct bnxt_napi *bnapi = cpr->bnapi; 2486 u32 raw_cons = cpr->cp_raw_cons; 2487 u32 cons; 2488 int tx_pkts = 0; 2489 int rx_pkts = 0; 2490 u8 event = 0; 2491 struct tx_cmp *txcmp; 2492 2493 cpr->has_more_work = 0; 2494 cpr->had_work_done = 1; 2495 while (1) { 2496 int rc; 2497 2498 cons = RING_CMP(raw_cons); 2499 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]; 2500 2501 if (!TX_CMP_VALID(txcmp, raw_cons)) 2502 break; 2503 2504 /* The valid test of the entry must be done first before 2505 * reading any further. 2506 */ 2507 dma_rmb(); 2508 if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) { 2509 tx_pkts++; 2510 /* return full budget so NAPI will complete. */ 2511 if (unlikely(tx_pkts >= bp->tx_wake_thresh)) { 2512 rx_pkts = budget; 2513 raw_cons = NEXT_RAW_CMP(raw_cons); 2514 if (budget) 2515 cpr->has_more_work = 1; 2516 break; 2517 } 2518 } else if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) { 2519 if (likely(budget)) 2520 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event); 2521 else 2522 rc = bnxt_force_rx_discard(bp, cpr, &raw_cons, 2523 &event); 2524 if (likely(rc >= 0)) 2525 rx_pkts += rc; 2526 /* Increment rx_pkts when rc is -ENOMEM to count towards 2527 * the NAPI budget. Otherwise, we may potentially loop 2528 * here forever if we consistently cannot allocate 2529 * buffers. 2530 */ 2531 else if (rc == -ENOMEM && budget) 2532 rx_pkts++; 2533 else if (rc == -EBUSY) /* partial completion */ 2534 break; 2535 } else if (unlikely((TX_CMP_TYPE(txcmp) == 2536 CMPL_BASE_TYPE_HWRM_DONE) || 2537 (TX_CMP_TYPE(txcmp) == 2538 CMPL_BASE_TYPE_HWRM_FWD_REQ) || 2539 (TX_CMP_TYPE(txcmp) == 2540 CMPL_BASE_TYPE_HWRM_ASYNC_EVENT))) { 2541 bnxt_hwrm_handler(bp, txcmp); 2542 } 2543 raw_cons = NEXT_RAW_CMP(raw_cons); 2544 2545 if (rx_pkts && rx_pkts == budget) { 2546 cpr->has_more_work = 1; 2547 break; 2548 } 2549 } 2550 2551 if (event & BNXT_REDIRECT_EVENT) 2552 xdp_do_flush(); 2553 2554 if (event & BNXT_TX_EVENT) { 2555 struct bnxt_tx_ring_info *txr = bnapi->tx_ring; 2556 u16 prod = txr->tx_prod; 2557 2558 /* Sync BD data before updating doorbell */ 2559 wmb(); 2560 2561 bnxt_db_write_relaxed(bp, &txr->tx_db, prod); 2562 } 2563 2564 cpr->cp_raw_cons = raw_cons; 2565 bnapi->tx_pkts += tx_pkts; 2566 bnapi->events |= event; 2567 return rx_pkts; 2568 } 2569 2570 static void __bnxt_poll_work_done(struct bnxt *bp, struct bnxt_napi *bnapi, 2571 int budget) 2572 { 2573 if (bnapi->tx_pkts && !bnapi->tx_fault) 2574 bnapi->tx_int(bp, bnapi, budget); 2575 2576 if ((bnapi->events & BNXT_RX_EVENT) && !(bnapi->in_reset)) { 2577 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 2578 2579 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 2580 } 2581 if (bnapi->events & BNXT_AGG_EVENT) { 2582 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 2583 2584 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); 2585 } 2586 bnapi->events = 0; 2587 } 2588 2589 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 2590 int budget) 2591 { 2592 struct bnxt_napi *bnapi = cpr->bnapi; 2593 int rx_pkts; 2594 2595 rx_pkts = __bnxt_poll_work(bp, cpr, budget); 2596 2597 /* ACK completion ring before freeing tx ring and producing new 2598 * buffers in rx/agg rings to prevent overflowing the completion 2599 * ring. 2600 */ 2601 bnxt_db_cq(bp, &cpr->cp_db, cpr->cp_raw_cons); 2602 2603 __bnxt_poll_work_done(bp, bnapi, budget); 2604 return rx_pkts; 2605 } 2606 2607 static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget) 2608 { 2609 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi); 2610 struct bnxt *bp = bnapi->bp; 2611 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2612 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 2613 struct tx_cmp *txcmp; 2614 struct rx_cmp_ext *rxcmp1; 2615 u32 cp_cons, tmp_raw_cons; 2616 u32 raw_cons = cpr->cp_raw_cons; 2617 u32 rx_pkts = 0; 2618 u8 event = 0; 2619 2620 while (1) { 2621 int rc; 2622 2623 cp_cons = RING_CMP(raw_cons); 2624 txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 2625 2626 if (!TX_CMP_VALID(txcmp, raw_cons)) 2627 break; 2628 2629 /* The valid test of the entry must be done first before 2630 * reading any further. 2631 */ 2632 dma_rmb(); 2633 if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) { 2634 tmp_raw_cons = NEXT_RAW_CMP(raw_cons); 2635 cp_cons = RING_CMP(tmp_raw_cons); 2636 rxcmp1 = (struct rx_cmp_ext *) 2637 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 2638 2639 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons)) 2640 break; 2641 2642 /* force an error to recycle the buffer */ 2643 rxcmp1->rx_cmp_cfa_code_errors_v2 |= 2644 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR); 2645 2646 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event); 2647 if (likely(rc == -EIO) && budget) 2648 rx_pkts++; 2649 else if (rc == -EBUSY) /* partial completion */ 2650 break; 2651 } else if (unlikely(TX_CMP_TYPE(txcmp) == 2652 CMPL_BASE_TYPE_HWRM_DONE)) { 2653 bnxt_hwrm_handler(bp, txcmp); 2654 } else { 2655 netdev_err(bp->dev, 2656 "Invalid completion received on special ring\n"); 2657 } 2658 raw_cons = NEXT_RAW_CMP(raw_cons); 2659 2660 if (rx_pkts == budget) 2661 break; 2662 } 2663 2664 cpr->cp_raw_cons = raw_cons; 2665 BNXT_DB_CQ(&cpr->cp_db, cpr->cp_raw_cons); 2666 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 2667 2668 if (event & BNXT_AGG_EVENT) 2669 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); 2670 2671 if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) { 2672 napi_complete_done(napi, rx_pkts); 2673 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons); 2674 } 2675 return rx_pkts; 2676 } 2677 2678 static int bnxt_poll(struct napi_struct *napi, int budget) 2679 { 2680 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi); 2681 struct bnxt *bp = bnapi->bp; 2682 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2683 int work_done = 0; 2684 2685 if (unlikely(test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))) { 2686 napi_complete(napi); 2687 return 0; 2688 } 2689 while (1) { 2690 work_done += bnxt_poll_work(bp, cpr, budget - work_done); 2691 2692 if (work_done >= budget) { 2693 if (!budget) 2694 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons); 2695 break; 2696 } 2697 2698 if (!bnxt_has_work(bp, cpr)) { 2699 if (napi_complete_done(napi, work_done)) 2700 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons); 2701 break; 2702 } 2703 } 2704 if (bp->flags & BNXT_FLAG_DIM) { 2705 struct dim_sample dim_sample = {}; 2706 2707 dim_update_sample(cpr->event_ctr, 2708 cpr->rx_packets, 2709 cpr->rx_bytes, 2710 &dim_sample); 2711 net_dim(&cpr->dim, dim_sample); 2712 } 2713 return work_done; 2714 } 2715 2716 static int __bnxt_poll_cqs(struct bnxt *bp, struct bnxt_napi *bnapi, int budget) 2717 { 2718 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2719 int i, work_done = 0; 2720 2721 for (i = 0; i < 2; i++) { 2722 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i]; 2723 2724 if (cpr2) { 2725 work_done += __bnxt_poll_work(bp, cpr2, 2726 budget - work_done); 2727 cpr->has_more_work |= cpr2->has_more_work; 2728 } 2729 } 2730 return work_done; 2731 } 2732 2733 static void __bnxt_poll_cqs_done(struct bnxt *bp, struct bnxt_napi *bnapi, 2734 u64 dbr_type, int budget) 2735 { 2736 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2737 int i; 2738 2739 for (i = 0; i < 2; i++) { 2740 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i]; 2741 struct bnxt_db_info *db; 2742 2743 if (cpr2 && cpr2->had_work_done) { 2744 db = &cpr2->cp_db; 2745 bnxt_writeq(bp, db->db_key64 | dbr_type | 2746 RING_CMP(cpr2->cp_raw_cons), db->doorbell); 2747 cpr2->had_work_done = 0; 2748 } 2749 } 2750 __bnxt_poll_work_done(bp, bnapi, budget); 2751 } 2752 2753 static int bnxt_poll_p5(struct napi_struct *napi, int budget) 2754 { 2755 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi); 2756 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2757 struct bnxt_cp_ring_info *cpr_rx; 2758 u32 raw_cons = cpr->cp_raw_cons; 2759 struct bnxt *bp = bnapi->bp; 2760 struct nqe_cn *nqcmp; 2761 int work_done = 0; 2762 u32 cons; 2763 2764 if (unlikely(test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))) { 2765 napi_complete(napi); 2766 return 0; 2767 } 2768 if (cpr->has_more_work) { 2769 cpr->has_more_work = 0; 2770 work_done = __bnxt_poll_cqs(bp, bnapi, budget); 2771 } 2772 while (1) { 2773 cons = RING_CMP(raw_cons); 2774 nqcmp = &cpr->nq_desc_ring[CP_RING(cons)][CP_IDX(cons)]; 2775 2776 if (!NQ_CMP_VALID(nqcmp, raw_cons)) { 2777 if (cpr->has_more_work) 2778 break; 2779 2780 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ_ARMALL, 2781 budget); 2782 cpr->cp_raw_cons = raw_cons; 2783 if (napi_complete_done(napi, work_done)) 2784 BNXT_DB_NQ_ARM_P5(&cpr->cp_db, 2785 cpr->cp_raw_cons); 2786 goto poll_done; 2787 } 2788 2789 /* The valid test of the entry must be done first before 2790 * reading any further. 2791 */ 2792 dma_rmb(); 2793 2794 if (nqcmp->type == cpu_to_le16(NQ_CN_TYPE_CQ_NOTIFICATION)) { 2795 u32 idx = le32_to_cpu(nqcmp->cq_handle_low); 2796 struct bnxt_cp_ring_info *cpr2; 2797 2798 /* No more budget for RX work */ 2799 if (budget && work_done >= budget && idx == BNXT_RX_HDL) 2800 break; 2801 2802 cpr2 = cpr->cp_ring_arr[idx]; 2803 work_done += __bnxt_poll_work(bp, cpr2, 2804 budget - work_done); 2805 cpr->has_more_work |= cpr2->has_more_work; 2806 } else { 2807 bnxt_hwrm_handler(bp, (struct tx_cmp *)nqcmp); 2808 } 2809 raw_cons = NEXT_RAW_CMP(raw_cons); 2810 } 2811 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ, budget); 2812 if (raw_cons != cpr->cp_raw_cons) { 2813 cpr->cp_raw_cons = raw_cons; 2814 BNXT_DB_NQ_P5(&cpr->cp_db, raw_cons); 2815 } 2816 poll_done: 2817 cpr_rx = cpr->cp_ring_arr[BNXT_RX_HDL]; 2818 if (cpr_rx && (bp->flags & BNXT_FLAG_DIM)) { 2819 struct dim_sample dim_sample = {}; 2820 2821 dim_update_sample(cpr->event_ctr, 2822 cpr_rx->rx_packets, 2823 cpr_rx->rx_bytes, 2824 &dim_sample); 2825 net_dim(&cpr->dim, dim_sample); 2826 } 2827 return work_done; 2828 } 2829 2830 static void bnxt_free_tx_skbs(struct bnxt *bp) 2831 { 2832 int i, max_idx; 2833 struct pci_dev *pdev = bp->pdev; 2834 2835 if (!bp->tx_ring) 2836 return; 2837 2838 max_idx = bp->tx_nr_pages * TX_DESC_CNT; 2839 for (i = 0; i < bp->tx_nr_rings; i++) { 2840 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 2841 int j; 2842 2843 if (!txr->tx_buf_ring) 2844 continue; 2845 2846 for (j = 0; j < max_idx;) { 2847 struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j]; 2848 struct sk_buff *skb; 2849 int k, last; 2850 2851 if (i < bp->tx_nr_rings_xdp && 2852 tx_buf->action == XDP_REDIRECT) { 2853 dma_unmap_single(&pdev->dev, 2854 dma_unmap_addr(tx_buf, mapping), 2855 dma_unmap_len(tx_buf, len), 2856 DMA_TO_DEVICE); 2857 xdp_return_frame(tx_buf->xdpf); 2858 tx_buf->action = 0; 2859 tx_buf->xdpf = NULL; 2860 j++; 2861 continue; 2862 } 2863 2864 skb = tx_buf->skb; 2865 if (!skb) { 2866 j++; 2867 continue; 2868 } 2869 2870 tx_buf->skb = NULL; 2871 2872 if (tx_buf->is_push) { 2873 dev_kfree_skb(skb); 2874 j += 2; 2875 continue; 2876 } 2877 2878 dma_unmap_single(&pdev->dev, 2879 dma_unmap_addr(tx_buf, mapping), 2880 skb_headlen(skb), 2881 DMA_TO_DEVICE); 2882 2883 last = tx_buf->nr_frags; 2884 j += 2; 2885 for (k = 0; k < last; k++, j++) { 2886 int ring_idx = j & bp->tx_ring_mask; 2887 skb_frag_t *frag = &skb_shinfo(skb)->frags[k]; 2888 2889 tx_buf = &txr->tx_buf_ring[ring_idx]; 2890 dma_unmap_page( 2891 &pdev->dev, 2892 dma_unmap_addr(tx_buf, mapping), 2893 skb_frag_size(frag), DMA_TO_DEVICE); 2894 } 2895 dev_kfree_skb(skb); 2896 } 2897 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i)); 2898 } 2899 } 2900 2901 static void bnxt_free_one_rx_ring_skbs(struct bnxt *bp, int ring_nr) 2902 { 2903 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr]; 2904 struct pci_dev *pdev = bp->pdev; 2905 struct bnxt_tpa_idx_map *map; 2906 int i, max_idx, max_agg_idx; 2907 2908 max_idx = bp->rx_nr_pages * RX_DESC_CNT; 2909 max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT; 2910 if (!rxr->rx_tpa) 2911 goto skip_rx_tpa_free; 2912 2913 for (i = 0; i < bp->max_tpa; i++) { 2914 struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[i]; 2915 u8 *data = tpa_info->data; 2916 2917 if (!data) 2918 continue; 2919 2920 dma_unmap_single_attrs(&pdev->dev, tpa_info->mapping, 2921 bp->rx_buf_use_size, bp->rx_dir, 2922 DMA_ATTR_WEAK_ORDERING); 2923 2924 tpa_info->data = NULL; 2925 2926 skb_free_frag(data); 2927 } 2928 2929 skip_rx_tpa_free: 2930 if (!rxr->rx_buf_ring) 2931 goto skip_rx_buf_free; 2932 2933 for (i = 0; i < max_idx; i++) { 2934 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[i]; 2935 dma_addr_t mapping = rx_buf->mapping; 2936 void *data = rx_buf->data; 2937 2938 if (!data) 2939 continue; 2940 2941 rx_buf->data = NULL; 2942 if (BNXT_RX_PAGE_MODE(bp)) { 2943 page_pool_recycle_direct(rxr->page_pool, data); 2944 } else { 2945 dma_unmap_single_attrs(&pdev->dev, mapping, 2946 bp->rx_buf_use_size, bp->rx_dir, 2947 DMA_ATTR_WEAK_ORDERING); 2948 skb_free_frag(data); 2949 } 2950 } 2951 2952 skip_rx_buf_free: 2953 if (!rxr->rx_agg_ring) 2954 goto skip_rx_agg_free; 2955 2956 for (i = 0; i < max_agg_idx; i++) { 2957 struct bnxt_sw_rx_agg_bd *rx_agg_buf = &rxr->rx_agg_ring[i]; 2958 struct page *page = rx_agg_buf->page; 2959 2960 if (!page) 2961 continue; 2962 2963 rx_agg_buf->page = NULL; 2964 __clear_bit(i, rxr->rx_agg_bmap); 2965 2966 page_pool_recycle_direct(rxr->page_pool, page); 2967 } 2968 2969 skip_rx_agg_free: 2970 map = rxr->rx_tpa_idx_map; 2971 if (map) 2972 memset(map->agg_idx_bmap, 0, sizeof(map->agg_idx_bmap)); 2973 } 2974 2975 static void bnxt_free_rx_skbs(struct bnxt *bp) 2976 { 2977 int i; 2978 2979 if (!bp->rx_ring) 2980 return; 2981 2982 for (i = 0; i < bp->rx_nr_rings; i++) 2983 bnxt_free_one_rx_ring_skbs(bp, i); 2984 } 2985 2986 static void bnxt_free_skbs(struct bnxt *bp) 2987 { 2988 bnxt_free_tx_skbs(bp); 2989 bnxt_free_rx_skbs(bp); 2990 } 2991 2992 static void bnxt_init_ctx_mem(struct bnxt_mem_init *mem_init, void *p, int len) 2993 { 2994 u8 init_val = mem_init->init_val; 2995 u16 offset = mem_init->offset; 2996 u8 *p2 = p; 2997 int i; 2998 2999 if (!init_val) 3000 return; 3001 if (offset == BNXT_MEM_INVALID_OFFSET) { 3002 memset(p, init_val, len); 3003 return; 3004 } 3005 for (i = 0; i < len; i += mem_init->size) 3006 *(p2 + i + offset) = init_val; 3007 } 3008 3009 static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem) 3010 { 3011 struct pci_dev *pdev = bp->pdev; 3012 int i; 3013 3014 if (!rmem->pg_arr) 3015 goto skip_pages; 3016 3017 for (i = 0; i < rmem->nr_pages; i++) { 3018 if (!rmem->pg_arr[i]) 3019 continue; 3020 3021 dma_free_coherent(&pdev->dev, rmem->page_size, 3022 rmem->pg_arr[i], rmem->dma_arr[i]); 3023 3024 rmem->pg_arr[i] = NULL; 3025 } 3026 skip_pages: 3027 if (rmem->pg_tbl) { 3028 size_t pg_tbl_size = rmem->nr_pages * 8; 3029 3030 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG) 3031 pg_tbl_size = rmem->page_size; 3032 dma_free_coherent(&pdev->dev, pg_tbl_size, 3033 rmem->pg_tbl, rmem->pg_tbl_map); 3034 rmem->pg_tbl = NULL; 3035 } 3036 if (rmem->vmem_size && *rmem->vmem) { 3037 vfree(*rmem->vmem); 3038 *rmem->vmem = NULL; 3039 } 3040 } 3041 3042 static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem) 3043 { 3044 struct pci_dev *pdev = bp->pdev; 3045 u64 valid_bit = 0; 3046 int i; 3047 3048 if (rmem->flags & (BNXT_RMEM_VALID_PTE_FLAG | BNXT_RMEM_RING_PTE_FLAG)) 3049 valid_bit = PTU_PTE_VALID; 3050 if ((rmem->nr_pages > 1 || rmem->depth > 0) && !rmem->pg_tbl) { 3051 size_t pg_tbl_size = rmem->nr_pages * 8; 3052 3053 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG) 3054 pg_tbl_size = rmem->page_size; 3055 rmem->pg_tbl = dma_alloc_coherent(&pdev->dev, pg_tbl_size, 3056 &rmem->pg_tbl_map, 3057 GFP_KERNEL); 3058 if (!rmem->pg_tbl) 3059 return -ENOMEM; 3060 } 3061 3062 for (i = 0; i < rmem->nr_pages; i++) { 3063 u64 extra_bits = valid_bit; 3064 3065 rmem->pg_arr[i] = dma_alloc_coherent(&pdev->dev, 3066 rmem->page_size, 3067 &rmem->dma_arr[i], 3068 GFP_KERNEL); 3069 if (!rmem->pg_arr[i]) 3070 return -ENOMEM; 3071 3072 if (rmem->mem_init) 3073 bnxt_init_ctx_mem(rmem->mem_init, rmem->pg_arr[i], 3074 rmem->page_size); 3075 if (rmem->nr_pages > 1 || rmem->depth > 0) { 3076 if (i == rmem->nr_pages - 2 && 3077 (rmem->flags & BNXT_RMEM_RING_PTE_FLAG)) 3078 extra_bits |= PTU_PTE_NEXT_TO_LAST; 3079 else if (i == rmem->nr_pages - 1 && 3080 (rmem->flags & BNXT_RMEM_RING_PTE_FLAG)) 3081 extra_bits |= PTU_PTE_LAST; 3082 rmem->pg_tbl[i] = 3083 cpu_to_le64(rmem->dma_arr[i] | extra_bits); 3084 } 3085 } 3086 3087 if (rmem->vmem_size) { 3088 *rmem->vmem = vzalloc(rmem->vmem_size); 3089 if (!(*rmem->vmem)) 3090 return -ENOMEM; 3091 } 3092 return 0; 3093 } 3094 3095 static void bnxt_free_tpa_info(struct bnxt *bp) 3096 { 3097 int i, j; 3098 3099 for (i = 0; i < bp->rx_nr_rings; i++) { 3100 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 3101 3102 kfree(rxr->rx_tpa_idx_map); 3103 rxr->rx_tpa_idx_map = NULL; 3104 if (rxr->rx_tpa) { 3105 for (j = 0; j < bp->max_tpa; j++) { 3106 kfree(rxr->rx_tpa[j].agg_arr); 3107 rxr->rx_tpa[j].agg_arr = NULL; 3108 } 3109 } 3110 kfree(rxr->rx_tpa); 3111 rxr->rx_tpa = NULL; 3112 } 3113 } 3114 3115 static int bnxt_alloc_tpa_info(struct bnxt *bp) 3116 { 3117 int i, j; 3118 3119 bp->max_tpa = MAX_TPA; 3120 if (bp->flags & BNXT_FLAG_CHIP_P5) { 3121 if (!bp->max_tpa_v2) 3122 return 0; 3123 bp->max_tpa = max_t(u16, bp->max_tpa_v2, MAX_TPA_P5); 3124 } 3125 3126 for (i = 0; i < bp->rx_nr_rings; i++) { 3127 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 3128 struct rx_agg_cmp *agg; 3129 3130 rxr->rx_tpa = kcalloc(bp->max_tpa, sizeof(struct bnxt_tpa_info), 3131 GFP_KERNEL); 3132 if (!rxr->rx_tpa) 3133 return -ENOMEM; 3134 3135 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 3136 continue; 3137 for (j = 0; j < bp->max_tpa; j++) { 3138 agg = kcalloc(MAX_SKB_FRAGS, sizeof(*agg), GFP_KERNEL); 3139 if (!agg) 3140 return -ENOMEM; 3141 rxr->rx_tpa[j].agg_arr = agg; 3142 } 3143 rxr->rx_tpa_idx_map = kzalloc(sizeof(*rxr->rx_tpa_idx_map), 3144 GFP_KERNEL); 3145 if (!rxr->rx_tpa_idx_map) 3146 return -ENOMEM; 3147 } 3148 return 0; 3149 } 3150 3151 static void bnxt_free_rx_rings(struct bnxt *bp) 3152 { 3153 int i; 3154 3155 if (!bp->rx_ring) 3156 return; 3157 3158 bnxt_free_tpa_info(bp); 3159 for (i = 0; i < bp->rx_nr_rings; i++) { 3160 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 3161 struct bnxt_ring_struct *ring; 3162 3163 if (rxr->xdp_prog) 3164 bpf_prog_put(rxr->xdp_prog); 3165 3166 if (xdp_rxq_info_is_reg(&rxr->xdp_rxq)) 3167 xdp_rxq_info_unreg(&rxr->xdp_rxq); 3168 3169 page_pool_destroy(rxr->page_pool); 3170 rxr->page_pool = NULL; 3171 3172 kfree(rxr->rx_agg_bmap); 3173 rxr->rx_agg_bmap = NULL; 3174 3175 ring = &rxr->rx_ring_struct; 3176 bnxt_free_ring(bp, &ring->ring_mem); 3177 3178 ring = &rxr->rx_agg_ring_struct; 3179 bnxt_free_ring(bp, &ring->ring_mem); 3180 } 3181 } 3182 3183 static int bnxt_alloc_rx_page_pool(struct bnxt *bp, 3184 struct bnxt_rx_ring_info *rxr) 3185 { 3186 struct page_pool_params pp = { 0 }; 3187 3188 pp.pool_size = bp->rx_agg_ring_size; 3189 if (BNXT_RX_PAGE_MODE(bp)) 3190 pp.pool_size += bp->rx_ring_size; 3191 pp.nid = dev_to_node(&bp->pdev->dev); 3192 pp.napi = &rxr->bnapi->napi; 3193 pp.dev = &bp->pdev->dev; 3194 pp.dma_dir = bp->rx_dir; 3195 pp.max_len = PAGE_SIZE; 3196 pp.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; 3197 if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) 3198 pp.flags |= PP_FLAG_PAGE_FRAG; 3199 3200 rxr->page_pool = page_pool_create(&pp); 3201 if (IS_ERR(rxr->page_pool)) { 3202 int err = PTR_ERR(rxr->page_pool); 3203 3204 rxr->page_pool = NULL; 3205 return err; 3206 } 3207 return 0; 3208 } 3209 3210 static int bnxt_alloc_rx_rings(struct bnxt *bp) 3211 { 3212 int i, rc = 0, agg_rings = 0; 3213 3214 if (!bp->rx_ring) 3215 return -ENOMEM; 3216 3217 if (bp->flags & BNXT_FLAG_AGG_RINGS) 3218 agg_rings = 1; 3219 3220 for (i = 0; i < bp->rx_nr_rings; i++) { 3221 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 3222 struct bnxt_ring_struct *ring; 3223 3224 ring = &rxr->rx_ring_struct; 3225 3226 rc = bnxt_alloc_rx_page_pool(bp, rxr); 3227 if (rc) 3228 return rc; 3229 3230 rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i, 0); 3231 if (rc < 0) 3232 return rc; 3233 3234 rc = xdp_rxq_info_reg_mem_model(&rxr->xdp_rxq, 3235 MEM_TYPE_PAGE_POOL, 3236 rxr->page_pool); 3237 if (rc) { 3238 xdp_rxq_info_unreg(&rxr->xdp_rxq); 3239 return rc; 3240 } 3241 3242 rc = bnxt_alloc_ring(bp, &ring->ring_mem); 3243 if (rc) 3244 return rc; 3245 3246 ring->grp_idx = i; 3247 if (agg_rings) { 3248 u16 mem_size; 3249 3250 ring = &rxr->rx_agg_ring_struct; 3251 rc = bnxt_alloc_ring(bp, &ring->ring_mem); 3252 if (rc) 3253 return rc; 3254 3255 ring->grp_idx = i; 3256 rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1; 3257 mem_size = rxr->rx_agg_bmap_size / 8; 3258 rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL); 3259 if (!rxr->rx_agg_bmap) 3260 return -ENOMEM; 3261 } 3262 } 3263 if (bp->flags & BNXT_FLAG_TPA) 3264 rc = bnxt_alloc_tpa_info(bp); 3265 return rc; 3266 } 3267 3268 static void bnxt_free_tx_rings(struct bnxt *bp) 3269 { 3270 int i; 3271 struct pci_dev *pdev = bp->pdev; 3272 3273 if (!bp->tx_ring) 3274 return; 3275 3276 for (i = 0; i < bp->tx_nr_rings; i++) { 3277 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 3278 struct bnxt_ring_struct *ring; 3279 3280 if (txr->tx_push) { 3281 dma_free_coherent(&pdev->dev, bp->tx_push_size, 3282 txr->tx_push, txr->tx_push_mapping); 3283 txr->tx_push = NULL; 3284 } 3285 3286 ring = &txr->tx_ring_struct; 3287 3288 bnxt_free_ring(bp, &ring->ring_mem); 3289 } 3290 } 3291 3292 static int bnxt_alloc_tx_rings(struct bnxt *bp) 3293 { 3294 int i, j, rc; 3295 struct pci_dev *pdev = bp->pdev; 3296 3297 bp->tx_push_size = 0; 3298 if (bp->tx_push_thresh) { 3299 int push_size; 3300 3301 push_size = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) + 3302 bp->tx_push_thresh); 3303 3304 if (push_size > 256) { 3305 push_size = 0; 3306 bp->tx_push_thresh = 0; 3307 } 3308 3309 bp->tx_push_size = push_size; 3310 } 3311 3312 for (i = 0, j = 0; i < bp->tx_nr_rings; i++) { 3313 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 3314 struct bnxt_ring_struct *ring; 3315 u8 qidx; 3316 3317 ring = &txr->tx_ring_struct; 3318 3319 rc = bnxt_alloc_ring(bp, &ring->ring_mem); 3320 if (rc) 3321 return rc; 3322 3323 ring->grp_idx = txr->bnapi->index; 3324 if (bp->tx_push_size) { 3325 dma_addr_t mapping; 3326 3327 /* One pre-allocated DMA buffer to backup 3328 * TX push operation 3329 */ 3330 txr->tx_push = dma_alloc_coherent(&pdev->dev, 3331 bp->tx_push_size, 3332 &txr->tx_push_mapping, 3333 GFP_KERNEL); 3334 3335 if (!txr->tx_push) 3336 return -ENOMEM; 3337 3338 mapping = txr->tx_push_mapping + 3339 sizeof(struct tx_push_bd); 3340 txr->data_mapping = cpu_to_le64(mapping); 3341 } 3342 qidx = bp->tc_to_qidx[j]; 3343 ring->queue_id = bp->q_info[qidx].queue_id; 3344 spin_lock_init(&txr->xdp_tx_lock); 3345 if (i < bp->tx_nr_rings_xdp) 3346 continue; 3347 if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1)) 3348 j++; 3349 } 3350 return 0; 3351 } 3352 3353 static void bnxt_free_cp_arrays(struct bnxt_cp_ring_info *cpr) 3354 { 3355 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 3356 3357 kfree(cpr->cp_desc_ring); 3358 cpr->cp_desc_ring = NULL; 3359 ring->ring_mem.pg_arr = NULL; 3360 kfree(cpr->cp_desc_mapping); 3361 cpr->cp_desc_mapping = NULL; 3362 ring->ring_mem.dma_arr = NULL; 3363 } 3364 3365 static int bnxt_alloc_cp_arrays(struct bnxt_cp_ring_info *cpr, int n) 3366 { 3367 cpr->cp_desc_ring = kcalloc(n, sizeof(*cpr->cp_desc_ring), GFP_KERNEL); 3368 if (!cpr->cp_desc_ring) 3369 return -ENOMEM; 3370 cpr->cp_desc_mapping = kcalloc(n, sizeof(*cpr->cp_desc_mapping), 3371 GFP_KERNEL); 3372 if (!cpr->cp_desc_mapping) 3373 return -ENOMEM; 3374 return 0; 3375 } 3376 3377 static void bnxt_free_all_cp_arrays(struct bnxt *bp) 3378 { 3379 int i; 3380 3381 if (!bp->bnapi) 3382 return; 3383 for (i = 0; i < bp->cp_nr_rings; i++) { 3384 struct bnxt_napi *bnapi = bp->bnapi[i]; 3385 3386 if (!bnapi) 3387 continue; 3388 bnxt_free_cp_arrays(&bnapi->cp_ring); 3389 } 3390 } 3391 3392 static int bnxt_alloc_all_cp_arrays(struct bnxt *bp) 3393 { 3394 int i, n = bp->cp_nr_pages; 3395 3396 for (i = 0; i < bp->cp_nr_rings; i++) { 3397 struct bnxt_napi *bnapi = bp->bnapi[i]; 3398 int rc; 3399 3400 if (!bnapi) 3401 continue; 3402 rc = bnxt_alloc_cp_arrays(&bnapi->cp_ring, n); 3403 if (rc) 3404 return rc; 3405 } 3406 return 0; 3407 } 3408 3409 static void bnxt_free_cp_rings(struct bnxt *bp) 3410 { 3411 int i; 3412 3413 if (!bp->bnapi) 3414 return; 3415 3416 for (i = 0; i < bp->cp_nr_rings; i++) { 3417 struct bnxt_napi *bnapi = bp->bnapi[i]; 3418 struct bnxt_cp_ring_info *cpr; 3419 struct bnxt_ring_struct *ring; 3420 int j; 3421 3422 if (!bnapi) 3423 continue; 3424 3425 cpr = &bnapi->cp_ring; 3426 ring = &cpr->cp_ring_struct; 3427 3428 bnxt_free_ring(bp, &ring->ring_mem); 3429 3430 for (j = 0; j < 2; j++) { 3431 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j]; 3432 3433 if (cpr2) { 3434 ring = &cpr2->cp_ring_struct; 3435 bnxt_free_ring(bp, &ring->ring_mem); 3436 bnxt_free_cp_arrays(cpr2); 3437 kfree(cpr2); 3438 cpr->cp_ring_arr[j] = NULL; 3439 } 3440 } 3441 } 3442 } 3443 3444 static struct bnxt_cp_ring_info *bnxt_alloc_cp_sub_ring(struct bnxt *bp) 3445 { 3446 struct bnxt_ring_mem_info *rmem; 3447 struct bnxt_ring_struct *ring; 3448 struct bnxt_cp_ring_info *cpr; 3449 int rc; 3450 3451 cpr = kzalloc(sizeof(*cpr), GFP_KERNEL); 3452 if (!cpr) 3453 return NULL; 3454 3455 rc = bnxt_alloc_cp_arrays(cpr, bp->cp_nr_pages); 3456 if (rc) { 3457 bnxt_free_cp_arrays(cpr); 3458 kfree(cpr); 3459 return NULL; 3460 } 3461 ring = &cpr->cp_ring_struct; 3462 rmem = &ring->ring_mem; 3463 rmem->nr_pages = bp->cp_nr_pages; 3464 rmem->page_size = HW_CMPD_RING_SIZE; 3465 rmem->pg_arr = (void **)cpr->cp_desc_ring; 3466 rmem->dma_arr = cpr->cp_desc_mapping; 3467 rmem->flags = BNXT_RMEM_RING_PTE_FLAG; 3468 rc = bnxt_alloc_ring(bp, rmem); 3469 if (rc) { 3470 bnxt_free_ring(bp, rmem); 3471 bnxt_free_cp_arrays(cpr); 3472 kfree(cpr); 3473 cpr = NULL; 3474 } 3475 return cpr; 3476 } 3477 3478 static int bnxt_alloc_cp_rings(struct bnxt *bp) 3479 { 3480 bool sh = !!(bp->flags & BNXT_FLAG_SHARED_RINGS); 3481 int i, rc, ulp_base_vec, ulp_msix; 3482 3483 ulp_msix = bnxt_get_ulp_msix_num(bp); 3484 ulp_base_vec = bnxt_get_ulp_msix_base(bp); 3485 for (i = 0; i < bp->cp_nr_rings; i++) { 3486 struct bnxt_napi *bnapi = bp->bnapi[i]; 3487 struct bnxt_cp_ring_info *cpr; 3488 struct bnxt_ring_struct *ring; 3489 3490 if (!bnapi) 3491 continue; 3492 3493 cpr = &bnapi->cp_ring; 3494 cpr->bnapi = bnapi; 3495 ring = &cpr->cp_ring_struct; 3496 3497 rc = bnxt_alloc_ring(bp, &ring->ring_mem); 3498 if (rc) 3499 return rc; 3500 3501 if (ulp_msix && i >= ulp_base_vec) 3502 ring->map_idx = i + ulp_msix; 3503 else 3504 ring->map_idx = i; 3505 3506 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 3507 continue; 3508 3509 if (i < bp->rx_nr_rings) { 3510 struct bnxt_cp_ring_info *cpr2 = 3511 bnxt_alloc_cp_sub_ring(bp); 3512 3513 cpr->cp_ring_arr[BNXT_RX_HDL] = cpr2; 3514 if (!cpr2) 3515 return -ENOMEM; 3516 cpr2->bnapi = bnapi; 3517 } 3518 if ((sh && i < bp->tx_nr_rings) || 3519 (!sh && i >= bp->rx_nr_rings)) { 3520 struct bnxt_cp_ring_info *cpr2 = 3521 bnxt_alloc_cp_sub_ring(bp); 3522 3523 cpr->cp_ring_arr[BNXT_TX_HDL] = cpr2; 3524 if (!cpr2) 3525 return -ENOMEM; 3526 cpr2->bnapi = bnapi; 3527 } 3528 } 3529 return 0; 3530 } 3531 3532 static void bnxt_init_ring_struct(struct bnxt *bp) 3533 { 3534 int i; 3535 3536 for (i = 0; i < bp->cp_nr_rings; i++) { 3537 struct bnxt_napi *bnapi = bp->bnapi[i]; 3538 struct bnxt_ring_mem_info *rmem; 3539 struct bnxt_cp_ring_info *cpr; 3540 struct bnxt_rx_ring_info *rxr; 3541 struct bnxt_tx_ring_info *txr; 3542 struct bnxt_ring_struct *ring; 3543 3544 if (!bnapi) 3545 continue; 3546 3547 cpr = &bnapi->cp_ring; 3548 ring = &cpr->cp_ring_struct; 3549 rmem = &ring->ring_mem; 3550 rmem->nr_pages = bp->cp_nr_pages; 3551 rmem->page_size = HW_CMPD_RING_SIZE; 3552 rmem->pg_arr = (void **)cpr->cp_desc_ring; 3553 rmem->dma_arr = cpr->cp_desc_mapping; 3554 rmem->vmem_size = 0; 3555 3556 rxr = bnapi->rx_ring; 3557 if (!rxr) 3558 goto skip_rx; 3559 3560 ring = &rxr->rx_ring_struct; 3561 rmem = &ring->ring_mem; 3562 rmem->nr_pages = bp->rx_nr_pages; 3563 rmem->page_size = HW_RXBD_RING_SIZE; 3564 rmem->pg_arr = (void **)rxr->rx_desc_ring; 3565 rmem->dma_arr = rxr->rx_desc_mapping; 3566 rmem->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages; 3567 rmem->vmem = (void **)&rxr->rx_buf_ring; 3568 3569 ring = &rxr->rx_agg_ring_struct; 3570 rmem = &ring->ring_mem; 3571 rmem->nr_pages = bp->rx_agg_nr_pages; 3572 rmem->page_size = HW_RXBD_RING_SIZE; 3573 rmem->pg_arr = (void **)rxr->rx_agg_desc_ring; 3574 rmem->dma_arr = rxr->rx_agg_desc_mapping; 3575 rmem->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages; 3576 rmem->vmem = (void **)&rxr->rx_agg_ring; 3577 3578 skip_rx: 3579 txr = bnapi->tx_ring; 3580 if (!txr) 3581 continue; 3582 3583 ring = &txr->tx_ring_struct; 3584 rmem = &ring->ring_mem; 3585 rmem->nr_pages = bp->tx_nr_pages; 3586 rmem->page_size = HW_RXBD_RING_SIZE; 3587 rmem->pg_arr = (void **)txr->tx_desc_ring; 3588 rmem->dma_arr = txr->tx_desc_mapping; 3589 rmem->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages; 3590 rmem->vmem = (void **)&txr->tx_buf_ring; 3591 } 3592 } 3593 3594 static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type) 3595 { 3596 int i; 3597 u32 prod; 3598 struct rx_bd **rx_buf_ring; 3599 3600 rx_buf_ring = (struct rx_bd **)ring->ring_mem.pg_arr; 3601 for (i = 0, prod = 0; i < ring->ring_mem.nr_pages; i++) { 3602 int j; 3603 struct rx_bd *rxbd; 3604 3605 rxbd = rx_buf_ring[i]; 3606 if (!rxbd) 3607 continue; 3608 3609 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) { 3610 rxbd->rx_bd_len_flags_type = cpu_to_le32(type); 3611 rxbd->rx_bd_opaque = prod; 3612 } 3613 } 3614 } 3615 3616 static int bnxt_alloc_one_rx_ring(struct bnxt *bp, int ring_nr) 3617 { 3618 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr]; 3619 struct net_device *dev = bp->dev; 3620 u32 prod; 3621 int i; 3622 3623 prod = rxr->rx_prod; 3624 for (i = 0; i < bp->rx_ring_size; i++) { 3625 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL)) { 3626 netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n", 3627 ring_nr, i, bp->rx_ring_size); 3628 break; 3629 } 3630 prod = NEXT_RX(prod); 3631 } 3632 rxr->rx_prod = prod; 3633 3634 if (!(bp->flags & BNXT_FLAG_AGG_RINGS)) 3635 return 0; 3636 3637 prod = rxr->rx_agg_prod; 3638 for (i = 0; i < bp->rx_agg_ring_size; i++) { 3639 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL)) { 3640 netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n", 3641 ring_nr, i, bp->rx_ring_size); 3642 break; 3643 } 3644 prod = NEXT_RX_AGG(prod); 3645 } 3646 rxr->rx_agg_prod = prod; 3647 3648 if (rxr->rx_tpa) { 3649 dma_addr_t mapping; 3650 u8 *data; 3651 3652 for (i = 0; i < bp->max_tpa; i++) { 3653 data = __bnxt_alloc_rx_frag(bp, &mapping, GFP_KERNEL); 3654 if (!data) 3655 return -ENOMEM; 3656 3657 rxr->rx_tpa[i].data = data; 3658 rxr->rx_tpa[i].data_ptr = data + bp->rx_offset; 3659 rxr->rx_tpa[i].mapping = mapping; 3660 } 3661 } 3662 return 0; 3663 } 3664 3665 static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr) 3666 { 3667 struct bnxt_rx_ring_info *rxr; 3668 struct bnxt_ring_struct *ring; 3669 u32 type; 3670 3671 type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) | 3672 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP; 3673 3674 if (NET_IP_ALIGN == 2) 3675 type |= RX_BD_FLAGS_SOP; 3676 3677 rxr = &bp->rx_ring[ring_nr]; 3678 ring = &rxr->rx_ring_struct; 3679 bnxt_init_rxbd_pages(ring, type); 3680 3681 if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) { 3682 bpf_prog_add(bp->xdp_prog, 1); 3683 rxr->xdp_prog = bp->xdp_prog; 3684 } 3685 ring->fw_ring_id = INVALID_HW_RING_ID; 3686 3687 ring = &rxr->rx_agg_ring_struct; 3688 ring->fw_ring_id = INVALID_HW_RING_ID; 3689 3690 if ((bp->flags & BNXT_FLAG_AGG_RINGS)) { 3691 type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) | 3692 RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP; 3693 3694 bnxt_init_rxbd_pages(ring, type); 3695 } 3696 3697 return bnxt_alloc_one_rx_ring(bp, ring_nr); 3698 } 3699 3700 static void bnxt_init_cp_rings(struct bnxt *bp) 3701 { 3702 int i, j; 3703 3704 for (i = 0; i < bp->cp_nr_rings; i++) { 3705 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring; 3706 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 3707 3708 ring->fw_ring_id = INVALID_HW_RING_ID; 3709 cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks; 3710 cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs; 3711 for (j = 0; j < 2; j++) { 3712 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j]; 3713 3714 if (!cpr2) 3715 continue; 3716 3717 ring = &cpr2->cp_ring_struct; 3718 ring->fw_ring_id = INVALID_HW_RING_ID; 3719 cpr2->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks; 3720 cpr2->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs; 3721 } 3722 } 3723 } 3724 3725 static int bnxt_init_rx_rings(struct bnxt *bp) 3726 { 3727 int i, rc = 0; 3728 3729 if (BNXT_RX_PAGE_MODE(bp)) { 3730 bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM; 3731 bp->rx_dma_offset = XDP_PACKET_HEADROOM; 3732 } else { 3733 bp->rx_offset = BNXT_RX_OFFSET; 3734 bp->rx_dma_offset = BNXT_RX_DMA_OFFSET; 3735 } 3736 3737 for (i = 0; i < bp->rx_nr_rings; i++) { 3738 rc = bnxt_init_one_rx_ring(bp, i); 3739 if (rc) 3740 break; 3741 } 3742 3743 return rc; 3744 } 3745 3746 static int bnxt_init_tx_rings(struct bnxt *bp) 3747 { 3748 u16 i; 3749 3750 bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2, 3751 BNXT_MIN_TX_DESC_CNT); 3752 3753 for (i = 0; i < bp->tx_nr_rings; i++) { 3754 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 3755 struct bnxt_ring_struct *ring = &txr->tx_ring_struct; 3756 3757 ring->fw_ring_id = INVALID_HW_RING_ID; 3758 } 3759 3760 return 0; 3761 } 3762 3763 static void bnxt_free_ring_grps(struct bnxt *bp) 3764 { 3765 kfree(bp->grp_info); 3766 bp->grp_info = NULL; 3767 } 3768 3769 static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init) 3770 { 3771 int i; 3772 3773 if (irq_re_init) { 3774 bp->grp_info = kcalloc(bp->cp_nr_rings, 3775 sizeof(struct bnxt_ring_grp_info), 3776 GFP_KERNEL); 3777 if (!bp->grp_info) 3778 return -ENOMEM; 3779 } 3780 for (i = 0; i < bp->cp_nr_rings; i++) { 3781 if (irq_re_init) 3782 bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID; 3783 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID; 3784 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID; 3785 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID; 3786 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID; 3787 } 3788 return 0; 3789 } 3790 3791 static void bnxt_free_vnics(struct bnxt *bp) 3792 { 3793 kfree(bp->vnic_info); 3794 bp->vnic_info = NULL; 3795 bp->nr_vnics = 0; 3796 } 3797 3798 static int bnxt_alloc_vnics(struct bnxt *bp) 3799 { 3800 int num_vnics = 1; 3801 3802 #ifdef CONFIG_RFS_ACCEL 3803 if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS) 3804 num_vnics += bp->rx_nr_rings; 3805 #endif 3806 3807 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 3808 num_vnics++; 3809 3810 bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info), 3811 GFP_KERNEL); 3812 if (!bp->vnic_info) 3813 return -ENOMEM; 3814 3815 bp->nr_vnics = num_vnics; 3816 return 0; 3817 } 3818 3819 static void bnxt_init_vnics(struct bnxt *bp) 3820 { 3821 int i; 3822 3823 for (i = 0; i < bp->nr_vnics; i++) { 3824 struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; 3825 int j; 3826 3827 vnic->fw_vnic_id = INVALID_HW_RING_ID; 3828 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) 3829 vnic->fw_rss_cos_lb_ctx[j] = INVALID_HW_RING_ID; 3830 3831 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID; 3832 3833 if (bp->vnic_info[i].rss_hash_key) { 3834 if (i == 0) 3835 get_random_bytes(vnic->rss_hash_key, 3836 HW_HASH_KEY_SIZE); 3837 else 3838 memcpy(vnic->rss_hash_key, 3839 bp->vnic_info[0].rss_hash_key, 3840 HW_HASH_KEY_SIZE); 3841 } 3842 } 3843 } 3844 3845 static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg) 3846 { 3847 int pages; 3848 3849 pages = ring_size / desc_per_pg; 3850 3851 if (!pages) 3852 return 1; 3853 3854 pages++; 3855 3856 while (pages & (pages - 1)) 3857 pages++; 3858 3859 return pages; 3860 } 3861 3862 void bnxt_set_tpa_flags(struct bnxt *bp) 3863 { 3864 bp->flags &= ~BNXT_FLAG_TPA; 3865 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS) 3866 return; 3867 if (bp->dev->features & NETIF_F_LRO) 3868 bp->flags |= BNXT_FLAG_LRO; 3869 else if (bp->dev->features & NETIF_F_GRO_HW) 3870 bp->flags |= BNXT_FLAG_GRO; 3871 } 3872 3873 /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must 3874 * be set on entry. 3875 */ 3876 void bnxt_set_ring_params(struct bnxt *bp) 3877 { 3878 u32 ring_size, rx_size, rx_space, max_rx_cmpl; 3879 u32 agg_factor = 0, agg_ring_size = 0; 3880 3881 /* 8 for CRC and VLAN */ 3882 rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8); 3883 3884 rx_space = rx_size + ALIGN(max(NET_SKB_PAD, XDP_PACKET_HEADROOM), 8) + 3885 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 3886 3887 bp->rx_copy_thresh = BNXT_RX_COPY_THRESH; 3888 ring_size = bp->rx_ring_size; 3889 bp->rx_agg_ring_size = 0; 3890 bp->rx_agg_nr_pages = 0; 3891 3892 if (bp->flags & BNXT_FLAG_TPA) 3893 agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE); 3894 3895 bp->flags &= ~BNXT_FLAG_JUMBO; 3896 if (rx_space > PAGE_SIZE && !(bp->flags & BNXT_FLAG_NO_AGG_RINGS)) { 3897 u32 jumbo_factor; 3898 3899 bp->flags |= BNXT_FLAG_JUMBO; 3900 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT; 3901 if (jumbo_factor > agg_factor) 3902 agg_factor = jumbo_factor; 3903 } 3904 if (agg_factor) { 3905 if (ring_size > BNXT_MAX_RX_DESC_CNT_JUM_ENA) { 3906 ring_size = BNXT_MAX_RX_DESC_CNT_JUM_ENA; 3907 netdev_warn(bp->dev, "RX ring size reduced from %d to %d because the jumbo ring is now enabled\n", 3908 bp->rx_ring_size, ring_size); 3909 bp->rx_ring_size = ring_size; 3910 } 3911 agg_ring_size = ring_size * agg_factor; 3912 3913 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size, 3914 RX_DESC_CNT); 3915 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) { 3916 u32 tmp = agg_ring_size; 3917 3918 bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES; 3919 agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1; 3920 netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n", 3921 tmp, agg_ring_size); 3922 } 3923 bp->rx_agg_ring_size = agg_ring_size; 3924 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1; 3925 3926 if (BNXT_RX_PAGE_MODE(bp)) { 3927 rx_space = PAGE_SIZE; 3928 rx_size = PAGE_SIZE - 3929 ALIGN(max(NET_SKB_PAD, XDP_PACKET_HEADROOM), 8) - 3930 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 3931 } else { 3932 rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN); 3933 rx_space = rx_size + NET_SKB_PAD + 3934 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 3935 } 3936 } 3937 3938 bp->rx_buf_use_size = rx_size; 3939 bp->rx_buf_size = rx_space; 3940 3941 bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT); 3942 bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1; 3943 3944 ring_size = bp->tx_ring_size; 3945 bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT); 3946 bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1; 3947 3948 max_rx_cmpl = bp->rx_ring_size; 3949 /* MAX TPA needs to be added because TPA_START completions are 3950 * immediately recycled, so the TPA completions are not bound by 3951 * the RX ring size. 3952 */ 3953 if (bp->flags & BNXT_FLAG_TPA) 3954 max_rx_cmpl += bp->max_tpa; 3955 /* RX and TPA completions are 32-byte, all others are 16-byte */ 3956 ring_size = max_rx_cmpl * 2 + agg_ring_size + bp->tx_ring_size; 3957 bp->cp_ring_size = ring_size; 3958 3959 bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT); 3960 if (bp->cp_nr_pages > MAX_CP_PAGES) { 3961 bp->cp_nr_pages = MAX_CP_PAGES; 3962 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1; 3963 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n", 3964 ring_size, bp->cp_ring_size); 3965 } 3966 bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT; 3967 bp->cp_ring_mask = bp->cp_bit - 1; 3968 } 3969 3970 /* Changing allocation mode of RX rings. 3971 * TODO: Update when extending xdp_rxq_info to support allocation modes. 3972 */ 3973 int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode) 3974 { 3975 struct net_device *dev = bp->dev; 3976 3977 if (page_mode) { 3978 bp->flags &= ~BNXT_FLAG_AGG_RINGS; 3979 bp->flags |= BNXT_FLAG_RX_PAGE_MODE; 3980 3981 if (bp->xdp_prog->aux->xdp_has_frags) 3982 dev->max_mtu = min_t(u16, bp->max_mtu, BNXT_MAX_MTU); 3983 else 3984 dev->max_mtu = 3985 min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU); 3986 if (dev->mtu > BNXT_MAX_PAGE_MODE_MTU) { 3987 bp->flags |= BNXT_FLAG_JUMBO; 3988 bp->rx_skb_func = bnxt_rx_multi_page_skb; 3989 } else { 3990 bp->flags |= BNXT_FLAG_NO_AGG_RINGS; 3991 bp->rx_skb_func = bnxt_rx_page_skb; 3992 } 3993 bp->rx_dir = DMA_BIDIRECTIONAL; 3994 /* Disable LRO or GRO_HW */ 3995 netdev_update_features(dev); 3996 } else { 3997 dev->max_mtu = bp->max_mtu; 3998 bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE; 3999 bp->rx_dir = DMA_FROM_DEVICE; 4000 bp->rx_skb_func = bnxt_rx_skb; 4001 } 4002 return 0; 4003 } 4004 4005 static void bnxt_free_vnic_attributes(struct bnxt *bp) 4006 { 4007 int i; 4008 struct bnxt_vnic_info *vnic; 4009 struct pci_dev *pdev = bp->pdev; 4010 4011 if (!bp->vnic_info) 4012 return; 4013 4014 for (i = 0; i < bp->nr_vnics; i++) { 4015 vnic = &bp->vnic_info[i]; 4016 4017 kfree(vnic->fw_grp_ids); 4018 vnic->fw_grp_ids = NULL; 4019 4020 kfree(vnic->uc_list); 4021 vnic->uc_list = NULL; 4022 4023 if (vnic->mc_list) { 4024 dma_free_coherent(&pdev->dev, vnic->mc_list_size, 4025 vnic->mc_list, vnic->mc_list_mapping); 4026 vnic->mc_list = NULL; 4027 } 4028 4029 if (vnic->rss_table) { 4030 dma_free_coherent(&pdev->dev, vnic->rss_table_size, 4031 vnic->rss_table, 4032 vnic->rss_table_dma_addr); 4033 vnic->rss_table = NULL; 4034 } 4035 4036 vnic->rss_hash_key = NULL; 4037 vnic->flags = 0; 4038 } 4039 } 4040 4041 static int bnxt_alloc_vnic_attributes(struct bnxt *bp) 4042 { 4043 int i, rc = 0, size; 4044 struct bnxt_vnic_info *vnic; 4045 struct pci_dev *pdev = bp->pdev; 4046 int max_rings; 4047 4048 for (i = 0; i < bp->nr_vnics; i++) { 4049 vnic = &bp->vnic_info[i]; 4050 4051 if (vnic->flags & BNXT_VNIC_UCAST_FLAG) { 4052 int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN; 4053 4054 if (mem_size > 0) { 4055 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL); 4056 if (!vnic->uc_list) { 4057 rc = -ENOMEM; 4058 goto out; 4059 } 4060 } 4061 } 4062 4063 if (vnic->flags & BNXT_VNIC_MCAST_FLAG) { 4064 vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN; 4065 vnic->mc_list = 4066 dma_alloc_coherent(&pdev->dev, 4067 vnic->mc_list_size, 4068 &vnic->mc_list_mapping, 4069 GFP_KERNEL); 4070 if (!vnic->mc_list) { 4071 rc = -ENOMEM; 4072 goto out; 4073 } 4074 } 4075 4076 if (bp->flags & BNXT_FLAG_CHIP_P5) 4077 goto vnic_skip_grps; 4078 4079 if (vnic->flags & BNXT_VNIC_RSS_FLAG) 4080 max_rings = bp->rx_nr_rings; 4081 else 4082 max_rings = 1; 4083 4084 vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL); 4085 if (!vnic->fw_grp_ids) { 4086 rc = -ENOMEM; 4087 goto out; 4088 } 4089 vnic_skip_grps: 4090 if ((bp->flags & BNXT_FLAG_NEW_RSS_CAP) && 4091 !(vnic->flags & BNXT_VNIC_RSS_FLAG)) 4092 continue; 4093 4094 /* Allocate rss table and hash key */ 4095 size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16)); 4096 if (bp->flags & BNXT_FLAG_CHIP_P5) 4097 size = L1_CACHE_ALIGN(BNXT_MAX_RSS_TABLE_SIZE_P5); 4098 4099 vnic->rss_table_size = size + HW_HASH_KEY_SIZE; 4100 vnic->rss_table = dma_alloc_coherent(&pdev->dev, 4101 vnic->rss_table_size, 4102 &vnic->rss_table_dma_addr, 4103 GFP_KERNEL); 4104 if (!vnic->rss_table) { 4105 rc = -ENOMEM; 4106 goto out; 4107 } 4108 4109 vnic->rss_hash_key = ((void *)vnic->rss_table) + size; 4110 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size; 4111 } 4112 return 0; 4113 4114 out: 4115 return rc; 4116 } 4117 4118 static void bnxt_free_hwrm_resources(struct bnxt *bp) 4119 { 4120 struct bnxt_hwrm_wait_token *token; 4121 4122 dma_pool_destroy(bp->hwrm_dma_pool); 4123 bp->hwrm_dma_pool = NULL; 4124 4125 rcu_read_lock(); 4126 hlist_for_each_entry_rcu(token, &bp->hwrm_pending_list, node) 4127 WRITE_ONCE(token->state, BNXT_HWRM_CANCELLED); 4128 rcu_read_unlock(); 4129 } 4130 4131 static int bnxt_alloc_hwrm_resources(struct bnxt *bp) 4132 { 4133 bp->hwrm_dma_pool = dma_pool_create("bnxt_hwrm", &bp->pdev->dev, 4134 BNXT_HWRM_DMA_SIZE, 4135 BNXT_HWRM_DMA_ALIGN, 0); 4136 if (!bp->hwrm_dma_pool) 4137 return -ENOMEM; 4138 4139 INIT_HLIST_HEAD(&bp->hwrm_pending_list); 4140 4141 return 0; 4142 } 4143 4144 static void bnxt_free_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats) 4145 { 4146 kfree(stats->hw_masks); 4147 stats->hw_masks = NULL; 4148 kfree(stats->sw_stats); 4149 stats->sw_stats = NULL; 4150 if (stats->hw_stats) { 4151 dma_free_coherent(&bp->pdev->dev, stats->len, stats->hw_stats, 4152 stats->hw_stats_map); 4153 stats->hw_stats = NULL; 4154 } 4155 } 4156 4157 static int bnxt_alloc_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats, 4158 bool alloc_masks) 4159 { 4160 stats->hw_stats = dma_alloc_coherent(&bp->pdev->dev, stats->len, 4161 &stats->hw_stats_map, GFP_KERNEL); 4162 if (!stats->hw_stats) 4163 return -ENOMEM; 4164 4165 stats->sw_stats = kzalloc(stats->len, GFP_KERNEL); 4166 if (!stats->sw_stats) 4167 goto stats_mem_err; 4168 4169 if (alloc_masks) { 4170 stats->hw_masks = kzalloc(stats->len, GFP_KERNEL); 4171 if (!stats->hw_masks) 4172 goto stats_mem_err; 4173 } 4174 return 0; 4175 4176 stats_mem_err: 4177 bnxt_free_stats_mem(bp, stats); 4178 return -ENOMEM; 4179 } 4180 4181 static void bnxt_fill_masks(u64 *mask_arr, u64 mask, int count) 4182 { 4183 int i; 4184 4185 for (i = 0; i < count; i++) 4186 mask_arr[i] = mask; 4187 } 4188 4189 static void bnxt_copy_hw_masks(u64 *mask_arr, __le64 *hw_mask_arr, int count) 4190 { 4191 int i; 4192 4193 for (i = 0; i < count; i++) 4194 mask_arr[i] = le64_to_cpu(hw_mask_arr[i]); 4195 } 4196 4197 static int bnxt_hwrm_func_qstat_ext(struct bnxt *bp, 4198 struct bnxt_stats_mem *stats) 4199 { 4200 struct hwrm_func_qstats_ext_output *resp; 4201 struct hwrm_func_qstats_ext_input *req; 4202 __le64 *hw_masks; 4203 int rc; 4204 4205 if (!(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED) || 4206 !(bp->flags & BNXT_FLAG_CHIP_P5)) 4207 return -EOPNOTSUPP; 4208 4209 rc = hwrm_req_init(bp, req, HWRM_FUNC_QSTATS_EXT); 4210 if (rc) 4211 return rc; 4212 4213 req->fid = cpu_to_le16(0xffff); 4214 req->flags = FUNC_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK; 4215 4216 resp = hwrm_req_hold(bp, req); 4217 rc = hwrm_req_send(bp, req); 4218 if (!rc) { 4219 hw_masks = &resp->rx_ucast_pkts; 4220 bnxt_copy_hw_masks(stats->hw_masks, hw_masks, stats->len / 8); 4221 } 4222 hwrm_req_drop(bp, req); 4223 return rc; 4224 } 4225 4226 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags); 4227 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags); 4228 4229 static void bnxt_init_stats(struct bnxt *bp) 4230 { 4231 struct bnxt_napi *bnapi = bp->bnapi[0]; 4232 struct bnxt_cp_ring_info *cpr; 4233 struct bnxt_stats_mem *stats; 4234 __le64 *rx_stats, *tx_stats; 4235 int rc, rx_count, tx_count; 4236 u64 *rx_masks, *tx_masks; 4237 u64 mask; 4238 u8 flags; 4239 4240 cpr = &bnapi->cp_ring; 4241 stats = &cpr->stats; 4242 rc = bnxt_hwrm_func_qstat_ext(bp, stats); 4243 if (rc) { 4244 if (bp->flags & BNXT_FLAG_CHIP_P5) 4245 mask = (1ULL << 48) - 1; 4246 else 4247 mask = -1ULL; 4248 bnxt_fill_masks(stats->hw_masks, mask, stats->len / 8); 4249 } 4250 if (bp->flags & BNXT_FLAG_PORT_STATS) { 4251 stats = &bp->port_stats; 4252 rx_stats = stats->hw_stats; 4253 rx_masks = stats->hw_masks; 4254 rx_count = sizeof(struct rx_port_stats) / 8; 4255 tx_stats = rx_stats + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 4256 tx_masks = rx_masks + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 4257 tx_count = sizeof(struct tx_port_stats) / 8; 4258 4259 flags = PORT_QSTATS_REQ_FLAGS_COUNTER_MASK; 4260 rc = bnxt_hwrm_port_qstats(bp, flags); 4261 if (rc) { 4262 mask = (1ULL << 40) - 1; 4263 4264 bnxt_fill_masks(rx_masks, mask, rx_count); 4265 bnxt_fill_masks(tx_masks, mask, tx_count); 4266 } else { 4267 bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count); 4268 bnxt_copy_hw_masks(tx_masks, tx_stats, tx_count); 4269 bnxt_hwrm_port_qstats(bp, 0); 4270 } 4271 } 4272 if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) { 4273 stats = &bp->rx_port_stats_ext; 4274 rx_stats = stats->hw_stats; 4275 rx_masks = stats->hw_masks; 4276 rx_count = sizeof(struct rx_port_stats_ext) / 8; 4277 stats = &bp->tx_port_stats_ext; 4278 tx_stats = stats->hw_stats; 4279 tx_masks = stats->hw_masks; 4280 tx_count = sizeof(struct tx_port_stats_ext) / 8; 4281 4282 flags = PORT_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK; 4283 rc = bnxt_hwrm_port_qstats_ext(bp, flags); 4284 if (rc) { 4285 mask = (1ULL << 40) - 1; 4286 4287 bnxt_fill_masks(rx_masks, mask, rx_count); 4288 if (tx_stats) 4289 bnxt_fill_masks(tx_masks, mask, tx_count); 4290 } else { 4291 bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count); 4292 if (tx_stats) 4293 bnxt_copy_hw_masks(tx_masks, tx_stats, 4294 tx_count); 4295 bnxt_hwrm_port_qstats_ext(bp, 0); 4296 } 4297 } 4298 } 4299 4300 static void bnxt_free_port_stats(struct bnxt *bp) 4301 { 4302 bp->flags &= ~BNXT_FLAG_PORT_STATS; 4303 bp->flags &= ~BNXT_FLAG_PORT_STATS_EXT; 4304 4305 bnxt_free_stats_mem(bp, &bp->port_stats); 4306 bnxt_free_stats_mem(bp, &bp->rx_port_stats_ext); 4307 bnxt_free_stats_mem(bp, &bp->tx_port_stats_ext); 4308 } 4309 4310 static void bnxt_free_ring_stats(struct bnxt *bp) 4311 { 4312 int i; 4313 4314 if (!bp->bnapi) 4315 return; 4316 4317 for (i = 0; i < bp->cp_nr_rings; i++) { 4318 struct bnxt_napi *bnapi = bp->bnapi[i]; 4319 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 4320 4321 bnxt_free_stats_mem(bp, &cpr->stats); 4322 } 4323 } 4324 4325 static int bnxt_alloc_stats(struct bnxt *bp) 4326 { 4327 u32 size, i; 4328 int rc; 4329 4330 size = bp->hw_ring_stats_size; 4331 4332 for (i = 0; i < bp->cp_nr_rings; i++) { 4333 struct bnxt_napi *bnapi = bp->bnapi[i]; 4334 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 4335 4336 cpr->stats.len = size; 4337 rc = bnxt_alloc_stats_mem(bp, &cpr->stats, !i); 4338 if (rc) 4339 return rc; 4340 4341 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID; 4342 } 4343 4344 if (BNXT_VF(bp) || bp->chip_num == CHIP_NUM_58700) 4345 return 0; 4346 4347 if (bp->port_stats.hw_stats) 4348 goto alloc_ext_stats; 4349 4350 bp->port_stats.len = BNXT_PORT_STATS_SIZE; 4351 rc = bnxt_alloc_stats_mem(bp, &bp->port_stats, true); 4352 if (rc) 4353 return rc; 4354 4355 bp->flags |= BNXT_FLAG_PORT_STATS; 4356 4357 alloc_ext_stats: 4358 /* Display extended statistics only if FW supports it */ 4359 if (bp->hwrm_spec_code < 0x10804 || bp->hwrm_spec_code == 0x10900) 4360 if (!(bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED)) 4361 return 0; 4362 4363 if (bp->rx_port_stats_ext.hw_stats) 4364 goto alloc_tx_ext_stats; 4365 4366 bp->rx_port_stats_ext.len = sizeof(struct rx_port_stats_ext); 4367 rc = bnxt_alloc_stats_mem(bp, &bp->rx_port_stats_ext, true); 4368 /* Extended stats are optional */ 4369 if (rc) 4370 return 0; 4371 4372 alloc_tx_ext_stats: 4373 if (bp->tx_port_stats_ext.hw_stats) 4374 return 0; 4375 4376 if (bp->hwrm_spec_code >= 0x10902 || 4377 (bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED)) { 4378 bp->tx_port_stats_ext.len = sizeof(struct tx_port_stats_ext); 4379 rc = bnxt_alloc_stats_mem(bp, &bp->tx_port_stats_ext, true); 4380 /* Extended stats are optional */ 4381 if (rc) 4382 return 0; 4383 } 4384 bp->flags |= BNXT_FLAG_PORT_STATS_EXT; 4385 return 0; 4386 } 4387 4388 static void bnxt_clear_ring_indices(struct bnxt *bp) 4389 { 4390 int i; 4391 4392 if (!bp->bnapi) 4393 return; 4394 4395 for (i = 0; i < bp->cp_nr_rings; i++) { 4396 struct bnxt_napi *bnapi = bp->bnapi[i]; 4397 struct bnxt_cp_ring_info *cpr; 4398 struct bnxt_rx_ring_info *rxr; 4399 struct bnxt_tx_ring_info *txr; 4400 4401 if (!bnapi) 4402 continue; 4403 4404 cpr = &bnapi->cp_ring; 4405 cpr->cp_raw_cons = 0; 4406 4407 txr = bnapi->tx_ring; 4408 if (txr) { 4409 txr->tx_prod = 0; 4410 txr->tx_cons = 0; 4411 } 4412 4413 rxr = bnapi->rx_ring; 4414 if (rxr) { 4415 rxr->rx_prod = 0; 4416 rxr->rx_agg_prod = 0; 4417 rxr->rx_sw_agg_prod = 0; 4418 rxr->rx_next_cons = 0; 4419 } 4420 } 4421 } 4422 4423 static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool irq_reinit) 4424 { 4425 #ifdef CONFIG_RFS_ACCEL 4426 int i; 4427 4428 /* Under rtnl_lock and all our NAPIs have been disabled. It's 4429 * safe to delete the hash table. 4430 */ 4431 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) { 4432 struct hlist_head *head; 4433 struct hlist_node *tmp; 4434 struct bnxt_ntuple_filter *fltr; 4435 4436 head = &bp->ntp_fltr_hash_tbl[i]; 4437 hlist_for_each_entry_safe(fltr, tmp, head, hash) { 4438 hlist_del(&fltr->hash); 4439 kfree(fltr); 4440 } 4441 } 4442 if (irq_reinit) { 4443 bitmap_free(bp->ntp_fltr_bmap); 4444 bp->ntp_fltr_bmap = NULL; 4445 } 4446 bp->ntp_fltr_count = 0; 4447 #endif 4448 } 4449 4450 static int bnxt_alloc_ntp_fltrs(struct bnxt *bp) 4451 { 4452 #ifdef CONFIG_RFS_ACCEL 4453 int i, rc = 0; 4454 4455 if (!(bp->flags & BNXT_FLAG_RFS)) 4456 return 0; 4457 4458 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) 4459 INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]); 4460 4461 bp->ntp_fltr_count = 0; 4462 bp->ntp_fltr_bmap = bitmap_zalloc(BNXT_NTP_FLTR_MAX_FLTR, GFP_KERNEL); 4463 4464 if (!bp->ntp_fltr_bmap) 4465 rc = -ENOMEM; 4466 4467 return rc; 4468 #else 4469 return 0; 4470 #endif 4471 } 4472 4473 static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init) 4474 { 4475 bnxt_free_vnic_attributes(bp); 4476 bnxt_free_tx_rings(bp); 4477 bnxt_free_rx_rings(bp); 4478 bnxt_free_cp_rings(bp); 4479 bnxt_free_all_cp_arrays(bp); 4480 bnxt_free_ntp_fltrs(bp, irq_re_init); 4481 if (irq_re_init) { 4482 bnxt_free_ring_stats(bp); 4483 if (!(bp->phy_flags & BNXT_PHY_FL_PORT_STATS_NO_RESET) || 4484 test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 4485 bnxt_free_port_stats(bp); 4486 bnxt_free_ring_grps(bp); 4487 bnxt_free_vnics(bp); 4488 kfree(bp->tx_ring_map); 4489 bp->tx_ring_map = NULL; 4490 kfree(bp->tx_ring); 4491 bp->tx_ring = NULL; 4492 kfree(bp->rx_ring); 4493 bp->rx_ring = NULL; 4494 kfree(bp->bnapi); 4495 bp->bnapi = NULL; 4496 } else { 4497 bnxt_clear_ring_indices(bp); 4498 } 4499 } 4500 4501 static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init) 4502 { 4503 int i, j, rc, size, arr_size; 4504 void *bnapi; 4505 4506 if (irq_re_init) { 4507 /* Allocate bnapi mem pointer array and mem block for 4508 * all queues 4509 */ 4510 arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) * 4511 bp->cp_nr_rings); 4512 size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi)); 4513 bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL); 4514 if (!bnapi) 4515 return -ENOMEM; 4516 4517 bp->bnapi = bnapi; 4518 bnapi += arr_size; 4519 for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) { 4520 bp->bnapi[i] = bnapi; 4521 bp->bnapi[i]->index = i; 4522 bp->bnapi[i]->bp = bp; 4523 if (bp->flags & BNXT_FLAG_CHIP_P5) { 4524 struct bnxt_cp_ring_info *cpr = 4525 &bp->bnapi[i]->cp_ring; 4526 4527 cpr->cp_ring_struct.ring_mem.flags = 4528 BNXT_RMEM_RING_PTE_FLAG; 4529 } 4530 } 4531 4532 bp->rx_ring = kcalloc(bp->rx_nr_rings, 4533 sizeof(struct bnxt_rx_ring_info), 4534 GFP_KERNEL); 4535 if (!bp->rx_ring) 4536 return -ENOMEM; 4537 4538 for (i = 0; i < bp->rx_nr_rings; i++) { 4539 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 4540 4541 if (bp->flags & BNXT_FLAG_CHIP_P5) { 4542 rxr->rx_ring_struct.ring_mem.flags = 4543 BNXT_RMEM_RING_PTE_FLAG; 4544 rxr->rx_agg_ring_struct.ring_mem.flags = 4545 BNXT_RMEM_RING_PTE_FLAG; 4546 } 4547 rxr->bnapi = bp->bnapi[i]; 4548 bp->bnapi[i]->rx_ring = &bp->rx_ring[i]; 4549 } 4550 4551 bp->tx_ring = kcalloc(bp->tx_nr_rings, 4552 sizeof(struct bnxt_tx_ring_info), 4553 GFP_KERNEL); 4554 if (!bp->tx_ring) 4555 return -ENOMEM; 4556 4557 bp->tx_ring_map = kcalloc(bp->tx_nr_rings, sizeof(u16), 4558 GFP_KERNEL); 4559 4560 if (!bp->tx_ring_map) 4561 return -ENOMEM; 4562 4563 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 4564 j = 0; 4565 else 4566 j = bp->rx_nr_rings; 4567 4568 for (i = 0; i < bp->tx_nr_rings; i++, j++) { 4569 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 4570 4571 if (bp->flags & BNXT_FLAG_CHIP_P5) 4572 txr->tx_ring_struct.ring_mem.flags = 4573 BNXT_RMEM_RING_PTE_FLAG; 4574 txr->bnapi = bp->bnapi[j]; 4575 bp->bnapi[j]->tx_ring = txr; 4576 bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i; 4577 if (i >= bp->tx_nr_rings_xdp) { 4578 txr->txq_index = i - bp->tx_nr_rings_xdp; 4579 bp->bnapi[j]->tx_int = bnxt_tx_int; 4580 } else { 4581 bp->bnapi[j]->flags |= BNXT_NAPI_FLAG_XDP; 4582 bp->bnapi[j]->tx_int = bnxt_tx_int_xdp; 4583 } 4584 } 4585 4586 rc = bnxt_alloc_stats(bp); 4587 if (rc) 4588 goto alloc_mem_err; 4589 bnxt_init_stats(bp); 4590 4591 rc = bnxt_alloc_ntp_fltrs(bp); 4592 if (rc) 4593 goto alloc_mem_err; 4594 4595 rc = bnxt_alloc_vnics(bp); 4596 if (rc) 4597 goto alloc_mem_err; 4598 } 4599 4600 rc = bnxt_alloc_all_cp_arrays(bp); 4601 if (rc) 4602 goto alloc_mem_err; 4603 4604 bnxt_init_ring_struct(bp); 4605 4606 rc = bnxt_alloc_rx_rings(bp); 4607 if (rc) 4608 goto alloc_mem_err; 4609 4610 rc = bnxt_alloc_tx_rings(bp); 4611 if (rc) 4612 goto alloc_mem_err; 4613 4614 rc = bnxt_alloc_cp_rings(bp); 4615 if (rc) 4616 goto alloc_mem_err; 4617 4618 bp->vnic_info[0].flags |= BNXT_VNIC_RSS_FLAG | BNXT_VNIC_MCAST_FLAG | 4619 BNXT_VNIC_UCAST_FLAG; 4620 rc = bnxt_alloc_vnic_attributes(bp); 4621 if (rc) 4622 goto alloc_mem_err; 4623 return 0; 4624 4625 alloc_mem_err: 4626 bnxt_free_mem(bp, true); 4627 return rc; 4628 } 4629 4630 static void bnxt_disable_int(struct bnxt *bp) 4631 { 4632 int i; 4633 4634 if (!bp->bnapi) 4635 return; 4636 4637 for (i = 0; i < bp->cp_nr_rings; i++) { 4638 struct bnxt_napi *bnapi = bp->bnapi[i]; 4639 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 4640 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 4641 4642 if (ring->fw_ring_id != INVALID_HW_RING_ID) 4643 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons); 4644 } 4645 } 4646 4647 static int bnxt_cp_num_to_irq_num(struct bnxt *bp, int n) 4648 { 4649 struct bnxt_napi *bnapi = bp->bnapi[n]; 4650 struct bnxt_cp_ring_info *cpr; 4651 4652 cpr = &bnapi->cp_ring; 4653 return cpr->cp_ring_struct.map_idx; 4654 } 4655 4656 static void bnxt_disable_int_sync(struct bnxt *bp) 4657 { 4658 int i; 4659 4660 if (!bp->irq_tbl) 4661 return; 4662 4663 atomic_inc(&bp->intr_sem); 4664 4665 bnxt_disable_int(bp); 4666 for (i = 0; i < bp->cp_nr_rings; i++) { 4667 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 4668 4669 synchronize_irq(bp->irq_tbl[map_idx].vector); 4670 } 4671 } 4672 4673 static void bnxt_enable_int(struct bnxt *bp) 4674 { 4675 int i; 4676 4677 atomic_set(&bp->intr_sem, 0); 4678 for (i = 0; i < bp->cp_nr_rings; i++) { 4679 struct bnxt_napi *bnapi = bp->bnapi[i]; 4680 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 4681 4682 bnxt_db_nq_arm(bp, &cpr->cp_db, cpr->cp_raw_cons); 4683 } 4684 } 4685 4686 int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp, unsigned long *bmap, int bmap_size, 4687 bool async_only) 4688 { 4689 DECLARE_BITMAP(async_events_bmap, 256); 4690 u32 *events = (u32 *)async_events_bmap; 4691 struct hwrm_func_drv_rgtr_output *resp; 4692 struct hwrm_func_drv_rgtr_input *req; 4693 u32 flags; 4694 int rc, i; 4695 4696 rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_RGTR); 4697 if (rc) 4698 return rc; 4699 4700 req->enables = cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE | 4701 FUNC_DRV_RGTR_REQ_ENABLES_VER | 4702 FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD); 4703 4704 req->os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX); 4705 flags = FUNC_DRV_RGTR_REQ_FLAGS_16BIT_VER_MODE; 4706 if (bp->fw_cap & BNXT_FW_CAP_HOT_RESET) 4707 flags |= FUNC_DRV_RGTR_REQ_FLAGS_HOT_RESET_SUPPORT; 4708 if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) 4709 flags |= FUNC_DRV_RGTR_REQ_FLAGS_ERROR_RECOVERY_SUPPORT | 4710 FUNC_DRV_RGTR_REQ_FLAGS_MASTER_SUPPORT; 4711 req->flags = cpu_to_le32(flags); 4712 req->ver_maj_8b = DRV_VER_MAJ; 4713 req->ver_min_8b = DRV_VER_MIN; 4714 req->ver_upd_8b = DRV_VER_UPD; 4715 req->ver_maj = cpu_to_le16(DRV_VER_MAJ); 4716 req->ver_min = cpu_to_le16(DRV_VER_MIN); 4717 req->ver_upd = cpu_to_le16(DRV_VER_UPD); 4718 4719 if (BNXT_PF(bp)) { 4720 u32 data[8]; 4721 int i; 4722 4723 memset(data, 0, sizeof(data)); 4724 for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) { 4725 u16 cmd = bnxt_vf_req_snif[i]; 4726 unsigned int bit, idx; 4727 4728 idx = cmd / 32; 4729 bit = cmd % 32; 4730 data[idx] |= 1 << bit; 4731 } 4732 4733 for (i = 0; i < 8; i++) 4734 req->vf_req_fwd[i] = cpu_to_le32(data[i]); 4735 4736 req->enables |= 4737 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD); 4738 } 4739 4740 if (bp->fw_cap & BNXT_FW_CAP_OVS_64BIT_HANDLE) 4741 req->flags |= cpu_to_le32( 4742 FUNC_DRV_RGTR_REQ_FLAGS_FLOW_HANDLE_64BIT_MODE); 4743 4744 memset(async_events_bmap, 0, sizeof(async_events_bmap)); 4745 for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++) { 4746 u16 event_id = bnxt_async_events_arr[i]; 4747 4748 if (event_id == ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY && 4749 !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) 4750 continue; 4751 if (event_id == ASYNC_EVENT_CMPL_EVENT_ID_PHC_UPDATE && 4752 !bp->ptp_cfg) 4753 continue; 4754 __set_bit(bnxt_async_events_arr[i], async_events_bmap); 4755 } 4756 if (bmap && bmap_size) { 4757 for (i = 0; i < bmap_size; i++) { 4758 if (test_bit(i, bmap)) 4759 __set_bit(i, async_events_bmap); 4760 } 4761 } 4762 for (i = 0; i < 8; i++) 4763 req->async_event_fwd[i] |= cpu_to_le32(events[i]); 4764 4765 if (async_only) 4766 req->enables = 4767 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD); 4768 4769 resp = hwrm_req_hold(bp, req); 4770 rc = hwrm_req_send(bp, req); 4771 if (!rc) { 4772 set_bit(BNXT_STATE_DRV_REGISTERED, &bp->state); 4773 if (resp->flags & 4774 cpu_to_le32(FUNC_DRV_RGTR_RESP_FLAGS_IF_CHANGE_SUPPORTED)) 4775 bp->fw_cap |= BNXT_FW_CAP_IF_CHANGE; 4776 } 4777 hwrm_req_drop(bp, req); 4778 return rc; 4779 } 4780 4781 int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp) 4782 { 4783 struct hwrm_func_drv_unrgtr_input *req; 4784 int rc; 4785 4786 if (!test_and_clear_bit(BNXT_STATE_DRV_REGISTERED, &bp->state)) 4787 return 0; 4788 4789 rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_UNRGTR); 4790 if (rc) 4791 return rc; 4792 return hwrm_req_send(bp, req); 4793 } 4794 4795 static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type) 4796 { 4797 struct hwrm_tunnel_dst_port_free_input *req; 4798 int rc; 4799 4800 if (tunnel_type == TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN && 4801 bp->vxlan_fw_dst_port_id == INVALID_HW_RING_ID) 4802 return 0; 4803 if (tunnel_type == TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE && 4804 bp->nge_fw_dst_port_id == INVALID_HW_RING_ID) 4805 return 0; 4806 4807 rc = hwrm_req_init(bp, req, HWRM_TUNNEL_DST_PORT_FREE); 4808 if (rc) 4809 return rc; 4810 4811 req->tunnel_type = tunnel_type; 4812 4813 switch (tunnel_type) { 4814 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN: 4815 req->tunnel_dst_port_id = cpu_to_le16(bp->vxlan_fw_dst_port_id); 4816 bp->vxlan_port = 0; 4817 bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID; 4818 break; 4819 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE: 4820 req->tunnel_dst_port_id = cpu_to_le16(bp->nge_fw_dst_port_id); 4821 bp->nge_port = 0; 4822 bp->nge_fw_dst_port_id = INVALID_HW_RING_ID; 4823 break; 4824 default: 4825 break; 4826 } 4827 4828 rc = hwrm_req_send(bp, req); 4829 if (rc) 4830 netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n", 4831 rc); 4832 return rc; 4833 } 4834 4835 static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port, 4836 u8 tunnel_type) 4837 { 4838 struct hwrm_tunnel_dst_port_alloc_output *resp; 4839 struct hwrm_tunnel_dst_port_alloc_input *req; 4840 int rc; 4841 4842 rc = hwrm_req_init(bp, req, HWRM_TUNNEL_DST_PORT_ALLOC); 4843 if (rc) 4844 return rc; 4845 4846 req->tunnel_type = tunnel_type; 4847 req->tunnel_dst_port_val = port; 4848 4849 resp = hwrm_req_hold(bp, req); 4850 rc = hwrm_req_send(bp, req); 4851 if (rc) { 4852 netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n", 4853 rc); 4854 goto err_out; 4855 } 4856 4857 switch (tunnel_type) { 4858 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN: 4859 bp->vxlan_port = port; 4860 bp->vxlan_fw_dst_port_id = 4861 le16_to_cpu(resp->tunnel_dst_port_id); 4862 break; 4863 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE: 4864 bp->nge_port = port; 4865 bp->nge_fw_dst_port_id = le16_to_cpu(resp->tunnel_dst_port_id); 4866 break; 4867 default: 4868 break; 4869 } 4870 4871 err_out: 4872 hwrm_req_drop(bp, req); 4873 return rc; 4874 } 4875 4876 static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id) 4877 { 4878 struct hwrm_cfa_l2_set_rx_mask_input *req; 4879 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 4880 int rc; 4881 4882 rc = hwrm_req_init(bp, req, HWRM_CFA_L2_SET_RX_MASK); 4883 if (rc) 4884 return rc; 4885 4886 req->vnic_id = cpu_to_le32(vnic->fw_vnic_id); 4887 if (vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_MCAST) { 4888 req->num_mc_entries = cpu_to_le32(vnic->mc_list_count); 4889 req->mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping); 4890 } 4891 req->mask = cpu_to_le32(vnic->rx_mask); 4892 return hwrm_req_send_silent(bp, req); 4893 } 4894 4895 #ifdef CONFIG_RFS_ACCEL 4896 static int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp, 4897 struct bnxt_ntuple_filter *fltr) 4898 { 4899 struct hwrm_cfa_ntuple_filter_free_input *req; 4900 int rc; 4901 4902 rc = hwrm_req_init(bp, req, HWRM_CFA_NTUPLE_FILTER_FREE); 4903 if (rc) 4904 return rc; 4905 4906 req->ntuple_filter_id = fltr->filter_id; 4907 return hwrm_req_send(bp, req); 4908 } 4909 4910 #define BNXT_NTP_FLTR_FLAGS \ 4911 (CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID | \ 4912 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE | \ 4913 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR | \ 4914 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE | \ 4915 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR | \ 4916 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK | \ 4917 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR | \ 4918 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK | \ 4919 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL | \ 4920 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT | \ 4921 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK | \ 4922 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT | \ 4923 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK | \ 4924 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID) 4925 4926 #define BNXT_NTP_TUNNEL_FLTR_FLAG \ 4927 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE 4928 4929 static int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp, 4930 struct bnxt_ntuple_filter *fltr) 4931 { 4932 struct hwrm_cfa_ntuple_filter_alloc_output *resp; 4933 struct hwrm_cfa_ntuple_filter_alloc_input *req; 4934 struct flow_keys *keys = &fltr->fkeys; 4935 struct bnxt_vnic_info *vnic; 4936 u32 flags = 0; 4937 int rc; 4938 4939 rc = hwrm_req_init(bp, req, HWRM_CFA_NTUPLE_FILTER_ALLOC); 4940 if (rc) 4941 return rc; 4942 4943 req->l2_filter_id = bp->vnic_info[0].fw_l2_filter_id[fltr->l2_fltr_idx]; 4944 4945 if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2) { 4946 flags = CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DEST_RFS_RING_IDX; 4947 req->dst_id = cpu_to_le16(fltr->rxq); 4948 } else { 4949 vnic = &bp->vnic_info[fltr->rxq + 1]; 4950 req->dst_id = cpu_to_le16(vnic->fw_vnic_id); 4951 } 4952 req->flags = cpu_to_le32(flags); 4953 req->enables = cpu_to_le32(BNXT_NTP_FLTR_FLAGS); 4954 4955 req->ethertype = htons(ETH_P_IP); 4956 memcpy(req->src_macaddr, fltr->src_mac_addr, ETH_ALEN); 4957 req->ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4; 4958 req->ip_protocol = keys->basic.ip_proto; 4959 4960 if (keys->basic.n_proto == htons(ETH_P_IPV6)) { 4961 int i; 4962 4963 req->ethertype = htons(ETH_P_IPV6); 4964 req->ip_addr_type = 4965 CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6; 4966 *(struct in6_addr *)&req->src_ipaddr[0] = 4967 keys->addrs.v6addrs.src; 4968 *(struct in6_addr *)&req->dst_ipaddr[0] = 4969 keys->addrs.v6addrs.dst; 4970 for (i = 0; i < 4; i++) { 4971 req->src_ipaddr_mask[i] = cpu_to_be32(0xffffffff); 4972 req->dst_ipaddr_mask[i] = cpu_to_be32(0xffffffff); 4973 } 4974 } else { 4975 req->src_ipaddr[0] = keys->addrs.v4addrs.src; 4976 req->src_ipaddr_mask[0] = cpu_to_be32(0xffffffff); 4977 req->dst_ipaddr[0] = keys->addrs.v4addrs.dst; 4978 req->dst_ipaddr_mask[0] = cpu_to_be32(0xffffffff); 4979 } 4980 if (keys->control.flags & FLOW_DIS_ENCAPSULATION) { 4981 req->enables |= cpu_to_le32(BNXT_NTP_TUNNEL_FLTR_FLAG); 4982 req->tunnel_type = 4983 CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL; 4984 } 4985 4986 req->src_port = keys->ports.src; 4987 req->src_port_mask = cpu_to_be16(0xffff); 4988 req->dst_port = keys->ports.dst; 4989 req->dst_port_mask = cpu_to_be16(0xffff); 4990 4991 resp = hwrm_req_hold(bp, req); 4992 rc = hwrm_req_send(bp, req); 4993 if (!rc) 4994 fltr->filter_id = resp->ntuple_filter_id; 4995 hwrm_req_drop(bp, req); 4996 return rc; 4997 } 4998 #endif 4999 5000 static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx, 5001 const u8 *mac_addr) 5002 { 5003 struct hwrm_cfa_l2_filter_alloc_output *resp; 5004 struct hwrm_cfa_l2_filter_alloc_input *req; 5005 int rc; 5006 5007 rc = hwrm_req_init(bp, req, HWRM_CFA_L2_FILTER_ALLOC); 5008 if (rc) 5009 return rc; 5010 5011 req->flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX); 5012 if (!BNXT_CHIP_TYPE_NITRO_A0(bp)) 5013 req->flags |= 5014 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST); 5015 req->dst_id = cpu_to_le16(bp->vnic_info[vnic_id].fw_vnic_id); 5016 req->enables = 5017 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR | 5018 CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID | 5019 CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK); 5020 memcpy(req->l2_addr, mac_addr, ETH_ALEN); 5021 req->l2_addr_mask[0] = 0xff; 5022 req->l2_addr_mask[1] = 0xff; 5023 req->l2_addr_mask[2] = 0xff; 5024 req->l2_addr_mask[3] = 0xff; 5025 req->l2_addr_mask[4] = 0xff; 5026 req->l2_addr_mask[5] = 0xff; 5027 5028 resp = hwrm_req_hold(bp, req); 5029 rc = hwrm_req_send(bp, req); 5030 if (!rc) 5031 bp->vnic_info[vnic_id].fw_l2_filter_id[idx] = 5032 resp->l2_filter_id; 5033 hwrm_req_drop(bp, req); 5034 return rc; 5035 } 5036 5037 static int bnxt_hwrm_clear_vnic_filter(struct bnxt *bp) 5038 { 5039 struct hwrm_cfa_l2_filter_free_input *req; 5040 u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */ 5041 int rc; 5042 5043 /* Any associated ntuple filters will also be cleared by firmware. */ 5044 rc = hwrm_req_init(bp, req, HWRM_CFA_L2_FILTER_FREE); 5045 if (rc) 5046 return rc; 5047 hwrm_req_hold(bp, req); 5048 for (i = 0; i < num_of_vnics; i++) { 5049 struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; 5050 5051 for (j = 0; j < vnic->uc_filter_count; j++) { 5052 req->l2_filter_id = vnic->fw_l2_filter_id[j]; 5053 5054 rc = hwrm_req_send(bp, req); 5055 } 5056 vnic->uc_filter_count = 0; 5057 } 5058 hwrm_req_drop(bp, req); 5059 return rc; 5060 } 5061 5062 static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags) 5063 { 5064 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5065 u16 max_aggs = VNIC_TPA_CFG_REQ_MAX_AGGS_MAX; 5066 struct hwrm_vnic_tpa_cfg_input *req; 5067 int rc; 5068 5069 if (vnic->fw_vnic_id == INVALID_HW_RING_ID) 5070 return 0; 5071 5072 rc = hwrm_req_init(bp, req, HWRM_VNIC_TPA_CFG); 5073 if (rc) 5074 return rc; 5075 5076 if (tpa_flags) { 5077 u16 mss = bp->dev->mtu - 40; 5078 u32 nsegs, n, segs = 0, flags; 5079 5080 flags = VNIC_TPA_CFG_REQ_FLAGS_TPA | 5081 VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA | 5082 VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE | 5083 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN | 5084 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ; 5085 if (tpa_flags & BNXT_FLAG_GRO) 5086 flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO; 5087 5088 req->flags = cpu_to_le32(flags); 5089 5090 req->enables = 5091 cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS | 5092 VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS | 5093 VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN); 5094 5095 /* Number of segs are log2 units, and first packet is not 5096 * included as part of this units. 5097 */ 5098 if (mss <= BNXT_RX_PAGE_SIZE) { 5099 n = BNXT_RX_PAGE_SIZE / mss; 5100 nsegs = (MAX_SKB_FRAGS - 1) * n; 5101 } else { 5102 n = mss / BNXT_RX_PAGE_SIZE; 5103 if (mss & (BNXT_RX_PAGE_SIZE - 1)) 5104 n++; 5105 nsegs = (MAX_SKB_FRAGS - n) / n; 5106 } 5107 5108 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5109 segs = MAX_TPA_SEGS_P5; 5110 max_aggs = bp->max_tpa; 5111 } else { 5112 segs = ilog2(nsegs); 5113 } 5114 req->max_agg_segs = cpu_to_le16(segs); 5115 req->max_aggs = cpu_to_le16(max_aggs); 5116 5117 req->min_agg_len = cpu_to_le32(512); 5118 } 5119 req->vnic_id = cpu_to_le16(vnic->fw_vnic_id); 5120 5121 return hwrm_req_send(bp, req); 5122 } 5123 5124 static u16 bnxt_cp_ring_from_grp(struct bnxt *bp, struct bnxt_ring_struct *ring) 5125 { 5126 struct bnxt_ring_grp_info *grp_info; 5127 5128 grp_info = &bp->grp_info[ring->grp_idx]; 5129 return grp_info->cp_fw_ring_id; 5130 } 5131 5132 static u16 bnxt_cp_ring_for_rx(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) 5133 { 5134 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5135 struct bnxt_napi *bnapi = rxr->bnapi; 5136 struct bnxt_cp_ring_info *cpr; 5137 5138 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_RX_HDL]; 5139 return cpr->cp_ring_struct.fw_ring_id; 5140 } else { 5141 return bnxt_cp_ring_from_grp(bp, &rxr->rx_ring_struct); 5142 } 5143 } 5144 5145 static u16 bnxt_cp_ring_for_tx(struct bnxt *bp, struct bnxt_tx_ring_info *txr) 5146 { 5147 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5148 struct bnxt_napi *bnapi = txr->bnapi; 5149 struct bnxt_cp_ring_info *cpr; 5150 5151 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_TX_HDL]; 5152 return cpr->cp_ring_struct.fw_ring_id; 5153 } else { 5154 return bnxt_cp_ring_from_grp(bp, &txr->tx_ring_struct); 5155 } 5156 } 5157 5158 static int bnxt_alloc_rss_indir_tbl(struct bnxt *bp) 5159 { 5160 int entries; 5161 5162 if (bp->flags & BNXT_FLAG_CHIP_P5) 5163 entries = BNXT_MAX_RSS_TABLE_ENTRIES_P5; 5164 else 5165 entries = HW_HASH_INDEX_SIZE; 5166 5167 bp->rss_indir_tbl_entries = entries; 5168 bp->rss_indir_tbl = kmalloc_array(entries, sizeof(*bp->rss_indir_tbl), 5169 GFP_KERNEL); 5170 if (!bp->rss_indir_tbl) 5171 return -ENOMEM; 5172 return 0; 5173 } 5174 5175 static void bnxt_set_dflt_rss_indir_tbl(struct bnxt *bp) 5176 { 5177 u16 max_rings, max_entries, pad, i; 5178 5179 if (!bp->rx_nr_rings) 5180 return; 5181 5182 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 5183 max_rings = bp->rx_nr_rings - 1; 5184 else 5185 max_rings = bp->rx_nr_rings; 5186 5187 max_entries = bnxt_get_rxfh_indir_size(bp->dev); 5188 5189 for (i = 0; i < max_entries; i++) 5190 bp->rss_indir_tbl[i] = ethtool_rxfh_indir_default(i, max_rings); 5191 5192 pad = bp->rss_indir_tbl_entries - max_entries; 5193 if (pad) 5194 memset(&bp->rss_indir_tbl[i], 0, pad * sizeof(u16)); 5195 } 5196 5197 static u16 bnxt_get_max_rss_ring(struct bnxt *bp) 5198 { 5199 u16 i, tbl_size, max_ring = 0; 5200 5201 if (!bp->rss_indir_tbl) 5202 return 0; 5203 5204 tbl_size = bnxt_get_rxfh_indir_size(bp->dev); 5205 for (i = 0; i < tbl_size; i++) 5206 max_ring = max(max_ring, bp->rss_indir_tbl[i]); 5207 return max_ring; 5208 } 5209 5210 int bnxt_get_nr_rss_ctxs(struct bnxt *bp, int rx_rings) 5211 { 5212 if (bp->flags & BNXT_FLAG_CHIP_P5) 5213 return DIV_ROUND_UP(rx_rings, BNXT_RSS_TABLE_ENTRIES_P5); 5214 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 5215 return 2; 5216 return 1; 5217 } 5218 5219 static void bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic) 5220 { 5221 bool no_rss = !(vnic->flags & BNXT_VNIC_RSS_FLAG); 5222 u16 i, j; 5223 5224 /* Fill the RSS indirection table with ring group ids */ 5225 for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++) { 5226 if (!no_rss) 5227 j = bp->rss_indir_tbl[i]; 5228 vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]); 5229 } 5230 } 5231 5232 static void bnxt_fill_hw_rss_tbl_p5(struct bnxt *bp, 5233 struct bnxt_vnic_info *vnic) 5234 { 5235 __le16 *ring_tbl = vnic->rss_table; 5236 struct bnxt_rx_ring_info *rxr; 5237 u16 tbl_size, i; 5238 5239 tbl_size = bnxt_get_rxfh_indir_size(bp->dev); 5240 5241 for (i = 0; i < tbl_size; i++) { 5242 u16 ring_id, j; 5243 5244 j = bp->rss_indir_tbl[i]; 5245 rxr = &bp->rx_ring[j]; 5246 5247 ring_id = rxr->rx_ring_struct.fw_ring_id; 5248 *ring_tbl++ = cpu_to_le16(ring_id); 5249 ring_id = bnxt_cp_ring_for_rx(bp, rxr); 5250 *ring_tbl++ = cpu_to_le16(ring_id); 5251 } 5252 } 5253 5254 static void 5255 __bnxt_hwrm_vnic_set_rss(struct bnxt *bp, struct hwrm_vnic_rss_cfg_input *req, 5256 struct bnxt_vnic_info *vnic) 5257 { 5258 if (bp->flags & BNXT_FLAG_CHIP_P5) 5259 bnxt_fill_hw_rss_tbl_p5(bp, vnic); 5260 else 5261 bnxt_fill_hw_rss_tbl(bp, vnic); 5262 5263 if (bp->rss_hash_delta) { 5264 req->hash_type = cpu_to_le32(bp->rss_hash_delta); 5265 if (bp->rss_hash_cfg & bp->rss_hash_delta) 5266 req->flags |= VNIC_RSS_CFG_REQ_FLAGS_HASH_TYPE_INCLUDE; 5267 else 5268 req->flags |= VNIC_RSS_CFG_REQ_FLAGS_HASH_TYPE_EXCLUDE; 5269 } else { 5270 req->hash_type = cpu_to_le32(bp->rss_hash_cfg); 5271 } 5272 req->hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT; 5273 req->ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr); 5274 req->hash_key_tbl_addr = cpu_to_le64(vnic->rss_hash_key_dma_addr); 5275 } 5276 5277 static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss) 5278 { 5279 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5280 struct hwrm_vnic_rss_cfg_input *req; 5281 int rc; 5282 5283 if ((bp->flags & BNXT_FLAG_CHIP_P5) || 5284 vnic->fw_rss_cos_lb_ctx[0] == INVALID_HW_RING_ID) 5285 return 0; 5286 5287 rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_CFG); 5288 if (rc) 5289 return rc; 5290 5291 if (set_rss) 5292 __bnxt_hwrm_vnic_set_rss(bp, req, vnic); 5293 req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]); 5294 return hwrm_req_send(bp, req); 5295 } 5296 5297 static int bnxt_hwrm_vnic_set_rss_p5(struct bnxt *bp, u16 vnic_id, bool set_rss) 5298 { 5299 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5300 struct hwrm_vnic_rss_cfg_input *req; 5301 dma_addr_t ring_tbl_map; 5302 u32 i, nr_ctxs; 5303 int rc; 5304 5305 rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_CFG); 5306 if (rc) 5307 return rc; 5308 5309 req->vnic_id = cpu_to_le16(vnic->fw_vnic_id); 5310 if (!set_rss) 5311 return hwrm_req_send(bp, req); 5312 5313 __bnxt_hwrm_vnic_set_rss(bp, req, vnic); 5314 ring_tbl_map = vnic->rss_table_dma_addr; 5315 nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings); 5316 5317 hwrm_req_hold(bp, req); 5318 for (i = 0; i < nr_ctxs; ring_tbl_map += BNXT_RSS_TABLE_SIZE_P5, i++) { 5319 req->ring_grp_tbl_addr = cpu_to_le64(ring_tbl_map); 5320 req->ring_table_pair_index = i; 5321 req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[i]); 5322 rc = hwrm_req_send(bp, req); 5323 if (rc) 5324 goto exit; 5325 } 5326 5327 exit: 5328 hwrm_req_drop(bp, req); 5329 return rc; 5330 } 5331 5332 static void bnxt_hwrm_update_rss_hash_cfg(struct bnxt *bp) 5333 { 5334 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 5335 struct hwrm_vnic_rss_qcfg_output *resp; 5336 struct hwrm_vnic_rss_qcfg_input *req; 5337 5338 if (hwrm_req_init(bp, req, HWRM_VNIC_RSS_QCFG)) 5339 return; 5340 5341 req->vnic_id = cpu_to_le16(vnic->fw_vnic_id); 5342 /* all contexts configured to same hash_type, zero always exists */ 5343 req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]); 5344 resp = hwrm_req_hold(bp, req); 5345 if (!hwrm_req_send(bp, req)) { 5346 bp->rss_hash_cfg = le32_to_cpu(resp->hash_type) ?: bp->rss_hash_cfg; 5347 bp->rss_hash_delta = 0; 5348 } 5349 hwrm_req_drop(bp, req); 5350 } 5351 5352 static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id) 5353 { 5354 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5355 struct hwrm_vnic_plcmodes_cfg_input *req; 5356 int rc; 5357 5358 rc = hwrm_req_init(bp, req, HWRM_VNIC_PLCMODES_CFG); 5359 if (rc) 5360 return rc; 5361 5362 req->flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT); 5363 req->enables = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID); 5364 5365 if (BNXT_RX_PAGE_MODE(bp)) { 5366 req->jumbo_thresh = cpu_to_le16(bp->rx_buf_use_size); 5367 } else { 5368 req->flags |= cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 | 5369 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6); 5370 req->enables |= 5371 cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID); 5372 req->jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh); 5373 req->hds_threshold = cpu_to_le16(bp->rx_copy_thresh); 5374 } 5375 req->vnic_id = cpu_to_le32(vnic->fw_vnic_id); 5376 return hwrm_req_send(bp, req); 5377 } 5378 5379 static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id, 5380 u16 ctx_idx) 5381 { 5382 struct hwrm_vnic_rss_cos_lb_ctx_free_input *req; 5383 5384 if (hwrm_req_init(bp, req, HWRM_VNIC_RSS_COS_LB_CTX_FREE)) 5385 return; 5386 5387 req->rss_cos_lb_ctx_id = 5388 cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx]); 5389 5390 hwrm_req_send(bp, req); 5391 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID; 5392 } 5393 5394 static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp) 5395 { 5396 int i, j; 5397 5398 for (i = 0; i < bp->nr_vnics; i++) { 5399 struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; 5400 5401 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) { 5402 if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID) 5403 bnxt_hwrm_vnic_ctx_free_one(bp, i, j); 5404 } 5405 } 5406 bp->rsscos_nr_ctxs = 0; 5407 } 5408 5409 static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id, u16 ctx_idx) 5410 { 5411 struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp; 5412 struct hwrm_vnic_rss_cos_lb_ctx_alloc_input *req; 5413 int rc; 5414 5415 rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC); 5416 if (rc) 5417 return rc; 5418 5419 resp = hwrm_req_hold(bp, req); 5420 rc = hwrm_req_send(bp, req); 5421 if (!rc) 5422 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = 5423 le16_to_cpu(resp->rss_cos_lb_ctx_id); 5424 hwrm_req_drop(bp, req); 5425 5426 return rc; 5427 } 5428 5429 static u32 bnxt_get_roce_vnic_mode(struct bnxt *bp) 5430 { 5431 if (bp->flags & BNXT_FLAG_ROCE_MIRROR_CAP) 5432 return VNIC_CFG_REQ_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE; 5433 return VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE; 5434 } 5435 5436 int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id) 5437 { 5438 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5439 struct hwrm_vnic_cfg_input *req; 5440 unsigned int ring = 0, grp_idx; 5441 u16 def_vlan = 0; 5442 int rc; 5443 5444 rc = hwrm_req_init(bp, req, HWRM_VNIC_CFG); 5445 if (rc) 5446 return rc; 5447 5448 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5449 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0]; 5450 5451 req->default_rx_ring_id = 5452 cpu_to_le16(rxr->rx_ring_struct.fw_ring_id); 5453 req->default_cmpl_ring_id = 5454 cpu_to_le16(bnxt_cp_ring_for_rx(bp, rxr)); 5455 req->enables = 5456 cpu_to_le32(VNIC_CFG_REQ_ENABLES_DEFAULT_RX_RING_ID | 5457 VNIC_CFG_REQ_ENABLES_DEFAULT_CMPL_RING_ID); 5458 goto vnic_mru; 5459 } 5460 req->enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP); 5461 /* Only RSS support for now TBD: COS & LB */ 5462 if (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID) { 5463 req->rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]); 5464 req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE | 5465 VNIC_CFG_REQ_ENABLES_MRU); 5466 } else if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) { 5467 req->rss_rule = 5468 cpu_to_le16(bp->vnic_info[0].fw_rss_cos_lb_ctx[0]); 5469 req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE | 5470 VNIC_CFG_REQ_ENABLES_MRU); 5471 req->flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE); 5472 } else { 5473 req->rss_rule = cpu_to_le16(0xffff); 5474 } 5475 5476 if (BNXT_CHIP_TYPE_NITRO_A0(bp) && 5477 (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID)) { 5478 req->cos_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[1]); 5479 req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_COS_RULE); 5480 } else { 5481 req->cos_rule = cpu_to_le16(0xffff); 5482 } 5483 5484 if (vnic->flags & BNXT_VNIC_RSS_FLAG) 5485 ring = 0; 5486 else if (vnic->flags & BNXT_VNIC_RFS_FLAG) 5487 ring = vnic_id - 1; 5488 else if ((vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp)) 5489 ring = bp->rx_nr_rings - 1; 5490 5491 grp_idx = bp->rx_ring[ring].bnapi->index; 5492 req->dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id); 5493 req->lb_rule = cpu_to_le16(0xffff); 5494 vnic_mru: 5495 req->mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + VLAN_HLEN); 5496 5497 req->vnic_id = cpu_to_le16(vnic->fw_vnic_id); 5498 #ifdef CONFIG_BNXT_SRIOV 5499 if (BNXT_VF(bp)) 5500 def_vlan = bp->vf.vlan; 5501 #endif 5502 if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan) 5503 req->flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE); 5504 if (!vnic_id && bnxt_ulp_registered(bp->edev)) 5505 req->flags |= cpu_to_le32(bnxt_get_roce_vnic_mode(bp)); 5506 5507 return hwrm_req_send(bp, req); 5508 } 5509 5510 static void bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id) 5511 { 5512 if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) { 5513 struct hwrm_vnic_free_input *req; 5514 5515 if (hwrm_req_init(bp, req, HWRM_VNIC_FREE)) 5516 return; 5517 5518 req->vnic_id = 5519 cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id); 5520 5521 hwrm_req_send(bp, req); 5522 bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID; 5523 } 5524 } 5525 5526 static void bnxt_hwrm_vnic_free(struct bnxt *bp) 5527 { 5528 u16 i; 5529 5530 for (i = 0; i < bp->nr_vnics; i++) 5531 bnxt_hwrm_vnic_free_one(bp, i); 5532 } 5533 5534 static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id, 5535 unsigned int start_rx_ring_idx, 5536 unsigned int nr_rings) 5537 { 5538 unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings; 5539 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5540 struct hwrm_vnic_alloc_output *resp; 5541 struct hwrm_vnic_alloc_input *req; 5542 int rc; 5543 5544 rc = hwrm_req_init(bp, req, HWRM_VNIC_ALLOC); 5545 if (rc) 5546 return rc; 5547 5548 if (bp->flags & BNXT_FLAG_CHIP_P5) 5549 goto vnic_no_ring_grps; 5550 5551 /* map ring groups to this vnic */ 5552 for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) { 5553 grp_idx = bp->rx_ring[i].bnapi->index; 5554 if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) { 5555 netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n", 5556 j, nr_rings); 5557 break; 5558 } 5559 vnic->fw_grp_ids[j] = bp->grp_info[grp_idx].fw_grp_id; 5560 } 5561 5562 vnic_no_ring_grps: 5563 for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++) 5564 vnic->fw_rss_cos_lb_ctx[i] = INVALID_HW_RING_ID; 5565 if (vnic_id == 0) 5566 req->flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT); 5567 5568 resp = hwrm_req_hold(bp, req); 5569 rc = hwrm_req_send(bp, req); 5570 if (!rc) 5571 vnic->fw_vnic_id = le32_to_cpu(resp->vnic_id); 5572 hwrm_req_drop(bp, req); 5573 return rc; 5574 } 5575 5576 static int bnxt_hwrm_vnic_qcaps(struct bnxt *bp) 5577 { 5578 struct hwrm_vnic_qcaps_output *resp; 5579 struct hwrm_vnic_qcaps_input *req; 5580 int rc; 5581 5582 bp->hw_ring_stats_size = sizeof(struct ctx_hw_stats); 5583 bp->flags &= ~(BNXT_FLAG_NEW_RSS_CAP | BNXT_FLAG_ROCE_MIRROR_CAP); 5584 if (bp->hwrm_spec_code < 0x10600) 5585 return 0; 5586 5587 rc = hwrm_req_init(bp, req, HWRM_VNIC_QCAPS); 5588 if (rc) 5589 return rc; 5590 5591 resp = hwrm_req_hold(bp, req); 5592 rc = hwrm_req_send(bp, req); 5593 if (!rc) { 5594 u32 flags = le32_to_cpu(resp->flags); 5595 5596 if (!(bp->flags & BNXT_FLAG_CHIP_P5) && 5597 (flags & VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP)) 5598 bp->flags |= BNXT_FLAG_NEW_RSS_CAP; 5599 if (flags & 5600 VNIC_QCAPS_RESP_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_CAP) 5601 bp->flags |= BNXT_FLAG_ROCE_MIRROR_CAP; 5602 5603 /* Older P5 fw before EXT_HW_STATS support did not set 5604 * VLAN_STRIP_CAP properly. 5605 */ 5606 if ((flags & VNIC_QCAPS_RESP_FLAGS_VLAN_STRIP_CAP) || 5607 (BNXT_CHIP_P5_THOR(bp) && 5608 !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))) 5609 bp->fw_cap |= BNXT_FW_CAP_VLAN_RX_STRIP; 5610 if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_HASH_TYPE_DELTA_CAP) 5611 bp->fw_cap |= BNXT_FW_CAP_RSS_HASH_TYPE_DELTA; 5612 bp->max_tpa_v2 = le16_to_cpu(resp->max_aggs_supported); 5613 if (bp->max_tpa_v2) { 5614 if (BNXT_CHIP_P5_THOR(bp)) 5615 bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P5; 5616 else 5617 bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P5_SR2; 5618 } 5619 } 5620 hwrm_req_drop(bp, req); 5621 return rc; 5622 } 5623 5624 static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp) 5625 { 5626 struct hwrm_ring_grp_alloc_output *resp; 5627 struct hwrm_ring_grp_alloc_input *req; 5628 int rc; 5629 u16 i; 5630 5631 if (bp->flags & BNXT_FLAG_CHIP_P5) 5632 return 0; 5633 5634 rc = hwrm_req_init(bp, req, HWRM_RING_GRP_ALLOC); 5635 if (rc) 5636 return rc; 5637 5638 resp = hwrm_req_hold(bp, req); 5639 for (i = 0; i < bp->rx_nr_rings; i++) { 5640 unsigned int grp_idx = bp->rx_ring[i].bnapi->index; 5641 5642 req->cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id); 5643 req->rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id); 5644 req->ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id); 5645 req->sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx); 5646 5647 rc = hwrm_req_send(bp, req); 5648 5649 if (rc) 5650 break; 5651 5652 bp->grp_info[grp_idx].fw_grp_id = 5653 le32_to_cpu(resp->ring_group_id); 5654 } 5655 hwrm_req_drop(bp, req); 5656 return rc; 5657 } 5658 5659 static void bnxt_hwrm_ring_grp_free(struct bnxt *bp) 5660 { 5661 struct hwrm_ring_grp_free_input *req; 5662 u16 i; 5663 5664 if (!bp->grp_info || (bp->flags & BNXT_FLAG_CHIP_P5)) 5665 return; 5666 5667 if (hwrm_req_init(bp, req, HWRM_RING_GRP_FREE)) 5668 return; 5669 5670 hwrm_req_hold(bp, req); 5671 for (i = 0; i < bp->cp_nr_rings; i++) { 5672 if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID) 5673 continue; 5674 req->ring_group_id = 5675 cpu_to_le32(bp->grp_info[i].fw_grp_id); 5676 5677 hwrm_req_send(bp, req); 5678 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID; 5679 } 5680 hwrm_req_drop(bp, req); 5681 } 5682 5683 static int hwrm_ring_alloc_send_msg(struct bnxt *bp, 5684 struct bnxt_ring_struct *ring, 5685 u32 ring_type, u32 map_index) 5686 { 5687 struct hwrm_ring_alloc_output *resp; 5688 struct hwrm_ring_alloc_input *req; 5689 struct bnxt_ring_mem_info *rmem = &ring->ring_mem; 5690 struct bnxt_ring_grp_info *grp_info; 5691 int rc, err = 0; 5692 u16 ring_id; 5693 5694 rc = hwrm_req_init(bp, req, HWRM_RING_ALLOC); 5695 if (rc) 5696 goto exit; 5697 5698 req->enables = 0; 5699 if (rmem->nr_pages > 1) { 5700 req->page_tbl_addr = cpu_to_le64(rmem->pg_tbl_map); 5701 /* Page size is in log2 units */ 5702 req->page_size = BNXT_PAGE_SHIFT; 5703 req->page_tbl_depth = 1; 5704 } else { 5705 req->page_tbl_addr = cpu_to_le64(rmem->dma_arr[0]); 5706 } 5707 req->fbo = 0; 5708 /* Association of ring index with doorbell index and MSIX number */ 5709 req->logical_id = cpu_to_le16(map_index); 5710 5711 switch (ring_type) { 5712 case HWRM_RING_ALLOC_TX: { 5713 struct bnxt_tx_ring_info *txr; 5714 5715 txr = container_of(ring, struct bnxt_tx_ring_info, 5716 tx_ring_struct); 5717 req->ring_type = RING_ALLOC_REQ_RING_TYPE_TX; 5718 /* Association of transmit ring with completion ring */ 5719 grp_info = &bp->grp_info[ring->grp_idx]; 5720 req->cmpl_ring_id = cpu_to_le16(bnxt_cp_ring_for_tx(bp, txr)); 5721 req->length = cpu_to_le32(bp->tx_ring_mask + 1); 5722 req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx); 5723 req->queue_id = cpu_to_le16(ring->queue_id); 5724 break; 5725 } 5726 case HWRM_RING_ALLOC_RX: 5727 req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX; 5728 req->length = cpu_to_le32(bp->rx_ring_mask + 1); 5729 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5730 u16 flags = 0; 5731 5732 /* Association of rx ring with stats context */ 5733 grp_info = &bp->grp_info[ring->grp_idx]; 5734 req->rx_buf_size = cpu_to_le16(bp->rx_buf_use_size); 5735 req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx); 5736 req->enables |= cpu_to_le32( 5737 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID); 5738 if (NET_IP_ALIGN == 2) 5739 flags = RING_ALLOC_REQ_FLAGS_RX_SOP_PAD; 5740 req->flags = cpu_to_le16(flags); 5741 } 5742 break; 5743 case HWRM_RING_ALLOC_AGG: 5744 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5745 req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX_AGG; 5746 /* Association of agg ring with rx ring */ 5747 grp_info = &bp->grp_info[ring->grp_idx]; 5748 req->rx_ring_id = cpu_to_le16(grp_info->rx_fw_ring_id); 5749 req->rx_buf_size = cpu_to_le16(BNXT_RX_PAGE_SIZE); 5750 req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx); 5751 req->enables |= cpu_to_le32( 5752 RING_ALLOC_REQ_ENABLES_RX_RING_ID_VALID | 5753 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID); 5754 } else { 5755 req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX; 5756 } 5757 req->length = cpu_to_le32(bp->rx_agg_ring_mask + 1); 5758 break; 5759 case HWRM_RING_ALLOC_CMPL: 5760 req->ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL; 5761 req->length = cpu_to_le32(bp->cp_ring_mask + 1); 5762 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5763 /* Association of cp ring with nq */ 5764 grp_info = &bp->grp_info[map_index]; 5765 req->nq_ring_id = cpu_to_le16(grp_info->cp_fw_ring_id); 5766 req->cq_handle = cpu_to_le64(ring->handle); 5767 req->enables |= cpu_to_le32( 5768 RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID); 5769 } else if (bp->flags & BNXT_FLAG_USING_MSIX) { 5770 req->int_mode = RING_ALLOC_REQ_INT_MODE_MSIX; 5771 } 5772 break; 5773 case HWRM_RING_ALLOC_NQ: 5774 req->ring_type = RING_ALLOC_REQ_RING_TYPE_NQ; 5775 req->length = cpu_to_le32(bp->cp_ring_mask + 1); 5776 if (bp->flags & BNXT_FLAG_USING_MSIX) 5777 req->int_mode = RING_ALLOC_REQ_INT_MODE_MSIX; 5778 break; 5779 default: 5780 netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n", 5781 ring_type); 5782 return -1; 5783 } 5784 5785 resp = hwrm_req_hold(bp, req); 5786 rc = hwrm_req_send(bp, req); 5787 err = le16_to_cpu(resp->error_code); 5788 ring_id = le16_to_cpu(resp->ring_id); 5789 hwrm_req_drop(bp, req); 5790 5791 exit: 5792 if (rc || err) { 5793 netdev_err(bp->dev, "hwrm_ring_alloc type %d failed. rc:%x err:%x\n", 5794 ring_type, rc, err); 5795 return -EIO; 5796 } 5797 ring->fw_ring_id = ring_id; 5798 return rc; 5799 } 5800 5801 static int bnxt_hwrm_set_async_event_cr(struct bnxt *bp, int idx) 5802 { 5803 int rc; 5804 5805 if (BNXT_PF(bp)) { 5806 struct hwrm_func_cfg_input *req; 5807 5808 rc = hwrm_req_init(bp, req, HWRM_FUNC_CFG); 5809 if (rc) 5810 return rc; 5811 5812 req->fid = cpu_to_le16(0xffff); 5813 req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR); 5814 req->async_event_cr = cpu_to_le16(idx); 5815 return hwrm_req_send(bp, req); 5816 } else { 5817 struct hwrm_func_vf_cfg_input *req; 5818 5819 rc = hwrm_req_init(bp, req, HWRM_FUNC_VF_CFG); 5820 if (rc) 5821 return rc; 5822 5823 req->enables = 5824 cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR); 5825 req->async_event_cr = cpu_to_le16(idx); 5826 return hwrm_req_send(bp, req); 5827 } 5828 } 5829 5830 static void bnxt_set_db(struct bnxt *bp, struct bnxt_db_info *db, u32 ring_type, 5831 u32 map_idx, u32 xid) 5832 { 5833 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5834 if (BNXT_PF(bp)) 5835 db->doorbell = bp->bar1 + DB_PF_OFFSET_P5; 5836 else 5837 db->doorbell = bp->bar1 + DB_VF_OFFSET_P5; 5838 switch (ring_type) { 5839 case HWRM_RING_ALLOC_TX: 5840 db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SQ; 5841 break; 5842 case HWRM_RING_ALLOC_RX: 5843 case HWRM_RING_ALLOC_AGG: 5844 db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SRQ; 5845 break; 5846 case HWRM_RING_ALLOC_CMPL: 5847 db->db_key64 = DBR_PATH_L2; 5848 break; 5849 case HWRM_RING_ALLOC_NQ: 5850 db->db_key64 = DBR_PATH_L2; 5851 break; 5852 } 5853 db->db_key64 |= (u64)xid << DBR_XID_SFT; 5854 } else { 5855 db->doorbell = bp->bar1 + map_idx * 0x80; 5856 switch (ring_type) { 5857 case HWRM_RING_ALLOC_TX: 5858 db->db_key32 = DB_KEY_TX; 5859 break; 5860 case HWRM_RING_ALLOC_RX: 5861 case HWRM_RING_ALLOC_AGG: 5862 db->db_key32 = DB_KEY_RX; 5863 break; 5864 case HWRM_RING_ALLOC_CMPL: 5865 db->db_key32 = DB_KEY_CP; 5866 break; 5867 } 5868 } 5869 } 5870 5871 static int bnxt_hwrm_ring_alloc(struct bnxt *bp) 5872 { 5873 bool agg_rings = !!(bp->flags & BNXT_FLAG_AGG_RINGS); 5874 int i, rc = 0; 5875 u32 type; 5876 5877 if (bp->flags & BNXT_FLAG_CHIP_P5) 5878 type = HWRM_RING_ALLOC_NQ; 5879 else 5880 type = HWRM_RING_ALLOC_CMPL; 5881 for (i = 0; i < bp->cp_nr_rings; i++) { 5882 struct bnxt_napi *bnapi = bp->bnapi[i]; 5883 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 5884 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 5885 u32 map_idx = ring->map_idx; 5886 unsigned int vector; 5887 5888 vector = bp->irq_tbl[map_idx].vector; 5889 disable_irq_nosync(vector); 5890 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 5891 if (rc) { 5892 enable_irq(vector); 5893 goto err_out; 5894 } 5895 bnxt_set_db(bp, &cpr->cp_db, type, map_idx, ring->fw_ring_id); 5896 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons); 5897 enable_irq(vector); 5898 bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id; 5899 5900 if (!i) { 5901 rc = bnxt_hwrm_set_async_event_cr(bp, ring->fw_ring_id); 5902 if (rc) 5903 netdev_warn(bp->dev, "Failed to set async event completion ring.\n"); 5904 } 5905 } 5906 5907 type = HWRM_RING_ALLOC_TX; 5908 for (i = 0; i < bp->tx_nr_rings; i++) { 5909 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 5910 struct bnxt_ring_struct *ring; 5911 u32 map_idx; 5912 5913 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5914 struct bnxt_napi *bnapi = txr->bnapi; 5915 struct bnxt_cp_ring_info *cpr, *cpr2; 5916 u32 type2 = HWRM_RING_ALLOC_CMPL; 5917 5918 cpr = &bnapi->cp_ring; 5919 cpr2 = cpr->cp_ring_arr[BNXT_TX_HDL]; 5920 ring = &cpr2->cp_ring_struct; 5921 ring->handle = BNXT_TX_HDL; 5922 map_idx = bnapi->index; 5923 rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx); 5924 if (rc) 5925 goto err_out; 5926 bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx, 5927 ring->fw_ring_id); 5928 bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons); 5929 } 5930 ring = &txr->tx_ring_struct; 5931 map_idx = i; 5932 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 5933 if (rc) 5934 goto err_out; 5935 bnxt_set_db(bp, &txr->tx_db, type, map_idx, ring->fw_ring_id); 5936 } 5937 5938 type = HWRM_RING_ALLOC_RX; 5939 for (i = 0; i < bp->rx_nr_rings; i++) { 5940 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 5941 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct; 5942 struct bnxt_napi *bnapi = rxr->bnapi; 5943 u32 map_idx = bnapi->index; 5944 5945 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 5946 if (rc) 5947 goto err_out; 5948 bnxt_set_db(bp, &rxr->rx_db, type, map_idx, ring->fw_ring_id); 5949 /* If we have agg rings, post agg buffers first. */ 5950 if (!agg_rings) 5951 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 5952 bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id; 5953 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5954 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 5955 u32 type2 = HWRM_RING_ALLOC_CMPL; 5956 struct bnxt_cp_ring_info *cpr2; 5957 5958 cpr2 = cpr->cp_ring_arr[BNXT_RX_HDL]; 5959 ring = &cpr2->cp_ring_struct; 5960 ring->handle = BNXT_RX_HDL; 5961 rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx); 5962 if (rc) 5963 goto err_out; 5964 bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx, 5965 ring->fw_ring_id); 5966 bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons); 5967 } 5968 } 5969 5970 if (agg_rings) { 5971 type = HWRM_RING_ALLOC_AGG; 5972 for (i = 0; i < bp->rx_nr_rings; i++) { 5973 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 5974 struct bnxt_ring_struct *ring = 5975 &rxr->rx_agg_ring_struct; 5976 u32 grp_idx = ring->grp_idx; 5977 u32 map_idx = grp_idx + bp->rx_nr_rings; 5978 5979 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 5980 if (rc) 5981 goto err_out; 5982 5983 bnxt_set_db(bp, &rxr->rx_agg_db, type, map_idx, 5984 ring->fw_ring_id); 5985 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); 5986 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 5987 bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id; 5988 } 5989 } 5990 err_out: 5991 return rc; 5992 } 5993 5994 static int hwrm_ring_free_send_msg(struct bnxt *bp, 5995 struct bnxt_ring_struct *ring, 5996 u32 ring_type, int cmpl_ring_id) 5997 { 5998 struct hwrm_ring_free_output *resp; 5999 struct hwrm_ring_free_input *req; 6000 u16 error_code = 0; 6001 int rc; 6002 6003 if (BNXT_NO_FW_ACCESS(bp)) 6004 return 0; 6005 6006 rc = hwrm_req_init(bp, req, HWRM_RING_FREE); 6007 if (rc) 6008 goto exit; 6009 6010 req->cmpl_ring = cpu_to_le16(cmpl_ring_id); 6011 req->ring_type = ring_type; 6012 req->ring_id = cpu_to_le16(ring->fw_ring_id); 6013 6014 resp = hwrm_req_hold(bp, req); 6015 rc = hwrm_req_send(bp, req); 6016 error_code = le16_to_cpu(resp->error_code); 6017 hwrm_req_drop(bp, req); 6018 exit: 6019 if (rc || error_code) { 6020 netdev_err(bp->dev, "hwrm_ring_free type %d failed. rc:%x err:%x\n", 6021 ring_type, rc, error_code); 6022 return -EIO; 6023 } 6024 return 0; 6025 } 6026 6027 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path) 6028 { 6029 u32 type; 6030 int i; 6031 6032 if (!bp->bnapi) 6033 return; 6034 6035 for (i = 0; i < bp->tx_nr_rings; i++) { 6036 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 6037 struct bnxt_ring_struct *ring = &txr->tx_ring_struct; 6038 6039 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 6040 u32 cmpl_ring_id = bnxt_cp_ring_for_tx(bp, txr); 6041 6042 hwrm_ring_free_send_msg(bp, ring, 6043 RING_FREE_REQ_RING_TYPE_TX, 6044 close_path ? cmpl_ring_id : 6045 INVALID_HW_RING_ID); 6046 ring->fw_ring_id = INVALID_HW_RING_ID; 6047 } 6048 } 6049 6050 for (i = 0; i < bp->rx_nr_rings; i++) { 6051 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 6052 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct; 6053 u32 grp_idx = rxr->bnapi->index; 6054 6055 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 6056 u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr); 6057 6058 hwrm_ring_free_send_msg(bp, ring, 6059 RING_FREE_REQ_RING_TYPE_RX, 6060 close_path ? cmpl_ring_id : 6061 INVALID_HW_RING_ID); 6062 ring->fw_ring_id = INVALID_HW_RING_ID; 6063 bp->grp_info[grp_idx].rx_fw_ring_id = 6064 INVALID_HW_RING_ID; 6065 } 6066 } 6067 6068 if (bp->flags & BNXT_FLAG_CHIP_P5) 6069 type = RING_FREE_REQ_RING_TYPE_RX_AGG; 6070 else 6071 type = RING_FREE_REQ_RING_TYPE_RX; 6072 for (i = 0; i < bp->rx_nr_rings; i++) { 6073 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 6074 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct; 6075 u32 grp_idx = rxr->bnapi->index; 6076 6077 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 6078 u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr); 6079 6080 hwrm_ring_free_send_msg(bp, ring, type, 6081 close_path ? cmpl_ring_id : 6082 INVALID_HW_RING_ID); 6083 ring->fw_ring_id = INVALID_HW_RING_ID; 6084 bp->grp_info[grp_idx].agg_fw_ring_id = 6085 INVALID_HW_RING_ID; 6086 } 6087 } 6088 6089 /* The completion rings are about to be freed. After that the 6090 * IRQ doorbell will not work anymore. So we need to disable 6091 * IRQ here. 6092 */ 6093 bnxt_disable_int_sync(bp); 6094 6095 if (bp->flags & BNXT_FLAG_CHIP_P5) 6096 type = RING_FREE_REQ_RING_TYPE_NQ; 6097 else 6098 type = RING_FREE_REQ_RING_TYPE_L2_CMPL; 6099 for (i = 0; i < bp->cp_nr_rings; i++) { 6100 struct bnxt_napi *bnapi = bp->bnapi[i]; 6101 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 6102 struct bnxt_ring_struct *ring; 6103 int j; 6104 6105 for (j = 0; j < 2; j++) { 6106 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j]; 6107 6108 if (cpr2) { 6109 ring = &cpr2->cp_ring_struct; 6110 if (ring->fw_ring_id == INVALID_HW_RING_ID) 6111 continue; 6112 hwrm_ring_free_send_msg(bp, ring, 6113 RING_FREE_REQ_RING_TYPE_L2_CMPL, 6114 INVALID_HW_RING_ID); 6115 ring->fw_ring_id = INVALID_HW_RING_ID; 6116 } 6117 } 6118 ring = &cpr->cp_ring_struct; 6119 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 6120 hwrm_ring_free_send_msg(bp, ring, type, 6121 INVALID_HW_RING_ID); 6122 ring->fw_ring_id = INVALID_HW_RING_ID; 6123 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID; 6124 } 6125 } 6126 } 6127 6128 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max, 6129 bool shared); 6130 6131 static int bnxt_hwrm_get_rings(struct bnxt *bp) 6132 { 6133 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 6134 struct hwrm_func_qcfg_output *resp; 6135 struct hwrm_func_qcfg_input *req; 6136 int rc; 6137 6138 if (bp->hwrm_spec_code < 0x10601) 6139 return 0; 6140 6141 rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG); 6142 if (rc) 6143 return rc; 6144 6145 req->fid = cpu_to_le16(0xffff); 6146 resp = hwrm_req_hold(bp, req); 6147 rc = hwrm_req_send(bp, req); 6148 if (rc) { 6149 hwrm_req_drop(bp, req); 6150 return rc; 6151 } 6152 6153 hw_resc->resv_tx_rings = le16_to_cpu(resp->alloc_tx_rings); 6154 if (BNXT_NEW_RM(bp)) { 6155 u16 cp, stats; 6156 6157 hw_resc->resv_rx_rings = le16_to_cpu(resp->alloc_rx_rings); 6158 hw_resc->resv_hw_ring_grps = 6159 le32_to_cpu(resp->alloc_hw_ring_grps); 6160 hw_resc->resv_vnics = le16_to_cpu(resp->alloc_vnics); 6161 cp = le16_to_cpu(resp->alloc_cmpl_rings); 6162 stats = le16_to_cpu(resp->alloc_stat_ctx); 6163 hw_resc->resv_irqs = cp; 6164 if (bp->flags & BNXT_FLAG_CHIP_P5) { 6165 int rx = hw_resc->resv_rx_rings; 6166 int tx = hw_resc->resv_tx_rings; 6167 6168 if (bp->flags & BNXT_FLAG_AGG_RINGS) 6169 rx >>= 1; 6170 if (cp < (rx + tx)) { 6171 bnxt_trim_rings(bp, &rx, &tx, cp, false); 6172 if (bp->flags & BNXT_FLAG_AGG_RINGS) 6173 rx <<= 1; 6174 hw_resc->resv_rx_rings = rx; 6175 hw_resc->resv_tx_rings = tx; 6176 } 6177 hw_resc->resv_irqs = le16_to_cpu(resp->alloc_msix); 6178 hw_resc->resv_hw_ring_grps = rx; 6179 } 6180 hw_resc->resv_cp_rings = cp; 6181 hw_resc->resv_stat_ctxs = stats; 6182 } 6183 hwrm_req_drop(bp, req); 6184 return 0; 6185 } 6186 6187 int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings) 6188 { 6189 struct hwrm_func_qcfg_output *resp; 6190 struct hwrm_func_qcfg_input *req; 6191 int rc; 6192 6193 if (bp->hwrm_spec_code < 0x10601) 6194 return 0; 6195 6196 rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG); 6197 if (rc) 6198 return rc; 6199 6200 req->fid = cpu_to_le16(fid); 6201 resp = hwrm_req_hold(bp, req); 6202 rc = hwrm_req_send(bp, req); 6203 if (!rc) 6204 *tx_rings = le16_to_cpu(resp->alloc_tx_rings); 6205 6206 hwrm_req_drop(bp, req); 6207 return rc; 6208 } 6209 6210 static bool bnxt_rfs_supported(struct bnxt *bp); 6211 6212 static struct hwrm_func_cfg_input * 6213 __bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6214 int ring_grps, int cp_rings, int stats, int vnics) 6215 { 6216 struct hwrm_func_cfg_input *req; 6217 u32 enables = 0; 6218 6219 if (hwrm_req_init(bp, req, HWRM_FUNC_CFG)) 6220 return NULL; 6221 6222 req->fid = cpu_to_le16(0xffff); 6223 enables |= tx_rings ? FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS : 0; 6224 req->num_tx_rings = cpu_to_le16(tx_rings); 6225 if (BNXT_NEW_RM(bp)) { 6226 enables |= rx_rings ? FUNC_CFG_REQ_ENABLES_NUM_RX_RINGS : 0; 6227 enables |= stats ? FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0; 6228 if (bp->flags & BNXT_FLAG_CHIP_P5) { 6229 enables |= cp_rings ? FUNC_CFG_REQ_ENABLES_NUM_MSIX : 0; 6230 enables |= tx_rings + ring_grps ? 6231 FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0; 6232 enables |= rx_rings ? 6233 FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0; 6234 } else { 6235 enables |= cp_rings ? 6236 FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0; 6237 enables |= ring_grps ? 6238 FUNC_CFG_REQ_ENABLES_NUM_HW_RING_GRPS | 6239 FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0; 6240 } 6241 enables |= vnics ? FUNC_CFG_REQ_ENABLES_NUM_VNICS : 0; 6242 6243 req->num_rx_rings = cpu_to_le16(rx_rings); 6244 if (bp->flags & BNXT_FLAG_CHIP_P5) { 6245 req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps); 6246 req->num_msix = cpu_to_le16(cp_rings); 6247 req->num_rsscos_ctxs = 6248 cpu_to_le16(DIV_ROUND_UP(ring_grps, 64)); 6249 } else { 6250 req->num_cmpl_rings = cpu_to_le16(cp_rings); 6251 req->num_hw_ring_grps = cpu_to_le16(ring_grps); 6252 req->num_rsscos_ctxs = cpu_to_le16(1); 6253 if (!(bp->flags & BNXT_FLAG_NEW_RSS_CAP) && 6254 bnxt_rfs_supported(bp)) 6255 req->num_rsscos_ctxs = 6256 cpu_to_le16(ring_grps + 1); 6257 } 6258 req->num_stat_ctxs = cpu_to_le16(stats); 6259 req->num_vnics = cpu_to_le16(vnics); 6260 } 6261 req->enables = cpu_to_le32(enables); 6262 return req; 6263 } 6264 6265 static struct hwrm_func_vf_cfg_input * 6266 __bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6267 int ring_grps, int cp_rings, int stats, int vnics) 6268 { 6269 struct hwrm_func_vf_cfg_input *req; 6270 u32 enables = 0; 6271 6272 if (hwrm_req_init(bp, req, HWRM_FUNC_VF_CFG)) 6273 return NULL; 6274 6275 enables |= tx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_TX_RINGS : 0; 6276 enables |= rx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_RX_RINGS | 6277 FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0; 6278 enables |= stats ? FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0; 6279 if (bp->flags & BNXT_FLAG_CHIP_P5) { 6280 enables |= tx_rings + ring_grps ? 6281 FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0; 6282 } else { 6283 enables |= cp_rings ? 6284 FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0; 6285 enables |= ring_grps ? 6286 FUNC_VF_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0; 6287 } 6288 enables |= vnics ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0; 6289 enables |= FUNC_VF_CFG_REQ_ENABLES_NUM_L2_CTXS; 6290 6291 req->num_l2_ctxs = cpu_to_le16(BNXT_VF_MAX_L2_CTX); 6292 req->num_tx_rings = cpu_to_le16(tx_rings); 6293 req->num_rx_rings = cpu_to_le16(rx_rings); 6294 if (bp->flags & BNXT_FLAG_CHIP_P5) { 6295 req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps); 6296 req->num_rsscos_ctxs = cpu_to_le16(DIV_ROUND_UP(ring_grps, 64)); 6297 } else { 6298 req->num_cmpl_rings = cpu_to_le16(cp_rings); 6299 req->num_hw_ring_grps = cpu_to_le16(ring_grps); 6300 req->num_rsscos_ctxs = cpu_to_le16(BNXT_VF_MAX_RSS_CTX); 6301 } 6302 req->num_stat_ctxs = cpu_to_le16(stats); 6303 req->num_vnics = cpu_to_le16(vnics); 6304 6305 req->enables = cpu_to_le32(enables); 6306 return req; 6307 } 6308 6309 static int 6310 bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6311 int ring_grps, int cp_rings, int stats, int vnics) 6312 { 6313 struct hwrm_func_cfg_input *req; 6314 int rc; 6315 6316 req = __bnxt_hwrm_reserve_pf_rings(bp, tx_rings, rx_rings, ring_grps, 6317 cp_rings, stats, vnics); 6318 if (!req) 6319 return -ENOMEM; 6320 6321 if (!req->enables) { 6322 hwrm_req_drop(bp, req); 6323 return 0; 6324 } 6325 6326 rc = hwrm_req_send(bp, req); 6327 if (rc) 6328 return rc; 6329 6330 if (bp->hwrm_spec_code < 0x10601) 6331 bp->hw_resc.resv_tx_rings = tx_rings; 6332 6333 return bnxt_hwrm_get_rings(bp); 6334 } 6335 6336 static int 6337 bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6338 int ring_grps, int cp_rings, int stats, int vnics) 6339 { 6340 struct hwrm_func_vf_cfg_input *req; 6341 int rc; 6342 6343 if (!BNXT_NEW_RM(bp)) { 6344 bp->hw_resc.resv_tx_rings = tx_rings; 6345 return 0; 6346 } 6347 6348 req = __bnxt_hwrm_reserve_vf_rings(bp, tx_rings, rx_rings, ring_grps, 6349 cp_rings, stats, vnics); 6350 if (!req) 6351 return -ENOMEM; 6352 6353 rc = hwrm_req_send(bp, req); 6354 if (rc) 6355 return rc; 6356 6357 return bnxt_hwrm_get_rings(bp); 6358 } 6359 6360 static int bnxt_hwrm_reserve_rings(struct bnxt *bp, int tx, int rx, int grp, 6361 int cp, int stat, int vnic) 6362 { 6363 if (BNXT_PF(bp)) 6364 return bnxt_hwrm_reserve_pf_rings(bp, tx, rx, grp, cp, stat, 6365 vnic); 6366 else 6367 return bnxt_hwrm_reserve_vf_rings(bp, tx, rx, grp, cp, stat, 6368 vnic); 6369 } 6370 6371 int bnxt_nq_rings_in_use(struct bnxt *bp) 6372 { 6373 int cp = bp->cp_nr_rings; 6374 int ulp_msix, ulp_base; 6375 6376 ulp_msix = bnxt_get_ulp_msix_num(bp); 6377 if (ulp_msix) { 6378 ulp_base = bnxt_get_ulp_msix_base(bp); 6379 cp += ulp_msix; 6380 if ((ulp_base + ulp_msix) > cp) 6381 cp = ulp_base + ulp_msix; 6382 } 6383 return cp; 6384 } 6385 6386 static int bnxt_cp_rings_in_use(struct bnxt *bp) 6387 { 6388 int cp; 6389 6390 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 6391 return bnxt_nq_rings_in_use(bp); 6392 6393 cp = bp->tx_nr_rings + bp->rx_nr_rings; 6394 return cp; 6395 } 6396 6397 static int bnxt_get_func_stat_ctxs(struct bnxt *bp) 6398 { 6399 int ulp_stat = bnxt_get_ulp_stat_ctxs(bp); 6400 int cp = bp->cp_nr_rings; 6401 6402 if (!ulp_stat) 6403 return cp; 6404 6405 if (bnxt_nq_rings_in_use(bp) > cp + bnxt_get_ulp_msix_num(bp)) 6406 return bnxt_get_ulp_msix_base(bp) + ulp_stat; 6407 6408 return cp + ulp_stat; 6409 } 6410 6411 /* Check if a default RSS map needs to be setup. This function is only 6412 * used on older firmware that does not require reserving RX rings. 6413 */ 6414 static void bnxt_check_rss_tbl_no_rmgr(struct bnxt *bp) 6415 { 6416 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 6417 6418 /* The RSS map is valid for RX rings set to resv_rx_rings */ 6419 if (hw_resc->resv_rx_rings != bp->rx_nr_rings) { 6420 hw_resc->resv_rx_rings = bp->rx_nr_rings; 6421 if (!netif_is_rxfh_configured(bp->dev)) 6422 bnxt_set_dflt_rss_indir_tbl(bp); 6423 } 6424 } 6425 6426 static bool bnxt_need_reserve_rings(struct bnxt *bp) 6427 { 6428 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 6429 int cp = bnxt_cp_rings_in_use(bp); 6430 int nq = bnxt_nq_rings_in_use(bp); 6431 int rx = bp->rx_nr_rings, stat; 6432 int vnic = 1, grp = rx; 6433 6434 if (hw_resc->resv_tx_rings != bp->tx_nr_rings && 6435 bp->hwrm_spec_code >= 0x10601) 6436 return true; 6437 6438 /* Old firmware does not need RX ring reservations but we still 6439 * need to setup a default RSS map when needed. With new firmware 6440 * we go through RX ring reservations first and then set up the 6441 * RSS map for the successfully reserved RX rings when needed. 6442 */ 6443 if (!BNXT_NEW_RM(bp)) { 6444 bnxt_check_rss_tbl_no_rmgr(bp); 6445 return false; 6446 } 6447 if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5)) 6448 vnic = rx + 1; 6449 if (bp->flags & BNXT_FLAG_AGG_RINGS) 6450 rx <<= 1; 6451 stat = bnxt_get_func_stat_ctxs(bp); 6452 if (hw_resc->resv_rx_rings != rx || hw_resc->resv_cp_rings != cp || 6453 hw_resc->resv_vnics != vnic || hw_resc->resv_stat_ctxs != stat || 6454 (hw_resc->resv_hw_ring_grps != grp && 6455 !(bp->flags & BNXT_FLAG_CHIP_P5))) 6456 return true; 6457 if ((bp->flags & BNXT_FLAG_CHIP_P5) && BNXT_PF(bp) && 6458 hw_resc->resv_irqs != nq) 6459 return true; 6460 return false; 6461 } 6462 6463 static int __bnxt_reserve_rings(struct bnxt *bp) 6464 { 6465 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 6466 int cp = bnxt_nq_rings_in_use(bp); 6467 int tx = bp->tx_nr_rings; 6468 int rx = bp->rx_nr_rings; 6469 int grp, rx_rings, rc; 6470 int vnic = 1, stat; 6471 bool sh = false; 6472 6473 if (!bnxt_need_reserve_rings(bp)) 6474 return 0; 6475 6476 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 6477 sh = true; 6478 if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5)) 6479 vnic = rx + 1; 6480 if (bp->flags & BNXT_FLAG_AGG_RINGS) 6481 rx <<= 1; 6482 grp = bp->rx_nr_rings; 6483 stat = bnxt_get_func_stat_ctxs(bp); 6484 6485 rc = bnxt_hwrm_reserve_rings(bp, tx, rx, grp, cp, stat, vnic); 6486 if (rc) 6487 return rc; 6488 6489 tx = hw_resc->resv_tx_rings; 6490 if (BNXT_NEW_RM(bp)) { 6491 rx = hw_resc->resv_rx_rings; 6492 cp = hw_resc->resv_irqs; 6493 grp = hw_resc->resv_hw_ring_grps; 6494 vnic = hw_resc->resv_vnics; 6495 stat = hw_resc->resv_stat_ctxs; 6496 } 6497 6498 rx_rings = rx; 6499 if (bp->flags & BNXT_FLAG_AGG_RINGS) { 6500 if (rx >= 2) { 6501 rx_rings = rx >> 1; 6502 } else { 6503 if (netif_running(bp->dev)) 6504 return -ENOMEM; 6505 6506 bp->flags &= ~BNXT_FLAG_AGG_RINGS; 6507 bp->flags |= BNXT_FLAG_NO_AGG_RINGS; 6508 bp->dev->hw_features &= ~NETIF_F_LRO; 6509 bp->dev->features &= ~NETIF_F_LRO; 6510 bnxt_set_ring_params(bp); 6511 } 6512 } 6513 rx_rings = min_t(int, rx_rings, grp); 6514 cp = min_t(int, cp, bp->cp_nr_rings); 6515 if (stat > bnxt_get_ulp_stat_ctxs(bp)) 6516 stat -= bnxt_get_ulp_stat_ctxs(bp); 6517 cp = min_t(int, cp, stat); 6518 rc = bnxt_trim_rings(bp, &rx_rings, &tx, cp, sh); 6519 if (bp->flags & BNXT_FLAG_AGG_RINGS) 6520 rx = rx_rings << 1; 6521 cp = sh ? max_t(int, tx, rx_rings) : tx + rx_rings; 6522 bp->tx_nr_rings = tx; 6523 6524 /* If we cannot reserve all the RX rings, reset the RSS map only 6525 * if absolutely necessary 6526 */ 6527 if (rx_rings != bp->rx_nr_rings) { 6528 netdev_warn(bp->dev, "Able to reserve only %d out of %d requested RX rings\n", 6529 rx_rings, bp->rx_nr_rings); 6530 if (netif_is_rxfh_configured(bp->dev) && 6531 (bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings) != 6532 bnxt_get_nr_rss_ctxs(bp, rx_rings) || 6533 bnxt_get_max_rss_ring(bp) >= rx_rings)) { 6534 netdev_warn(bp->dev, "RSS table entries reverting to default\n"); 6535 bp->dev->priv_flags &= ~IFF_RXFH_CONFIGURED; 6536 } 6537 } 6538 bp->rx_nr_rings = rx_rings; 6539 bp->cp_nr_rings = cp; 6540 6541 if (!tx || !rx || !cp || !grp || !vnic || !stat) 6542 return -ENOMEM; 6543 6544 if (!netif_is_rxfh_configured(bp->dev)) 6545 bnxt_set_dflt_rss_indir_tbl(bp); 6546 6547 return rc; 6548 } 6549 6550 static int bnxt_hwrm_check_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6551 int ring_grps, int cp_rings, int stats, 6552 int vnics) 6553 { 6554 struct hwrm_func_vf_cfg_input *req; 6555 u32 flags; 6556 6557 if (!BNXT_NEW_RM(bp)) 6558 return 0; 6559 6560 req = __bnxt_hwrm_reserve_vf_rings(bp, tx_rings, rx_rings, ring_grps, 6561 cp_rings, stats, vnics); 6562 flags = FUNC_VF_CFG_REQ_FLAGS_TX_ASSETS_TEST | 6563 FUNC_VF_CFG_REQ_FLAGS_RX_ASSETS_TEST | 6564 FUNC_VF_CFG_REQ_FLAGS_CMPL_ASSETS_TEST | 6565 FUNC_VF_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST | 6566 FUNC_VF_CFG_REQ_FLAGS_VNIC_ASSETS_TEST | 6567 FUNC_VF_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST; 6568 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 6569 flags |= FUNC_VF_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST; 6570 6571 req->flags = cpu_to_le32(flags); 6572 return hwrm_req_send_silent(bp, req); 6573 } 6574 6575 static int bnxt_hwrm_check_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6576 int ring_grps, int cp_rings, int stats, 6577 int vnics) 6578 { 6579 struct hwrm_func_cfg_input *req; 6580 u32 flags; 6581 6582 req = __bnxt_hwrm_reserve_pf_rings(bp, tx_rings, rx_rings, ring_grps, 6583 cp_rings, stats, vnics); 6584 flags = FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST; 6585 if (BNXT_NEW_RM(bp)) { 6586 flags |= FUNC_CFG_REQ_FLAGS_RX_ASSETS_TEST | 6587 FUNC_CFG_REQ_FLAGS_CMPL_ASSETS_TEST | 6588 FUNC_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST | 6589 FUNC_CFG_REQ_FLAGS_VNIC_ASSETS_TEST; 6590 if (bp->flags & BNXT_FLAG_CHIP_P5) 6591 flags |= FUNC_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST | 6592 FUNC_CFG_REQ_FLAGS_NQ_ASSETS_TEST; 6593 else 6594 flags |= FUNC_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST; 6595 } 6596 6597 req->flags = cpu_to_le32(flags); 6598 return hwrm_req_send_silent(bp, req); 6599 } 6600 6601 static int bnxt_hwrm_check_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6602 int ring_grps, int cp_rings, int stats, 6603 int vnics) 6604 { 6605 if (bp->hwrm_spec_code < 0x10801) 6606 return 0; 6607 6608 if (BNXT_PF(bp)) 6609 return bnxt_hwrm_check_pf_rings(bp, tx_rings, rx_rings, 6610 ring_grps, cp_rings, stats, 6611 vnics); 6612 6613 return bnxt_hwrm_check_vf_rings(bp, tx_rings, rx_rings, ring_grps, 6614 cp_rings, stats, vnics); 6615 } 6616 6617 static void bnxt_hwrm_coal_params_qcaps(struct bnxt *bp) 6618 { 6619 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 6620 struct hwrm_ring_aggint_qcaps_output *resp; 6621 struct hwrm_ring_aggint_qcaps_input *req; 6622 int rc; 6623 6624 coal_cap->cmpl_params = BNXT_LEGACY_COAL_CMPL_PARAMS; 6625 coal_cap->num_cmpl_dma_aggr_max = 63; 6626 coal_cap->num_cmpl_dma_aggr_during_int_max = 63; 6627 coal_cap->cmpl_aggr_dma_tmr_max = 65535; 6628 coal_cap->cmpl_aggr_dma_tmr_during_int_max = 65535; 6629 coal_cap->int_lat_tmr_min_max = 65535; 6630 coal_cap->int_lat_tmr_max_max = 65535; 6631 coal_cap->num_cmpl_aggr_int_max = 65535; 6632 coal_cap->timer_units = 80; 6633 6634 if (bp->hwrm_spec_code < 0x10902) 6635 return; 6636 6637 if (hwrm_req_init(bp, req, HWRM_RING_AGGINT_QCAPS)) 6638 return; 6639 6640 resp = hwrm_req_hold(bp, req); 6641 rc = hwrm_req_send_silent(bp, req); 6642 if (!rc) { 6643 coal_cap->cmpl_params = le32_to_cpu(resp->cmpl_params); 6644 coal_cap->nq_params = le32_to_cpu(resp->nq_params); 6645 coal_cap->num_cmpl_dma_aggr_max = 6646 le16_to_cpu(resp->num_cmpl_dma_aggr_max); 6647 coal_cap->num_cmpl_dma_aggr_during_int_max = 6648 le16_to_cpu(resp->num_cmpl_dma_aggr_during_int_max); 6649 coal_cap->cmpl_aggr_dma_tmr_max = 6650 le16_to_cpu(resp->cmpl_aggr_dma_tmr_max); 6651 coal_cap->cmpl_aggr_dma_tmr_during_int_max = 6652 le16_to_cpu(resp->cmpl_aggr_dma_tmr_during_int_max); 6653 coal_cap->int_lat_tmr_min_max = 6654 le16_to_cpu(resp->int_lat_tmr_min_max); 6655 coal_cap->int_lat_tmr_max_max = 6656 le16_to_cpu(resp->int_lat_tmr_max_max); 6657 coal_cap->num_cmpl_aggr_int_max = 6658 le16_to_cpu(resp->num_cmpl_aggr_int_max); 6659 coal_cap->timer_units = le16_to_cpu(resp->timer_units); 6660 } 6661 hwrm_req_drop(bp, req); 6662 } 6663 6664 static u16 bnxt_usec_to_coal_tmr(struct bnxt *bp, u16 usec) 6665 { 6666 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 6667 6668 return usec * 1000 / coal_cap->timer_units; 6669 } 6670 6671 static void bnxt_hwrm_set_coal_params(struct bnxt *bp, 6672 struct bnxt_coal *hw_coal, 6673 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req) 6674 { 6675 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 6676 u16 val, tmr, max, flags = hw_coal->flags; 6677 u32 cmpl_params = coal_cap->cmpl_params; 6678 6679 max = hw_coal->bufs_per_record * 128; 6680 if (hw_coal->budget) 6681 max = hw_coal->bufs_per_record * hw_coal->budget; 6682 max = min_t(u16, max, coal_cap->num_cmpl_aggr_int_max); 6683 6684 val = clamp_t(u16, hw_coal->coal_bufs, 1, max); 6685 req->num_cmpl_aggr_int = cpu_to_le16(val); 6686 6687 val = min_t(u16, val, coal_cap->num_cmpl_dma_aggr_max); 6688 req->num_cmpl_dma_aggr = cpu_to_le16(val); 6689 6690 val = clamp_t(u16, hw_coal->coal_bufs_irq, 1, 6691 coal_cap->num_cmpl_dma_aggr_during_int_max); 6692 req->num_cmpl_dma_aggr_during_int = cpu_to_le16(val); 6693 6694 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks); 6695 tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_max_max); 6696 req->int_lat_tmr_max = cpu_to_le16(tmr); 6697 6698 /* min timer set to 1/2 of interrupt timer */ 6699 if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_INT_LAT_TMR_MIN) { 6700 val = tmr / 2; 6701 val = clamp_t(u16, val, 1, coal_cap->int_lat_tmr_min_max); 6702 req->int_lat_tmr_min = cpu_to_le16(val); 6703 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE); 6704 } 6705 6706 /* buf timer set to 1/4 of interrupt timer */ 6707 val = clamp_t(u16, tmr / 4, 1, coal_cap->cmpl_aggr_dma_tmr_max); 6708 req->cmpl_aggr_dma_tmr = cpu_to_le16(val); 6709 6710 if (cmpl_params & 6711 RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_NUM_CMPL_DMA_AGGR_DURING_INT) { 6712 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks_irq); 6713 val = clamp_t(u16, tmr, 1, 6714 coal_cap->cmpl_aggr_dma_tmr_during_int_max); 6715 req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(val); 6716 req->enables |= 6717 cpu_to_le16(BNXT_COAL_CMPL_AGGR_TMR_DURING_INT_ENABLE); 6718 } 6719 6720 if ((cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_RING_IDLE) && 6721 hw_coal->idle_thresh && hw_coal->coal_ticks < hw_coal->idle_thresh) 6722 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE; 6723 req->flags = cpu_to_le16(flags); 6724 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_ENABLES); 6725 } 6726 6727 static int __bnxt_hwrm_set_coal_nq(struct bnxt *bp, struct bnxt_napi *bnapi, 6728 struct bnxt_coal *hw_coal) 6729 { 6730 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req; 6731 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 6732 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 6733 u32 nq_params = coal_cap->nq_params; 6734 u16 tmr; 6735 int rc; 6736 6737 if (!(nq_params & RING_AGGINT_QCAPS_RESP_NQ_PARAMS_INT_LAT_TMR_MIN)) 6738 return 0; 6739 6740 rc = hwrm_req_init(bp, req, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS); 6741 if (rc) 6742 return rc; 6743 6744 req->ring_id = cpu_to_le16(cpr->cp_ring_struct.fw_ring_id); 6745 req->flags = 6746 cpu_to_le16(RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_IS_NQ); 6747 6748 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks) / 2; 6749 tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_min_max); 6750 req->int_lat_tmr_min = cpu_to_le16(tmr); 6751 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE); 6752 return hwrm_req_send(bp, req); 6753 } 6754 6755 int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi) 6756 { 6757 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req_rx; 6758 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 6759 struct bnxt_coal coal; 6760 int rc; 6761 6762 /* Tick values in micro seconds. 6763 * 1 coal_buf x bufs_per_record = 1 completion record. 6764 */ 6765 memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal)); 6766 6767 coal.coal_ticks = cpr->rx_ring_coal.coal_ticks; 6768 coal.coal_bufs = cpr->rx_ring_coal.coal_bufs; 6769 6770 if (!bnapi->rx_ring) 6771 return -ENODEV; 6772 6773 rc = hwrm_req_init(bp, req_rx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS); 6774 if (rc) 6775 return rc; 6776 6777 bnxt_hwrm_set_coal_params(bp, &coal, req_rx); 6778 6779 req_rx->ring_id = cpu_to_le16(bnxt_cp_ring_for_rx(bp, bnapi->rx_ring)); 6780 6781 return hwrm_req_send(bp, req_rx); 6782 } 6783 6784 int bnxt_hwrm_set_coal(struct bnxt *bp) 6785 { 6786 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req_rx, *req_tx, 6787 *req; 6788 int i, rc; 6789 6790 rc = hwrm_req_init(bp, req_rx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS); 6791 if (rc) 6792 return rc; 6793 6794 rc = hwrm_req_init(bp, req_tx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS); 6795 if (rc) { 6796 hwrm_req_drop(bp, req_rx); 6797 return rc; 6798 } 6799 6800 bnxt_hwrm_set_coal_params(bp, &bp->rx_coal, req_rx); 6801 bnxt_hwrm_set_coal_params(bp, &bp->tx_coal, req_tx); 6802 6803 hwrm_req_hold(bp, req_rx); 6804 hwrm_req_hold(bp, req_tx); 6805 for (i = 0; i < bp->cp_nr_rings; i++) { 6806 struct bnxt_napi *bnapi = bp->bnapi[i]; 6807 struct bnxt_coal *hw_coal; 6808 u16 ring_id; 6809 6810 req = req_rx; 6811 if (!bnapi->rx_ring) { 6812 ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring); 6813 req = req_tx; 6814 } else { 6815 ring_id = bnxt_cp_ring_for_rx(bp, bnapi->rx_ring); 6816 } 6817 req->ring_id = cpu_to_le16(ring_id); 6818 6819 rc = hwrm_req_send(bp, req); 6820 if (rc) 6821 break; 6822 6823 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 6824 continue; 6825 6826 if (bnapi->rx_ring && bnapi->tx_ring) { 6827 req = req_tx; 6828 ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring); 6829 req->ring_id = cpu_to_le16(ring_id); 6830 rc = hwrm_req_send(bp, req); 6831 if (rc) 6832 break; 6833 } 6834 if (bnapi->rx_ring) 6835 hw_coal = &bp->rx_coal; 6836 else 6837 hw_coal = &bp->tx_coal; 6838 __bnxt_hwrm_set_coal_nq(bp, bnapi, hw_coal); 6839 } 6840 hwrm_req_drop(bp, req_rx); 6841 hwrm_req_drop(bp, req_tx); 6842 return rc; 6843 } 6844 6845 static void bnxt_hwrm_stat_ctx_free(struct bnxt *bp) 6846 { 6847 struct hwrm_stat_ctx_clr_stats_input *req0 = NULL; 6848 struct hwrm_stat_ctx_free_input *req; 6849 int i; 6850 6851 if (!bp->bnapi) 6852 return; 6853 6854 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 6855 return; 6856 6857 if (hwrm_req_init(bp, req, HWRM_STAT_CTX_FREE)) 6858 return; 6859 if (BNXT_FW_MAJ(bp) <= 20) { 6860 if (hwrm_req_init(bp, req0, HWRM_STAT_CTX_CLR_STATS)) { 6861 hwrm_req_drop(bp, req); 6862 return; 6863 } 6864 hwrm_req_hold(bp, req0); 6865 } 6866 hwrm_req_hold(bp, req); 6867 for (i = 0; i < bp->cp_nr_rings; i++) { 6868 struct bnxt_napi *bnapi = bp->bnapi[i]; 6869 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 6870 6871 if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) { 6872 req->stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id); 6873 if (req0) { 6874 req0->stat_ctx_id = req->stat_ctx_id; 6875 hwrm_req_send(bp, req0); 6876 } 6877 hwrm_req_send(bp, req); 6878 6879 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID; 6880 } 6881 } 6882 hwrm_req_drop(bp, req); 6883 if (req0) 6884 hwrm_req_drop(bp, req0); 6885 } 6886 6887 static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp) 6888 { 6889 struct hwrm_stat_ctx_alloc_output *resp; 6890 struct hwrm_stat_ctx_alloc_input *req; 6891 int rc, i; 6892 6893 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 6894 return 0; 6895 6896 rc = hwrm_req_init(bp, req, HWRM_STAT_CTX_ALLOC); 6897 if (rc) 6898 return rc; 6899 6900 req->stats_dma_length = cpu_to_le16(bp->hw_ring_stats_size); 6901 req->update_period_ms = cpu_to_le32(bp->stats_coal_ticks / 1000); 6902 6903 resp = hwrm_req_hold(bp, req); 6904 for (i = 0; i < bp->cp_nr_rings; i++) { 6905 struct bnxt_napi *bnapi = bp->bnapi[i]; 6906 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 6907 6908 req->stats_dma_addr = cpu_to_le64(cpr->stats.hw_stats_map); 6909 6910 rc = hwrm_req_send(bp, req); 6911 if (rc) 6912 break; 6913 6914 cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id); 6915 6916 bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id; 6917 } 6918 hwrm_req_drop(bp, req); 6919 return rc; 6920 } 6921 6922 static int bnxt_hwrm_func_qcfg(struct bnxt *bp) 6923 { 6924 struct hwrm_func_qcfg_output *resp; 6925 struct hwrm_func_qcfg_input *req; 6926 u32 min_db_offset = 0; 6927 u16 flags; 6928 int rc; 6929 6930 rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG); 6931 if (rc) 6932 return rc; 6933 6934 req->fid = cpu_to_le16(0xffff); 6935 resp = hwrm_req_hold(bp, req); 6936 rc = hwrm_req_send(bp, req); 6937 if (rc) 6938 goto func_qcfg_exit; 6939 6940 #ifdef CONFIG_BNXT_SRIOV 6941 if (BNXT_VF(bp)) { 6942 struct bnxt_vf_info *vf = &bp->vf; 6943 6944 vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK; 6945 } else { 6946 bp->pf.registered_vfs = le16_to_cpu(resp->registered_vfs); 6947 } 6948 #endif 6949 flags = le16_to_cpu(resp->flags); 6950 if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED | 6951 FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED)) { 6952 bp->fw_cap |= BNXT_FW_CAP_LLDP_AGENT; 6953 if (flags & FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED) 6954 bp->fw_cap |= BNXT_FW_CAP_DCBX_AGENT; 6955 } 6956 if (BNXT_PF(bp) && (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST)) 6957 bp->flags |= BNXT_FLAG_MULTI_HOST; 6958 6959 if (flags & FUNC_QCFG_RESP_FLAGS_RING_MONITOR_ENABLED) 6960 bp->fw_cap |= BNXT_FW_CAP_RING_MONITOR; 6961 6962 switch (resp->port_partition_type) { 6963 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0: 6964 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5: 6965 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0: 6966 bp->port_partition_type = resp->port_partition_type; 6967 break; 6968 } 6969 if (bp->hwrm_spec_code < 0x10707 || 6970 resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEB) 6971 bp->br_mode = BRIDGE_MODE_VEB; 6972 else if (resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEPA) 6973 bp->br_mode = BRIDGE_MODE_VEPA; 6974 else 6975 bp->br_mode = BRIDGE_MODE_UNDEF; 6976 6977 bp->max_mtu = le16_to_cpu(resp->max_mtu_configured); 6978 if (!bp->max_mtu) 6979 bp->max_mtu = BNXT_MAX_MTU; 6980 6981 if (bp->db_size) 6982 goto func_qcfg_exit; 6983 6984 if (bp->flags & BNXT_FLAG_CHIP_P5) { 6985 if (BNXT_PF(bp)) 6986 min_db_offset = DB_PF_OFFSET_P5; 6987 else 6988 min_db_offset = DB_VF_OFFSET_P5; 6989 } 6990 bp->db_size = PAGE_ALIGN(le16_to_cpu(resp->l2_doorbell_bar_size_kb) * 6991 1024); 6992 if (!bp->db_size || bp->db_size > pci_resource_len(bp->pdev, 2) || 6993 bp->db_size <= min_db_offset) 6994 bp->db_size = pci_resource_len(bp->pdev, 2); 6995 6996 func_qcfg_exit: 6997 hwrm_req_drop(bp, req); 6998 return rc; 6999 } 7000 7001 static void bnxt_init_ctx_initializer(struct bnxt_ctx_mem_info *ctx, 7002 struct hwrm_func_backing_store_qcaps_output *resp) 7003 { 7004 struct bnxt_mem_init *mem_init; 7005 u16 init_mask; 7006 u8 init_val; 7007 u8 *offset; 7008 int i; 7009 7010 init_val = resp->ctx_kind_initializer; 7011 init_mask = le16_to_cpu(resp->ctx_init_mask); 7012 offset = &resp->qp_init_offset; 7013 mem_init = &ctx->mem_init[BNXT_CTX_MEM_INIT_QP]; 7014 for (i = 0; i < BNXT_CTX_MEM_INIT_MAX; i++, mem_init++, offset++) { 7015 mem_init->init_val = init_val; 7016 mem_init->offset = BNXT_MEM_INVALID_OFFSET; 7017 if (!init_mask) 7018 continue; 7019 if (i == BNXT_CTX_MEM_INIT_STAT) 7020 offset = &resp->stat_init_offset; 7021 if (init_mask & (1 << i)) 7022 mem_init->offset = *offset * 4; 7023 else 7024 mem_init->init_val = 0; 7025 } 7026 ctx->mem_init[BNXT_CTX_MEM_INIT_QP].size = ctx->qp_entry_size; 7027 ctx->mem_init[BNXT_CTX_MEM_INIT_SRQ].size = ctx->srq_entry_size; 7028 ctx->mem_init[BNXT_CTX_MEM_INIT_CQ].size = ctx->cq_entry_size; 7029 ctx->mem_init[BNXT_CTX_MEM_INIT_VNIC].size = ctx->vnic_entry_size; 7030 ctx->mem_init[BNXT_CTX_MEM_INIT_STAT].size = ctx->stat_entry_size; 7031 ctx->mem_init[BNXT_CTX_MEM_INIT_MRAV].size = ctx->mrav_entry_size; 7032 } 7033 7034 static int bnxt_hwrm_func_backing_store_qcaps(struct bnxt *bp) 7035 { 7036 struct hwrm_func_backing_store_qcaps_output *resp; 7037 struct hwrm_func_backing_store_qcaps_input *req; 7038 int rc; 7039 7040 if (bp->hwrm_spec_code < 0x10902 || BNXT_VF(bp) || bp->ctx) 7041 return 0; 7042 7043 rc = hwrm_req_init(bp, req, HWRM_FUNC_BACKING_STORE_QCAPS); 7044 if (rc) 7045 return rc; 7046 7047 resp = hwrm_req_hold(bp, req); 7048 rc = hwrm_req_send_silent(bp, req); 7049 if (!rc) { 7050 struct bnxt_ctx_pg_info *ctx_pg; 7051 struct bnxt_ctx_mem_info *ctx; 7052 int i, tqm_rings; 7053 7054 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 7055 if (!ctx) { 7056 rc = -ENOMEM; 7057 goto ctx_err; 7058 } 7059 ctx->qp_max_entries = le32_to_cpu(resp->qp_max_entries); 7060 ctx->qp_min_qp1_entries = le16_to_cpu(resp->qp_min_qp1_entries); 7061 ctx->qp_max_l2_entries = le16_to_cpu(resp->qp_max_l2_entries); 7062 ctx->qp_entry_size = le16_to_cpu(resp->qp_entry_size); 7063 ctx->srq_max_l2_entries = le16_to_cpu(resp->srq_max_l2_entries); 7064 ctx->srq_max_entries = le32_to_cpu(resp->srq_max_entries); 7065 ctx->srq_entry_size = le16_to_cpu(resp->srq_entry_size); 7066 ctx->cq_max_l2_entries = le16_to_cpu(resp->cq_max_l2_entries); 7067 ctx->cq_max_entries = le32_to_cpu(resp->cq_max_entries); 7068 ctx->cq_entry_size = le16_to_cpu(resp->cq_entry_size); 7069 ctx->vnic_max_vnic_entries = 7070 le16_to_cpu(resp->vnic_max_vnic_entries); 7071 ctx->vnic_max_ring_table_entries = 7072 le16_to_cpu(resp->vnic_max_ring_table_entries); 7073 ctx->vnic_entry_size = le16_to_cpu(resp->vnic_entry_size); 7074 ctx->stat_max_entries = le32_to_cpu(resp->stat_max_entries); 7075 ctx->stat_entry_size = le16_to_cpu(resp->stat_entry_size); 7076 ctx->tqm_entry_size = le16_to_cpu(resp->tqm_entry_size); 7077 ctx->tqm_min_entries_per_ring = 7078 le32_to_cpu(resp->tqm_min_entries_per_ring); 7079 ctx->tqm_max_entries_per_ring = 7080 le32_to_cpu(resp->tqm_max_entries_per_ring); 7081 ctx->tqm_entries_multiple = resp->tqm_entries_multiple; 7082 if (!ctx->tqm_entries_multiple) 7083 ctx->tqm_entries_multiple = 1; 7084 ctx->mrav_max_entries = le32_to_cpu(resp->mrav_max_entries); 7085 ctx->mrav_entry_size = le16_to_cpu(resp->mrav_entry_size); 7086 ctx->mrav_num_entries_units = 7087 le16_to_cpu(resp->mrav_num_entries_units); 7088 ctx->tim_entry_size = le16_to_cpu(resp->tim_entry_size); 7089 ctx->tim_max_entries = le32_to_cpu(resp->tim_max_entries); 7090 7091 bnxt_init_ctx_initializer(ctx, resp); 7092 7093 ctx->tqm_fp_rings_count = resp->tqm_fp_rings_count; 7094 if (!ctx->tqm_fp_rings_count) 7095 ctx->tqm_fp_rings_count = bp->max_q; 7096 else if (ctx->tqm_fp_rings_count > BNXT_MAX_TQM_FP_RINGS) 7097 ctx->tqm_fp_rings_count = BNXT_MAX_TQM_FP_RINGS; 7098 7099 tqm_rings = ctx->tqm_fp_rings_count + BNXT_MAX_TQM_SP_RINGS; 7100 ctx_pg = kcalloc(tqm_rings, sizeof(*ctx_pg), GFP_KERNEL); 7101 if (!ctx_pg) { 7102 kfree(ctx); 7103 rc = -ENOMEM; 7104 goto ctx_err; 7105 } 7106 for (i = 0; i < tqm_rings; i++, ctx_pg++) 7107 ctx->tqm_mem[i] = ctx_pg; 7108 bp->ctx = ctx; 7109 } else { 7110 rc = 0; 7111 } 7112 ctx_err: 7113 hwrm_req_drop(bp, req); 7114 return rc; 7115 } 7116 7117 static void bnxt_hwrm_set_pg_attr(struct bnxt_ring_mem_info *rmem, u8 *pg_attr, 7118 __le64 *pg_dir) 7119 { 7120 if (!rmem->nr_pages) 7121 return; 7122 7123 BNXT_SET_CTX_PAGE_ATTR(*pg_attr); 7124 if (rmem->depth >= 1) { 7125 if (rmem->depth == 2) 7126 *pg_attr |= 2; 7127 else 7128 *pg_attr |= 1; 7129 *pg_dir = cpu_to_le64(rmem->pg_tbl_map); 7130 } else { 7131 *pg_dir = cpu_to_le64(rmem->dma_arr[0]); 7132 } 7133 } 7134 7135 #define FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES \ 7136 (FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP | \ 7137 FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ | \ 7138 FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ | \ 7139 FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC | \ 7140 FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) 7141 7142 static int bnxt_hwrm_func_backing_store_cfg(struct bnxt *bp, u32 enables) 7143 { 7144 struct hwrm_func_backing_store_cfg_input *req; 7145 struct bnxt_ctx_mem_info *ctx = bp->ctx; 7146 struct bnxt_ctx_pg_info *ctx_pg; 7147 void **__req = (void **)&req; 7148 u32 req_len = sizeof(*req); 7149 __le32 *num_entries; 7150 __le64 *pg_dir; 7151 u32 flags = 0; 7152 u8 *pg_attr; 7153 u32 ena; 7154 int rc; 7155 int i; 7156 7157 if (!ctx) 7158 return 0; 7159 7160 if (req_len > bp->hwrm_max_ext_req_len) 7161 req_len = BNXT_BACKING_STORE_CFG_LEGACY_LEN; 7162 rc = __hwrm_req_init(bp, __req, HWRM_FUNC_BACKING_STORE_CFG, req_len); 7163 if (rc) 7164 return rc; 7165 7166 req->enables = cpu_to_le32(enables); 7167 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP) { 7168 ctx_pg = &ctx->qp_mem; 7169 req->qp_num_entries = cpu_to_le32(ctx_pg->entries); 7170 req->qp_num_qp1_entries = cpu_to_le16(ctx->qp_min_qp1_entries); 7171 req->qp_num_l2_entries = cpu_to_le16(ctx->qp_max_l2_entries); 7172 req->qp_entry_size = cpu_to_le16(ctx->qp_entry_size); 7173 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 7174 &req->qpc_pg_size_qpc_lvl, 7175 &req->qpc_page_dir); 7176 } 7177 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ) { 7178 ctx_pg = &ctx->srq_mem; 7179 req->srq_num_entries = cpu_to_le32(ctx_pg->entries); 7180 req->srq_num_l2_entries = cpu_to_le16(ctx->srq_max_l2_entries); 7181 req->srq_entry_size = cpu_to_le16(ctx->srq_entry_size); 7182 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 7183 &req->srq_pg_size_srq_lvl, 7184 &req->srq_page_dir); 7185 } 7186 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ) { 7187 ctx_pg = &ctx->cq_mem; 7188 req->cq_num_entries = cpu_to_le32(ctx_pg->entries); 7189 req->cq_num_l2_entries = cpu_to_le16(ctx->cq_max_l2_entries); 7190 req->cq_entry_size = cpu_to_le16(ctx->cq_entry_size); 7191 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 7192 &req->cq_pg_size_cq_lvl, 7193 &req->cq_page_dir); 7194 } 7195 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC) { 7196 ctx_pg = &ctx->vnic_mem; 7197 req->vnic_num_vnic_entries = 7198 cpu_to_le16(ctx->vnic_max_vnic_entries); 7199 req->vnic_num_ring_table_entries = 7200 cpu_to_le16(ctx->vnic_max_ring_table_entries); 7201 req->vnic_entry_size = cpu_to_le16(ctx->vnic_entry_size); 7202 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 7203 &req->vnic_pg_size_vnic_lvl, 7204 &req->vnic_page_dir); 7205 } 7206 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) { 7207 ctx_pg = &ctx->stat_mem; 7208 req->stat_num_entries = cpu_to_le32(ctx->stat_max_entries); 7209 req->stat_entry_size = cpu_to_le16(ctx->stat_entry_size); 7210 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 7211 &req->stat_pg_size_stat_lvl, 7212 &req->stat_page_dir); 7213 } 7214 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV) { 7215 ctx_pg = &ctx->mrav_mem; 7216 req->mrav_num_entries = cpu_to_le32(ctx_pg->entries); 7217 if (ctx->mrav_num_entries_units) 7218 flags |= 7219 FUNC_BACKING_STORE_CFG_REQ_FLAGS_MRAV_RESERVATION_SPLIT; 7220 req->mrav_entry_size = cpu_to_le16(ctx->mrav_entry_size); 7221 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 7222 &req->mrav_pg_size_mrav_lvl, 7223 &req->mrav_page_dir); 7224 } 7225 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM) { 7226 ctx_pg = &ctx->tim_mem; 7227 req->tim_num_entries = cpu_to_le32(ctx_pg->entries); 7228 req->tim_entry_size = cpu_to_le16(ctx->tim_entry_size); 7229 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 7230 &req->tim_pg_size_tim_lvl, 7231 &req->tim_page_dir); 7232 } 7233 for (i = 0, num_entries = &req->tqm_sp_num_entries, 7234 pg_attr = &req->tqm_sp_pg_size_tqm_sp_lvl, 7235 pg_dir = &req->tqm_sp_page_dir, 7236 ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP; 7237 i < BNXT_MAX_TQM_RINGS; 7238 i++, num_entries++, pg_attr++, pg_dir++, ena <<= 1) { 7239 if (!(enables & ena)) 7240 continue; 7241 7242 req->tqm_entry_size = cpu_to_le16(ctx->tqm_entry_size); 7243 ctx_pg = ctx->tqm_mem[i]; 7244 *num_entries = cpu_to_le32(ctx_pg->entries); 7245 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, pg_attr, pg_dir); 7246 } 7247 req->flags = cpu_to_le32(flags); 7248 return hwrm_req_send(bp, req); 7249 } 7250 7251 static int bnxt_alloc_ctx_mem_blk(struct bnxt *bp, 7252 struct bnxt_ctx_pg_info *ctx_pg) 7253 { 7254 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem; 7255 7256 rmem->page_size = BNXT_PAGE_SIZE; 7257 rmem->pg_arr = ctx_pg->ctx_pg_arr; 7258 rmem->dma_arr = ctx_pg->ctx_dma_arr; 7259 rmem->flags = BNXT_RMEM_VALID_PTE_FLAG; 7260 if (rmem->depth >= 1) 7261 rmem->flags |= BNXT_RMEM_USE_FULL_PAGE_FLAG; 7262 return bnxt_alloc_ring(bp, rmem); 7263 } 7264 7265 static int bnxt_alloc_ctx_pg_tbls(struct bnxt *bp, 7266 struct bnxt_ctx_pg_info *ctx_pg, u32 mem_size, 7267 u8 depth, struct bnxt_mem_init *mem_init) 7268 { 7269 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem; 7270 int rc; 7271 7272 if (!mem_size) 7273 return -EINVAL; 7274 7275 ctx_pg->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE); 7276 if (ctx_pg->nr_pages > MAX_CTX_TOTAL_PAGES) { 7277 ctx_pg->nr_pages = 0; 7278 return -EINVAL; 7279 } 7280 if (ctx_pg->nr_pages > MAX_CTX_PAGES || depth > 1) { 7281 int nr_tbls, i; 7282 7283 rmem->depth = 2; 7284 ctx_pg->ctx_pg_tbl = kcalloc(MAX_CTX_PAGES, sizeof(ctx_pg), 7285 GFP_KERNEL); 7286 if (!ctx_pg->ctx_pg_tbl) 7287 return -ENOMEM; 7288 nr_tbls = DIV_ROUND_UP(ctx_pg->nr_pages, MAX_CTX_PAGES); 7289 rmem->nr_pages = nr_tbls; 7290 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg); 7291 if (rc) 7292 return rc; 7293 for (i = 0; i < nr_tbls; i++) { 7294 struct bnxt_ctx_pg_info *pg_tbl; 7295 7296 pg_tbl = kzalloc(sizeof(*pg_tbl), GFP_KERNEL); 7297 if (!pg_tbl) 7298 return -ENOMEM; 7299 ctx_pg->ctx_pg_tbl[i] = pg_tbl; 7300 rmem = &pg_tbl->ring_mem; 7301 rmem->pg_tbl = ctx_pg->ctx_pg_arr[i]; 7302 rmem->pg_tbl_map = ctx_pg->ctx_dma_arr[i]; 7303 rmem->depth = 1; 7304 rmem->nr_pages = MAX_CTX_PAGES; 7305 rmem->mem_init = mem_init; 7306 if (i == (nr_tbls - 1)) { 7307 int rem = ctx_pg->nr_pages % MAX_CTX_PAGES; 7308 7309 if (rem) 7310 rmem->nr_pages = rem; 7311 } 7312 rc = bnxt_alloc_ctx_mem_blk(bp, pg_tbl); 7313 if (rc) 7314 break; 7315 } 7316 } else { 7317 rmem->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE); 7318 if (rmem->nr_pages > 1 || depth) 7319 rmem->depth = 1; 7320 rmem->mem_init = mem_init; 7321 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg); 7322 } 7323 return rc; 7324 } 7325 7326 static void bnxt_free_ctx_pg_tbls(struct bnxt *bp, 7327 struct bnxt_ctx_pg_info *ctx_pg) 7328 { 7329 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem; 7330 7331 if (rmem->depth > 1 || ctx_pg->nr_pages > MAX_CTX_PAGES || 7332 ctx_pg->ctx_pg_tbl) { 7333 int i, nr_tbls = rmem->nr_pages; 7334 7335 for (i = 0; i < nr_tbls; i++) { 7336 struct bnxt_ctx_pg_info *pg_tbl; 7337 struct bnxt_ring_mem_info *rmem2; 7338 7339 pg_tbl = ctx_pg->ctx_pg_tbl[i]; 7340 if (!pg_tbl) 7341 continue; 7342 rmem2 = &pg_tbl->ring_mem; 7343 bnxt_free_ring(bp, rmem2); 7344 ctx_pg->ctx_pg_arr[i] = NULL; 7345 kfree(pg_tbl); 7346 ctx_pg->ctx_pg_tbl[i] = NULL; 7347 } 7348 kfree(ctx_pg->ctx_pg_tbl); 7349 ctx_pg->ctx_pg_tbl = NULL; 7350 } 7351 bnxt_free_ring(bp, rmem); 7352 ctx_pg->nr_pages = 0; 7353 } 7354 7355 void bnxt_free_ctx_mem(struct bnxt *bp) 7356 { 7357 struct bnxt_ctx_mem_info *ctx = bp->ctx; 7358 int i; 7359 7360 if (!ctx) 7361 return; 7362 7363 if (ctx->tqm_mem[0]) { 7364 for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++) 7365 bnxt_free_ctx_pg_tbls(bp, ctx->tqm_mem[i]); 7366 kfree(ctx->tqm_mem[0]); 7367 ctx->tqm_mem[0] = NULL; 7368 } 7369 7370 bnxt_free_ctx_pg_tbls(bp, &ctx->tim_mem); 7371 bnxt_free_ctx_pg_tbls(bp, &ctx->mrav_mem); 7372 bnxt_free_ctx_pg_tbls(bp, &ctx->stat_mem); 7373 bnxt_free_ctx_pg_tbls(bp, &ctx->vnic_mem); 7374 bnxt_free_ctx_pg_tbls(bp, &ctx->cq_mem); 7375 bnxt_free_ctx_pg_tbls(bp, &ctx->srq_mem); 7376 bnxt_free_ctx_pg_tbls(bp, &ctx->qp_mem); 7377 ctx->flags &= ~BNXT_CTX_FLAG_INITED; 7378 } 7379 7380 static int bnxt_alloc_ctx_mem(struct bnxt *bp) 7381 { 7382 struct bnxt_ctx_pg_info *ctx_pg; 7383 struct bnxt_ctx_mem_info *ctx; 7384 struct bnxt_mem_init *init; 7385 u32 mem_size, ena, entries; 7386 u32 entries_sp, min; 7387 u32 num_mr, num_ah; 7388 u32 extra_srqs = 0; 7389 u32 extra_qps = 0; 7390 u8 pg_lvl = 1; 7391 int i, rc; 7392 7393 rc = bnxt_hwrm_func_backing_store_qcaps(bp); 7394 if (rc) { 7395 netdev_err(bp->dev, "Failed querying context mem capability, rc = %d.\n", 7396 rc); 7397 return rc; 7398 } 7399 ctx = bp->ctx; 7400 if (!ctx || (ctx->flags & BNXT_CTX_FLAG_INITED)) 7401 return 0; 7402 7403 if ((bp->flags & BNXT_FLAG_ROCE_CAP) && !is_kdump_kernel()) { 7404 pg_lvl = 2; 7405 extra_qps = 65536; 7406 extra_srqs = 8192; 7407 } 7408 7409 ctx_pg = &ctx->qp_mem; 7410 ctx_pg->entries = ctx->qp_min_qp1_entries + ctx->qp_max_l2_entries + 7411 extra_qps; 7412 if (ctx->qp_entry_size) { 7413 mem_size = ctx->qp_entry_size * ctx_pg->entries; 7414 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_QP]; 7415 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, init); 7416 if (rc) 7417 return rc; 7418 } 7419 7420 ctx_pg = &ctx->srq_mem; 7421 ctx_pg->entries = ctx->srq_max_l2_entries + extra_srqs; 7422 if (ctx->srq_entry_size) { 7423 mem_size = ctx->srq_entry_size * ctx_pg->entries; 7424 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_SRQ]; 7425 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, init); 7426 if (rc) 7427 return rc; 7428 } 7429 7430 ctx_pg = &ctx->cq_mem; 7431 ctx_pg->entries = ctx->cq_max_l2_entries + extra_qps * 2; 7432 if (ctx->cq_entry_size) { 7433 mem_size = ctx->cq_entry_size * ctx_pg->entries; 7434 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_CQ]; 7435 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, init); 7436 if (rc) 7437 return rc; 7438 } 7439 7440 ctx_pg = &ctx->vnic_mem; 7441 ctx_pg->entries = ctx->vnic_max_vnic_entries + 7442 ctx->vnic_max_ring_table_entries; 7443 if (ctx->vnic_entry_size) { 7444 mem_size = ctx->vnic_entry_size * ctx_pg->entries; 7445 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_VNIC]; 7446 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, init); 7447 if (rc) 7448 return rc; 7449 } 7450 7451 ctx_pg = &ctx->stat_mem; 7452 ctx_pg->entries = ctx->stat_max_entries; 7453 if (ctx->stat_entry_size) { 7454 mem_size = ctx->stat_entry_size * ctx_pg->entries; 7455 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_STAT]; 7456 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, init); 7457 if (rc) 7458 return rc; 7459 } 7460 7461 ena = 0; 7462 if (!(bp->flags & BNXT_FLAG_ROCE_CAP)) 7463 goto skip_rdma; 7464 7465 ctx_pg = &ctx->mrav_mem; 7466 /* 128K extra is needed to accommodate static AH context 7467 * allocation by f/w. 7468 */ 7469 num_mr = 1024 * 256; 7470 num_ah = 1024 * 128; 7471 ctx_pg->entries = num_mr + num_ah; 7472 if (ctx->mrav_entry_size) { 7473 mem_size = ctx->mrav_entry_size * ctx_pg->entries; 7474 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_MRAV]; 7475 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 2, init); 7476 if (rc) 7477 return rc; 7478 } 7479 ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV; 7480 if (ctx->mrav_num_entries_units) 7481 ctx_pg->entries = 7482 ((num_mr / ctx->mrav_num_entries_units) << 16) | 7483 (num_ah / ctx->mrav_num_entries_units); 7484 7485 ctx_pg = &ctx->tim_mem; 7486 ctx_pg->entries = ctx->qp_mem.entries; 7487 if (ctx->tim_entry_size) { 7488 mem_size = ctx->tim_entry_size * ctx_pg->entries; 7489 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, NULL); 7490 if (rc) 7491 return rc; 7492 } 7493 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM; 7494 7495 skip_rdma: 7496 min = ctx->tqm_min_entries_per_ring; 7497 entries_sp = ctx->vnic_max_vnic_entries + ctx->qp_max_l2_entries + 7498 2 * (extra_qps + ctx->qp_min_qp1_entries) + min; 7499 entries_sp = roundup(entries_sp, ctx->tqm_entries_multiple); 7500 entries = ctx->qp_max_l2_entries + 2 * (extra_qps + ctx->qp_min_qp1_entries); 7501 entries = roundup(entries, ctx->tqm_entries_multiple); 7502 entries = clamp_t(u32, entries, min, ctx->tqm_max_entries_per_ring); 7503 for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++) { 7504 ctx_pg = ctx->tqm_mem[i]; 7505 ctx_pg->entries = i ? entries : entries_sp; 7506 if (ctx->tqm_entry_size) { 7507 mem_size = ctx->tqm_entry_size * ctx_pg->entries; 7508 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, 7509 NULL); 7510 if (rc) 7511 return rc; 7512 } 7513 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP << i; 7514 } 7515 ena |= FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES; 7516 rc = bnxt_hwrm_func_backing_store_cfg(bp, ena); 7517 if (rc) { 7518 netdev_err(bp->dev, "Failed configuring context mem, rc = %d.\n", 7519 rc); 7520 return rc; 7521 } 7522 ctx->flags |= BNXT_CTX_FLAG_INITED; 7523 return 0; 7524 } 7525 7526 int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp, bool all) 7527 { 7528 struct hwrm_func_resource_qcaps_output *resp; 7529 struct hwrm_func_resource_qcaps_input *req; 7530 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 7531 int rc; 7532 7533 rc = hwrm_req_init(bp, req, HWRM_FUNC_RESOURCE_QCAPS); 7534 if (rc) 7535 return rc; 7536 7537 req->fid = cpu_to_le16(0xffff); 7538 resp = hwrm_req_hold(bp, req); 7539 rc = hwrm_req_send_silent(bp, req); 7540 if (rc) 7541 goto hwrm_func_resc_qcaps_exit; 7542 7543 hw_resc->max_tx_sch_inputs = le16_to_cpu(resp->max_tx_scheduler_inputs); 7544 if (!all) 7545 goto hwrm_func_resc_qcaps_exit; 7546 7547 hw_resc->min_rsscos_ctxs = le16_to_cpu(resp->min_rsscos_ctx); 7548 hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx); 7549 hw_resc->min_cp_rings = le16_to_cpu(resp->min_cmpl_rings); 7550 hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings); 7551 hw_resc->min_tx_rings = le16_to_cpu(resp->min_tx_rings); 7552 hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings); 7553 hw_resc->min_rx_rings = le16_to_cpu(resp->min_rx_rings); 7554 hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings); 7555 hw_resc->min_hw_ring_grps = le16_to_cpu(resp->min_hw_ring_grps); 7556 hw_resc->max_hw_ring_grps = le16_to_cpu(resp->max_hw_ring_grps); 7557 hw_resc->min_l2_ctxs = le16_to_cpu(resp->min_l2_ctxs); 7558 hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs); 7559 hw_resc->min_vnics = le16_to_cpu(resp->min_vnics); 7560 hw_resc->max_vnics = le16_to_cpu(resp->max_vnics); 7561 hw_resc->min_stat_ctxs = le16_to_cpu(resp->min_stat_ctx); 7562 hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx); 7563 7564 if (bp->flags & BNXT_FLAG_CHIP_P5) { 7565 u16 max_msix = le16_to_cpu(resp->max_msix); 7566 7567 hw_resc->max_nqs = max_msix; 7568 hw_resc->max_hw_ring_grps = hw_resc->max_rx_rings; 7569 } 7570 7571 if (BNXT_PF(bp)) { 7572 struct bnxt_pf_info *pf = &bp->pf; 7573 7574 pf->vf_resv_strategy = 7575 le16_to_cpu(resp->vf_reservation_strategy); 7576 if (pf->vf_resv_strategy > BNXT_VF_RESV_STRATEGY_MINIMAL_STATIC) 7577 pf->vf_resv_strategy = BNXT_VF_RESV_STRATEGY_MAXIMAL; 7578 } 7579 hwrm_func_resc_qcaps_exit: 7580 hwrm_req_drop(bp, req); 7581 return rc; 7582 } 7583 7584 static int __bnxt_hwrm_ptp_qcfg(struct bnxt *bp) 7585 { 7586 struct hwrm_port_mac_ptp_qcfg_output *resp; 7587 struct hwrm_port_mac_ptp_qcfg_input *req; 7588 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; 7589 bool phc_cfg; 7590 u8 flags; 7591 int rc; 7592 7593 if (bp->hwrm_spec_code < 0x10801 || !BNXT_CHIP_P5_THOR(bp)) { 7594 rc = -ENODEV; 7595 goto no_ptp; 7596 } 7597 7598 rc = hwrm_req_init(bp, req, HWRM_PORT_MAC_PTP_QCFG); 7599 if (rc) 7600 goto no_ptp; 7601 7602 req->port_id = cpu_to_le16(bp->pf.port_id); 7603 resp = hwrm_req_hold(bp, req); 7604 rc = hwrm_req_send(bp, req); 7605 if (rc) 7606 goto exit; 7607 7608 flags = resp->flags; 7609 if (!(flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_HWRM_ACCESS)) { 7610 rc = -ENODEV; 7611 goto exit; 7612 } 7613 if (!ptp) { 7614 ptp = kzalloc(sizeof(*ptp), GFP_KERNEL); 7615 if (!ptp) { 7616 rc = -ENOMEM; 7617 goto exit; 7618 } 7619 ptp->bp = bp; 7620 bp->ptp_cfg = ptp; 7621 } 7622 if (flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_PARTIAL_DIRECT_ACCESS_REF_CLOCK) { 7623 ptp->refclk_regs[0] = le32_to_cpu(resp->ts_ref_clock_reg_lower); 7624 ptp->refclk_regs[1] = le32_to_cpu(resp->ts_ref_clock_reg_upper); 7625 } else if (bp->flags & BNXT_FLAG_CHIP_P5) { 7626 ptp->refclk_regs[0] = BNXT_TS_REG_TIMESYNC_TS0_LOWER; 7627 ptp->refclk_regs[1] = BNXT_TS_REG_TIMESYNC_TS0_UPPER; 7628 } else { 7629 rc = -ENODEV; 7630 goto exit; 7631 } 7632 phc_cfg = (flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_RTC_CONFIGURED) != 0; 7633 rc = bnxt_ptp_init(bp, phc_cfg); 7634 if (rc) 7635 netdev_warn(bp->dev, "PTP initialization failed.\n"); 7636 exit: 7637 hwrm_req_drop(bp, req); 7638 if (!rc) 7639 return 0; 7640 7641 no_ptp: 7642 bnxt_ptp_clear(bp); 7643 kfree(ptp); 7644 bp->ptp_cfg = NULL; 7645 return rc; 7646 } 7647 7648 static int __bnxt_hwrm_func_qcaps(struct bnxt *bp) 7649 { 7650 struct hwrm_func_qcaps_output *resp; 7651 struct hwrm_func_qcaps_input *req; 7652 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 7653 u32 flags, flags_ext, flags_ext2; 7654 int rc; 7655 7656 rc = hwrm_req_init(bp, req, HWRM_FUNC_QCAPS); 7657 if (rc) 7658 return rc; 7659 7660 req->fid = cpu_to_le16(0xffff); 7661 resp = hwrm_req_hold(bp, req); 7662 rc = hwrm_req_send(bp, req); 7663 if (rc) 7664 goto hwrm_func_qcaps_exit; 7665 7666 flags = le32_to_cpu(resp->flags); 7667 if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED) 7668 bp->flags |= BNXT_FLAG_ROCEV1_CAP; 7669 if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED) 7670 bp->flags |= BNXT_FLAG_ROCEV2_CAP; 7671 if (flags & FUNC_QCAPS_RESP_FLAGS_PCIE_STATS_SUPPORTED) 7672 bp->fw_cap |= BNXT_FW_CAP_PCIE_STATS_SUPPORTED; 7673 if (flags & FUNC_QCAPS_RESP_FLAGS_HOT_RESET_CAPABLE) 7674 bp->fw_cap |= BNXT_FW_CAP_HOT_RESET; 7675 if (flags & FUNC_QCAPS_RESP_FLAGS_EXT_STATS_SUPPORTED) 7676 bp->fw_cap |= BNXT_FW_CAP_EXT_STATS_SUPPORTED; 7677 if (flags & FUNC_QCAPS_RESP_FLAGS_ERROR_RECOVERY_CAPABLE) 7678 bp->fw_cap |= BNXT_FW_CAP_ERROR_RECOVERY; 7679 if (flags & FUNC_QCAPS_RESP_FLAGS_ERR_RECOVER_RELOAD) 7680 bp->fw_cap |= BNXT_FW_CAP_ERR_RECOVER_RELOAD; 7681 if (!(flags & FUNC_QCAPS_RESP_FLAGS_VLAN_ACCELERATION_TX_DISABLED)) 7682 bp->fw_cap |= BNXT_FW_CAP_VLAN_TX_INSERT; 7683 if (flags & FUNC_QCAPS_RESP_FLAGS_DBG_QCAPS_CMD_SUPPORTED) 7684 bp->fw_cap |= BNXT_FW_CAP_DBG_QCAPS; 7685 7686 flags_ext = le32_to_cpu(resp->flags_ext); 7687 if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_EXT_HW_STATS_SUPPORTED) 7688 bp->fw_cap |= BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED; 7689 if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_PTP_PPS_SUPPORTED)) 7690 bp->fw_cap |= BNXT_FW_CAP_PTP_PPS; 7691 if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_PTP_64BIT_RTC_SUPPORTED) 7692 bp->fw_cap |= BNXT_FW_CAP_PTP_RTC; 7693 if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_HOT_RESET_IF_SUPPORT)) 7694 bp->fw_cap |= BNXT_FW_CAP_HOT_RESET_IF; 7695 if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_FW_LIVEPATCH_SUPPORTED)) 7696 bp->fw_cap |= BNXT_FW_CAP_LIVEPATCH; 7697 7698 flags_ext2 = le32_to_cpu(resp->flags_ext2); 7699 if (flags_ext2 & FUNC_QCAPS_RESP_FLAGS_EXT2_RX_ALL_PKTS_TIMESTAMPS_SUPPORTED) 7700 bp->fw_cap |= BNXT_FW_CAP_RX_ALL_PKT_TS; 7701 7702 bp->tx_push_thresh = 0; 7703 if ((flags & FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED) && 7704 BNXT_FW_MAJ(bp) > 217) 7705 bp->tx_push_thresh = BNXT_TX_PUSH_THRESH; 7706 7707 hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx); 7708 hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings); 7709 hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings); 7710 hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings); 7711 hw_resc->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps); 7712 if (!hw_resc->max_hw_ring_grps) 7713 hw_resc->max_hw_ring_grps = hw_resc->max_tx_rings; 7714 hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs); 7715 hw_resc->max_vnics = le16_to_cpu(resp->max_vnics); 7716 hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx); 7717 7718 if (BNXT_PF(bp)) { 7719 struct bnxt_pf_info *pf = &bp->pf; 7720 7721 pf->fw_fid = le16_to_cpu(resp->fid); 7722 pf->port_id = le16_to_cpu(resp->port_id); 7723 memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN); 7724 pf->first_vf_id = le16_to_cpu(resp->first_vf_id); 7725 pf->max_vfs = le16_to_cpu(resp->max_vfs); 7726 pf->max_encap_records = le32_to_cpu(resp->max_encap_records); 7727 pf->max_decap_records = le32_to_cpu(resp->max_decap_records); 7728 pf->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows); 7729 pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows); 7730 pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows); 7731 pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows); 7732 bp->flags &= ~BNXT_FLAG_WOL_CAP; 7733 if (flags & FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED) 7734 bp->flags |= BNXT_FLAG_WOL_CAP; 7735 if (flags & FUNC_QCAPS_RESP_FLAGS_PTP_SUPPORTED) { 7736 bp->fw_cap |= BNXT_FW_CAP_PTP; 7737 } else { 7738 bnxt_ptp_clear(bp); 7739 kfree(bp->ptp_cfg); 7740 bp->ptp_cfg = NULL; 7741 } 7742 } else { 7743 #ifdef CONFIG_BNXT_SRIOV 7744 struct bnxt_vf_info *vf = &bp->vf; 7745 7746 vf->fw_fid = le16_to_cpu(resp->fid); 7747 memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN); 7748 #endif 7749 } 7750 7751 hwrm_func_qcaps_exit: 7752 hwrm_req_drop(bp, req); 7753 return rc; 7754 } 7755 7756 static void bnxt_hwrm_dbg_qcaps(struct bnxt *bp) 7757 { 7758 struct hwrm_dbg_qcaps_output *resp; 7759 struct hwrm_dbg_qcaps_input *req; 7760 int rc; 7761 7762 bp->fw_dbg_cap = 0; 7763 if (!(bp->fw_cap & BNXT_FW_CAP_DBG_QCAPS)) 7764 return; 7765 7766 rc = hwrm_req_init(bp, req, HWRM_DBG_QCAPS); 7767 if (rc) 7768 return; 7769 7770 req->fid = cpu_to_le16(0xffff); 7771 resp = hwrm_req_hold(bp, req); 7772 rc = hwrm_req_send(bp, req); 7773 if (rc) 7774 goto hwrm_dbg_qcaps_exit; 7775 7776 bp->fw_dbg_cap = le32_to_cpu(resp->flags); 7777 7778 hwrm_dbg_qcaps_exit: 7779 hwrm_req_drop(bp, req); 7780 } 7781 7782 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp); 7783 7784 int bnxt_hwrm_func_qcaps(struct bnxt *bp) 7785 { 7786 int rc; 7787 7788 rc = __bnxt_hwrm_func_qcaps(bp); 7789 if (rc) 7790 return rc; 7791 7792 bnxt_hwrm_dbg_qcaps(bp); 7793 7794 rc = bnxt_hwrm_queue_qportcfg(bp); 7795 if (rc) { 7796 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %d\n", rc); 7797 return rc; 7798 } 7799 if (bp->hwrm_spec_code >= 0x10803) { 7800 rc = bnxt_alloc_ctx_mem(bp); 7801 if (rc) 7802 return rc; 7803 rc = bnxt_hwrm_func_resc_qcaps(bp, true); 7804 if (!rc) 7805 bp->fw_cap |= BNXT_FW_CAP_NEW_RM; 7806 } 7807 return 0; 7808 } 7809 7810 static int bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(struct bnxt *bp) 7811 { 7812 struct hwrm_cfa_adv_flow_mgnt_qcaps_output *resp; 7813 struct hwrm_cfa_adv_flow_mgnt_qcaps_input *req; 7814 u32 flags; 7815 int rc; 7816 7817 if (!(bp->fw_cap & BNXT_FW_CAP_CFA_ADV_FLOW)) 7818 return 0; 7819 7820 rc = hwrm_req_init(bp, req, HWRM_CFA_ADV_FLOW_MGNT_QCAPS); 7821 if (rc) 7822 return rc; 7823 7824 resp = hwrm_req_hold(bp, req); 7825 rc = hwrm_req_send(bp, req); 7826 if (rc) 7827 goto hwrm_cfa_adv_qcaps_exit; 7828 7829 flags = le32_to_cpu(resp->flags); 7830 if (flags & 7831 CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_V2_SUPPORTED) 7832 bp->fw_cap |= BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2; 7833 7834 hwrm_cfa_adv_qcaps_exit: 7835 hwrm_req_drop(bp, req); 7836 return rc; 7837 } 7838 7839 static int __bnxt_alloc_fw_health(struct bnxt *bp) 7840 { 7841 if (bp->fw_health) 7842 return 0; 7843 7844 bp->fw_health = kzalloc(sizeof(*bp->fw_health), GFP_KERNEL); 7845 if (!bp->fw_health) 7846 return -ENOMEM; 7847 7848 mutex_init(&bp->fw_health->lock); 7849 return 0; 7850 } 7851 7852 static int bnxt_alloc_fw_health(struct bnxt *bp) 7853 { 7854 int rc; 7855 7856 if (!(bp->fw_cap & BNXT_FW_CAP_HOT_RESET) && 7857 !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) 7858 return 0; 7859 7860 rc = __bnxt_alloc_fw_health(bp); 7861 if (rc) { 7862 bp->fw_cap &= ~BNXT_FW_CAP_HOT_RESET; 7863 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY; 7864 return rc; 7865 } 7866 7867 return 0; 7868 } 7869 7870 static void __bnxt_map_fw_health_reg(struct bnxt *bp, u32 reg) 7871 { 7872 writel(reg & BNXT_GRC_BASE_MASK, bp->bar0 + 7873 BNXT_GRCPF_REG_WINDOW_BASE_OUT + 7874 BNXT_FW_HEALTH_WIN_MAP_OFF); 7875 } 7876 7877 static void bnxt_inv_fw_health_reg(struct bnxt *bp) 7878 { 7879 struct bnxt_fw_health *fw_health = bp->fw_health; 7880 u32 reg_type; 7881 7882 if (!fw_health) 7883 return; 7884 7885 reg_type = BNXT_FW_HEALTH_REG_TYPE(fw_health->regs[BNXT_FW_HEALTH_REG]); 7886 if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC) 7887 fw_health->status_reliable = false; 7888 7889 reg_type = BNXT_FW_HEALTH_REG_TYPE(fw_health->regs[BNXT_FW_RESET_CNT_REG]); 7890 if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC) 7891 fw_health->resets_reliable = false; 7892 } 7893 7894 static void bnxt_try_map_fw_health_reg(struct bnxt *bp) 7895 { 7896 void __iomem *hs; 7897 u32 status_loc; 7898 u32 reg_type; 7899 u32 sig; 7900 7901 if (bp->fw_health) 7902 bp->fw_health->status_reliable = false; 7903 7904 __bnxt_map_fw_health_reg(bp, HCOMM_STATUS_STRUCT_LOC); 7905 hs = bp->bar0 + BNXT_FW_HEALTH_WIN_OFF(HCOMM_STATUS_STRUCT_LOC); 7906 7907 sig = readl(hs + offsetof(struct hcomm_status, sig_ver)); 7908 if ((sig & HCOMM_STATUS_SIGNATURE_MASK) != HCOMM_STATUS_SIGNATURE_VAL) { 7909 if (!bp->chip_num) { 7910 __bnxt_map_fw_health_reg(bp, BNXT_GRC_REG_BASE); 7911 bp->chip_num = readl(bp->bar0 + 7912 BNXT_FW_HEALTH_WIN_BASE + 7913 BNXT_GRC_REG_CHIP_NUM); 7914 } 7915 if (!BNXT_CHIP_P5(bp)) 7916 return; 7917 7918 status_loc = BNXT_GRC_REG_STATUS_P5 | 7919 BNXT_FW_HEALTH_REG_TYPE_BAR0; 7920 } else { 7921 status_loc = readl(hs + offsetof(struct hcomm_status, 7922 fw_status_loc)); 7923 } 7924 7925 if (__bnxt_alloc_fw_health(bp)) { 7926 netdev_warn(bp->dev, "no memory for firmware status checks\n"); 7927 return; 7928 } 7929 7930 bp->fw_health->regs[BNXT_FW_HEALTH_REG] = status_loc; 7931 reg_type = BNXT_FW_HEALTH_REG_TYPE(status_loc); 7932 if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC) { 7933 __bnxt_map_fw_health_reg(bp, status_loc); 7934 bp->fw_health->mapped_regs[BNXT_FW_HEALTH_REG] = 7935 BNXT_FW_HEALTH_WIN_OFF(status_loc); 7936 } 7937 7938 bp->fw_health->status_reliable = true; 7939 } 7940 7941 static int bnxt_map_fw_health_regs(struct bnxt *bp) 7942 { 7943 struct bnxt_fw_health *fw_health = bp->fw_health; 7944 u32 reg_base = 0xffffffff; 7945 int i; 7946 7947 bp->fw_health->status_reliable = false; 7948 bp->fw_health->resets_reliable = false; 7949 /* Only pre-map the monitoring GRC registers using window 3 */ 7950 for (i = 0; i < 4; i++) { 7951 u32 reg = fw_health->regs[i]; 7952 7953 if (BNXT_FW_HEALTH_REG_TYPE(reg) != BNXT_FW_HEALTH_REG_TYPE_GRC) 7954 continue; 7955 if (reg_base == 0xffffffff) 7956 reg_base = reg & BNXT_GRC_BASE_MASK; 7957 if ((reg & BNXT_GRC_BASE_MASK) != reg_base) 7958 return -ERANGE; 7959 fw_health->mapped_regs[i] = BNXT_FW_HEALTH_WIN_OFF(reg); 7960 } 7961 bp->fw_health->status_reliable = true; 7962 bp->fw_health->resets_reliable = true; 7963 if (reg_base == 0xffffffff) 7964 return 0; 7965 7966 __bnxt_map_fw_health_reg(bp, reg_base); 7967 return 0; 7968 } 7969 7970 static void bnxt_remap_fw_health_regs(struct bnxt *bp) 7971 { 7972 if (!bp->fw_health) 7973 return; 7974 7975 if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) { 7976 bp->fw_health->status_reliable = true; 7977 bp->fw_health->resets_reliable = true; 7978 } else { 7979 bnxt_try_map_fw_health_reg(bp); 7980 } 7981 } 7982 7983 static int bnxt_hwrm_error_recovery_qcfg(struct bnxt *bp) 7984 { 7985 struct bnxt_fw_health *fw_health = bp->fw_health; 7986 struct hwrm_error_recovery_qcfg_output *resp; 7987 struct hwrm_error_recovery_qcfg_input *req; 7988 int rc, i; 7989 7990 if (!(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) 7991 return 0; 7992 7993 rc = hwrm_req_init(bp, req, HWRM_ERROR_RECOVERY_QCFG); 7994 if (rc) 7995 return rc; 7996 7997 resp = hwrm_req_hold(bp, req); 7998 rc = hwrm_req_send(bp, req); 7999 if (rc) 8000 goto err_recovery_out; 8001 fw_health->flags = le32_to_cpu(resp->flags); 8002 if ((fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) && 8003 !(bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL)) { 8004 rc = -EINVAL; 8005 goto err_recovery_out; 8006 } 8007 fw_health->polling_dsecs = le32_to_cpu(resp->driver_polling_freq); 8008 fw_health->master_func_wait_dsecs = 8009 le32_to_cpu(resp->master_func_wait_period); 8010 fw_health->normal_func_wait_dsecs = 8011 le32_to_cpu(resp->normal_func_wait_period); 8012 fw_health->post_reset_wait_dsecs = 8013 le32_to_cpu(resp->master_func_wait_period_after_reset); 8014 fw_health->post_reset_max_wait_dsecs = 8015 le32_to_cpu(resp->max_bailout_time_after_reset); 8016 fw_health->regs[BNXT_FW_HEALTH_REG] = 8017 le32_to_cpu(resp->fw_health_status_reg); 8018 fw_health->regs[BNXT_FW_HEARTBEAT_REG] = 8019 le32_to_cpu(resp->fw_heartbeat_reg); 8020 fw_health->regs[BNXT_FW_RESET_CNT_REG] = 8021 le32_to_cpu(resp->fw_reset_cnt_reg); 8022 fw_health->regs[BNXT_FW_RESET_INPROG_REG] = 8023 le32_to_cpu(resp->reset_inprogress_reg); 8024 fw_health->fw_reset_inprog_reg_mask = 8025 le32_to_cpu(resp->reset_inprogress_reg_mask); 8026 fw_health->fw_reset_seq_cnt = resp->reg_array_cnt; 8027 if (fw_health->fw_reset_seq_cnt >= 16) { 8028 rc = -EINVAL; 8029 goto err_recovery_out; 8030 } 8031 for (i = 0; i < fw_health->fw_reset_seq_cnt; i++) { 8032 fw_health->fw_reset_seq_regs[i] = 8033 le32_to_cpu(resp->reset_reg[i]); 8034 fw_health->fw_reset_seq_vals[i] = 8035 le32_to_cpu(resp->reset_reg_val[i]); 8036 fw_health->fw_reset_seq_delay_msec[i] = 8037 resp->delay_after_reset[i]; 8038 } 8039 err_recovery_out: 8040 hwrm_req_drop(bp, req); 8041 if (!rc) 8042 rc = bnxt_map_fw_health_regs(bp); 8043 if (rc) 8044 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY; 8045 return rc; 8046 } 8047 8048 static int bnxt_hwrm_func_reset(struct bnxt *bp) 8049 { 8050 struct hwrm_func_reset_input *req; 8051 int rc; 8052 8053 rc = hwrm_req_init(bp, req, HWRM_FUNC_RESET); 8054 if (rc) 8055 return rc; 8056 8057 req->enables = 0; 8058 hwrm_req_timeout(bp, req, HWRM_RESET_TIMEOUT); 8059 return hwrm_req_send(bp, req); 8060 } 8061 8062 static void bnxt_nvm_cfg_ver_get(struct bnxt *bp) 8063 { 8064 struct hwrm_nvm_get_dev_info_output nvm_info; 8065 8066 if (!bnxt_hwrm_nvm_get_dev_info(bp, &nvm_info)) 8067 snprintf(bp->nvm_cfg_ver, FW_VER_STR_LEN, "%d.%d.%d", 8068 nvm_info.nvm_cfg_ver_maj, nvm_info.nvm_cfg_ver_min, 8069 nvm_info.nvm_cfg_ver_upd); 8070 } 8071 8072 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp) 8073 { 8074 struct hwrm_queue_qportcfg_output *resp; 8075 struct hwrm_queue_qportcfg_input *req; 8076 u8 i, j, *qptr; 8077 bool no_rdma; 8078 int rc = 0; 8079 8080 rc = hwrm_req_init(bp, req, HWRM_QUEUE_QPORTCFG); 8081 if (rc) 8082 return rc; 8083 8084 resp = hwrm_req_hold(bp, req); 8085 rc = hwrm_req_send(bp, req); 8086 if (rc) 8087 goto qportcfg_exit; 8088 8089 if (!resp->max_configurable_queues) { 8090 rc = -EINVAL; 8091 goto qportcfg_exit; 8092 } 8093 bp->max_tc = resp->max_configurable_queues; 8094 bp->max_lltc = resp->max_configurable_lossless_queues; 8095 if (bp->max_tc > BNXT_MAX_QUEUE) 8096 bp->max_tc = BNXT_MAX_QUEUE; 8097 8098 no_rdma = !(bp->flags & BNXT_FLAG_ROCE_CAP); 8099 qptr = &resp->queue_id0; 8100 for (i = 0, j = 0; i < bp->max_tc; i++) { 8101 bp->q_info[j].queue_id = *qptr; 8102 bp->q_ids[i] = *qptr++; 8103 bp->q_info[j].queue_profile = *qptr++; 8104 bp->tc_to_qidx[j] = j; 8105 if (!BNXT_CNPQ(bp->q_info[j].queue_profile) || 8106 (no_rdma && BNXT_PF(bp))) 8107 j++; 8108 } 8109 bp->max_q = bp->max_tc; 8110 bp->max_tc = max_t(u8, j, 1); 8111 8112 if (resp->queue_cfg_info & QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG) 8113 bp->max_tc = 1; 8114 8115 if (bp->max_lltc > bp->max_tc) 8116 bp->max_lltc = bp->max_tc; 8117 8118 qportcfg_exit: 8119 hwrm_req_drop(bp, req); 8120 return rc; 8121 } 8122 8123 static int bnxt_hwrm_poll(struct bnxt *bp) 8124 { 8125 struct hwrm_ver_get_input *req; 8126 int rc; 8127 8128 rc = hwrm_req_init(bp, req, HWRM_VER_GET); 8129 if (rc) 8130 return rc; 8131 8132 req->hwrm_intf_maj = HWRM_VERSION_MAJOR; 8133 req->hwrm_intf_min = HWRM_VERSION_MINOR; 8134 req->hwrm_intf_upd = HWRM_VERSION_UPDATE; 8135 8136 hwrm_req_flags(bp, req, BNXT_HWRM_CTX_SILENT | BNXT_HWRM_FULL_WAIT); 8137 rc = hwrm_req_send(bp, req); 8138 return rc; 8139 } 8140 8141 static int bnxt_hwrm_ver_get(struct bnxt *bp) 8142 { 8143 struct hwrm_ver_get_output *resp; 8144 struct hwrm_ver_get_input *req; 8145 u16 fw_maj, fw_min, fw_bld, fw_rsv; 8146 u32 dev_caps_cfg, hwrm_ver; 8147 int rc, len; 8148 8149 rc = hwrm_req_init(bp, req, HWRM_VER_GET); 8150 if (rc) 8151 return rc; 8152 8153 hwrm_req_flags(bp, req, BNXT_HWRM_FULL_WAIT); 8154 bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN; 8155 req->hwrm_intf_maj = HWRM_VERSION_MAJOR; 8156 req->hwrm_intf_min = HWRM_VERSION_MINOR; 8157 req->hwrm_intf_upd = HWRM_VERSION_UPDATE; 8158 8159 resp = hwrm_req_hold(bp, req); 8160 rc = hwrm_req_send(bp, req); 8161 if (rc) 8162 goto hwrm_ver_get_exit; 8163 8164 memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output)); 8165 8166 bp->hwrm_spec_code = resp->hwrm_intf_maj_8b << 16 | 8167 resp->hwrm_intf_min_8b << 8 | 8168 resp->hwrm_intf_upd_8b; 8169 if (resp->hwrm_intf_maj_8b < 1) { 8170 netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n", 8171 resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b, 8172 resp->hwrm_intf_upd_8b); 8173 netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n"); 8174 } 8175 8176 hwrm_ver = HWRM_VERSION_MAJOR << 16 | HWRM_VERSION_MINOR << 8 | 8177 HWRM_VERSION_UPDATE; 8178 8179 if (bp->hwrm_spec_code > hwrm_ver) 8180 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d", 8181 HWRM_VERSION_MAJOR, HWRM_VERSION_MINOR, 8182 HWRM_VERSION_UPDATE); 8183 else 8184 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d", 8185 resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b, 8186 resp->hwrm_intf_upd_8b); 8187 8188 fw_maj = le16_to_cpu(resp->hwrm_fw_major); 8189 if (bp->hwrm_spec_code > 0x10803 && fw_maj) { 8190 fw_min = le16_to_cpu(resp->hwrm_fw_minor); 8191 fw_bld = le16_to_cpu(resp->hwrm_fw_build); 8192 fw_rsv = le16_to_cpu(resp->hwrm_fw_patch); 8193 len = FW_VER_STR_LEN; 8194 } else { 8195 fw_maj = resp->hwrm_fw_maj_8b; 8196 fw_min = resp->hwrm_fw_min_8b; 8197 fw_bld = resp->hwrm_fw_bld_8b; 8198 fw_rsv = resp->hwrm_fw_rsvd_8b; 8199 len = BC_HWRM_STR_LEN; 8200 } 8201 bp->fw_ver_code = BNXT_FW_VER_CODE(fw_maj, fw_min, fw_bld, fw_rsv); 8202 snprintf(bp->fw_ver_str, len, "%d.%d.%d.%d", fw_maj, fw_min, fw_bld, 8203 fw_rsv); 8204 8205 if (strlen(resp->active_pkg_name)) { 8206 int fw_ver_len = strlen(bp->fw_ver_str); 8207 8208 snprintf(bp->fw_ver_str + fw_ver_len, 8209 FW_VER_STR_LEN - fw_ver_len - 1, "/pkg %s", 8210 resp->active_pkg_name); 8211 bp->fw_cap |= BNXT_FW_CAP_PKG_VER; 8212 } 8213 8214 bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout); 8215 if (!bp->hwrm_cmd_timeout) 8216 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT; 8217 bp->hwrm_cmd_max_timeout = le16_to_cpu(resp->max_req_timeout) * 1000; 8218 if (!bp->hwrm_cmd_max_timeout) 8219 bp->hwrm_cmd_max_timeout = HWRM_CMD_MAX_TIMEOUT; 8220 else if (bp->hwrm_cmd_max_timeout > HWRM_CMD_MAX_TIMEOUT) 8221 netdev_warn(bp->dev, "Device requests max timeout of %d seconds, may trigger hung task watchdog\n", 8222 bp->hwrm_cmd_max_timeout / 1000); 8223 8224 if (resp->hwrm_intf_maj_8b >= 1) { 8225 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len); 8226 bp->hwrm_max_ext_req_len = le16_to_cpu(resp->max_ext_req_len); 8227 } 8228 if (bp->hwrm_max_ext_req_len < HWRM_MAX_REQ_LEN) 8229 bp->hwrm_max_ext_req_len = HWRM_MAX_REQ_LEN; 8230 8231 bp->chip_num = le16_to_cpu(resp->chip_num); 8232 bp->chip_rev = resp->chip_rev; 8233 if (bp->chip_num == CHIP_NUM_58700 && !resp->chip_rev && 8234 !resp->chip_metal) 8235 bp->flags |= BNXT_FLAG_CHIP_NITRO_A0; 8236 8237 dev_caps_cfg = le32_to_cpu(resp->dev_caps_cfg); 8238 if ((dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) && 8239 (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED)) 8240 bp->fw_cap |= BNXT_FW_CAP_SHORT_CMD; 8241 8242 if (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_KONG_MB_CHNL_SUPPORTED) 8243 bp->fw_cap |= BNXT_FW_CAP_KONG_MB_CHNL; 8244 8245 if (dev_caps_cfg & 8246 VER_GET_RESP_DEV_CAPS_CFG_FLOW_HANDLE_64BIT_SUPPORTED) 8247 bp->fw_cap |= BNXT_FW_CAP_OVS_64BIT_HANDLE; 8248 8249 if (dev_caps_cfg & 8250 VER_GET_RESP_DEV_CAPS_CFG_TRUSTED_VF_SUPPORTED) 8251 bp->fw_cap |= BNXT_FW_CAP_TRUSTED_VF; 8252 8253 if (dev_caps_cfg & 8254 VER_GET_RESP_DEV_CAPS_CFG_CFA_ADV_FLOW_MGNT_SUPPORTED) 8255 bp->fw_cap |= BNXT_FW_CAP_CFA_ADV_FLOW; 8256 8257 hwrm_ver_get_exit: 8258 hwrm_req_drop(bp, req); 8259 return rc; 8260 } 8261 8262 int bnxt_hwrm_fw_set_time(struct bnxt *bp) 8263 { 8264 struct hwrm_fw_set_time_input *req; 8265 struct tm tm; 8266 time64_t now = ktime_get_real_seconds(); 8267 int rc; 8268 8269 if ((BNXT_VF(bp) && bp->hwrm_spec_code < 0x10901) || 8270 bp->hwrm_spec_code < 0x10400) 8271 return -EOPNOTSUPP; 8272 8273 time64_to_tm(now, 0, &tm); 8274 rc = hwrm_req_init(bp, req, HWRM_FW_SET_TIME); 8275 if (rc) 8276 return rc; 8277 8278 req->year = cpu_to_le16(1900 + tm.tm_year); 8279 req->month = 1 + tm.tm_mon; 8280 req->day = tm.tm_mday; 8281 req->hour = tm.tm_hour; 8282 req->minute = tm.tm_min; 8283 req->second = tm.tm_sec; 8284 return hwrm_req_send(bp, req); 8285 } 8286 8287 static void bnxt_add_one_ctr(u64 hw, u64 *sw, u64 mask) 8288 { 8289 u64 sw_tmp; 8290 8291 hw &= mask; 8292 sw_tmp = (*sw & ~mask) | hw; 8293 if (hw < (*sw & mask)) 8294 sw_tmp += mask + 1; 8295 WRITE_ONCE(*sw, sw_tmp); 8296 } 8297 8298 static void __bnxt_accumulate_stats(__le64 *hw_stats, u64 *sw_stats, u64 *masks, 8299 int count, bool ignore_zero) 8300 { 8301 int i; 8302 8303 for (i = 0; i < count; i++) { 8304 u64 hw = le64_to_cpu(READ_ONCE(hw_stats[i])); 8305 8306 if (ignore_zero && !hw) 8307 continue; 8308 8309 if (masks[i] == -1ULL) 8310 sw_stats[i] = hw; 8311 else 8312 bnxt_add_one_ctr(hw, &sw_stats[i], masks[i]); 8313 } 8314 } 8315 8316 static void bnxt_accumulate_stats(struct bnxt_stats_mem *stats) 8317 { 8318 if (!stats->hw_stats) 8319 return; 8320 8321 __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats, 8322 stats->hw_masks, stats->len / 8, false); 8323 } 8324 8325 static void bnxt_accumulate_all_stats(struct bnxt *bp) 8326 { 8327 struct bnxt_stats_mem *ring0_stats; 8328 bool ignore_zero = false; 8329 int i; 8330 8331 /* Chip bug. Counter intermittently becomes 0. */ 8332 if (bp->flags & BNXT_FLAG_CHIP_P5) 8333 ignore_zero = true; 8334 8335 for (i = 0; i < bp->cp_nr_rings; i++) { 8336 struct bnxt_napi *bnapi = bp->bnapi[i]; 8337 struct bnxt_cp_ring_info *cpr; 8338 struct bnxt_stats_mem *stats; 8339 8340 cpr = &bnapi->cp_ring; 8341 stats = &cpr->stats; 8342 if (!i) 8343 ring0_stats = stats; 8344 __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats, 8345 ring0_stats->hw_masks, 8346 ring0_stats->len / 8, ignore_zero); 8347 } 8348 if (bp->flags & BNXT_FLAG_PORT_STATS) { 8349 struct bnxt_stats_mem *stats = &bp->port_stats; 8350 __le64 *hw_stats = stats->hw_stats; 8351 u64 *sw_stats = stats->sw_stats; 8352 u64 *masks = stats->hw_masks; 8353 int cnt; 8354 8355 cnt = sizeof(struct rx_port_stats) / 8; 8356 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false); 8357 8358 hw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 8359 sw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 8360 masks += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 8361 cnt = sizeof(struct tx_port_stats) / 8; 8362 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false); 8363 } 8364 if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) { 8365 bnxt_accumulate_stats(&bp->rx_port_stats_ext); 8366 bnxt_accumulate_stats(&bp->tx_port_stats_ext); 8367 } 8368 } 8369 8370 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags) 8371 { 8372 struct hwrm_port_qstats_input *req; 8373 struct bnxt_pf_info *pf = &bp->pf; 8374 int rc; 8375 8376 if (!(bp->flags & BNXT_FLAG_PORT_STATS)) 8377 return 0; 8378 8379 if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED)) 8380 return -EOPNOTSUPP; 8381 8382 rc = hwrm_req_init(bp, req, HWRM_PORT_QSTATS); 8383 if (rc) 8384 return rc; 8385 8386 req->flags = flags; 8387 req->port_id = cpu_to_le16(pf->port_id); 8388 req->tx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map + 8389 BNXT_TX_PORT_STATS_BYTE_OFFSET); 8390 req->rx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map); 8391 return hwrm_req_send(bp, req); 8392 } 8393 8394 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags) 8395 { 8396 struct hwrm_queue_pri2cos_qcfg_output *resp_qc; 8397 struct hwrm_queue_pri2cos_qcfg_input *req_qc; 8398 struct hwrm_port_qstats_ext_output *resp_qs; 8399 struct hwrm_port_qstats_ext_input *req_qs; 8400 struct bnxt_pf_info *pf = &bp->pf; 8401 u32 tx_stat_size; 8402 int rc; 8403 8404 if (!(bp->flags & BNXT_FLAG_PORT_STATS_EXT)) 8405 return 0; 8406 8407 if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED)) 8408 return -EOPNOTSUPP; 8409 8410 rc = hwrm_req_init(bp, req_qs, HWRM_PORT_QSTATS_EXT); 8411 if (rc) 8412 return rc; 8413 8414 req_qs->flags = flags; 8415 req_qs->port_id = cpu_to_le16(pf->port_id); 8416 req_qs->rx_stat_size = cpu_to_le16(sizeof(struct rx_port_stats_ext)); 8417 req_qs->rx_stat_host_addr = cpu_to_le64(bp->rx_port_stats_ext.hw_stats_map); 8418 tx_stat_size = bp->tx_port_stats_ext.hw_stats ? 8419 sizeof(struct tx_port_stats_ext) : 0; 8420 req_qs->tx_stat_size = cpu_to_le16(tx_stat_size); 8421 req_qs->tx_stat_host_addr = cpu_to_le64(bp->tx_port_stats_ext.hw_stats_map); 8422 resp_qs = hwrm_req_hold(bp, req_qs); 8423 rc = hwrm_req_send(bp, req_qs); 8424 if (!rc) { 8425 bp->fw_rx_stats_ext_size = 8426 le16_to_cpu(resp_qs->rx_stat_size) / 8; 8427 if (BNXT_FW_MAJ(bp) < 220 && 8428 bp->fw_rx_stats_ext_size > BNXT_RX_STATS_EXT_NUM_LEGACY) 8429 bp->fw_rx_stats_ext_size = BNXT_RX_STATS_EXT_NUM_LEGACY; 8430 8431 bp->fw_tx_stats_ext_size = tx_stat_size ? 8432 le16_to_cpu(resp_qs->tx_stat_size) / 8 : 0; 8433 } else { 8434 bp->fw_rx_stats_ext_size = 0; 8435 bp->fw_tx_stats_ext_size = 0; 8436 } 8437 hwrm_req_drop(bp, req_qs); 8438 8439 if (flags) 8440 return rc; 8441 8442 if (bp->fw_tx_stats_ext_size <= 8443 offsetof(struct tx_port_stats_ext, pfc_pri0_tx_duration_us) / 8) { 8444 bp->pri2cos_valid = 0; 8445 return rc; 8446 } 8447 8448 rc = hwrm_req_init(bp, req_qc, HWRM_QUEUE_PRI2COS_QCFG); 8449 if (rc) 8450 return rc; 8451 8452 req_qc->flags = cpu_to_le32(QUEUE_PRI2COS_QCFG_REQ_FLAGS_IVLAN); 8453 8454 resp_qc = hwrm_req_hold(bp, req_qc); 8455 rc = hwrm_req_send(bp, req_qc); 8456 if (!rc) { 8457 u8 *pri2cos; 8458 int i, j; 8459 8460 pri2cos = &resp_qc->pri0_cos_queue_id; 8461 for (i = 0; i < 8; i++) { 8462 u8 queue_id = pri2cos[i]; 8463 u8 queue_idx; 8464 8465 /* Per port queue IDs start from 0, 10, 20, etc */ 8466 queue_idx = queue_id % 10; 8467 if (queue_idx > BNXT_MAX_QUEUE) { 8468 bp->pri2cos_valid = false; 8469 hwrm_req_drop(bp, req_qc); 8470 return rc; 8471 } 8472 for (j = 0; j < bp->max_q; j++) { 8473 if (bp->q_ids[j] == queue_id) 8474 bp->pri2cos_idx[i] = queue_idx; 8475 } 8476 } 8477 bp->pri2cos_valid = true; 8478 } 8479 hwrm_req_drop(bp, req_qc); 8480 8481 return rc; 8482 } 8483 8484 static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp) 8485 { 8486 bnxt_hwrm_tunnel_dst_port_free(bp, 8487 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN); 8488 bnxt_hwrm_tunnel_dst_port_free(bp, 8489 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE); 8490 } 8491 8492 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa) 8493 { 8494 int rc, i; 8495 u32 tpa_flags = 0; 8496 8497 if (set_tpa) 8498 tpa_flags = bp->flags & BNXT_FLAG_TPA; 8499 else if (BNXT_NO_FW_ACCESS(bp)) 8500 return 0; 8501 for (i = 0; i < bp->nr_vnics; i++) { 8502 rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags); 8503 if (rc) { 8504 netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n", 8505 i, rc); 8506 return rc; 8507 } 8508 } 8509 return 0; 8510 } 8511 8512 static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp) 8513 { 8514 int i; 8515 8516 for (i = 0; i < bp->nr_vnics; i++) 8517 bnxt_hwrm_vnic_set_rss(bp, i, false); 8518 } 8519 8520 static void bnxt_clear_vnic(struct bnxt *bp) 8521 { 8522 if (!bp->vnic_info) 8523 return; 8524 8525 bnxt_hwrm_clear_vnic_filter(bp); 8526 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) { 8527 /* clear all RSS setting before free vnic ctx */ 8528 bnxt_hwrm_clear_vnic_rss(bp); 8529 bnxt_hwrm_vnic_ctx_free(bp); 8530 } 8531 /* before free the vnic, undo the vnic tpa settings */ 8532 if (bp->flags & BNXT_FLAG_TPA) 8533 bnxt_set_tpa(bp, false); 8534 bnxt_hwrm_vnic_free(bp); 8535 if (bp->flags & BNXT_FLAG_CHIP_P5) 8536 bnxt_hwrm_vnic_ctx_free(bp); 8537 } 8538 8539 static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path, 8540 bool irq_re_init) 8541 { 8542 bnxt_clear_vnic(bp); 8543 bnxt_hwrm_ring_free(bp, close_path); 8544 bnxt_hwrm_ring_grp_free(bp); 8545 if (irq_re_init) { 8546 bnxt_hwrm_stat_ctx_free(bp); 8547 bnxt_hwrm_free_tunnel_ports(bp); 8548 } 8549 } 8550 8551 static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode) 8552 { 8553 struct hwrm_func_cfg_input *req; 8554 u8 evb_mode; 8555 int rc; 8556 8557 if (br_mode == BRIDGE_MODE_VEB) 8558 evb_mode = FUNC_CFG_REQ_EVB_MODE_VEB; 8559 else if (br_mode == BRIDGE_MODE_VEPA) 8560 evb_mode = FUNC_CFG_REQ_EVB_MODE_VEPA; 8561 else 8562 return -EINVAL; 8563 8564 rc = hwrm_req_init(bp, req, HWRM_FUNC_CFG); 8565 if (rc) 8566 return rc; 8567 8568 req->fid = cpu_to_le16(0xffff); 8569 req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_EVB_MODE); 8570 req->evb_mode = evb_mode; 8571 return hwrm_req_send(bp, req); 8572 } 8573 8574 static int bnxt_hwrm_set_cache_line_size(struct bnxt *bp, int size) 8575 { 8576 struct hwrm_func_cfg_input *req; 8577 int rc; 8578 8579 if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10803) 8580 return 0; 8581 8582 rc = hwrm_req_init(bp, req, HWRM_FUNC_CFG); 8583 if (rc) 8584 return rc; 8585 8586 req->fid = cpu_to_le16(0xffff); 8587 req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_CACHE_LINESIZE); 8588 req->options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_64; 8589 if (size == 128) 8590 req->options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_128; 8591 8592 return hwrm_req_send(bp, req); 8593 } 8594 8595 static int __bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id) 8596 { 8597 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 8598 int rc; 8599 8600 if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) 8601 goto skip_rss_ctx; 8602 8603 /* allocate context for vnic */ 8604 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 0); 8605 if (rc) { 8606 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n", 8607 vnic_id, rc); 8608 goto vnic_setup_err; 8609 } 8610 bp->rsscos_nr_ctxs++; 8611 8612 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 8613 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 1); 8614 if (rc) { 8615 netdev_err(bp->dev, "hwrm vnic %d cos ctx alloc failure rc: %x\n", 8616 vnic_id, rc); 8617 goto vnic_setup_err; 8618 } 8619 bp->rsscos_nr_ctxs++; 8620 } 8621 8622 skip_rss_ctx: 8623 /* configure default vnic, ring grp */ 8624 rc = bnxt_hwrm_vnic_cfg(bp, vnic_id); 8625 if (rc) { 8626 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n", 8627 vnic_id, rc); 8628 goto vnic_setup_err; 8629 } 8630 8631 /* Enable RSS hashing on vnic */ 8632 rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true); 8633 if (rc) { 8634 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n", 8635 vnic_id, rc); 8636 goto vnic_setup_err; 8637 } 8638 8639 if (bp->flags & BNXT_FLAG_AGG_RINGS) { 8640 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id); 8641 if (rc) { 8642 netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n", 8643 vnic_id, rc); 8644 } 8645 } 8646 8647 vnic_setup_err: 8648 return rc; 8649 } 8650 8651 static int __bnxt_setup_vnic_p5(struct bnxt *bp, u16 vnic_id) 8652 { 8653 int rc, i, nr_ctxs; 8654 8655 nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings); 8656 for (i = 0; i < nr_ctxs; i++) { 8657 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, i); 8658 if (rc) { 8659 netdev_err(bp->dev, "hwrm vnic %d ctx %d alloc failure rc: %x\n", 8660 vnic_id, i, rc); 8661 break; 8662 } 8663 bp->rsscos_nr_ctxs++; 8664 } 8665 if (i < nr_ctxs) 8666 return -ENOMEM; 8667 8668 rc = bnxt_hwrm_vnic_set_rss_p5(bp, vnic_id, true); 8669 if (rc) { 8670 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %d\n", 8671 vnic_id, rc); 8672 return rc; 8673 } 8674 rc = bnxt_hwrm_vnic_cfg(bp, vnic_id); 8675 if (rc) { 8676 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n", 8677 vnic_id, rc); 8678 return rc; 8679 } 8680 if (bp->flags & BNXT_FLAG_AGG_RINGS) { 8681 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id); 8682 if (rc) { 8683 netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n", 8684 vnic_id, rc); 8685 } 8686 } 8687 return rc; 8688 } 8689 8690 static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id) 8691 { 8692 if (bp->flags & BNXT_FLAG_CHIP_P5) 8693 return __bnxt_setup_vnic_p5(bp, vnic_id); 8694 else 8695 return __bnxt_setup_vnic(bp, vnic_id); 8696 } 8697 8698 static int bnxt_alloc_rfs_vnics(struct bnxt *bp) 8699 { 8700 #ifdef CONFIG_RFS_ACCEL 8701 int i, rc = 0; 8702 8703 if (bp->flags & BNXT_FLAG_CHIP_P5) 8704 return 0; 8705 8706 for (i = 0; i < bp->rx_nr_rings; i++) { 8707 struct bnxt_vnic_info *vnic; 8708 u16 vnic_id = i + 1; 8709 u16 ring_id = i; 8710 8711 if (vnic_id >= bp->nr_vnics) 8712 break; 8713 8714 vnic = &bp->vnic_info[vnic_id]; 8715 vnic->flags |= BNXT_VNIC_RFS_FLAG; 8716 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP) 8717 vnic->flags |= BNXT_VNIC_RFS_NEW_RSS_FLAG; 8718 rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, ring_id, 1); 8719 if (rc) { 8720 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n", 8721 vnic_id, rc); 8722 break; 8723 } 8724 rc = bnxt_setup_vnic(bp, vnic_id); 8725 if (rc) 8726 break; 8727 } 8728 return rc; 8729 #else 8730 return 0; 8731 #endif 8732 } 8733 8734 /* Allow PF, trusted VFs and VFs with default VLAN to be in promiscuous mode */ 8735 static bool bnxt_promisc_ok(struct bnxt *bp) 8736 { 8737 #ifdef CONFIG_BNXT_SRIOV 8738 if (BNXT_VF(bp) && !bp->vf.vlan && !bnxt_is_trusted_vf(bp, &bp->vf)) 8739 return false; 8740 #endif 8741 return true; 8742 } 8743 8744 static int bnxt_setup_nitroa0_vnic(struct bnxt *bp) 8745 { 8746 unsigned int rc = 0; 8747 8748 rc = bnxt_hwrm_vnic_alloc(bp, 1, bp->rx_nr_rings - 1, 1); 8749 if (rc) { 8750 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n", 8751 rc); 8752 return rc; 8753 } 8754 8755 rc = bnxt_hwrm_vnic_cfg(bp, 1); 8756 if (rc) { 8757 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n", 8758 rc); 8759 return rc; 8760 } 8761 return rc; 8762 } 8763 8764 static int bnxt_cfg_rx_mode(struct bnxt *); 8765 static bool bnxt_mc_list_updated(struct bnxt *, u32 *); 8766 8767 static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init) 8768 { 8769 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 8770 int rc = 0; 8771 unsigned int rx_nr_rings = bp->rx_nr_rings; 8772 8773 if (irq_re_init) { 8774 rc = bnxt_hwrm_stat_ctx_alloc(bp); 8775 if (rc) { 8776 netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n", 8777 rc); 8778 goto err_out; 8779 } 8780 } 8781 8782 rc = bnxt_hwrm_ring_alloc(bp); 8783 if (rc) { 8784 netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc); 8785 goto err_out; 8786 } 8787 8788 rc = bnxt_hwrm_ring_grp_alloc(bp); 8789 if (rc) { 8790 netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc); 8791 goto err_out; 8792 } 8793 8794 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 8795 rx_nr_rings--; 8796 8797 /* default vnic 0 */ 8798 rc = bnxt_hwrm_vnic_alloc(bp, 0, 0, rx_nr_rings); 8799 if (rc) { 8800 netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc); 8801 goto err_out; 8802 } 8803 8804 if (BNXT_VF(bp)) 8805 bnxt_hwrm_func_qcfg(bp); 8806 8807 rc = bnxt_setup_vnic(bp, 0); 8808 if (rc) 8809 goto err_out; 8810 if (bp->fw_cap & BNXT_FW_CAP_RSS_HASH_TYPE_DELTA) 8811 bnxt_hwrm_update_rss_hash_cfg(bp); 8812 8813 if (bp->flags & BNXT_FLAG_RFS) { 8814 rc = bnxt_alloc_rfs_vnics(bp); 8815 if (rc) 8816 goto err_out; 8817 } 8818 8819 if (bp->flags & BNXT_FLAG_TPA) { 8820 rc = bnxt_set_tpa(bp, true); 8821 if (rc) 8822 goto err_out; 8823 } 8824 8825 if (BNXT_VF(bp)) 8826 bnxt_update_vf_mac(bp); 8827 8828 /* Filter for default vnic 0 */ 8829 rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr); 8830 if (rc) { 8831 if (BNXT_VF(bp) && rc == -ENODEV) 8832 netdev_err(bp->dev, "Cannot configure L2 filter while PF is unavailable\n"); 8833 else 8834 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc); 8835 goto err_out; 8836 } 8837 vnic->uc_filter_count = 1; 8838 8839 vnic->rx_mask = 0; 8840 if (test_bit(BNXT_STATE_HALF_OPEN, &bp->state)) 8841 goto skip_rx_mask; 8842 8843 if (bp->dev->flags & IFF_BROADCAST) 8844 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST; 8845 8846 if (bp->dev->flags & IFF_PROMISC) 8847 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 8848 8849 if (bp->dev->flags & IFF_ALLMULTI) { 8850 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 8851 vnic->mc_list_count = 0; 8852 } else if (bp->dev->flags & IFF_MULTICAST) { 8853 u32 mask = 0; 8854 8855 bnxt_mc_list_updated(bp, &mask); 8856 vnic->rx_mask |= mask; 8857 } 8858 8859 rc = bnxt_cfg_rx_mode(bp); 8860 if (rc) 8861 goto err_out; 8862 8863 skip_rx_mask: 8864 rc = bnxt_hwrm_set_coal(bp); 8865 if (rc) 8866 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n", 8867 rc); 8868 8869 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 8870 rc = bnxt_setup_nitroa0_vnic(bp); 8871 if (rc) 8872 netdev_err(bp->dev, "Special vnic setup failure for NS2 A0 rc: %x\n", 8873 rc); 8874 } 8875 8876 if (BNXT_VF(bp)) { 8877 bnxt_hwrm_func_qcfg(bp); 8878 netdev_update_features(bp->dev); 8879 } 8880 8881 return 0; 8882 8883 err_out: 8884 bnxt_hwrm_resource_free(bp, 0, true); 8885 8886 return rc; 8887 } 8888 8889 static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init) 8890 { 8891 bnxt_hwrm_resource_free(bp, 1, irq_re_init); 8892 return 0; 8893 } 8894 8895 static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init) 8896 { 8897 bnxt_init_cp_rings(bp); 8898 bnxt_init_rx_rings(bp); 8899 bnxt_init_tx_rings(bp); 8900 bnxt_init_ring_grps(bp, irq_re_init); 8901 bnxt_init_vnics(bp); 8902 8903 return bnxt_init_chip(bp, irq_re_init); 8904 } 8905 8906 static int bnxt_set_real_num_queues(struct bnxt *bp) 8907 { 8908 int rc; 8909 struct net_device *dev = bp->dev; 8910 8911 rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings - 8912 bp->tx_nr_rings_xdp); 8913 if (rc) 8914 return rc; 8915 8916 rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings); 8917 if (rc) 8918 return rc; 8919 8920 #ifdef CONFIG_RFS_ACCEL 8921 if (bp->flags & BNXT_FLAG_RFS) 8922 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings); 8923 #endif 8924 8925 return rc; 8926 } 8927 8928 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max, 8929 bool shared) 8930 { 8931 int _rx = *rx, _tx = *tx; 8932 8933 if (shared) { 8934 *rx = min_t(int, _rx, max); 8935 *tx = min_t(int, _tx, max); 8936 } else { 8937 if (max < 2) 8938 return -ENOMEM; 8939 8940 while (_rx + _tx > max) { 8941 if (_rx > _tx && _rx > 1) 8942 _rx--; 8943 else if (_tx > 1) 8944 _tx--; 8945 } 8946 *rx = _rx; 8947 *tx = _tx; 8948 } 8949 return 0; 8950 } 8951 8952 static void bnxt_setup_msix(struct bnxt *bp) 8953 { 8954 const int len = sizeof(bp->irq_tbl[0].name); 8955 struct net_device *dev = bp->dev; 8956 int tcs, i; 8957 8958 tcs = netdev_get_num_tc(dev); 8959 if (tcs) { 8960 int i, off, count; 8961 8962 for (i = 0; i < tcs; i++) { 8963 count = bp->tx_nr_rings_per_tc; 8964 off = i * count; 8965 netdev_set_tc_queue(dev, i, count, off); 8966 } 8967 } 8968 8969 for (i = 0; i < bp->cp_nr_rings; i++) { 8970 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 8971 char *attr; 8972 8973 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 8974 attr = "TxRx"; 8975 else if (i < bp->rx_nr_rings) 8976 attr = "rx"; 8977 else 8978 attr = "tx"; 8979 8980 snprintf(bp->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name, 8981 attr, i); 8982 bp->irq_tbl[map_idx].handler = bnxt_msix; 8983 } 8984 } 8985 8986 static void bnxt_setup_inta(struct bnxt *bp) 8987 { 8988 const int len = sizeof(bp->irq_tbl[0].name); 8989 8990 if (netdev_get_num_tc(bp->dev)) 8991 netdev_reset_tc(bp->dev); 8992 8993 snprintf(bp->irq_tbl[0].name, len, "%s-%s-%d", bp->dev->name, "TxRx", 8994 0); 8995 bp->irq_tbl[0].handler = bnxt_inta; 8996 } 8997 8998 static int bnxt_init_int_mode(struct bnxt *bp); 8999 9000 static int bnxt_setup_int_mode(struct bnxt *bp) 9001 { 9002 int rc; 9003 9004 if (!bp->irq_tbl) { 9005 rc = bnxt_init_int_mode(bp); 9006 if (rc || !bp->irq_tbl) 9007 return rc ?: -ENODEV; 9008 } 9009 9010 if (bp->flags & BNXT_FLAG_USING_MSIX) 9011 bnxt_setup_msix(bp); 9012 else 9013 bnxt_setup_inta(bp); 9014 9015 rc = bnxt_set_real_num_queues(bp); 9016 return rc; 9017 } 9018 9019 #ifdef CONFIG_RFS_ACCEL 9020 static unsigned int bnxt_get_max_func_rss_ctxs(struct bnxt *bp) 9021 { 9022 return bp->hw_resc.max_rsscos_ctxs; 9023 } 9024 9025 static unsigned int bnxt_get_max_func_vnics(struct bnxt *bp) 9026 { 9027 return bp->hw_resc.max_vnics; 9028 } 9029 #endif 9030 9031 unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp) 9032 { 9033 return bp->hw_resc.max_stat_ctxs; 9034 } 9035 9036 unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp) 9037 { 9038 return bp->hw_resc.max_cp_rings; 9039 } 9040 9041 static unsigned int bnxt_get_max_func_cp_rings_for_en(struct bnxt *bp) 9042 { 9043 unsigned int cp = bp->hw_resc.max_cp_rings; 9044 9045 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 9046 cp -= bnxt_get_ulp_msix_num(bp); 9047 9048 return cp; 9049 } 9050 9051 static unsigned int bnxt_get_max_func_irqs(struct bnxt *bp) 9052 { 9053 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 9054 9055 if (bp->flags & BNXT_FLAG_CHIP_P5) 9056 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_nqs); 9057 9058 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_cp_rings); 9059 } 9060 9061 static void bnxt_set_max_func_irqs(struct bnxt *bp, unsigned int max_irqs) 9062 { 9063 bp->hw_resc.max_irqs = max_irqs; 9064 } 9065 9066 unsigned int bnxt_get_avail_cp_rings_for_en(struct bnxt *bp) 9067 { 9068 unsigned int cp; 9069 9070 cp = bnxt_get_max_func_cp_rings_for_en(bp); 9071 if (bp->flags & BNXT_FLAG_CHIP_P5) 9072 return cp - bp->rx_nr_rings - bp->tx_nr_rings; 9073 else 9074 return cp - bp->cp_nr_rings; 9075 } 9076 9077 unsigned int bnxt_get_avail_stat_ctxs_for_en(struct bnxt *bp) 9078 { 9079 return bnxt_get_max_func_stat_ctxs(bp) - bnxt_get_func_stat_ctxs(bp); 9080 } 9081 9082 int bnxt_get_avail_msix(struct bnxt *bp, int num) 9083 { 9084 int max_cp = bnxt_get_max_func_cp_rings(bp); 9085 int max_irq = bnxt_get_max_func_irqs(bp); 9086 int total_req = bp->cp_nr_rings + num; 9087 int max_idx, avail_msix; 9088 9089 max_idx = bp->total_irqs; 9090 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 9091 max_idx = min_t(int, bp->total_irqs, max_cp); 9092 avail_msix = max_idx - bp->cp_nr_rings; 9093 if (!BNXT_NEW_RM(bp) || avail_msix >= num) 9094 return avail_msix; 9095 9096 if (max_irq < total_req) { 9097 num = max_irq - bp->cp_nr_rings; 9098 if (num <= 0) 9099 return 0; 9100 } 9101 return num; 9102 } 9103 9104 static int bnxt_get_num_msix(struct bnxt *bp) 9105 { 9106 if (!BNXT_NEW_RM(bp)) 9107 return bnxt_get_max_func_irqs(bp); 9108 9109 return bnxt_nq_rings_in_use(bp); 9110 } 9111 9112 static int bnxt_init_msix(struct bnxt *bp) 9113 { 9114 int i, total_vecs, max, rc = 0, min = 1, ulp_msix; 9115 struct msix_entry *msix_ent; 9116 9117 total_vecs = bnxt_get_num_msix(bp); 9118 max = bnxt_get_max_func_irqs(bp); 9119 if (total_vecs > max) 9120 total_vecs = max; 9121 9122 if (!total_vecs) 9123 return 0; 9124 9125 msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL); 9126 if (!msix_ent) 9127 return -ENOMEM; 9128 9129 for (i = 0; i < total_vecs; i++) { 9130 msix_ent[i].entry = i; 9131 msix_ent[i].vector = 0; 9132 } 9133 9134 if (!(bp->flags & BNXT_FLAG_SHARED_RINGS)) 9135 min = 2; 9136 9137 total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs); 9138 ulp_msix = bnxt_get_ulp_msix_num(bp); 9139 if (total_vecs < 0 || total_vecs < ulp_msix) { 9140 rc = -ENODEV; 9141 goto msix_setup_exit; 9142 } 9143 9144 bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL); 9145 if (bp->irq_tbl) { 9146 for (i = 0; i < total_vecs; i++) 9147 bp->irq_tbl[i].vector = msix_ent[i].vector; 9148 9149 bp->total_irqs = total_vecs; 9150 /* Trim rings based upon num of vectors allocated */ 9151 rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings, 9152 total_vecs - ulp_msix, min == 1); 9153 if (rc) 9154 goto msix_setup_exit; 9155 9156 bp->cp_nr_rings = (min == 1) ? 9157 max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) : 9158 bp->tx_nr_rings + bp->rx_nr_rings; 9159 9160 } else { 9161 rc = -ENOMEM; 9162 goto msix_setup_exit; 9163 } 9164 bp->flags |= BNXT_FLAG_USING_MSIX; 9165 kfree(msix_ent); 9166 return 0; 9167 9168 msix_setup_exit: 9169 netdev_err(bp->dev, "bnxt_init_msix err: %x\n", rc); 9170 kfree(bp->irq_tbl); 9171 bp->irq_tbl = NULL; 9172 pci_disable_msix(bp->pdev); 9173 kfree(msix_ent); 9174 return rc; 9175 } 9176 9177 static int bnxt_init_inta(struct bnxt *bp) 9178 { 9179 bp->irq_tbl = kzalloc(sizeof(struct bnxt_irq), GFP_KERNEL); 9180 if (!bp->irq_tbl) 9181 return -ENOMEM; 9182 9183 bp->total_irqs = 1; 9184 bp->rx_nr_rings = 1; 9185 bp->tx_nr_rings = 1; 9186 bp->cp_nr_rings = 1; 9187 bp->flags |= BNXT_FLAG_SHARED_RINGS; 9188 bp->irq_tbl[0].vector = bp->pdev->irq; 9189 return 0; 9190 } 9191 9192 static int bnxt_init_int_mode(struct bnxt *bp) 9193 { 9194 int rc = -ENODEV; 9195 9196 if (bp->flags & BNXT_FLAG_MSIX_CAP) 9197 rc = bnxt_init_msix(bp); 9198 9199 if (!(bp->flags & BNXT_FLAG_USING_MSIX) && BNXT_PF(bp)) { 9200 /* fallback to INTA */ 9201 rc = bnxt_init_inta(bp); 9202 } 9203 return rc; 9204 } 9205 9206 static void bnxt_clear_int_mode(struct bnxt *bp) 9207 { 9208 if (bp->flags & BNXT_FLAG_USING_MSIX) 9209 pci_disable_msix(bp->pdev); 9210 9211 kfree(bp->irq_tbl); 9212 bp->irq_tbl = NULL; 9213 bp->flags &= ~BNXT_FLAG_USING_MSIX; 9214 } 9215 9216 int bnxt_reserve_rings(struct bnxt *bp, bool irq_re_init) 9217 { 9218 int tcs = netdev_get_num_tc(bp->dev); 9219 bool irq_cleared = false; 9220 int rc; 9221 9222 if (!bnxt_need_reserve_rings(bp)) 9223 return 0; 9224 9225 if (irq_re_init && BNXT_NEW_RM(bp) && 9226 bnxt_get_num_msix(bp) != bp->total_irqs) { 9227 bnxt_ulp_irq_stop(bp); 9228 bnxt_clear_int_mode(bp); 9229 irq_cleared = true; 9230 } 9231 rc = __bnxt_reserve_rings(bp); 9232 if (irq_cleared) { 9233 if (!rc) 9234 rc = bnxt_init_int_mode(bp); 9235 bnxt_ulp_irq_restart(bp, rc); 9236 } 9237 if (rc) { 9238 netdev_err(bp->dev, "ring reservation/IRQ init failure rc: %d\n", rc); 9239 return rc; 9240 } 9241 if (tcs && (bp->tx_nr_rings_per_tc * tcs != 9242 bp->tx_nr_rings - bp->tx_nr_rings_xdp)) { 9243 netdev_err(bp->dev, "tx ring reservation failure\n"); 9244 netdev_reset_tc(bp->dev); 9245 if (bp->tx_nr_rings_xdp) 9246 bp->tx_nr_rings_per_tc = bp->tx_nr_rings_xdp; 9247 else 9248 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 9249 return -ENOMEM; 9250 } 9251 return 0; 9252 } 9253 9254 static void bnxt_free_irq(struct bnxt *bp) 9255 { 9256 struct bnxt_irq *irq; 9257 int i; 9258 9259 #ifdef CONFIG_RFS_ACCEL 9260 free_irq_cpu_rmap(bp->dev->rx_cpu_rmap); 9261 bp->dev->rx_cpu_rmap = NULL; 9262 #endif 9263 if (!bp->irq_tbl || !bp->bnapi) 9264 return; 9265 9266 for (i = 0; i < bp->cp_nr_rings; i++) { 9267 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 9268 9269 irq = &bp->irq_tbl[map_idx]; 9270 if (irq->requested) { 9271 if (irq->have_cpumask) { 9272 irq_set_affinity_hint(irq->vector, NULL); 9273 free_cpumask_var(irq->cpu_mask); 9274 irq->have_cpumask = 0; 9275 } 9276 free_irq(irq->vector, bp->bnapi[i]); 9277 } 9278 9279 irq->requested = 0; 9280 } 9281 } 9282 9283 static int bnxt_request_irq(struct bnxt *bp) 9284 { 9285 int i, j, rc = 0; 9286 unsigned long flags = 0; 9287 #ifdef CONFIG_RFS_ACCEL 9288 struct cpu_rmap *rmap; 9289 #endif 9290 9291 rc = bnxt_setup_int_mode(bp); 9292 if (rc) { 9293 netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n", 9294 rc); 9295 return rc; 9296 } 9297 #ifdef CONFIG_RFS_ACCEL 9298 rmap = bp->dev->rx_cpu_rmap; 9299 #endif 9300 if (!(bp->flags & BNXT_FLAG_USING_MSIX)) 9301 flags = IRQF_SHARED; 9302 9303 for (i = 0, j = 0; i < bp->cp_nr_rings; i++) { 9304 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 9305 struct bnxt_irq *irq = &bp->irq_tbl[map_idx]; 9306 9307 #ifdef CONFIG_RFS_ACCEL 9308 if (rmap && bp->bnapi[i]->rx_ring) { 9309 rc = irq_cpu_rmap_add(rmap, irq->vector); 9310 if (rc) 9311 netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n", 9312 j); 9313 j++; 9314 } 9315 #endif 9316 rc = request_irq(irq->vector, irq->handler, flags, irq->name, 9317 bp->bnapi[i]); 9318 if (rc) 9319 break; 9320 9321 irq->requested = 1; 9322 9323 if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) { 9324 int numa_node = dev_to_node(&bp->pdev->dev); 9325 9326 irq->have_cpumask = 1; 9327 cpumask_set_cpu(cpumask_local_spread(i, numa_node), 9328 irq->cpu_mask); 9329 rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask); 9330 if (rc) { 9331 netdev_warn(bp->dev, 9332 "Set affinity failed, IRQ = %d\n", 9333 irq->vector); 9334 break; 9335 } 9336 } 9337 } 9338 return rc; 9339 } 9340 9341 static void bnxt_del_napi(struct bnxt *bp) 9342 { 9343 int i; 9344 9345 if (!bp->bnapi) 9346 return; 9347 9348 for (i = 0; i < bp->cp_nr_rings; i++) { 9349 struct bnxt_napi *bnapi = bp->bnapi[i]; 9350 9351 __netif_napi_del(&bnapi->napi); 9352 } 9353 /* We called __netif_napi_del(), we need 9354 * to respect an RCU grace period before freeing napi structures. 9355 */ 9356 synchronize_net(); 9357 } 9358 9359 static void bnxt_init_napi(struct bnxt *bp) 9360 { 9361 int i; 9362 unsigned int cp_nr_rings = bp->cp_nr_rings; 9363 struct bnxt_napi *bnapi; 9364 9365 if (bp->flags & BNXT_FLAG_USING_MSIX) { 9366 int (*poll_fn)(struct napi_struct *, int) = bnxt_poll; 9367 9368 if (bp->flags & BNXT_FLAG_CHIP_P5) 9369 poll_fn = bnxt_poll_p5; 9370 else if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 9371 cp_nr_rings--; 9372 for (i = 0; i < cp_nr_rings; i++) { 9373 bnapi = bp->bnapi[i]; 9374 netif_napi_add(bp->dev, &bnapi->napi, poll_fn); 9375 } 9376 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 9377 bnapi = bp->bnapi[cp_nr_rings]; 9378 netif_napi_add(bp->dev, &bnapi->napi, 9379 bnxt_poll_nitroa0); 9380 } 9381 } else { 9382 bnapi = bp->bnapi[0]; 9383 netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll); 9384 } 9385 } 9386 9387 static void bnxt_disable_napi(struct bnxt *bp) 9388 { 9389 int i; 9390 9391 if (!bp->bnapi || 9392 test_and_set_bit(BNXT_STATE_NAPI_DISABLED, &bp->state)) 9393 return; 9394 9395 for (i = 0; i < bp->cp_nr_rings; i++) { 9396 struct bnxt_napi *bnapi = bp->bnapi[i]; 9397 struct bnxt_cp_ring_info *cpr; 9398 9399 cpr = &bnapi->cp_ring; 9400 if (bnapi->tx_fault) 9401 cpr->sw_stats.tx.tx_resets++; 9402 if (bnapi->in_reset) 9403 cpr->sw_stats.rx.rx_resets++; 9404 napi_disable(&bnapi->napi); 9405 if (bnapi->rx_ring) 9406 cancel_work_sync(&cpr->dim.work); 9407 } 9408 } 9409 9410 static void bnxt_enable_napi(struct bnxt *bp) 9411 { 9412 int i; 9413 9414 clear_bit(BNXT_STATE_NAPI_DISABLED, &bp->state); 9415 for (i = 0; i < bp->cp_nr_rings; i++) { 9416 struct bnxt_napi *bnapi = bp->bnapi[i]; 9417 struct bnxt_cp_ring_info *cpr; 9418 9419 bnapi->tx_fault = 0; 9420 9421 cpr = &bnapi->cp_ring; 9422 bnapi->in_reset = false; 9423 9424 bnapi->tx_pkts = 0; 9425 9426 if (bnapi->rx_ring) { 9427 INIT_WORK(&cpr->dim.work, bnxt_dim_work); 9428 cpr->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; 9429 } 9430 napi_enable(&bnapi->napi); 9431 } 9432 } 9433 9434 void bnxt_tx_disable(struct bnxt *bp) 9435 { 9436 int i; 9437 struct bnxt_tx_ring_info *txr; 9438 9439 if (bp->tx_ring) { 9440 for (i = 0; i < bp->tx_nr_rings; i++) { 9441 txr = &bp->tx_ring[i]; 9442 WRITE_ONCE(txr->dev_state, BNXT_DEV_STATE_CLOSING); 9443 } 9444 } 9445 /* Make sure napi polls see @dev_state change */ 9446 synchronize_net(); 9447 /* Drop carrier first to prevent TX timeout */ 9448 netif_carrier_off(bp->dev); 9449 /* Stop all TX queues */ 9450 netif_tx_disable(bp->dev); 9451 } 9452 9453 void bnxt_tx_enable(struct bnxt *bp) 9454 { 9455 int i; 9456 struct bnxt_tx_ring_info *txr; 9457 9458 for (i = 0; i < bp->tx_nr_rings; i++) { 9459 txr = &bp->tx_ring[i]; 9460 WRITE_ONCE(txr->dev_state, 0); 9461 } 9462 /* Make sure napi polls see @dev_state change */ 9463 synchronize_net(); 9464 netif_tx_wake_all_queues(bp->dev); 9465 if (BNXT_LINK_IS_UP(bp)) 9466 netif_carrier_on(bp->dev); 9467 } 9468 9469 static char *bnxt_report_fec(struct bnxt_link_info *link_info) 9470 { 9471 u8 active_fec = link_info->active_fec_sig_mode & 9472 PORT_PHY_QCFG_RESP_ACTIVE_FEC_MASK; 9473 9474 switch (active_fec) { 9475 default: 9476 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_NONE_ACTIVE: 9477 return "None"; 9478 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE74_ACTIVE: 9479 return "Clause 74 BaseR"; 9480 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE91_ACTIVE: 9481 return "Clause 91 RS(528,514)"; 9482 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_1XN_ACTIVE: 9483 return "Clause 91 RS544_1XN"; 9484 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_IEEE_ACTIVE: 9485 return "Clause 91 RS(544,514)"; 9486 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_1XN_ACTIVE: 9487 return "Clause 91 RS272_1XN"; 9488 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_IEEE_ACTIVE: 9489 return "Clause 91 RS(272,257)"; 9490 } 9491 } 9492 9493 void bnxt_report_link(struct bnxt *bp) 9494 { 9495 if (BNXT_LINK_IS_UP(bp)) { 9496 const char *signal = ""; 9497 const char *flow_ctrl; 9498 const char *duplex; 9499 u32 speed; 9500 u16 fec; 9501 9502 netif_carrier_on(bp->dev); 9503 speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed); 9504 if (speed == SPEED_UNKNOWN) { 9505 netdev_info(bp->dev, "NIC Link is Up, speed unknown\n"); 9506 return; 9507 } 9508 if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL) 9509 duplex = "full"; 9510 else 9511 duplex = "half"; 9512 if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH) 9513 flow_ctrl = "ON - receive & transmit"; 9514 else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX) 9515 flow_ctrl = "ON - transmit"; 9516 else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX) 9517 flow_ctrl = "ON - receive"; 9518 else 9519 flow_ctrl = "none"; 9520 if (bp->link_info.phy_qcfg_resp.option_flags & 9521 PORT_PHY_QCFG_RESP_OPTION_FLAGS_SIGNAL_MODE_KNOWN) { 9522 u8 sig_mode = bp->link_info.active_fec_sig_mode & 9523 PORT_PHY_QCFG_RESP_SIGNAL_MODE_MASK; 9524 switch (sig_mode) { 9525 case PORT_PHY_QCFG_RESP_SIGNAL_MODE_NRZ: 9526 signal = "(NRZ) "; 9527 break; 9528 case PORT_PHY_QCFG_RESP_SIGNAL_MODE_PAM4: 9529 signal = "(PAM4) "; 9530 break; 9531 default: 9532 break; 9533 } 9534 } 9535 netdev_info(bp->dev, "NIC Link is Up, %u Mbps %s%s duplex, Flow control: %s\n", 9536 speed, signal, duplex, flow_ctrl); 9537 if (bp->phy_flags & BNXT_PHY_FL_EEE_CAP) 9538 netdev_info(bp->dev, "EEE is %s\n", 9539 bp->eee.eee_active ? "active" : 9540 "not active"); 9541 fec = bp->link_info.fec_cfg; 9542 if (!(fec & PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED)) 9543 netdev_info(bp->dev, "FEC autoneg %s encoding: %s\n", 9544 (fec & BNXT_FEC_AUTONEG) ? "on" : "off", 9545 bnxt_report_fec(&bp->link_info)); 9546 } else { 9547 netif_carrier_off(bp->dev); 9548 netdev_err(bp->dev, "NIC Link is Down\n"); 9549 } 9550 } 9551 9552 static bool bnxt_phy_qcaps_no_speed(struct hwrm_port_phy_qcaps_output *resp) 9553 { 9554 if (!resp->supported_speeds_auto_mode && 9555 !resp->supported_speeds_force_mode && 9556 !resp->supported_pam4_speeds_auto_mode && 9557 !resp->supported_pam4_speeds_force_mode) 9558 return true; 9559 return false; 9560 } 9561 9562 static int bnxt_hwrm_phy_qcaps(struct bnxt *bp) 9563 { 9564 struct bnxt_link_info *link_info = &bp->link_info; 9565 struct hwrm_port_phy_qcaps_output *resp; 9566 struct hwrm_port_phy_qcaps_input *req; 9567 int rc = 0; 9568 9569 if (bp->hwrm_spec_code < 0x10201) 9570 return 0; 9571 9572 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_QCAPS); 9573 if (rc) 9574 return rc; 9575 9576 resp = hwrm_req_hold(bp, req); 9577 rc = hwrm_req_send(bp, req); 9578 if (rc) 9579 goto hwrm_phy_qcaps_exit; 9580 9581 bp->phy_flags = resp->flags | (le16_to_cpu(resp->flags2) << 8); 9582 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) { 9583 struct ethtool_eee *eee = &bp->eee; 9584 u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode); 9585 9586 eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0); 9587 bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) & 9588 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK; 9589 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) & 9590 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK; 9591 } 9592 9593 if (bp->hwrm_spec_code >= 0x10a01) { 9594 if (bnxt_phy_qcaps_no_speed(resp)) { 9595 link_info->phy_state = BNXT_PHY_STATE_DISABLED; 9596 netdev_warn(bp->dev, "Ethernet link disabled\n"); 9597 } else if (link_info->phy_state == BNXT_PHY_STATE_DISABLED) { 9598 link_info->phy_state = BNXT_PHY_STATE_ENABLED; 9599 netdev_info(bp->dev, "Ethernet link enabled\n"); 9600 /* Phy re-enabled, reprobe the speeds */ 9601 link_info->support_auto_speeds = 0; 9602 link_info->support_pam4_auto_speeds = 0; 9603 } 9604 } 9605 if (resp->supported_speeds_auto_mode) 9606 link_info->support_auto_speeds = 9607 le16_to_cpu(resp->supported_speeds_auto_mode); 9608 if (resp->supported_pam4_speeds_auto_mode) 9609 link_info->support_pam4_auto_speeds = 9610 le16_to_cpu(resp->supported_pam4_speeds_auto_mode); 9611 9612 bp->port_count = resp->port_cnt; 9613 9614 hwrm_phy_qcaps_exit: 9615 hwrm_req_drop(bp, req); 9616 return rc; 9617 } 9618 9619 static bool bnxt_support_dropped(u16 advertising, u16 supported) 9620 { 9621 u16 diff = advertising ^ supported; 9622 9623 return ((supported | diff) != supported); 9624 } 9625 9626 int bnxt_update_link(struct bnxt *bp, bool chng_link_state) 9627 { 9628 struct bnxt_link_info *link_info = &bp->link_info; 9629 struct hwrm_port_phy_qcfg_output *resp; 9630 struct hwrm_port_phy_qcfg_input *req; 9631 u8 link_state = link_info->link_state; 9632 bool support_changed = false; 9633 int rc; 9634 9635 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_QCFG); 9636 if (rc) 9637 return rc; 9638 9639 resp = hwrm_req_hold(bp, req); 9640 rc = hwrm_req_send(bp, req); 9641 if (rc) { 9642 hwrm_req_drop(bp, req); 9643 if (BNXT_VF(bp) && rc == -ENODEV) { 9644 netdev_warn(bp->dev, "Cannot obtain link state while PF unavailable.\n"); 9645 rc = 0; 9646 } 9647 return rc; 9648 } 9649 9650 memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp)); 9651 link_info->phy_link_status = resp->link; 9652 link_info->duplex = resp->duplex_cfg; 9653 if (bp->hwrm_spec_code >= 0x10800) 9654 link_info->duplex = resp->duplex_state; 9655 link_info->pause = resp->pause; 9656 link_info->auto_mode = resp->auto_mode; 9657 link_info->auto_pause_setting = resp->auto_pause; 9658 link_info->lp_pause = resp->link_partner_adv_pause; 9659 link_info->force_pause_setting = resp->force_pause; 9660 link_info->duplex_setting = resp->duplex_cfg; 9661 if (link_info->phy_link_status == BNXT_LINK_LINK) 9662 link_info->link_speed = le16_to_cpu(resp->link_speed); 9663 else 9664 link_info->link_speed = 0; 9665 link_info->force_link_speed = le16_to_cpu(resp->force_link_speed); 9666 link_info->force_pam4_link_speed = 9667 le16_to_cpu(resp->force_pam4_link_speed); 9668 link_info->support_speeds = le16_to_cpu(resp->support_speeds); 9669 link_info->support_pam4_speeds = le16_to_cpu(resp->support_pam4_speeds); 9670 link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask); 9671 link_info->auto_pam4_link_speeds = 9672 le16_to_cpu(resp->auto_pam4_link_speed_mask); 9673 link_info->lp_auto_link_speeds = 9674 le16_to_cpu(resp->link_partner_adv_speeds); 9675 link_info->lp_auto_pam4_link_speeds = 9676 resp->link_partner_pam4_adv_speeds; 9677 link_info->preemphasis = le32_to_cpu(resp->preemphasis); 9678 link_info->phy_ver[0] = resp->phy_maj; 9679 link_info->phy_ver[1] = resp->phy_min; 9680 link_info->phy_ver[2] = resp->phy_bld; 9681 link_info->media_type = resp->media_type; 9682 link_info->phy_type = resp->phy_type; 9683 link_info->transceiver = resp->xcvr_pkg_type; 9684 link_info->phy_addr = resp->eee_config_phy_addr & 9685 PORT_PHY_QCFG_RESP_PHY_ADDR_MASK; 9686 link_info->module_status = resp->module_status; 9687 9688 if (bp->phy_flags & BNXT_PHY_FL_EEE_CAP) { 9689 struct ethtool_eee *eee = &bp->eee; 9690 u16 fw_speeds; 9691 9692 eee->eee_active = 0; 9693 if (resp->eee_config_phy_addr & 9694 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) { 9695 eee->eee_active = 1; 9696 fw_speeds = le16_to_cpu( 9697 resp->link_partner_adv_eee_link_speed_mask); 9698 eee->lp_advertised = 9699 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0); 9700 } 9701 9702 /* Pull initial EEE config */ 9703 if (!chng_link_state) { 9704 if (resp->eee_config_phy_addr & 9705 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED) 9706 eee->eee_enabled = 1; 9707 9708 fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask); 9709 eee->advertised = 9710 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0); 9711 9712 if (resp->eee_config_phy_addr & 9713 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) { 9714 __le32 tmr; 9715 9716 eee->tx_lpi_enabled = 1; 9717 tmr = resp->xcvr_identifier_type_tx_lpi_timer; 9718 eee->tx_lpi_timer = le32_to_cpu(tmr) & 9719 PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK; 9720 } 9721 } 9722 } 9723 9724 link_info->fec_cfg = PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED; 9725 if (bp->hwrm_spec_code >= 0x10504) { 9726 link_info->fec_cfg = le16_to_cpu(resp->fec_cfg); 9727 link_info->active_fec_sig_mode = resp->active_fec_signal_mode; 9728 } 9729 /* TODO: need to add more logic to report VF link */ 9730 if (chng_link_state) { 9731 if (link_info->phy_link_status == BNXT_LINK_LINK) 9732 link_info->link_state = BNXT_LINK_STATE_UP; 9733 else 9734 link_info->link_state = BNXT_LINK_STATE_DOWN; 9735 if (link_state != link_info->link_state) 9736 bnxt_report_link(bp); 9737 } else { 9738 /* always link down if not require to update link state */ 9739 link_info->link_state = BNXT_LINK_STATE_DOWN; 9740 } 9741 hwrm_req_drop(bp, req); 9742 9743 if (!BNXT_PHY_CFG_ABLE(bp)) 9744 return 0; 9745 9746 /* Check if any advertised speeds are no longer supported. The caller 9747 * holds the link_lock mutex, so we can modify link_info settings. 9748 */ 9749 if (bnxt_support_dropped(link_info->advertising, 9750 link_info->support_auto_speeds)) { 9751 link_info->advertising = link_info->support_auto_speeds; 9752 support_changed = true; 9753 } 9754 if (bnxt_support_dropped(link_info->advertising_pam4, 9755 link_info->support_pam4_auto_speeds)) { 9756 link_info->advertising_pam4 = link_info->support_pam4_auto_speeds; 9757 support_changed = true; 9758 } 9759 if (support_changed && (link_info->autoneg & BNXT_AUTONEG_SPEED)) 9760 bnxt_hwrm_set_link_setting(bp, true, false); 9761 return 0; 9762 } 9763 9764 static void bnxt_get_port_module_status(struct bnxt *bp) 9765 { 9766 struct bnxt_link_info *link_info = &bp->link_info; 9767 struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp; 9768 u8 module_status; 9769 9770 if (bnxt_update_link(bp, true)) 9771 return; 9772 9773 module_status = link_info->module_status; 9774 switch (module_status) { 9775 case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX: 9776 case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN: 9777 case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG: 9778 netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n", 9779 bp->pf.port_id); 9780 if (bp->hwrm_spec_code >= 0x10201) { 9781 netdev_warn(bp->dev, "Module part number %s\n", 9782 resp->phy_vendor_partnumber); 9783 } 9784 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX) 9785 netdev_warn(bp->dev, "TX is disabled\n"); 9786 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN) 9787 netdev_warn(bp->dev, "SFP+ module is shutdown\n"); 9788 } 9789 } 9790 9791 static void 9792 bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req) 9793 { 9794 if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) { 9795 if (bp->hwrm_spec_code >= 0x10201) 9796 req->auto_pause = 9797 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE; 9798 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX) 9799 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX; 9800 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX) 9801 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX; 9802 req->enables |= 9803 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE); 9804 } else { 9805 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX) 9806 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX; 9807 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX) 9808 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX; 9809 req->enables |= 9810 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE); 9811 if (bp->hwrm_spec_code >= 0x10201) { 9812 req->auto_pause = req->force_pause; 9813 req->enables |= cpu_to_le32( 9814 PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE); 9815 } 9816 } 9817 } 9818 9819 static void bnxt_hwrm_set_link_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req) 9820 { 9821 if (bp->link_info.autoneg & BNXT_AUTONEG_SPEED) { 9822 req->auto_mode |= PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK; 9823 if (bp->link_info.advertising) { 9824 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK); 9825 req->auto_link_speed_mask = cpu_to_le16(bp->link_info.advertising); 9826 } 9827 if (bp->link_info.advertising_pam4) { 9828 req->enables |= 9829 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAM4_LINK_SPEED_MASK); 9830 req->auto_link_pam4_speed_mask = 9831 cpu_to_le16(bp->link_info.advertising_pam4); 9832 } 9833 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE); 9834 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG); 9835 } else { 9836 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE); 9837 if (bp->link_info.req_signal_mode == BNXT_SIG_MODE_PAM4) { 9838 req->force_pam4_link_speed = cpu_to_le16(bp->link_info.req_link_speed); 9839 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAM4_LINK_SPEED); 9840 } else { 9841 req->force_link_speed = cpu_to_le16(bp->link_info.req_link_speed); 9842 } 9843 } 9844 9845 /* tell chimp that the setting takes effect immediately */ 9846 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY); 9847 } 9848 9849 int bnxt_hwrm_set_pause(struct bnxt *bp) 9850 { 9851 struct hwrm_port_phy_cfg_input *req; 9852 int rc; 9853 9854 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG); 9855 if (rc) 9856 return rc; 9857 9858 bnxt_hwrm_set_pause_common(bp, req); 9859 9860 if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) || 9861 bp->link_info.force_link_chng) 9862 bnxt_hwrm_set_link_common(bp, req); 9863 9864 rc = hwrm_req_send(bp, req); 9865 if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) { 9866 /* since changing of pause setting doesn't trigger any link 9867 * change event, the driver needs to update the current pause 9868 * result upon successfully return of the phy_cfg command 9869 */ 9870 bp->link_info.pause = 9871 bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl; 9872 bp->link_info.auto_pause_setting = 0; 9873 if (!bp->link_info.force_link_chng) 9874 bnxt_report_link(bp); 9875 } 9876 bp->link_info.force_link_chng = false; 9877 return rc; 9878 } 9879 9880 static void bnxt_hwrm_set_eee(struct bnxt *bp, 9881 struct hwrm_port_phy_cfg_input *req) 9882 { 9883 struct ethtool_eee *eee = &bp->eee; 9884 9885 if (eee->eee_enabled) { 9886 u16 eee_speeds; 9887 u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE; 9888 9889 if (eee->tx_lpi_enabled) 9890 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE; 9891 else 9892 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE; 9893 9894 req->flags |= cpu_to_le32(flags); 9895 eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised); 9896 req->eee_link_speed_mask = cpu_to_le16(eee_speeds); 9897 req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer); 9898 } else { 9899 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE); 9900 } 9901 } 9902 9903 int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee) 9904 { 9905 struct hwrm_port_phy_cfg_input *req; 9906 int rc; 9907 9908 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG); 9909 if (rc) 9910 return rc; 9911 9912 if (set_pause) 9913 bnxt_hwrm_set_pause_common(bp, req); 9914 9915 bnxt_hwrm_set_link_common(bp, req); 9916 9917 if (set_eee) 9918 bnxt_hwrm_set_eee(bp, req); 9919 return hwrm_req_send(bp, req); 9920 } 9921 9922 static int bnxt_hwrm_shutdown_link(struct bnxt *bp) 9923 { 9924 struct hwrm_port_phy_cfg_input *req; 9925 int rc; 9926 9927 if (!BNXT_SINGLE_PF(bp)) 9928 return 0; 9929 9930 if (pci_num_vf(bp->pdev) && 9931 !(bp->phy_flags & BNXT_PHY_FL_FW_MANAGED_LKDN)) 9932 return 0; 9933 9934 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG); 9935 if (rc) 9936 return rc; 9937 9938 req->flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN); 9939 rc = hwrm_req_send(bp, req); 9940 if (!rc) { 9941 mutex_lock(&bp->link_lock); 9942 /* Device is not obliged link down in certain scenarios, even 9943 * when forced. Setting the state unknown is consistent with 9944 * driver startup and will force link state to be reported 9945 * during subsequent open based on PORT_PHY_QCFG. 9946 */ 9947 bp->link_info.link_state = BNXT_LINK_STATE_UNKNOWN; 9948 mutex_unlock(&bp->link_lock); 9949 } 9950 return rc; 9951 } 9952 9953 static int bnxt_fw_reset_via_optee(struct bnxt *bp) 9954 { 9955 #ifdef CONFIG_TEE_BNXT_FW 9956 int rc = tee_bnxt_fw_load(); 9957 9958 if (rc) 9959 netdev_err(bp->dev, "Failed FW reset via OP-TEE, rc=%d\n", rc); 9960 9961 return rc; 9962 #else 9963 netdev_err(bp->dev, "OP-TEE not supported\n"); 9964 return -ENODEV; 9965 #endif 9966 } 9967 9968 static int bnxt_try_recover_fw(struct bnxt *bp) 9969 { 9970 if (bp->fw_health && bp->fw_health->status_reliable) { 9971 int retry = 0, rc; 9972 u32 sts; 9973 9974 do { 9975 sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG); 9976 rc = bnxt_hwrm_poll(bp); 9977 if (!BNXT_FW_IS_BOOTING(sts) && 9978 !BNXT_FW_IS_RECOVERING(sts)) 9979 break; 9980 retry++; 9981 } while (rc == -EBUSY && retry < BNXT_FW_RETRY); 9982 9983 if (!BNXT_FW_IS_HEALTHY(sts)) { 9984 netdev_err(bp->dev, 9985 "Firmware not responding, status: 0x%x\n", 9986 sts); 9987 rc = -ENODEV; 9988 } 9989 if (sts & FW_STATUS_REG_CRASHED_NO_MASTER) { 9990 netdev_warn(bp->dev, "Firmware recover via OP-TEE requested\n"); 9991 return bnxt_fw_reset_via_optee(bp); 9992 } 9993 return rc; 9994 } 9995 9996 return -ENODEV; 9997 } 9998 9999 static void bnxt_clear_reservations(struct bnxt *bp, bool fw_reset) 10000 { 10001 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 10002 10003 if (!BNXT_NEW_RM(bp)) 10004 return; /* no resource reservations required */ 10005 10006 hw_resc->resv_cp_rings = 0; 10007 hw_resc->resv_stat_ctxs = 0; 10008 hw_resc->resv_irqs = 0; 10009 hw_resc->resv_tx_rings = 0; 10010 hw_resc->resv_rx_rings = 0; 10011 hw_resc->resv_hw_ring_grps = 0; 10012 hw_resc->resv_vnics = 0; 10013 if (!fw_reset) { 10014 bp->tx_nr_rings = 0; 10015 bp->rx_nr_rings = 0; 10016 } 10017 } 10018 10019 int bnxt_cancel_reservations(struct bnxt *bp, bool fw_reset) 10020 { 10021 int rc; 10022 10023 if (!BNXT_NEW_RM(bp)) 10024 return 0; /* no resource reservations required */ 10025 10026 rc = bnxt_hwrm_func_resc_qcaps(bp, true); 10027 if (rc) 10028 netdev_err(bp->dev, "resc_qcaps failed\n"); 10029 10030 bnxt_clear_reservations(bp, fw_reset); 10031 10032 return rc; 10033 } 10034 10035 static int bnxt_hwrm_if_change(struct bnxt *bp, bool up) 10036 { 10037 struct hwrm_func_drv_if_change_output *resp; 10038 struct hwrm_func_drv_if_change_input *req; 10039 bool fw_reset = !bp->irq_tbl; 10040 bool resc_reinit = false; 10041 int rc, retry = 0; 10042 u32 flags = 0; 10043 10044 if (!(bp->fw_cap & BNXT_FW_CAP_IF_CHANGE)) 10045 return 0; 10046 10047 rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_IF_CHANGE); 10048 if (rc) 10049 return rc; 10050 10051 if (up) 10052 req->flags = cpu_to_le32(FUNC_DRV_IF_CHANGE_REQ_FLAGS_UP); 10053 resp = hwrm_req_hold(bp, req); 10054 10055 hwrm_req_flags(bp, req, BNXT_HWRM_FULL_WAIT); 10056 while (retry < BNXT_FW_IF_RETRY) { 10057 rc = hwrm_req_send(bp, req); 10058 if (rc != -EAGAIN) 10059 break; 10060 10061 msleep(50); 10062 retry++; 10063 } 10064 10065 if (rc == -EAGAIN) { 10066 hwrm_req_drop(bp, req); 10067 return rc; 10068 } else if (!rc) { 10069 flags = le32_to_cpu(resp->flags); 10070 } else if (up) { 10071 rc = bnxt_try_recover_fw(bp); 10072 fw_reset = true; 10073 } 10074 hwrm_req_drop(bp, req); 10075 if (rc) 10076 return rc; 10077 10078 if (!up) { 10079 bnxt_inv_fw_health_reg(bp); 10080 return 0; 10081 } 10082 10083 if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_RESC_CHANGE) 10084 resc_reinit = true; 10085 if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_HOT_FW_RESET_DONE || 10086 test_bit(BNXT_STATE_FW_RESET_DET, &bp->state)) 10087 fw_reset = true; 10088 else 10089 bnxt_remap_fw_health_regs(bp); 10090 10091 if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state) && !fw_reset) { 10092 netdev_err(bp->dev, "RESET_DONE not set during FW reset.\n"); 10093 set_bit(BNXT_STATE_ABORT_ERR, &bp->state); 10094 return -ENODEV; 10095 } 10096 if (resc_reinit || fw_reset) { 10097 if (fw_reset) { 10098 set_bit(BNXT_STATE_FW_RESET_DET, &bp->state); 10099 if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 10100 bnxt_ulp_stop(bp); 10101 bnxt_free_ctx_mem(bp); 10102 kfree(bp->ctx); 10103 bp->ctx = NULL; 10104 bnxt_dcb_free(bp); 10105 rc = bnxt_fw_init_one(bp); 10106 if (rc) { 10107 clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state); 10108 set_bit(BNXT_STATE_ABORT_ERR, &bp->state); 10109 return rc; 10110 } 10111 bnxt_clear_int_mode(bp); 10112 rc = bnxt_init_int_mode(bp); 10113 if (rc) { 10114 clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state); 10115 netdev_err(bp->dev, "init int mode failed\n"); 10116 return rc; 10117 } 10118 } 10119 rc = bnxt_cancel_reservations(bp, fw_reset); 10120 } 10121 return rc; 10122 } 10123 10124 static int bnxt_hwrm_port_led_qcaps(struct bnxt *bp) 10125 { 10126 struct hwrm_port_led_qcaps_output *resp; 10127 struct hwrm_port_led_qcaps_input *req; 10128 struct bnxt_pf_info *pf = &bp->pf; 10129 int rc; 10130 10131 bp->num_leds = 0; 10132 if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10601) 10133 return 0; 10134 10135 rc = hwrm_req_init(bp, req, HWRM_PORT_LED_QCAPS); 10136 if (rc) 10137 return rc; 10138 10139 req->port_id = cpu_to_le16(pf->port_id); 10140 resp = hwrm_req_hold(bp, req); 10141 rc = hwrm_req_send(bp, req); 10142 if (rc) { 10143 hwrm_req_drop(bp, req); 10144 return rc; 10145 } 10146 if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) { 10147 int i; 10148 10149 bp->num_leds = resp->num_leds; 10150 memcpy(bp->leds, &resp->led0_id, sizeof(bp->leds[0]) * 10151 bp->num_leds); 10152 for (i = 0; i < bp->num_leds; i++) { 10153 struct bnxt_led_info *led = &bp->leds[i]; 10154 __le16 caps = led->led_state_caps; 10155 10156 if (!led->led_group_id || 10157 !BNXT_LED_ALT_BLINK_CAP(caps)) { 10158 bp->num_leds = 0; 10159 break; 10160 } 10161 } 10162 } 10163 hwrm_req_drop(bp, req); 10164 return 0; 10165 } 10166 10167 int bnxt_hwrm_alloc_wol_fltr(struct bnxt *bp) 10168 { 10169 struct hwrm_wol_filter_alloc_output *resp; 10170 struct hwrm_wol_filter_alloc_input *req; 10171 int rc; 10172 10173 rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_ALLOC); 10174 if (rc) 10175 return rc; 10176 10177 req->port_id = cpu_to_le16(bp->pf.port_id); 10178 req->wol_type = WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT; 10179 req->enables = cpu_to_le32(WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS); 10180 memcpy(req->mac_address, bp->dev->dev_addr, ETH_ALEN); 10181 10182 resp = hwrm_req_hold(bp, req); 10183 rc = hwrm_req_send(bp, req); 10184 if (!rc) 10185 bp->wol_filter_id = resp->wol_filter_id; 10186 hwrm_req_drop(bp, req); 10187 return rc; 10188 } 10189 10190 int bnxt_hwrm_free_wol_fltr(struct bnxt *bp) 10191 { 10192 struct hwrm_wol_filter_free_input *req; 10193 int rc; 10194 10195 rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_FREE); 10196 if (rc) 10197 return rc; 10198 10199 req->port_id = cpu_to_le16(bp->pf.port_id); 10200 req->enables = cpu_to_le32(WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID); 10201 req->wol_filter_id = bp->wol_filter_id; 10202 10203 return hwrm_req_send(bp, req); 10204 } 10205 10206 static u16 bnxt_hwrm_get_wol_fltrs(struct bnxt *bp, u16 handle) 10207 { 10208 struct hwrm_wol_filter_qcfg_output *resp; 10209 struct hwrm_wol_filter_qcfg_input *req; 10210 u16 next_handle = 0; 10211 int rc; 10212 10213 rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_QCFG); 10214 if (rc) 10215 return rc; 10216 10217 req->port_id = cpu_to_le16(bp->pf.port_id); 10218 req->handle = cpu_to_le16(handle); 10219 resp = hwrm_req_hold(bp, req); 10220 rc = hwrm_req_send(bp, req); 10221 if (!rc) { 10222 next_handle = le16_to_cpu(resp->next_handle); 10223 if (next_handle != 0) { 10224 if (resp->wol_type == 10225 WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT) { 10226 bp->wol = 1; 10227 bp->wol_filter_id = resp->wol_filter_id; 10228 } 10229 } 10230 } 10231 hwrm_req_drop(bp, req); 10232 return next_handle; 10233 } 10234 10235 static void bnxt_get_wol_settings(struct bnxt *bp) 10236 { 10237 u16 handle = 0; 10238 10239 bp->wol = 0; 10240 if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_WOL_CAP)) 10241 return; 10242 10243 do { 10244 handle = bnxt_hwrm_get_wol_fltrs(bp, handle); 10245 } while (handle && handle != 0xffff); 10246 } 10247 10248 #ifdef CONFIG_BNXT_HWMON 10249 static ssize_t bnxt_show_temp(struct device *dev, 10250 struct device_attribute *devattr, char *buf) 10251 { 10252 struct hwrm_temp_monitor_query_output *resp; 10253 struct hwrm_temp_monitor_query_input *req; 10254 struct bnxt *bp = dev_get_drvdata(dev); 10255 u32 len = 0; 10256 int rc; 10257 10258 rc = hwrm_req_init(bp, req, HWRM_TEMP_MONITOR_QUERY); 10259 if (rc) 10260 return rc; 10261 resp = hwrm_req_hold(bp, req); 10262 rc = hwrm_req_send(bp, req); 10263 if (!rc) 10264 len = sprintf(buf, "%u\n", resp->temp * 1000); /* display millidegree */ 10265 hwrm_req_drop(bp, req); 10266 if (rc) 10267 return rc; 10268 return len; 10269 } 10270 static SENSOR_DEVICE_ATTR(temp1_input, 0444, bnxt_show_temp, NULL, 0); 10271 10272 static struct attribute *bnxt_attrs[] = { 10273 &sensor_dev_attr_temp1_input.dev_attr.attr, 10274 NULL 10275 }; 10276 ATTRIBUTE_GROUPS(bnxt); 10277 10278 static void bnxt_hwmon_close(struct bnxt *bp) 10279 { 10280 if (bp->hwmon_dev) { 10281 hwmon_device_unregister(bp->hwmon_dev); 10282 bp->hwmon_dev = NULL; 10283 } 10284 } 10285 10286 static void bnxt_hwmon_open(struct bnxt *bp) 10287 { 10288 struct hwrm_temp_monitor_query_input *req; 10289 struct pci_dev *pdev = bp->pdev; 10290 int rc; 10291 10292 rc = hwrm_req_init(bp, req, HWRM_TEMP_MONITOR_QUERY); 10293 if (!rc) 10294 rc = hwrm_req_send_silent(bp, req); 10295 if (rc == -EACCES || rc == -EOPNOTSUPP) { 10296 bnxt_hwmon_close(bp); 10297 return; 10298 } 10299 10300 if (bp->hwmon_dev) 10301 return; 10302 10303 bp->hwmon_dev = hwmon_device_register_with_groups(&pdev->dev, 10304 DRV_MODULE_NAME, bp, 10305 bnxt_groups); 10306 if (IS_ERR(bp->hwmon_dev)) { 10307 bp->hwmon_dev = NULL; 10308 dev_warn(&pdev->dev, "Cannot register hwmon device\n"); 10309 } 10310 } 10311 #else 10312 static void bnxt_hwmon_close(struct bnxt *bp) 10313 { 10314 } 10315 10316 static void bnxt_hwmon_open(struct bnxt *bp) 10317 { 10318 } 10319 #endif 10320 10321 static bool bnxt_eee_config_ok(struct bnxt *bp) 10322 { 10323 struct ethtool_eee *eee = &bp->eee; 10324 struct bnxt_link_info *link_info = &bp->link_info; 10325 10326 if (!(bp->phy_flags & BNXT_PHY_FL_EEE_CAP)) 10327 return true; 10328 10329 if (eee->eee_enabled) { 10330 u32 advertising = 10331 _bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0); 10332 10333 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) { 10334 eee->eee_enabled = 0; 10335 return false; 10336 } 10337 if (eee->advertised & ~advertising) { 10338 eee->advertised = advertising & eee->supported; 10339 return false; 10340 } 10341 } 10342 return true; 10343 } 10344 10345 static int bnxt_update_phy_setting(struct bnxt *bp) 10346 { 10347 int rc; 10348 bool update_link = false; 10349 bool update_pause = false; 10350 bool update_eee = false; 10351 struct bnxt_link_info *link_info = &bp->link_info; 10352 10353 rc = bnxt_update_link(bp, true); 10354 if (rc) { 10355 netdev_err(bp->dev, "failed to update link (rc: %x)\n", 10356 rc); 10357 return rc; 10358 } 10359 if (!BNXT_SINGLE_PF(bp)) 10360 return 0; 10361 10362 if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) && 10363 (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) != 10364 link_info->req_flow_ctrl) 10365 update_pause = true; 10366 if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) && 10367 link_info->force_pause_setting != link_info->req_flow_ctrl) 10368 update_pause = true; 10369 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) { 10370 if (BNXT_AUTO_MODE(link_info->auto_mode)) 10371 update_link = true; 10372 if (link_info->req_signal_mode == BNXT_SIG_MODE_NRZ && 10373 link_info->req_link_speed != link_info->force_link_speed) 10374 update_link = true; 10375 else if (link_info->req_signal_mode == BNXT_SIG_MODE_PAM4 && 10376 link_info->req_link_speed != link_info->force_pam4_link_speed) 10377 update_link = true; 10378 if (link_info->req_duplex != link_info->duplex_setting) 10379 update_link = true; 10380 } else { 10381 if (link_info->auto_mode == BNXT_LINK_AUTO_NONE) 10382 update_link = true; 10383 if (link_info->advertising != link_info->auto_link_speeds || 10384 link_info->advertising_pam4 != link_info->auto_pam4_link_speeds) 10385 update_link = true; 10386 } 10387 10388 /* The last close may have shutdown the link, so need to call 10389 * PHY_CFG to bring it back up. 10390 */ 10391 if (!BNXT_LINK_IS_UP(bp)) 10392 update_link = true; 10393 10394 if (!bnxt_eee_config_ok(bp)) 10395 update_eee = true; 10396 10397 if (update_link) 10398 rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee); 10399 else if (update_pause) 10400 rc = bnxt_hwrm_set_pause(bp); 10401 if (rc) { 10402 netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n", 10403 rc); 10404 return rc; 10405 } 10406 10407 return rc; 10408 } 10409 10410 /* Common routine to pre-map certain register block to different GRC window. 10411 * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows 10412 * in PF and 3 windows in VF that can be customized to map in different 10413 * register blocks. 10414 */ 10415 static void bnxt_preset_reg_win(struct bnxt *bp) 10416 { 10417 if (BNXT_PF(bp)) { 10418 /* CAG registers map to GRC window #4 */ 10419 writel(BNXT_CAG_REG_BASE, 10420 bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12); 10421 } 10422 } 10423 10424 static int bnxt_init_dflt_ring_mode(struct bnxt *bp); 10425 10426 static int bnxt_reinit_after_abort(struct bnxt *bp) 10427 { 10428 int rc; 10429 10430 if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 10431 return -EBUSY; 10432 10433 if (bp->dev->reg_state == NETREG_UNREGISTERED) 10434 return -ENODEV; 10435 10436 rc = bnxt_fw_init_one(bp); 10437 if (!rc) { 10438 bnxt_clear_int_mode(bp); 10439 rc = bnxt_init_int_mode(bp); 10440 if (!rc) { 10441 clear_bit(BNXT_STATE_ABORT_ERR, &bp->state); 10442 set_bit(BNXT_STATE_FW_RESET_DET, &bp->state); 10443 } 10444 } 10445 return rc; 10446 } 10447 10448 static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) 10449 { 10450 int rc = 0; 10451 10452 bnxt_preset_reg_win(bp); 10453 netif_carrier_off(bp->dev); 10454 if (irq_re_init) { 10455 /* Reserve rings now if none were reserved at driver probe. */ 10456 rc = bnxt_init_dflt_ring_mode(bp); 10457 if (rc) { 10458 netdev_err(bp->dev, "Failed to reserve default rings at open\n"); 10459 return rc; 10460 } 10461 } 10462 rc = bnxt_reserve_rings(bp, irq_re_init); 10463 if (rc) 10464 return rc; 10465 if ((bp->flags & BNXT_FLAG_RFS) && 10466 !(bp->flags & BNXT_FLAG_USING_MSIX)) { 10467 /* disable RFS if falling back to INTA */ 10468 bp->dev->hw_features &= ~NETIF_F_NTUPLE; 10469 bp->flags &= ~BNXT_FLAG_RFS; 10470 } 10471 10472 rc = bnxt_alloc_mem(bp, irq_re_init); 10473 if (rc) { 10474 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc); 10475 goto open_err_free_mem; 10476 } 10477 10478 if (irq_re_init) { 10479 bnxt_init_napi(bp); 10480 rc = bnxt_request_irq(bp); 10481 if (rc) { 10482 netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc); 10483 goto open_err_irq; 10484 } 10485 } 10486 10487 rc = bnxt_init_nic(bp, irq_re_init); 10488 if (rc) { 10489 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc); 10490 goto open_err_irq; 10491 } 10492 10493 bnxt_enable_napi(bp); 10494 bnxt_debug_dev_init(bp); 10495 10496 if (link_re_init) { 10497 mutex_lock(&bp->link_lock); 10498 rc = bnxt_update_phy_setting(bp); 10499 mutex_unlock(&bp->link_lock); 10500 if (rc) { 10501 netdev_warn(bp->dev, "failed to update phy settings\n"); 10502 if (BNXT_SINGLE_PF(bp)) { 10503 bp->link_info.phy_retry = true; 10504 bp->link_info.phy_retry_expires = 10505 jiffies + 5 * HZ; 10506 } 10507 } 10508 } 10509 10510 if (irq_re_init) 10511 udp_tunnel_nic_reset_ntf(bp->dev); 10512 10513 if (bp->tx_nr_rings_xdp < num_possible_cpus()) { 10514 if (!static_key_enabled(&bnxt_xdp_locking_key)) 10515 static_branch_enable(&bnxt_xdp_locking_key); 10516 } else if (static_key_enabled(&bnxt_xdp_locking_key)) { 10517 static_branch_disable(&bnxt_xdp_locking_key); 10518 } 10519 set_bit(BNXT_STATE_OPEN, &bp->state); 10520 bnxt_enable_int(bp); 10521 /* Enable TX queues */ 10522 bnxt_tx_enable(bp); 10523 mod_timer(&bp->timer, jiffies + bp->current_interval); 10524 /* Poll link status and check for SFP+ module status */ 10525 mutex_lock(&bp->link_lock); 10526 bnxt_get_port_module_status(bp); 10527 mutex_unlock(&bp->link_lock); 10528 10529 /* VF-reps may need to be re-opened after the PF is re-opened */ 10530 if (BNXT_PF(bp)) 10531 bnxt_vf_reps_open(bp); 10532 bnxt_ptp_init_rtc(bp, true); 10533 bnxt_ptp_cfg_tstamp_filters(bp); 10534 return 0; 10535 10536 open_err_irq: 10537 bnxt_del_napi(bp); 10538 10539 open_err_free_mem: 10540 bnxt_free_skbs(bp); 10541 bnxt_free_irq(bp); 10542 bnxt_free_mem(bp, true); 10543 return rc; 10544 } 10545 10546 /* rtnl_lock held */ 10547 int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) 10548 { 10549 int rc = 0; 10550 10551 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) 10552 rc = -EIO; 10553 if (!rc) 10554 rc = __bnxt_open_nic(bp, irq_re_init, link_re_init); 10555 if (rc) { 10556 netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc); 10557 dev_close(bp->dev); 10558 } 10559 return rc; 10560 } 10561 10562 /* rtnl_lock held, open the NIC half way by allocating all resources, but 10563 * NAPI, IRQ, and TX are not enabled. This is mainly used for offline 10564 * self tests. 10565 */ 10566 int bnxt_half_open_nic(struct bnxt *bp) 10567 { 10568 int rc = 0; 10569 10570 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) { 10571 netdev_err(bp->dev, "A previous firmware reset has not completed, aborting half open\n"); 10572 rc = -ENODEV; 10573 goto half_open_err; 10574 } 10575 10576 rc = bnxt_alloc_mem(bp, true); 10577 if (rc) { 10578 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc); 10579 goto half_open_err; 10580 } 10581 set_bit(BNXT_STATE_HALF_OPEN, &bp->state); 10582 rc = bnxt_init_nic(bp, true); 10583 if (rc) { 10584 clear_bit(BNXT_STATE_HALF_OPEN, &bp->state); 10585 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc); 10586 goto half_open_err; 10587 } 10588 return 0; 10589 10590 half_open_err: 10591 bnxt_free_skbs(bp); 10592 bnxt_free_mem(bp, true); 10593 dev_close(bp->dev); 10594 return rc; 10595 } 10596 10597 /* rtnl_lock held, this call can only be made after a previous successful 10598 * call to bnxt_half_open_nic(). 10599 */ 10600 void bnxt_half_close_nic(struct bnxt *bp) 10601 { 10602 bnxt_hwrm_resource_free(bp, false, true); 10603 bnxt_free_skbs(bp); 10604 bnxt_free_mem(bp, true); 10605 clear_bit(BNXT_STATE_HALF_OPEN, &bp->state); 10606 } 10607 10608 void bnxt_reenable_sriov(struct bnxt *bp) 10609 { 10610 if (BNXT_PF(bp)) { 10611 struct bnxt_pf_info *pf = &bp->pf; 10612 int n = pf->active_vfs; 10613 10614 if (n) 10615 bnxt_cfg_hw_sriov(bp, &n, true); 10616 } 10617 } 10618 10619 static int bnxt_open(struct net_device *dev) 10620 { 10621 struct bnxt *bp = netdev_priv(dev); 10622 int rc; 10623 10624 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) { 10625 rc = bnxt_reinit_after_abort(bp); 10626 if (rc) { 10627 if (rc == -EBUSY) 10628 netdev_err(bp->dev, "A previous firmware reset has not completed, aborting\n"); 10629 else 10630 netdev_err(bp->dev, "Failed to reinitialize after aborted firmware reset\n"); 10631 return -ENODEV; 10632 } 10633 } 10634 10635 rc = bnxt_hwrm_if_change(bp, true); 10636 if (rc) 10637 return rc; 10638 10639 rc = __bnxt_open_nic(bp, true, true); 10640 if (rc) { 10641 bnxt_hwrm_if_change(bp, false); 10642 } else { 10643 if (test_and_clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state)) { 10644 if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) { 10645 bnxt_ulp_start(bp, 0); 10646 bnxt_reenable_sriov(bp); 10647 } 10648 } 10649 bnxt_hwmon_open(bp); 10650 } 10651 10652 return rc; 10653 } 10654 10655 static bool bnxt_drv_busy(struct bnxt *bp) 10656 { 10657 return (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state) || 10658 test_bit(BNXT_STATE_READ_STATS, &bp->state)); 10659 } 10660 10661 static void bnxt_get_ring_stats(struct bnxt *bp, 10662 struct rtnl_link_stats64 *stats); 10663 10664 static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init, 10665 bool link_re_init) 10666 { 10667 /* Close the VF-reps before closing PF */ 10668 if (BNXT_PF(bp)) 10669 bnxt_vf_reps_close(bp); 10670 10671 /* Change device state to avoid TX queue wake up's */ 10672 bnxt_tx_disable(bp); 10673 10674 clear_bit(BNXT_STATE_OPEN, &bp->state); 10675 smp_mb__after_atomic(); 10676 while (bnxt_drv_busy(bp)) 10677 msleep(20); 10678 10679 /* Flush rings and disable interrupts */ 10680 bnxt_shutdown_nic(bp, irq_re_init); 10681 10682 /* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */ 10683 10684 bnxt_debug_dev_exit(bp); 10685 bnxt_disable_napi(bp); 10686 del_timer_sync(&bp->timer); 10687 bnxt_free_skbs(bp); 10688 10689 /* Save ring stats before shutdown */ 10690 if (bp->bnapi && irq_re_init) { 10691 bnxt_get_ring_stats(bp, &bp->net_stats_prev); 10692 bnxt_get_ring_err_stats(bp, &bp->ring_err_stats_prev); 10693 } 10694 if (irq_re_init) { 10695 bnxt_free_irq(bp); 10696 bnxt_del_napi(bp); 10697 } 10698 bnxt_free_mem(bp, irq_re_init); 10699 } 10700 10701 int bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) 10702 { 10703 int rc = 0; 10704 10705 if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) { 10706 /* If we get here, it means firmware reset is in progress 10707 * while we are trying to close. We can safely proceed with 10708 * the close because we are holding rtnl_lock(). Some firmware 10709 * messages may fail as we proceed to close. We set the 10710 * ABORT_ERR flag here so that the FW reset thread will later 10711 * abort when it gets the rtnl_lock() and sees the flag. 10712 */ 10713 netdev_warn(bp->dev, "FW reset in progress during close, FW reset will be aborted\n"); 10714 set_bit(BNXT_STATE_ABORT_ERR, &bp->state); 10715 } 10716 10717 #ifdef CONFIG_BNXT_SRIOV 10718 if (bp->sriov_cfg) { 10719 rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait, 10720 !bp->sriov_cfg, 10721 BNXT_SRIOV_CFG_WAIT_TMO); 10722 if (rc) 10723 netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete!\n"); 10724 } 10725 #endif 10726 __bnxt_close_nic(bp, irq_re_init, link_re_init); 10727 return rc; 10728 } 10729 10730 static int bnxt_close(struct net_device *dev) 10731 { 10732 struct bnxt *bp = netdev_priv(dev); 10733 10734 bnxt_hwmon_close(bp); 10735 bnxt_close_nic(bp, true, true); 10736 bnxt_hwrm_shutdown_link(bp); 10737 bnxt_hwrm_if_change(bp, false); 10738 return 0; 10739 } 10740 10741 static int bnxt_hwrm_port_phy_read(struct bnxt *bp, u16 phy_addr, u16 reg, 10742 u16 *val) 10743 { 10744 struct hwrm_port_phy_mdio_read_output *resp; 10745 struct hwrm_port_phy_mdio_read_input *req; 10746 int rc; 10747 10748 if (bp->hwrm_spec_code < 0x10a00) 10749 return -EOPNOTSUPP; 10750 10751 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_MDIO_READ); 10752 if (rc) 10753 return rc; 10754 10755 req->port_id = cpu_to_le16(bp->pf.port_id); 10756 req->phy_addr = phy_addr; 10757 req->reg_addr = cpu_to_le16(reg & 0x1f); 10758 if (mdio_phy_id_is_c45(phy_addr)) { 10759 req->cl45_mdio = 1; 10760 req->phy_addr = mdio_phy_id_prtad(phy_addr); 10761 req->dev_addr = mdio_phy_id_devad(phy_addr); 10762 req->reg_addr = cpu_to_le16(reg); 10763 } 10764 10765 resp = hwrm_req_hold(bp, req); 10766 rc = hwrm_req_send(bp, req); 10767 if (!rc) 10768 *val = le16_to_cpu(resp->reg_data); 10769 hwrm_req_drop(bp, req); 10770 return rc; 10771 } 10772 10773 static int bnxt_hwrm_port_phy_write(struct bnxt *bp, u16 phy_addr, u16 reg, 10774 u16 val) 10775 { 10776 struct hwrm_port_phy_mdio_write_input *req; 10777 int rc; 10778 10779 if (bp->hwrm_spec_code < 0x10a00) 10780 return -EOPNOTSUPP; 10781 10782 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_MDIO_WRITE); 10783 if (rc) 10784 return rc; 10785 10786 req->port_id = cpu_to_le16(bp->pf.port_id); 10787 req->phy_addr = phy_addr; 10788 req->reg_addr = cpu_to_le16(reg & 0x1f); 10789 if (mdio_phy_id_is_c45(phy_addr)) { 10790 req->cl45_mdio = 1; 10791 req->phy_addr = mdio_phy_id_prtad(phy_addr); 10792 req->dev_addr = mdio_phy_id_devad(phy_addr); 10793 req->reg_addr = cpu_to_le16(reg); 10794 } 10795 req->reg_data = cpu_to_le16(val); 10796 10797 return hwrm_req_send(bp, req); 10798 } 10799 10800 /* rtnl_lock held */ 10801 static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) 10802 { 10803 struct mii_ioctl_data *mdio = if_mii(ifr); 10804 struct bnxt *bp = netdev_priv(dev); 10805 int rc; 10806 10807 switch (cmd) { 10808 case SIOCGMIIPHY: 10809 mdio->phy_id = bp->link_info.phy_addr; 10810 10811 fallthrough; 10812 case SIOCGMIIREG: { 10813 u16 mii_regval = 0; 10814 10815 if (!netif_running(dev)) 10816 return -EAGAIN; 10817 10818 rc = bnxt_hwrm_port_phy_read(bp, mdio->phy_id, mdio->reg_num, 10819 &mii_regval); 10820 mdio->val_out = mii_regval; 10821 return rc; 10822 } 10823 10824 case SIOCSMIIREG: 10825 if (!netif_running(dev)) 10826 return -EAGAIN; 10827 10828 return bnxt_hwrm_port_phy_write(bp, mdio->phy_id, mdio->reg_num, 10829 mdio->val_in); 10830 10831 case SIOCSHWTSTAMP: 10832 return bnxt_hwtstamp_set(dev, ifr); 10833 10834 case SIOCGHWTSTAMP: 10835 return bnxt_hwtstamp_get(dev, ifr); 10836 10837 default: 10838 /* do nothing */ 10839 break; 10840 } 10841 return -EOPNOTSUPP; 10842 } 10843 10844 static void bnxt_get_ring_stats(struct bnxt *bp, 10845 struct rtnl_link_stats64 *stats) 10846 { 10847 int i; 10848 10849 for (i = 0; i < bp->cp_nr_rings; i++) { 10850 struct bnxt_napi *bnapi = bp->bnapi[i]; 10851 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 10852 u64 *sw = cpr->stats.sw_stats; 10853 10854 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_ucast_pkts); 10855 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts); 10856 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_bcast_pkts); 10857 10858 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_ucast_pkts); 10859 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_mcast_pkts); 10860 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_bcast_pkts); 10861 10862 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_ucast_bytes); 10863 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_mcast_bytes); 10864 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_bcast_bytes); 10865 10866 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_ucast_bytes); 10867 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_mcast_bytes); 10868 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_bcast_bytes); 10869 10870 stats->rx_missed_errors += 10871 BNXT_GET_RING_STATS64(sw, rx_discard_pkts); 10872 10873 stats->multicast += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts); 10874 10875 stats->tx_dropped += BNXT_GET_RING_STATS64(sw, tx_error_pkts); 10876 10877 stats->rx_dropped += 10878 cpr->sw_stats.rx.rx_netpoll_discards + 10879 cpr->sw_stats.rx.rx_oom_discards; 10880 } 10881 } 10882 10883 static void bnxt_add_prev_stats(struct bnxt *bp, 10884 struct rtnl_link_stats64 *stats) 10885 { 10886 struct rtnl_link_stats64 *prev_stats = &bp->net_stats_prev; 10887 10888 stats->rx_packets += prev_stats->rx_packets; 10889 stats->tx_packets += prev_stats->tx_packets; 10890 stats->rx_bytes += prev_stats->rx_bytes; 10891 stats->tx_bytes += prev_stats->tx_bytes; 10892 stats->rx_missed_errors += prev_stats->rx_missed_errors; 10893 stats->multicast += prev_stats->multicast; 10894 stats->rx_dropped += prev_stats->rx_dropped; 10895 stats->tx_dropped += prev_stats->tx_dropped; 10896 } 10897 10898 static void 10899 bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) 10900 { 10901 struct bnxt *bp = netdev_priv(dev); 10902 10903 set_bit(BNXT_STATE_READ_STATS, &bp->state); 10904 /* Make sure bnxt_close_nic() sees that we are reading stats before 10905 * we check the BNXT_STATE_OPEN flag. 10906 */ 10907 smp_mb__after_atomic(); 10908 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 10909 clear_bit(BNXT_STATE_READ_STATS, &bp->state); 10910 *stats = bp->net_stats_prev; 10911 return; 10912 } 10913 10914 bnxt_get_ring_stats(bp, stats); 10915 bnxt_add_prev_stats(bp, stats); 10916 10917 if (bp->flags & BNXT_FLAG_PORT_STATS) { 10918 u64 *rx = bp->port_stats.sw_stats; 10919 u64 *tx = bp->port_stats.sw_stats + 10920 BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 10921 10922 stats->rx_crc_errors = 10923 BNXT_GET_RX_PORT_STATS64(rx, rx_fcs_err_frames); 10924 stats->rx_frame_errors = 10925 BNXT_GET_RX_PORT_STATS64(rx, rx_align_err_frames); 10926 stats->rx_length_errors = 10927 BNXT_GET_RX_PORT_STATS64(rx, rx_undrsz_frames) + 10928 BNXT_GET_RX_PORT_STATS64(rx, rx_ovrsz_frames) + 10929 BNXT_GET_RX_PORT_STATS64(rx, rx_runt_frames); 10930 stats->rx_errors = 10931 BNXT_GET_RX_PORT_STATS64(rx, rx_false_carrier_frames) + 10932 BNXT_GET_RX_PORT_STATS64(rx, rx_jbr_frames); 10933 stats->collisions = 10934 BNXT_GET_TX_PORT_STATS64(tx, tx_total_collisions); 10935 stats->tx_fifo_errors = 10936 BNXT_GET_TX_PORT_STATS64(tx, tx_fifo_underruns); 10937 stats->tx_errors = BNXT_GET_TX_PORT_STATS64(tx, tx_err); 10938 } 10939 clear_bit(BNXT_STATE_READ_STATS, &bp->state); 10940 } 10941 10942 static void bnxt_get_one_ring_err_stats(struct bnxt *bp, 10943 struct bnxt_total_ring_err_stats *stats, 10944 struct bnxt_cp_ring_info *cpr) 10945 { 10946 struct bnxt_sw_stats *sw_stats = &cpr->sw_stats; 10947 u64 *hw_stats = cpr->stats.sw_stats; 10948 10949 stats->rx_total_l4_csum_errors += sw_stats->rx.rx_l4_csum_errors; 10950 stats->rx_total_resets += sw_stats->rx.rx_resets; 10951 stats->rx_total_buf_errors += sw_stats->rx.rx_buf_errors; 10952 stats->rx_total_oom_discards += sw_stats->rx.rx_oom_discards; 10953 stats->rx_total_netpoll_discards += sw_stats->rx.rx_netpoll_discards; 10954 stats->rx_total_ring_discards += 10955 BNXT_GET_RING_STATS64(hw_stats, rx_discard_pkts); 10956 stats->tx_total_resets += sw_stats->tx.tx_resets; 10957 stats->tx_total_ring_discards += 10958 BNXT_GET_RING_STATS64(hw_stats, tx_discard_pkts); 10959 stats->total_missed_irqs += sw_stats->cmn.missed_irqs; 10960 } 10961 10962 void bnxt_get_ring_err_stats(struct bnxt *bp, 10963 struct bnxt_total_ring_err_stats *stats) 10964 { 10965 int i; 10966 10967 for (i = 0; i < bp->cp_nr_rings; i++) 10968 bnxt_get_one_ring_err_stats(bp, stats, &bp->bnapi[i]->cp_ring); 10969 } 10970 10971 static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask) 10972 { 10973 struct net_device *dev = bp->dev; 10974 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 10975 struct netdev_hw_addr *ha; 10976 u8 *haddr; 10977 int mc_count = 0; 10978 bool update = false; 10979 int off = 0; 10980 10981 netdev_for_each_mc_addr(ha, dev) { 10982 if (mc_count >= BNXT_MAX_MC_ADDRS) { 10983 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 10984 vnic->mc_list_count = 0; 10985 return false; 10986 } 10987 haddr = ha->addr; 10988 if (!ether_addr_equal(haddr, vnic->mc_list + off)) { 10989 memcpy(vnic->mc_list + off, haddr, ETH_ALEN); 10990 update = true; 10991 } 10992 off += ETH_ALEN; 10993 mc_count++; 10994 } 10995 if (mc_count) 10996 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST; 10997 10998 if (mc_count != vnic->mc_list_count) { 10999 vnic->mc_list_count = mc_count; 11000 update = true; 11001 } 11002 return update; 11003 } 11004 11005 static bool bnxt_uc_list_updated(struct bnxt *bp) 11006 { 11007 struct net_device *dev = bp->dev; 11008 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 11009 struct netdev_hw_addr *ha; 11010 int off = 0; 11011 11012 if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1)) 11013 return true; 11014 11015 netdev_for_each_uc_addr(ha, dev) { 11016 if (!ether_addr_equal(ha->addr, vnic->uc_list + off)) 11017 return true; 11018 11019 off += ETH_ALEN; 11020 } 11021 return false; 11022 } 11023 11024 static void bnxt_set_rx_mode(struct net_device *dev) 11025 { 11026 struct bnxt *bp = netdev_priv(dev); 11027 struct bnxt_vnic_info *vnic; 11028 bool mc_update = false; 11029 bool uc_update; 11030 u32 mask; 11031 11032 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) 11033 return; 11034 11035 vnic = &bp->vnic_info[0]; 11036 mask = vnic->rx_mask; 11037 mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS | 11038 CFA_L2_SET_RX_MASK_REQ_MASK_MCAST | 11039 CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST | 11040 CFA_L2_SET_RX_MASK_REQ_MASK_BCAST); 11041 11042 if (dev->flags & IFF_PROMISC) 11043 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 11044 11045 uc_update = bnxt_uc_list_updated(bp); 11046 11047 if (dev->flags & IFF_BROADCAST) 11048 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST; 11049 if (dev->flags & IFF_ALLMULTI) { 11050 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 11051 vnic->mc_list_count = 0; 11052 } else if (dev->flags & IFF_MULTICAST) { 11053 mc_update = bnxt_mc_list_updated(bp, &mask); 11054 } 11055 11056 if (mask != vnic->rx_mask || uc_update || mc_update) { 11057 vnic->rx_mask = mask; 11058 11059 bnxt_queue_sp_work(bp, BNXT_RX_MASK_SP_EVENT); 11060 } 11061 } 11062 11063 static int bnxt_cfg_rx_mode(struct bnxt *bp) 11064 { 11065 struct net_device *dev = bp->dev; 11066 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 11067 struct hwrm_cfa_l2_filter_free_input *req; 11068 struct netdev_hw_addr *ha; 11069 int i, off = 0, rc; 11070 bool uc_update; 11071 11072 netif_addr_lock_bh(dev); 11073 uc_update = bnxt_uc_list_updated(bp); 11074 netif_addr_unlock_bh(dev); 11075 11076 if (!uc_update) 11077 goto skip_uc; 11078 11079 rc = hwrm_req_init(bp, req, HWRM_CFA_L2_FILTER_FREE); 11080 if (rc) 11081 return rc; 11082 hwrm_req_hold(bp, req); 11083 for (i = 1; i < vnic->uc_filter_count; i++) { 11084 req->l2_filter_id = vnic->fw_l2_filter_id[i]; 11085 11086 rc = hwrm_req_send(bp, req); 11087 } 11088 hwrm_req_drop(bp, req); 11089 11090 vnic->uc_filter_count = 1; 11091 11092 netif_addr_lock_bh(dev); 11093 if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) { 11094 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 11095 } else { 11096 netdev_for_each_uc_addr(ha, dev) { 11097 memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN); 11098 off += ETH_ALEN; 11099 vnic->uc_filter_count++; 11100 } 11101 } 11102 netif_addr_unlock_bh(dev); 11103 11104 for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) { 11105 rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off); 11106 if (rc) { 11107 if (BNXT_VF(bp) && rc == -ENODEV) { 11108 if (!test_and_set_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state)) 11109 netdev_warn(bp->dev, "Cannot configure L2 filters while PF is unavailable, will retry\n"); 11110 else 11111 netdev_dbg(bp->dev, "PF still unavailable while configuring L2 filters.\n"); 11112 rc = 0; 11113 } else { 11114 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc); 11115 } 11116 vnic->uc_filter_count = i; 11117 return rc; 11118 } 11119 } 11120 if (test_and_clear_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state)) 11121 netdev_notice(bp->dev, "Retry of L2 filter configuration successful.\n"); 11122 11123 skip_uc: 11124 if ((vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS) && 11125 !bnxt_promisc_ok(bp)) 11126 vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 11127 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0); 11128 if (rc && (vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_MCAST)) { 11129 netdev_info(bp->dev, "Failed setting MC filters rc: %d, turning on ALL_MCAST mode\n", 11130 rc); 11131 vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_MCAST; 11132 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 11133 vnic->mc_list_count = 0; 11134 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0); 11135 } 11136 if (rc) 11137 netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %d\n", 11138 rc); 11139 11140 return rc; 11141 } 11142 11143 static bool bnxt_can_reserve_rings(struct bnxt *bp) 11144 { 11145 #ifdef CONFIG_BNXT_SRIOV 11146 if (BNXT_NEW_RM(bp) && BNXT_VF(bp)) { 11147 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 11148 11149 /* No minimum rings were provisioned by the PF. Don't 11150 * reserve rings by default when device is down. 11151 */ 11152 if (hw_resc->min_tx_rings || hw_resc->resv_tx_rings) 11153 return true; 11154 11155 if (!netif_running(bp->dev)) 11156 return false; 11157 } 11158 #endif 11159 return true; 11160 } 11161 11162 /* If the chip and firmware supports RFS */ 11163 static bool bnxt_rfs_supported(struct bnxt *bp) 11164 { 11165 if (bp->flags & BNXT_FLAG_CHIP_P5) { 11166 if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2) 11167 return true; 11168 return false; 11169 } 11170 /* 212 firmware is broken for aRFS */ 11171 if (BNXT_FW_MAJ(bp) == 212) 11172 return false; 11173 if (BNXT_PF(bp) && !BNXT_CHIP_TYPE_NITRO_A0(bp)) 11174 return true; 11175 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP) 11176 return true; 11177 return false; 11178 } 11179 11180 /* If runtime conditions support RFS */ 11181 static bool bnxt_rfs_capable(struct bnxt *bp) 11182 { 11183 #ifdef CONFIG_RFS_ACCEL 11184 int vnics, max_vnics, max_rss_ctxs; 11185 11186 if (bp->flags & BNXT_FLAG_CHIP_P5) 11187 return bnxt_rfs_supported(bp); 11188 if (!(bp->flags & BNXT_FLAG_MSIX_CAP) || !bnxt_can_reserve_rings(bp) || !bp->rx_nr_rings) 11189 return false; 11190 11191 vnics = 1 + bp->rx_nr_rings; 11192 max_vnics = bnxt_get_max_func_vnics(bp); 11193 max_rss_ctxs = bnxt_get_max_func_rss_ctxs(bp); 11194 11195 /* RSS contexts not a limiting factor */ 11196 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP) 11197 max_rss_ctxs = max_vnics; 11198 if (vnics > max_vnics || vnics > max_rss_ctxs) { 11199 if (bp->rx_nr_rings > 1) 11200 netdev_warn(bp->dev, 11201 "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n", 11202 min(max_rss_ctxs - 1, max_vnics - 1)); 11203 return false; 11204 } 11205 11206 if (!BNXT_NEW_RM(bp)) 11207 return true; 11208 11209 if (vnics == bp->hw_resc.resv_vnics) 11210 return true; 11211 11212 bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, vnics); 11213 if (vnics <= bp->hw_resc.resv_vnics) 11214 return true; 11215 11216 netdev_warn(bp->dev, "Unable to reserve resources to support NTUPLE filters.\n"); 11217 bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, 1); 11218 return false; 11219 #else 11220 return false; 11221 #endif 11222 } 11223 11224 static netdev_features_t bnxt_fix_features(struct net_device *dev, 11225 netdev_features_t features) 11226 { 11227 struct bnxt *bp = netdev_priv(dev); 11228 netdev_features_t vlan_features; 11229 11230 if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp)) 11231 features &= ~NETIF_F_NTUPLE; 11232 11233 if ((bp->flags & BNXT_FLAG_NO_AGG_RINGS) || bp->xdp_prog) 11234 features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); 11235 11236 if (!(features & NETIF_F_GRO)) 11237 features &= ~NETIF_F_GRO_HW; 11238 11239 if (features & NETIF_F_GRO_HW) 11240 features &= ~NETIF_F_LRO; 11241 11242 /* Both CTAG and STAG VLAN accelaration on the RX side have to be 11243 * turned on or off together. 11244 */ 11245 vlan_features = features & BNXT_HW_FEATURE_VLAN_ALL_RX; 11246 if (vlan_features != BNXT_HW_FEATURE_VLAN_ALL_RX) { 11247 if (dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX) 11248 features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX; 11249 else if (vlan_features) 11250 features |= BNXT_HW_FEATURE_VLAN_ALL_RX; 11251 } 11252 #ifdef CONFIG_BNXT_SRIOV 11253 if (BNXT_VF(bp) && bp->vf.vlan) 11254 features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX; 11255 #endif 11256 return features; 11257 } 11258 11259 static int bnxt_set_features(struct net_device *dev, netdev_features_t features) 11260 { 11261 struct bnxt *bp = netdev_priv(dev); 11262 u32 flags = bp->flags; 11263 u32 changes; 11264 int rc = 0; 11265 bool re_init = false; 11266 bool update_tpa = false; 11267 11268 flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS; 11269 if (features & NETIF_F_GRO_HW) 11270 flags |= BNXT_FLAG_GRO; 11271 else if (features & NETIF_F_LRO) 11272 flags |= BNXT_FLAG_LRO; 11273 11274 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS) 11275 flags &= ~BNXT_FLAG_TPA; 11276 11277 if (features & BNXT_HW_FEATURE_VLAN_ALL_RX) 11278 flags |= BNXT_FLAG_STRIP_VLAN; 11279 11280 if (features & NETIF_F_NTUPLE) 11281 flags |= BNXT_FLAG_RFS; 11282 11283 changes = flags ^ bp->flags; 11284 if (changes & BNXT_FLAG_TPA) { 11285 update_tpa = true; 11286 if ((bp->flags & BNXT_FLAG_TPA) == 0 || 11287 (flags & BNXT_FLAG_TPA) == 0 || 11288 (bp->flags & BNXT_FLAG_CHIP_P5)) 11289 re_init = true; 11290 } 11291 11292 if (changes & ~BNXT_FLAG_TPA) 11293 re_init = true; 11294 11295 if (flags != bp->flags) { 11296 u32 old_flags = bp->flags; 11297 11298 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 11299 bp->flags = flags; 11300 if (update_tpa) 11301 bnxt_set_ring_params(bp); 11302 return rc; 11303 } 11304 11305 if (re_init) { 11306 bnxt_close_nic(bp, false, false); 11307 bp->flags = flags; 11308 if (update_tpa) 11309 bnxt_set_ring_params(bp); 11310 11311 return bnxt_open_nic(bp, false, false); 11312 } 11313 if (update_tpa) { 11314 bp->flags = flags; 11315 rc = bnxt_set_tpa(bp, 11316 (flags & BNXT_FLAG_TPA) ? 11317 true : false); 11318 if (rc) 11319 bp->flags = old_flags; 11320 } 11321 } 11322 return rc; 11323 } 11324 11325 static bool bnxt_exthdr_check(struct bnxt *bp, struct sk_buff *skb, int nw_off, 11326 u8 **nextp) 11327 { 11328 struct ipv6hdr *ip6h = (struct ipv6hdr *)(skb->data + nw_off); 11329 struct hop_jumbo_hdr *jhdr; 11330 int hdr_count = 0; 11331 u8 *nexthdr; 11332 int start; 11333 11334 /* Check that there are at most 2 IPv6 extension headers, no 11335 * fragment header, and each is <= 64 bytes. 11336 */ 11337 start = nw_off + sizeof(*ip6h); 11338 nexthdr = &ip6h->nexthdr; 11339 while (ipv6_ext_hdr(*nexthdr)) { 11340 struct ipv6_opt_hdr *hp; 11341 int hdrlen; 11342 11343 if (hdr_count >= 3 || *nexthdr == NEXTHDR_NONE || 11344 *nexthdr == NEXTHDR_FRAGMENT) 11345 return false; 11346 hp = __skb_header_pointer(NULL, start, sizeof(*hp), skb->data, 11347 skb_headlen(skb), NULL); 11348 if (!hp) 11349 return false; 11350 if (*nexthdr == NEXTHDR_AUTH) 11351 hdrlen = ipv6_authlen(hp); 11352 else 11353 hdrlen = ipv6_optlen(hp); 11354 11355 if (hdrlen > 64) 11356 return false; 11357 11358 /* The ext header may be a hop-by-hop header inserted for 11359 * big TCP purposes. This will be removed before sending 11360 * from NIC, so do not count it. 11361 */ 11362 if (*nexthdr == NEXTHDR_HOP) { 11363 if (likely(skb->len <= GRO_LEGACY_MAX_SIZE)) 11364 goto increment_hdr; 11365 11366 jhdr = (struct hop_jumbo_hdr *)hp; 11367 if (jhdr->tlv_type != IPV6_TLV_JUMBO || jhdr->hdrlen != 0 || 11368 jhdr->nexthdr != IPPROTO_TCP) 11369 goto increment_hdr; 11370 11371 goto next_hdr; 11372 } 11373 increment_hdr: 11374 hdr_count++; 11375 next_hdr: 11376 nexthdr = &hp->nexthdr; 11377 start += hdrlen; 11378 } 11379 if (nextp) { 11380 /* Caller will check inner protocol */ 11381 if (skb->encapsulation) { 11382 *nextp = nexthdr; 11383 return true; 11384 } 11385 *nextp = NULL; 11386 } 11387 /* Only support TCP/UDP for non-tunneled ipv6 and inner ipv6 */ 11388 return *nexthdr == IPPROTO_TCP || *nexthdr == IPPROTO_UDP; 11389 } 11390 11391 /* For UDP, we can only handle 1 Vxlan port and 1 Geneve port. */ 11392 static bool bnxt_udp_tunl_check(struct bnxt *bp, struct sk_buff *skb) 11393 { 11394 struct udphdr *uh = udp_hdr(skb); 11395 __be16 udp_port = uh->dest; 11396 11397 if (udp_port != bp->vxlan_port && udp_port != bp->nge_port) 11398 return false; 11399 if (skb->inner_protocol_type == ENCAP_TYPE_ETHER) { 11400 struct ethhdr *eh = inner_eth_hdr(skb); 11401 11402 switch (eh->h_proto) { 11403 case htons(ETH_P_IP): 11404 return true; 11405 case htons(ETH_P_IPV6): 11406 return bnxt_exthdr_check(bp, skb, 11407 skb_inner_network_offset(skb), 11408 NULL); 11409 } 11410 } 11411 return false; 11412 } 11413 11414 static bool bnxt_tunl_check(struct bnxt *bp, struct sk_buff *skb, u8 l4_proto) 11415 { 11416 switch (l4_proto) { 11417 case IPPROTO_UDP: 11418 return bnxt_udp_tunl_check(bp, skb); 11419 case IPPROTO_IPIP: 11420 return true; 11421 case IPPROTO_GRE: { 11422 switch (skb->inner_protocol) { 11423 default: 11424 return false; 11425 case htons(ETH_P_IP): 11426 return true; 11427 case htons(ETH_P_IPV6): 11428 fallthrough; 11429 } 11430 } 11431 case IPPROTO_IPV6: 11432 /* Check ext headers of inner ipv6 */ 11433 return bnxt_exthdr_check(bp, skb, skb_inner_network_offset(skb), 11434 NULL); 11435 } 11436 return false; 11437 } 11438 11439 static netdev_features_t bnxt_features_check(struct sk_buff *skb, 11440 struct net_device *dev, 11441 netdev_features_t features) 11442 { 11443 struct bnxt *bp = netdev_priv(dev); 11444 u8 *l4_proto; 11445 11446 features = vlan_features_check(skb, features); 11447 switch (vlan_get_protocol(skb)) { 11448 case htons(ETH_P_IP): 11449 if (!skb->encapsulation) 11450 return features; 11451 l4_proto = &ip_hdr(skb)->protocol; 11452 if (bnxt_tunl_check(bp, skb, *l4_proto)) 11453 return features; 11454 break; 11455 case htons(ETH_P_IPV6): 11456 if (!bnxt_exthdr_check(bp, skb, skb_network_offset(skb), 11457 &l4_proto)) 11458 break; 11459 if (!l4_proto || bnxt_tunl_check(bp, skb, *l4_proto)) 11460 return features; 11461 break; 11462 } 11463 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 11464 } 11465 11466 int bnxt_dbg_hwrm_rd_reg(struct bnxt *bp, u32 reg_off, u16 num_words, 11467 u32 *reg_buf) 11468 { 11469 struct hwrm_dbg_read_direct_output *resp; 11470 struct hwrm_dbg_read_direct_input *req; 11471 __le32 *dbg_reg_buf; 11472 dma_addr_t mapping; 11473 int rc, i; 11474 11475 rc = hwrm_req_init(bp, req, HWRM_DBG_READ_DIRECT); 11476 if (rc) 11477 return rc; 11478 11479 dbg_reg_buf = hwrm_req_dma_slice(bp, req, num_words * 4, 11480 &mapping); 11481 if (!dbg_reg_buf) { 11482 rc = -ENOMEM; 11483 goto dbg_rd_reg_exit; 11484 } 11485 11486 req->host_dest_addr = cpu_to_le64(mapping); 11487 11488 resp = hwrm_req_hold(bp, req); 11489 req->read_addr = cpu_to_le32(reg_off + CHIMP_REG_VIEW_ADDR); 11490 req->read_len32 = cpu_to_le32(num_words); 11491 11492 rc = hwrm_req_send(bp, req); 11493 if (rc || resp->error_code) { 11494 rc = -EIO; 11495 goto dbg_rd_reg_exit; 11496 } 11497 for (i = 0; i < num_words; i++) 11498 reg_buf[i] = le32_to_cpu(dbg_reg_buf[i]); 11499 11500 dbg_rd_reg_exit: 11501 hwrm_req_drop(bp, req); 11502 return rc; 11503 } 11504 11505 static int bnxt_dbg_hwrm_ring_info_get(struct bnxt *bp, u8 ring_type, 11506 u32 ring_id, u32 *prod, u32 *cons) 11507 { 11508 struct hwrm_dbg_ring_info_get_output *resp; 11509 struct hwrm_dbg_ring_info_get_input *req; 11510 int rc; 11511 11512 rc = hwrm_req_init(bp, req, HWRM_DBG_RING_INFO_GET); 11513 if (rc) 11514 return rc; 11515 11516 req->ring_type = ring_type; 11517 req->fw_ring_id = cpu_to_le32(ring_id); 11518 resp = hwrm_req_hold(bp, req); 11519 rc = hwrm_req_send(bp, req); 11520 if (!rc) { 11521 *prod = le32_to_cpu(resp->producer_index); 11522 *cons = le32_to_cpu(resp->consumer_index); 11523 } 11524 hwrm_req_drop(bp, req); 11525 return rc; 11526 } 11527 11528 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi) 11529 { 11530 struct bnxt_tx_ring_info *txr = bnapi->tx_ring; 11531 int i = bnapi->index; 11532 11533 if (!txr) 11534 return; 11535 11536 netdev_info(bnapi->bp->dev, "[%d]: tx{fw_ring: %d prod: %x cons: %x}\n", 11537 i, txr->tx_ring_struct.fw_ring_id, txr->tx_prod, 11538 txr->tx_cons); 11539 } 11540 11541 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi) 11542 { 11543 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 11544 int i = bnapi->index; 11545 11546 if (!rxr) 11547 return; 11548 11549 netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n", 11550 i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod, 11551 rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod, 11552 rxr->rx_sw_agg_prod); 11553 } 11554 11555 static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi) 11556 { 11557 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 11558 int i = bnapi->index; 11559 11560 netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n", 11561 i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons); 11562 } 11563 11564 static void bnxt_dbg_dump_states(struct bnxt *bp) 11565 { 11566 int i; 11567 struct bnxt_napi *bnapi; 11568 11569 for (i = 0; i < bp->cp_nr_rings; i++) { 11570 bnapi = bp->bnapi[i]; 11571 if (netif_msg_drv(bp)) { 11572 bnxt_dump_tx_sw_state(bnapi); 11573 bnxt_dump_rx_sw_state(bnapi); 11574 bnxt_dump_cp_sw_state(bnapi); 11575 } 11576 } 11577 } 11578 11579 static int bnxt_hwrm_rx_ring_reset(struct bnxt *bp, int ring_nr) 11580 { 11581 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr]; 11582 struct hwrm_ring_reset_input *req; 11583 struct bnxt_napi *bnapi = rxr->bnapi; 11584 struct bnxt_cp_ring_info *cpr; 11585 u16 cp_ring_id; 11586 int rc; 11587 11588 rc = hwrm_req_init(bp, req, HWRM_RING_RESET); 11589 if (rc) 11590 return rc; 11591 11592 cpr = &bnapi->cp_ring; 11593 cp_ring_id = cpr->cp_ring_struct.fw_ring_id; 11594 req->cmpl_ring = cpu_to_le16(cp_ring_id); 11595 req->ring_type = RING_RESET_REQ_RING_TYPE_RX_RING_GRP; 11596 req->ring_id = cpu_to_le16(bp->grp_info[bnapi->index].fw_grp_id); 11597 return hwrm_req_send_silent(bp, req); 11598 } 11599 11600 static void bnxt_reset_task(struct bnxt *bp, bool silent) 11601 { 11602 if (!silent) 11603 bnxt_dbg_dump_states(bp); 11604 if (netif_running(bp->dev)) { 11605 int rc; 11606 11607 if (silent) { 11608 bnxt_close_nic(bp, false, false); 11609 bnxt_open_nic(bp, false, false); 11610 } else { 11611 bnxt_ulp_stop(bp); 11612 bnxt_close_nic(bp, true, false); 11613 rc = bnxt_open_nic(bp, true, false); 11614 bnxt_ulp_start(bp, rc); 11615 } 11616 } 11617 } 11618 11619 static void bnxt_tx_timeout(struct net_device *dev, unsigned int txqueue) 11620 { 11621 struct bnxt *bp = netdev_priv(dev); 11622 11623 netdev_err(bp->dev, "TX timeout detected, starting reset task!\n"); 11624 bnxt_queue_sp_work(bp, BNXT_RESET_TASK_SP_EVENT); 11625 } 11626 11627 static void bnxt_fw_health_check(struct bnxt *bp) 11628 { 11629 struct bnxt_fw_health *fw_health = bp->fw_health; 11630 struct pci_dev *pdev = bp->pdev; 11631 u32 val; 11632 11633 if (!fw_health->enabled || test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 11634 return; 11635 11636 /* Make sure it is enabled before checking the tmr_counter. */ 11637 smp_rmb(); 11638 if (fw_health->tmr_counter) { 11639 fw_health->tmr_counter--; 11640 return; 11641 } 11642 11643 val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG); 11644 if (val == fw_health->last_fw_heartbeat && pci_device_is_present(pdev)) { 11645 fw_health->arrests++; 11646 goto fw_reset; 11647 } 11648 11649 fw_health->last_fw_heartbeat = val; 11650 11651 val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG); 11652 if (val != fw_health->last_fw_reset_cnt && pci_device_is_present(pdev)) { 11653 fw_health->discoveries++; 11654 goto fw_reset; 11655 } 11656 11657 fw_health->tmr_counter = fw_health->tmr_multiplier; 11658 return; 11659 11660 fw_reset: 11661 bnxt_queue_sp_work(bp, BNXT_FW_EXCEPTION_SP_EVENT); 11662 } 11663 11664 static void bnxt_timer(struct timer_list *t) 11665 { 11666 struct bnxt *bp = from_timer(bp, t, timer); 11667 struct net_device *dev = bp->dev; 11668 11669 if (!netif_running(dev) || !test_bit(BNXT_STATE_OPEN, &bp->state)) 11670 return; 11671 11672 if (atomic_read(&bp->intr_sem) != 0) 11673 goto bnxt_restart_timer; 11674 11675 if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) 11676 bnxt_fw_health_check(bp); 11677 11678 if (BNXT_LINK_IS_UP(bp) && bp->stats_coal_ticks) 11679 bnxt_queue_sp_work(bp, BNXT_PERIODIC_STATS_SP_EVENT); 11680 11681 if (bnxt_tc_flower_enabled(bp)) 11682 bnxt_queue_sp_work(bp, BNXT_FLOW_STATS_SP_EVENT); 11683 11684 #ifdef CONFIG_RFS_ACCEL 11685 if ((bp->flags & BNXT_FLAG_RFS) && bp->ntp_fltr_count) 11686 bnxt_queue_sp_work(bp, BNXT_RX_NTP_FLTR_SP_EVENT); 11687 #endif /*CONFIG_RFS_ACCEL*/ 11688 11689 if (bp->link_info.phy_retry) { 11690 if (time_after(jiffies, bp->link_info.phy_retry_expires)) { 11691 bp->link_info.phy_retry = false; 11692 netdev_warn(bp->dev, "failed to update phy settings after maximum retries.\n"); 11693 } else { 11694 bnxt_queue_sp_work(bp, BNXT_UPDATE_PHY_SP_EVENT); 11695 } 11696 } 11697 11698 if (test_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state)) 11699 bnxt_queue_sp_work(bp, BNXT_RX_MASK_SP_EVENT); 11700 11701 if ((bp->flags & BNXT_FLAG_CHIP_P5) && !bp->chip_rev && 11702 netif_carrier_ok(dev)) 11703 bnxt_queue_sp_work(bp, BNXT_RING_COAL_NOW_SP_EVENT); 11704 11705 bnxt_restart_timer: 11706 mod_timer(&bp->timer, jiffies + bp->current_interval); 11707 } 11708 11709 static void bnxt_rtnl_lock_sp(struct bnxt *bp) 11710 { 11711 /* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK 11712 * set. If the device is being closed, bnxt_close() may be holding 11713 * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear. So we 11714 * must clear BNXT_STATE_IN_SP_TASK before holding rtnl(). 11715 */ 11716 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 11717 rtnl_lock(); 11718 } 11719 11720 static void bnxt_rtnl_unlock_sp(struct bnxt *bp) 11721 { 11722 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 11723 rtnl_unlock(); 11724 } 11725 11726 /* Only called from bnxt_sp_task() */ 11727 static void bnxt_reset(struct bnxt *bp, bool silent) 11728 { 11729 bnxt_rtnl_lock_sp(bp); 11730 if (test_bit(BNXT_STATE_OPEN, &bp->state)) 11731 bnxt_reset_task(bp, silent); 11732 bnxt_rtnl_unlock_sp(bp); 11733 } 11734 11735 /* Only called from bnxt_sp_task() */ 11736 static void bnxt_rx_ring_reset(struct bnxt *bp) 11737 { 11738 int i; 11739 11740 bnxt_rtnl_lock_sp(bp); 11741 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 11742 bnxt_rtnl_unlock_sp(bp); 11743 return; 11744 } 11745 /* Disable and flush TPA before resetting the RX ring */ 11746 if (bp->flags & BNXT_FLAG_TPA) 11747 bnxt_set_tpa(bp, false); 11748 for (i = 0; i < bp->rx_nr_rings; i++) { 11749 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 11750 struct bnxt_cp_ring_info *cpr; 11751 int rc; 11752 11753 if (!rxr->bnapi->in_reset) 11754 continue; 11755 11756 rc = bnxt_hwrm_rx_ring_reset(bp, i); 11757 if (rc) { 11758 if (rc == -EINVAL || rc == -EOPNOTSUPP) 11759 netdev_info_once(bp->dev, "RX ring reset not supported by firmware, falling back to global reset\n"); 11760 else 11761 netdev_warn(bp->dev, "RX ring reset failed, rc = %d, falling back to global reset\n", 11762 rc); 11763 bnxt_reset_task(bp, true); 11764 break; 11765 } 11766 bnxt_free_one_rx_ring_skbs(bp, i); 11767 rxr->rx_prod = 0; 11768 rxr->rx_agg_prod = 0; 11769 rxr->rx_sw_agg_prod = 0; 11770 rxr->rx_next_cons = 0; 11771 rxr->bnapi->in_reset = false; 11772 bnxt_alloc_one_rx_ring(bp, i); 11773 cpr = &rxr->bnapi->cp_ring; 11774 cpr->sw_stats.rx.rx_resets++; 11775 if (bp->flags & BNXT_FLAG_AGG_RINGS) 11776 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); 11777 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 11778 } 11779 if (bp->flags & BNXT_FLAG_TPA) 11780 bnxt_set_tpa(bp, true); 11781 bnxt_rtnl_unlock_sp(bp); 11782 } 11783 11784 static void bnxt_fw_reset_close(struct bnxt *bp) 11785 { 11786 bnxt_ulp_stop(bp); 11787 /* When firmware is in fatal state, quiesce device and disable 11788 * bus master to prevent any potential bad DMAs before freeing 11789 * kernel memory. 11790 */ 11791 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) { 11792 u16 val = 0; 11793 11794 pci_read_config_word(bp->pdev, PCI_SUBSYSTEM_ID, &val); 11795 if (val == 0xffff) 11796 bp->fw_reset_min_dsecs = 0; 11797 bnxt_tx_disable(bp); 11798 bnxt_disable_napi(bp); 11799 bnxt_disable_int_sync(bp); 11800 bnxt_free_irq(bp); 11801 bnxt_clear_int_mode(bp); 11802 pci_disable_device(bp->pdev); 11803 } 11804 __bnxt_close_nic(bp, true, false); 11805 bnxt_vf_reps_free(bp); 11806 bnxt_clear_int_mode(bp); 11807 bnxt_hwrm_func_drv_unrgtr(bp); 11808 if (pci_is_enabled(bp->pdev)) 11809 pci_disable_device(bp->pdev); 11810 bnxt_free_ctx_mem(bp); 11811 kfree(bp->ctx); 11812 bp->ctx = NULL; 11813 } 11814 11815 static bool is_bnxt_fw_ok(struct bnxt *bp) 11816 { 11817 struct bnxt_fw_health *fw_health = bp->fw_health; 11818 bool no_heartbeat = false, has_reset = false; 11819 u32 val; 11820 11821 val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG); 11822 if (val == fw_health->last_fw_heartbeat) 11823 no_heartbeat = true; 11824 11825 val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG); 11826 if (val != fw_health->last_fw_reset_cnt) 11827 has_reset = true; 11828 11829 if (!no_heartbeat && has_reset) 11830 return true; 11831 11832 return false; 11833 } 11834 11835 /* rtnl_lock is acquired before calling this function */ 11836 static void bnxt_force_fw_reset(struct bnxt *bp) 11837 { 11838 struct bnxt_fw_health *fw_health = bp->fw_health; 11839 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; 11840 u32 wait_dsecs; 11841 11842 if (!test_bit(BNXT_STATE_OPEN, &bp->state) || 11843 test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 11844 return; 11845 11846 if (ptp) { 11847 spin_lock_bh(&ptp->ptp_lock); 11848 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 11849 spin_unlock_bh(&ptp->ptp_lock); 11850 } else { 11851 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 11852 } 11853 bnxt_fw_reset_close(bp); 11854 wait_dsecs = fw_health->master_func_wait_dsecs; 11855 if (fw_health->primary) { 11856 if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) 11857 wait_dsecs = 0; 11858 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW; 11859 } else { 11860 bp->fw_reset_timestamp = jiffies + wait_dsecs * HZ / 10; 11861 wait_dsecs = fw_health->normal_func_wait_dsecs; 11862 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 11863 } 11864 11865 bp->fw_reset_min_dsecs = fw_health->post_reset_wait_dsecs; 11866 bp->fw_reset_max_dsecs = fw_health->post_reset_max_wait_dsecs; 11867 bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10); 11868 } 11869 11870 void bnxt_fw_exception(struct bnxt *bp) 11871 { 11872 netdev_warn(bp->dev, "Detected firmware fatal condition, initiating reset\n"); 11873 set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state); 11874 bnxt_rtnl_lock_sp(bp); 11875 bnxt_force_fw_reset(bp); 11876 bnxt_rtnl_unlock_sp(bp); 11877 } 11878 11879 /* Returns the number of registered VFs, or 1 if VF configuration is pending, or 11880 * < 0 on error. 11881 */ 11882 static int bnxt_get_registered_vfs(struct bnxt *bp) 11883 { 11884 #ifdef CONFIG_BNXT_SRIOV 11885 int rc; 11886 11887 if (!BNXT_PF(bp)) 11888 return 0; 11889 11890 rc = bnxt_hwrm_func_qcfg(bp); 11891 if (rc) { 11892 netdev_err(bp->dev, "func_qcfg cmd failed, rc = %d\n", rc); 11893 return rc; 11894 } 11895 if (bp->pf.registered_vfs) 11896 return bp->pf.registered_vfs; 11897 if (bp->sriov_cfg) 11898 return 1; 11899 #endif 11900 return 0; 11901 } 11902 11903 void bnxt_fw_reset(struct bnxt *bp) 11904 { 11905 bnxt_rtnl_lock_sp(bp); 11906 if (test_bit(BNXT_STATE_OPEN, &bp->state) && 11907 !test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) { 11908 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; 11909 int n = 0, tmo; 11910 11911 if (ptp) { 11912 spin_lock_bh(&ptp->ptp_lock); 11913 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 11914 spin_unlock_bh(&ptp->ptp_lock); 11915 } else { 11916 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 11917 } 11918 if (bp->pf.active_vfs && 11919 !test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) 11920 n = bnxt_get_registered_vfs(bp); 11921 if (n < 0) { 11922 netdev_err(bp->dev, "Firmware reset aborted, rc = %d\n", 11923 n); 11924 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 11925 dev_close(bp->dev); 11926 goto fw_reset_exit; 11927 } else if (n > 0) { 11928 u16 vf_tmo_dsecs = n * 10; 11929 11930 if (bp->fw_reset_max_dsecs < vf_tmo_dsecs) 11931 bp->fw_reset_max_dsecs = vf_tmo_dsecs; 11932 bp->fw_reset_state = 11933 BNXT_FW_RESET_STATE_POLL_VF; 11934 bnxt_queue_fw_reset_work(bp, HZ / 10); 11935 goto fw_reset_exit; 11936 } 11937 bnxt_fw_reset_close(bp); 11938 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) { 11939 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN; 11940 tmo = HZ / 10; 11941 } else { 11942 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 11943 tmo = bp->fw_reset_min_dsecs * HZ / 10; 11944 } 11945 bnxt_queue_fw_reset_work(bp, tmo); 11946 } 11947 fw_reset_exit: 11948 bnxt_rtnl_unlock_sp(bp); 11949 } 11950 11951 static void bnxt_chk_missed_irq(struct bnxt *bp) 11952 { 11953 int i; 11954 11955 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 11956 return; 11957 11958 for (i = 0; i < bp->cp_nr_rings; i++) { 11959 struct bnxt_napi *bnapi = bp->bnapi[i]; 11960 struct bnxt_cp_ring_info *cpr; 11961 u32 fw_ring_id; 11962 int j; 11963 11964 if (!bnapi) 11965 continue; 11966 11967 cpr = &bnapi->cp_ring; 11968 for (j = 0; j < 2; j++) { 11969 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j]; 11970 u32 val[2]; 11971 11972 if (!cpr2 || cpr2->has_more_work || 11973 !bnxt_has_work(bp, cpr2)) 11974 continue; 11975 11976 if (cpr2->cp_raw_cons != cpr2->last_cp_raw_cons) { 11977 cpr2->last_cp_raw_cons = cpr2->cp_raw_cons; 11978 continue; 11979 } 11980 fw_ring_id = cpr2->cp_ring_struct.fw_ring_id; 11981 bnxt_dbg_hwrm_ring_info_get(bp, 11982 DBG_RING_INFO_GET_REQ_RING_TYPE_L2_CMPL, 11983 fw_ring_id, &val[0], &val[1]); 11984 cpr->sw_stats.cmn.missed_irqs++; 11985 } 11986 } 11987 } 11988 11989 static void bnxt_cfg_ntp_filters(struct bnxt *); 11990 11991 static void bnxt_init_ethtool_link_settings(struct bnxt *bp) 11992 { 11993 struct bnxt_link_info *link_info = &bp->link_info; 11994 11995 if (BNXT_AUTO_MODE(link_info->auto_mode)) { 11996 link_info->autoneg = BNXT_AUTONEG_SPEED; 11997 if (bp->hwrm_spec_code >= 0x10201) { 11998 if (link_info->auto_pause_setting & 11999 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE) 12000 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL; 12001 } else { 12002 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL; 12003 } 12004 link_info->advertising = link_info->auto_link_speeds; 12005 link_info->advertising_pam4 = link_info->auto_pam4_link_speeds; 12006 } else { 12007 link_info->req_link_speed = link_info->force_link_speed; 12008 link_info->req_signal_mode = BNXT_SIG_MODE_NRZ; 12009 if (link_info->force_pam4_link_speed) { 12010 link_info->req_link_speed = 12011 link_info->force_pam4_link_speed; 12012 link_info->req_signal_mode = BNXT_SIG_MODE_PAM4; 12013 } 12014 link_info->req_duplex = link_info->duplex_setting; 12015 } 12016 if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) 12017 link_info->req_flow_ctrl = 12018 link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH; 12019 else 12020 link_info->req_flow_ctrl = link_info->force_pause_setting; 12021 } 12022 12023 static void bnxt_fw_echo_reply(struct bnxt *bp) 12024 { 12025 struct bnxt_fw_health *fw_health = bp->fw_health; 12026 struct hwrm_func_echo_response_input *req; 12027 int rc; 12028 12029 rc = hwrm_req_init(bp, req, HWRM_FUNC_ECHO_RESPONSE); 12030 if (rc) 12031 return; 12032 req->event_data1 = cpu_to_le32(fw_health->echo_req_data1); 12033 req->event_data2 = cpu_to_le32(fw_health->echo_req_data2); 12034 hwrm_req_send(bp, req); 12035 } 12036 12037 static void bnxt_sp_task(struct work_struct *work) 12038 { 12039 struct bnxt *bp = container_of(work, struct bnxt, sp_task); 12040 12041 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 12042 smp_mb__after_atomic(); 12043 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 12044 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 12045 return; 12046 } 12047 12048 if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event)) 12049 bnxt_cfg_rx_mode(bp); 12050 12051 if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event)) 12052 bnxt_cfg_ntp_filters(bp); 12053 if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event)) 12054 bnxt_hwrm_exec_fwd_req(bp); 12055 if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event)) { 12056 bnxt_hwrm_port_qstats(bp, 0); 12057 bnxt_hwrm_port_qstats_ext(bp, 0); 12058 bnxt_accumulate_all_stats(bp); 12059 } 12060 12061 if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) { 12062 int rc; 12063 12064 mutex_lock(&bp->link_lock); 12065 if (test_and_clear_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, 12066 &bp->sp_event)) 12067 bnxt_hwrm_phy_qcaps(bp); 12068 12069 rc = bnxt_update_link(bp, true); 12070 if (rc) 12071 netdev_err(bp->dev, "SP task can't update link (rc: %x)\n", 12072 rc); 12073 12074 if (test_and_clear_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT, 12075 &bp->sp_event)) 12076 bnxt_init_ethtool_link_settings(bp); 12077 mutex_unlock(&bp->link_lock); 12078 } 12079 if (test_and_clear_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event)) { 12080 int rc; 12081 12082 mutex_lock(&bp->link_lock); 12083 rc = bnxt_update_phy_setting(bp); 12084 mutex_unlock(&bp->link_lock); 12085 if (rc) { 12086 netdev_warn(bp->dev, "update phy settings retry failed\n"); 12087 } else { 12088 bp->link_info.phy_retry = false; 12089 netdev_info(bp->dev, "update phy settings retry succeeded\n"); 12090 } 12091 } 12092 if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event)) { 12093 mutex_lock(&bp->link_lock); 12094 bnxt_get_port_module_status(bp); 12095 mutex_unlock(&bp->link_lock); 12096 } 12097 12098 if (test_and_clear_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event)) 12099 bnxt_tc_flow_stats_work(bp); 12100 12101 if (test_and_clear_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event)) 12102 bnxt_chk_missed_irq(bp); 12103 12104 if (test_and_clear_bit(BNXT_FW_ECHO_REQUEST_SP_EVENT, &bp->sp_event)) 12105 bnxt_fw_echo_reply(bp); 12106 12107 /* These functions below will clear BNXT_STATE_IN_SP_TASK. They 12108 * must be the last functions to be called before exiting. 12109 */ 12110 if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event)) 12111 bnxt_reset(bp, false); 12112 12113 if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event)) 12114 bnxt_reset(bp, true); 12115 12116 if (test_and_clear_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event)) 12117 bnxt_rx_ring_reset(bp); 12118 12119 if (test_and_clear_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event)) { 12120 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state) || 12121 test_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state)) 12122 bnxt_devlink_health_fw_report(bp); 12123 else 12124 bnxt_fw_reset(bp); 12125 } 12126 12127 if (test_and_clear_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event)) { 12128 if (!is_bnxt_fw_ok(bp)) 12129 bnxt_devlink_health_fw_report(bp); 12130 } 12131 12132 smp_mb__before_atomic(); 12133 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 12134 } 12135 12136 /* Under rtnl_lock */ 12137 int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs, 12138 int tx_xdp) 12139 { 12140 int max_rx, max_tx, tx_sets = 1; 12141 int tx_rings_needed, stats; 12142 int rx_rings = rx; 12143 int cp, vnics, rc; 12144 12145 if (tcs) 12146 tx_sets = tcs; 12147 12148 rc = bnxt_get_max_rings(bp, &max_rx, &max_tx, sh); 12149 if (rc) 12150 return rc; 12151 12152 if (max_rx < rx) 12153 return -ENOMEM; 12154 12155 tx_rings_needed = tx * tx_sets + tx_xdp; 12156 if (max_tx < tx_rings_needed) 12157 return -ENOMEM; 12158 12159 vnics = 1; 12160 if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS) 12161 vnics += rx_rings; 12162 12163 if (bp->flags & BNXT_FLAG_AGG_RINGS) 12164 rx_rings <<= 1; 12165 cp = sh ? max_t(int, tx_rings_needed, rx) : tx_rings_needed + rx; 12166 stats = cp; 12167 if (BNXT_NEW_RM(bp)) { 12168 cp += bnxt_get_ulp_msix_num(bp); 12169 stats += bnxt_get_ulp_stat_ctxs(bp); 12170 } 12171 return bnxt_hwrm_check_rings(bp, tx_rings_needed, rx_rings, rx, cp, 12172 stats, vnics); 12173 } 12174 12175 static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev) 12176 { 12177 if (bp->bar2) { 12178 pci_iounmap(pdev, bp->bar2); 12179 bp->bar2 = NULL; 12180 } 12181 12182 if (bp->bar1) { 12183 pci_iounmap(pdev, bp->bar1); 12184 bp->bar1 = NULL; 12185 } 12186 12187 if (bp->bar0) { 12188 pci_iounmap(pdev, bp->bar0); 12189 bp->bar0 = NULL; 12190 } 12191 } 12192 12193 static void bnxt_cleanup_pci(struct bnxt *bp) 12194 { 12195 bnxt_unmap_bars(bp, bp->pdev); 12196 pci_release_regions(bp->pdev); 12197 if (pci_is_enabled(bp->pdev)) 12198 pci_disable_device(bp->pdev); 12199 } 12200 12201 static void bnxt_init_dflt_coal(struct bnxt *bp) 12202 { 12203 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 12204 struct bnxt_coal *coal; 12205 u16 flags = 0; 12206 12207 if (coal_cap->cmpl_params & 12208 RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_TIMER_RESET) 12209 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET; 12210 12211 /* Tick values in micro seconds. 12212 * 1 coal_buf x bufs_per_record = 1 completion record. 12213 */ 12214 coal = &bp->rx_coal; 12215 coal->coal_ticks = 10; 12216 coal->coal_bufs = 30; 12217 coal->coal_ticks_irq = 1; 12218 coal->coal_bufs_irq = 2; 12219 coal->idle_thresh = 50; 12220 coal->bufs_per_record = 2; 12221 coal->budget = 64; /* NAPI budget */ 12222 coal->flags = flags; 12223 12224 coal = &bp->tx_coal; 12225 coal->coal_ticks = 28; 12226 coal->coal_bufs = 30; 12227 coal->coal_ticks_irq = 2; 12228 coal->coal_bufs_irq = 2; 12229 coal->bufs_per_record = 1; 12230 coal->flags = flags; 12231 12232 bp->stats_coal_ticks = BNXT_DEF_STATS_COAL_TICKS; 12233 } 12234 12235 static int bnxt_fw_init_one_p1(struct bnxt *bp) 12236 { 12237 int rc; 12238 12239 bp->fw_cap = 0; 12240 rc = bnxt_hwrm_ver_get(bp); 12241 bnxt_try_map_fw_health_reg(bp); 12242 if (rc) { 12243 rc = bnxt_try_recover_fw(bp); 12244 if (rc) 12245 return rc; 12246 rc = bnxt_hwrm_ver_get(bp); 12247 if (rc) 12248 return rc; 12249 } 12250 12251 bnxt_nvm_cfg_ver_get(bp); 12252 12253 rc = bnxt_hwrm_func_reset(bp); 12254 if (rc) 12255 return -ENODEV; 12256 12257 bnxt_hwrm_fw_set_time(bp); 12258 return 0; 12259 } 12260 12261 static int bnxt_fw_init_one_p2(struct bnxt *bp) 12262 { 12263 int rc; 12264 12265 /* Get the MAX capabilities for this function */ 12266 rc = bnxt_hwrm_func_qcaps(bp); 12267 if (rc) { 12268 netdev_err(bp->dev, "hwrm query capability failure rc: %x\n", 12269 rc); 12270 return -ENODEV; 12271 } 12272 12273 rc = bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(bp); 12274 if (rc) 12275 netdev_warn(bp->dev, "hwrm query adv flow mgnt failure rc: %d\n", 12276 rc); 12277 12278 if (bnxt_alloc_fw_health(bp)) { 12279 netdev_warn(bp->dev, "no memory for firmware error recovery\n"); 12280 } else { 12281 rc = bnxt_hwrm_error_recovery_qcfg(bp); 12282 if (rc) 12283 netdev_warn(bp->dev, "hwrm query error recovery failure rc: %d\n", 12284 rc); 12285 } 12286 12287 rc = bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false); 12288 if (rc) 12289 return -ENODEV; 12290 12291 bnxt_hwrm_func_qcfg(bp); 12292 bnxt_hwrm_vnic_qcaps(bp); 12293 bnxt_hwrm_port_led_qcaps(bp); 12294 bnxt_ethtool_init(bp); 12295 if (bp->fw_cap & BNXT_FW_CAP_PTP) 12296 __bnxt_hwrm_ptp_qcfg(bp); 12297 bnxt_dcb_init(bp); 12298 return 0; 12299 } 12300 12301 static void bnxt_set_dflt_rss_hash_type(struct bnxt *bp) 12302 { 12303 bp->flags &= ~BNXT_FLAG_UDP_RSS_CAP; 12304 bp->rss_hash_cfg = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 | 12305 VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 | 12306 VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 | 12307 VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6; 12308 if (bp->fw_cap & BNXT_FW_CAP_RSS_HASH_TYPE_DELTA) 12309 bp->rss_hash_delta = bp->rss_hash_cfg; 12310 if (BNXT_CHIP_P4_PLUS(bp) && bp->hwrm_spec_code >= 0x10501) { 12311 bp->flags |= BNXT_FLAG_UDP_RSS_CAP; 12312 bp->rss_hash_cfg |= VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 | 12313 VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6; 12314 } 12315 } 12316 12317 static void bnxt_set_dflt_rfs(struct bnxt *bp) 12318 { 12319 struct net_device *dev = bp->dev; 12320 12321 dev->hw_features &= ~NETIF_F_NTUPLE; 12322 dev->features &= ~NETIF_F_NTUPLE; 12323 bp->flags &= ~BNXT_FLAG_RFS; 12324 if (bnxt_rfs_supported(bp)) { 12325 dev->hw_features |= NETIF_F_NTUPLE; 12326 if (bnxt_rfs_capable(bp)) { 12327 bp->flags |= BNXT_FLAG_RFS; 12328 dev->features |= NETIF_F_NTUPLE; 12329 } 12330 } 12331 } 12332 12333 static void bnxt_fw_init_one_p3(struct bnxt *bp) 12334 { 12335 struct pci_dev *pdev = bp->pdev; 12336 12337 bnxt_set_dflt_rss_hash_type(bp); 12338 bnxt_set_dflt_rfs(bp); 12339 12340 bnxt_get_wol_settings(bp); 12341 if (bp->flags & BNXT_FLAG_WOL_CAP) 12342 device_set_wakeup_enable(&pdev->dev, bp->wol); 12343 else 12344 device_set_wakeup_capable(&pdev->dev, false); 12345 12346 bnxt_hwrm_set_cache_line_size(bp, cache_line_size()); 12347 bnxt_hwrm_coal_params_qcaps(bp); 12348 } 12349 12350 static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt); 12351 12352 int bnxt_fw_init_one(struct bnxt *bp) 12353 { 12354 int rc; 12355 12356 rc = bnxt_fw_init_one_p1(bp); 12357 if (rc) { 12358 netdev_err(bp->dev, "Firmware init phase 1 failed\n"); 12359 return rc; 12360 } 12361 rc = bnxt_fw_init_one_p2(bp); 12362 if (rc) { 12363 netdev_err(bp->dev, "Firmware init phase 2 failed\n"); 12364 return rc; 12365 } 12366 rc = bnxt_probe_phy(bp, false); 12367 if (rc) 12368 return rc; 12369 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, false); 12370 if (rc) 12371 return rc; 12372 12373 bnxt_fw_init_one_p3(bp); 12374 return 0; 12375 } 12376 12377 static void bnxt_fw_reset_writel(struct bnxt *bp, int reg_idx) 12378 { 12379 struct bnxt_fw_health *fw_health = bp->fw_health; 12380 u32 reg = fw_health->fw_reset_seq_regs[reg_idx]; 12381 u32 val = fw_health->fw_reset_seq_vals[reg_idx]; 12382 u32 reg_type, reg_off, delay_msecs; 12383 12384 delay_msecs = fw_health->fw_reset_seq_delay_msec[reg_idx]; 12385 reg_type = BNXT_FW_HEALTH_REG_TYPE(reg); 12386 reg_off = BNXT_FW_HEALTH_REG_OFF(reg); 12387 switch (reg_type) { 12388 case BNXT_FW_HEALTH_REG_TYPE_CFG: 12389 pci_write_config_dword(bp->pdev, reg_off, val); 12390 break; 12391 case BNXT_FW_HEALTH_REG_TYPE_GRC: 12392 writel(reg_off & BNXT_GRC_BASE_MASK, 12393 bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4); 12394 reg_off = (reg_off & BNXT_GRC_OFFSET_MASK) + 0x2000; 12395 fallthrough; 12396 case BNXT_FW_HEALTH_REG_TYPE_BAR0: 12397 writel(val, bp->bar0 + reg_off); 12398 break; 12399 case BNXT_FW_HEALTH_REG_TYPE_BAR1: 12400 writel(val, bp->bar1 + reg_off); 12401 break; 12402 } 12403 if (delay_msecs) { 12404 pci_read_config_dword(bp->pdev, 0, &val); 12405 msleep(delay_msecs); 12406 } 12407 } 12408 12409 bool bnxt_hwrm_reset_permitted(struct bnxt *bp) 12410 { 12411 struct hwrm_func_qcfg_output *resp; 12412 struct hwrm_func_qcfg_input *req; 12413 bool result = true; /* firmware will enforce if unknown */ 12414 12415 if (~bp->fw_cap & BNXT_FW_CAP_HOT_RESET_IF) 12416 return result; 12417 12418 if (hwrm_req_init(bp, req, HWRM_FUNC_QCFG)) 12419 return result; 12420 12421 req->fid = cpu_to_le16(0xffff); 12422 resp = hwrm_req_hold(bp, req); 12423 if (!hwrm_req_send(bp, req)) 12424 result = !!(le16_to_cpu(resp->flags) & 12425 FUNC_QCFG_RESP_FLAGS_HOT_RESET_ALLOWED); 12426 hwrm_req_drop(bp, req); 12427 return result; 12428 } 12429 12430 static void bnxt_reset_all(struct bnxt *bp) 12431 { 12432 struct bnxt_fw_health *fw_health = bp->fw_health; 12433 int i, rc; 12434 12435 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) { 12436 bnxt_fw_reset_via_optee(bp); 12437 bp->fw_reset_timestamp = jiffies; 12438 return; 12439 } 12440 12441 if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_HOST) { 12442 for (i = 0; i < fw_health->fw_reset_seq_cnt; i++) 12443 bnxt_fw_reset_writel(bp, i); 12444 } else if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) { 12445 struct hwrm_fw_reset_input *req; 12446 12447 rc = hwrm_req_init(bp, req, HWRM_FW_RESET); 12448 if (!rc) { 12449 req->target_id = cpu_to_le16(HWRM_TARGET_ID_KONG); 12450 req->embedded_proc_type = FW_RESET_REQ_EMBEDDED_PROC_TYPE_CHIP; 12451 req->selfrst_status = FW_RESET_REQ_SELFRST_STATUS_SELFRSTASAP; 12452 req->flags = FW_RESET_REQ_FLAGS_RESET_GRACEFUL; 12453 rc = hwrm_req_send(bp, req); 12454 } 12455 if (rc != -ENODEV) 12456 netdev_warn(bp->dev, "Unable to reset FW rc=%d\n", rc); 12457 } 12458 bp->fw_reset_timestamp = jiffies; 12459 } 12460 12461 static bool bnxt_fw_reset_timeout(struct bnxt *bp) 12462 { 12463 return time_after(jiffies, bp->fw_reset_timestamp + 12464 (bp->fw_reset_max_dsecs * HZ / 10)); 12465 } 12466 12467 static void bnxt_fw_reset_abort(struct bnxt *bp, int rc) 12468 { 12469 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 12470 if (bp->fw_reset_state != BNXT_FW_RESET_STATE_POLL_VF) { 12471 bnxt_ulp_start(bp, rc); 12472 bnxt_dl_health_fw_status_update(bp, false); 12473 } 12474 bp->fw_reset_state = 0; 12475 dev_close(bp->dev); 12476 } 12477 12478 static void bnxt_fw_reset_task(struct work_struct *work) 12479 { 12480 struct bnxt *bp = container_of(work, struct bnxt, fw_reset_task.work); 12481 int rc = 0; 12482 12483 if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) { 12484 netdev_err(bp->dev, "bnxt_fw_reset_task() called when not in fw reset mode!\n"); 12485 return; 12486 } 12487 12488 switch (bp->fw_reset_state) { 12489 case BNXT_FW_RESET_STATE_POLL_VF: { 12490 int n = bnxt_get_registered_vfs(bp); 12491 int tmo; 12492 12493 if (n < 0) { 12494 netdev_err(bp->dev, "Firmware reset aborted, subsequent func_qcfg cmd failed, rc = %d, %d msecs since reset timestamp\n", 12495 n, jiffies_to_msecs(jiffies - 12496 bp->fw_reset_timestamp)); 12497 goto fw_reset_abort; 12498 } else if (n > 0) { 12499 if (bnxt_fw_reset_timeout(bp)) { 12500 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 12501 bp->fw_reset_state = 0; 12502 netdev_err(bp->dev, "Firmware reset aborted, bnxt_get_registered_vfs() returns %d\n", 12503 n); 12504 return; 12505 } 12506 bnxt_queue_fw_reset_work(bp, HZ / 10); 12507 return; 12508 } 12509 bp->fw_reset_timestamp = jiffies; 12510 rtnl_lock(); 12511 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) { 12512 bnxt_fw_reset_abort(bp, rc); 12513 rtnl_unlock(); 12514 return; 12515 } 12516 bnxt_fw_reset_close(bp); 12517 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) { 12518 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN; 12519 tmo = HZ / 10; 12520 } else { 12521 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 12522 tmo = bp->fw_reset_min_dsecs * HZ / 10; 12523 } 12524 rtnl_unlock(); 12525 bnxt_queue_fw_reset_work(bp, tmo); 12526 return; 12527 } 12528 case BNXT_FW_RESET_STATE_POLL_FW_DOWN: { 12529 u32 val; 12530 12531 val = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG); 12532 if (!(val & BNXT_FW_STATUS_SHUTDOWN) && 12533 !bnxt_fw_reset_timeout(bp)) { 12534 bnxt_queue_fw_reset_work(bp, HZ / 5); 12535 return; 12536 } 12537 12538 if (!bp->fw_health->primary) { 12539 u32 wait_dsecs = bp->fw_health->normal_func_wait_dsecs; 12540 12541 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 12542 bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10); 12543 return; 12544 } 12545 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW; 12546 } 12547 fallthrough; 12548 case BNXT_FW_RESET_STATE_RESET_FW: 12549 bnxt_reset_all(bp); 12550 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 12551 bnxt_queue_fw_reset_work(bp, bp->fw_reset_min_dsecs * HZ / 10); 12552 return; 12553 case BNXT_FW_RESET_STATE_ENABLE_DEV: 12554 bnxt_inv_fw_health_reg(bp); 12555 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state) && 12556 !bp->fw_reset_min_dsecs) { 12557 u16 val; 12558 12559 pci_read_config_word(bp->pdev, PCI_SUBSYSTEM_ID, &val); 12560 if (val == 0xffff) { 12561 if (bnxt_fw_reset_timeout(bp)) { 12562 netdev_err(bp->dev, "Firmware reset aborted, PCI config space invalid\n"); 12563 rc = -ETIMEDOUT; 12564 goto fw_reset_abort; 12565 } 12566 bnxt_queue_fw_reset_work(bp, HZ / 1000); 12567 return; 12568 } 12569 } 12570 clear_bit(BNXT_STATE_FW_FATAL_COND, &bp->state); 12571 clear_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state); 12572 if (test_and_clear_bit(BNXT_STATE_FW_ACTIVATE_RESET, &bp->state) && 12573 !test_bit(BNXT_STATE_FW_ACTIVATE, &bp->state)) 12574 bnxt_dl_remote_reload(bp); 12575 if (pci_enable_device(bp->pdev)) { 12576 netdev_err(bp->dev, "Cannot re-enable PCI device\n"); 12577 rc = -ENODEV; 12578 goto fw_reset_abort; 12579 } 12580 pci_set_master(bp->pdev); 12581 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW; 12582 fallthrough; 12583 case BNXT_FW_RESET_STATE_POLL_FW: 12584 bp->hwrm_cmd_timeout = SHORT_HWRM_CMD_TIMEOUT; 12585 rc = bnxt_hwrm_poll(bp); 12586 if (rc) { 12587 if (bnxt_fw_reset_timeout(bp)) { 12588 netdev_err(bp->dev, "Firmware reset aborted\n"); 12589 goto fw_reset_abort_status; 12590 } 12591 bnxt_queue_fw_reset_work(bp, HZ / 5); 12592 return; 12593 } 12594 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT; 12595 bp->fw_reset_state = BNXT_FW_RESET_STATE_OPENING; 12596 fallthrough; 12597 case BNXT_FW_RESET_STATE_OPENING: 12598 while (!rtnl_trylock()) { 12599 bnxt_queue_fw_reset_work(bp, HZ / 10); 12600 return; 12601 } 12602 rc = bnxt_open(bp->dev); 12603 if (rc) { 12604 netdev_err(bp->dev, "bnxt_open() failed during FW reset\n"); 12605 bnxt_fw_reset_abort(bp, rc); 12606 rtnl_unlock(); 12607 return; 12608 } 12609 12610 if ((bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) && 12611 bp->fw_health->enabled) { 12612 bp->fw_health->last_fw_reset_cnt = 12613 bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG); 12614 } 12615 bp->fw_reset_state = 0; 12616 /* Make sure fw_reset_state is 0 before clearing the flag */ 12617 smp_mb__before_atomic(); 12618 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 12619 bnxt_ulp_start(bp, 0); 12620 bnxt_reenable_sriov(bp); 12621 bnxt_vf_reps_alloc(bp); 12622 bnxt_vf_reps_open(bp); 12623 bnxt_ptp_reapply_pps(bp); 12624 clear_bit(BNXT_STATE_FW_ACTIVATE, &bp->state); 12625 if (test_and_clear_bit(BNXT_STATE_RECOVER, &bp->state)) { 12626 bnxt_dl_health_fw_recovery_done(bp); 12627 bnxt_dl_health_fw_status_update(bp, true); 12628 } 12629 rtnl_unlock(); 12630 break; 12631 } 12632 return; 12633 12634 fw_reset_abort_status: 12635 if (bp->fw_health->status_reliable || 12636 (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) { 12637 u32 sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG); 12638 12639 netdev_err(bp->dev, "fw_health_status 0x%x\n", sts); 12640 } 12641 fw_reset_abort: 12642 rtnl_lock(); 12643 bnxt_fw_reset_abort(bp, rc); 12644 rtnl_unlock(); 12645 } 12646 12647 static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev) 12648 { 12649 int rc; 12650 struct bnxt *bp = netdev_priv(dev); 12651 12652 SET_NETDEV_DEV(dev, &pdev->dev); 12653 12654 /* enable device (incl. PCI PM wakeup), and bus-mastering */ 12655 rc = pci_enable_device(pdev); 12656 if (rc) { 12657 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n"); 12658 goto init_err; 12659 } 12660 12661 if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) { 12662 dev_err(&pdev->dev, 12663 "Cannot find PCI device base address, aborting\n"); 12664 rc = -ENODEV; 12665 goto init_err_disable; 12666 } 12667 12668 rc = pci_request_regions(pdev, DRV_MODULE_NAME); 12669 if (rc) { 12670 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n"); 12671 goto init_err_disable; 12672 } 12673 12674 if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 && 12675 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) { 12676 dev_err(&pdev->dev, "System does not support DMA, aborting\n"); 12677 rc = -EIO; 12678 goto init_err_release; 12679 } 12680 12681 pci_set_master(pdev); 12682 12683 bp->dev = dev; 12684 bp->pdev = pdev; 12685 12686 /* Doorbell BAR bp->bar1 is mapped after bnxt_fw_init_one_p2() 12687 * determines the BAR size. 12688 */ 12689 bp->bar0 = pci_ioremap_bar(pdev, 0); 12690 if (!bp->bar0) { 12691 dev_err(&pdev->dev, "Cannot map device registers, aborting\n"); 12692 rc = -ENOMEM; 12693 goto init_err_release; 12694 } 12695 12696 bp->bar2 = pci_ioremap_bar(pdev, 4); 12697 if (!bp->bar2) { 12698 dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n"); 12699 rc = -ENOMEM; 12700 goto init_err_release; 12701 } 12702 12703 INIT_WORK(&bp->sp_task, bnxt_sp_task); 12704 INIT_DELAYED_WORK(&bp->fw_reset_task, bnxt_fw_reset_task); 12705 12706 spin_lock_init(&bp->ntp_fltr_lock); 12707 #if BITS_PER_LONG == 32 12708 spin_lock_init(&bp->db_lock); 12709 #endif 12710 12711 bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE; 12712 bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE; 12713 12714 timer_setup(&bp->timer, bnxt_timer, 0); 12715 bp->current_interval = BNXT_TIMER_INTERVAL; 12716 12717 bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID; 12718 bp->nge_fw_dst_port_id = INVALID_HW_RING_ID; 12719 12720 clear_bit(BNXT_STATE_OPEN, &bp->state); 12721 return 0; 12722 12723 init_err_release: 12724 bnxt_unmap_bars(bp, pdev); 12725 pci_release_regions(pdev); 12726 12727 init_err_disable: 12728 pci_disable_device(pdev); 12729 12730 init_err: 12731 return rc; 12732 } 12733 12734 /* rtnl_lock held */ 12735 static int bnxt_change_mac_addr(struct net_device *dev, void *p) 12736 { 12737 struct sockaddr *addr = p; 12738 struct bnxt *bp = netdev_priv(dev); 12739 int rc = 0; 12740 12741 if (!is_valid_ether_addr(addr->sa_data)) 12742 return -EADDRNOTAVAIL; 12743 12744 if (ether_addr_equal(addr->sa_data, dev->dev_addr)) 12745 return 0; 12746 12747 rc = bnxt_approve_mac(bp, addr->sa_data, true); 12748 if (rc) 12749 return rc; 12750 12751 eth_hw_addr_set(dev, addr->sa_data); 12752 if (netif_running(dev)) { 12753 bnxt_close_nic(bp, false, false); 12754 rc = bnxt_open_nic(bp, false, false); 12755 } 12756 12757 return rc; 12758 } 12759 12760 /* rtnl_lock held */ 12761 static int bnxt_change_mtu(struct net_device *dev, int new_mtu) 12762 { 12763 struct bnxt *bp = netdev_priv(dev); 12764 12765 if (netif_running(dev)) 12766 bnxt_close_nic(bp, true, false); 12767 12768 dev->mtu = new_mtu; 12769 bnxt_set_ring_params(bp); 12770 12771 if (netif_running(dev)) 12772 return bnxt_open_nic(bp, true, false); 12773 12774 return 0; 12775 } 12776 12777 int bnxt_setup_mq_tc(struct net_device *dev, u8 tc) 12778 { 12779 struct bnxt *bp = netdev_priv(dev); 12780 bool sh = false; 12781 int rc; 12782 12783 if (tc > bp->max_tc) { 12784 netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n", 12785 tc, bp->max_tc); 12786 return -EINVAL; 12787 } 12788 12789 if (netdev_get_num_tc(dev) == tc) 12790 return 0; 12791 12792 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 12793 sh = true; 12794 12795 rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings, 12796 sh, tc, bp->tx_nr_rings_xdp); 12797 if (rc) 12798 return rc; 12799 12800 /* Needs to close the device and do hw resource re-allocations */ 12801 if (netif_running(bp->dev)) 12802 bnxt_close_nic(bp, true, false); 12803 12804 if (tc) { 12805 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc; 12806 netdev_set_num_tc(dev, tc); 12807 } else { 12808 bp->tx_nr_rings = bp->tx_nr_rings_per_tc; 12809 netdev_reset_tc(dev); 12810 } 12811 bp->tx_nr_rings += bp->tx_nr_rings_xdp; 12812 bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) : 12813 bp->tx_nr_rings + bp->rx_nr_rings; 12814 12815 if (netif_running(bp->dev)) 12816 return bnxt_open_nic(bp, true, false); 12817 12818 return 0; 12819 } 12820 12821 static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 12822 void *cb_priv) 12823 { 12824 struct bnxt *bp = cb_priv; 12825 12826 if (!bnxt_tc_flower_enabled(bp) || 12827 !tc_cls_can_offload_and_chain0(bp->dev, type_data)) 12828 return -EOPNOTSUPP; 12829 12830 switch (type) { 12831 case TC_SETUP_CLSFLOWER: 12832 return bnxt_tc_setup_flower(bp, bp->pf.fw_fid, type_data); 12833 default: 12834 return -EOPNOTSUPP; 12835 } 12836 } 12837 12838 LIST_HEAD(bnxt_block_cb_list); 12839 12840 static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type, 12841 void *type_data) 12842 { 12843 struct bnxt *bp = netdev_priv(dev); 12844 12845 switch (type) { 12846 case TC_SETUP_BLOCK: 12847 return flow_block_cb_setup_simple(type_data, 12848 &bnxt_block_cb_list, 12849 bnxt_setup_tc_block_cb, 12850 bp, bp, true); 12851 case TC_SETUP_QDISC_MQPRIO: { 12852 struct tc_mqprio_qopt *mqprio = type_data; 12853 12854 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 12855 12856 return bnxt_setup_mq_tc(dev, mqprio->num_tc); 12857 } 12858 default: 12859 return -EOPNOTSUPP; 12860 } 12861 } 12862 12863 #ifdef CONFIG_RFS_ACCEL 12864 static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1, 12865 struct bnxt_ntuple_filter *f2) 12866 { 12867 struct flow_keys *keys1 = &f1->fkeys; 12868 struct flow_keys *keys2 = &f2->fkeys; 12869 12870 if (keys1->basic.n_proto != keys2->basic.n_proto || 12871 keys1->basic.ip_proto != keys2->basic.ip_proto) 12872 return false; 12873 12874 if (keys1->basic.n_proto == htons(ETH_P_IP)) { 12875 if (keys1->addrs.v4addrs.src != keys2->addrs.v4addrs.src || 12876 keys1->addrs.v4addrs.dst != keys2->addrs.v4addrs.dst) 12877 return false; 12878 } else { 12879 if (memcmp(&keys1->addrs.v6addrs.src, &keys2->addrs.v6addrs.src, 12880 sizeof(keys1->addrs.v6addrs.src)) || 12881 memcmp(&keys1->addrs.v6addrs.dst, &keys2->addrs.v6addrs.dst, 12882 sizeof(keys1->addrs.v6addrs.dst))) 12883 return false; 12884 } 12885 12886 if (keys1->ports.ports == keys2->ports.ports && 12887 keys1->control.flags == keys2->control.flags && 12888 ether_addr_equal(f1->src_mac_addr, f2->src_mac_addr) && 12889 ether_addr_equal(f1->dst_mac_addr, f2->dst_mac_addr)) 12890 return true; 12891 12892 return false; 12893 } 12894 12895 static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb, 12896 u16 rxq_index, u32 flow_id) 12897 { 12898 struct bnxt *bp = netdev_priv(dev); 12899 struct bnxt_ntuple_filter *fltr, *new_fltr; 12900 struct flow_keys *fkeys; 12901 struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb); 12902 int rc = 0, idx, bit_id, l2_idx = 0; 12903 struct hlist_head *head; 12904 u32 flags; 12905 12906 if (!ether_addr_equal(dev->dev_addr, eth->h_dest)) { 12907 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 12908 int off = 0, j; 12909 12910 netif_addr_lock_bh(dev); 12911 for (j = 0; j < vnic->uc_filter_count; j++, off += ETH_ALEN) { 12912 if (ether_addr_equal(eth->h_dest, 12913 vnic->uc_list + off)) { 12914 l2_idx = j + 1; 12915 break; 12916 } 12917 } 12918 netif_addr_unlock_bh(dev); 12919 if (!l2_idx) 12920 return -EINVAL; 12921 } 12922 new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC); 12923 if (!new_fltr) 12924 return -ENOMEM; 12925 12926 fkeys = &new_fltr->fkeys; 12927 if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) { 12928 rc = -EPROTONOSUPPORT; 12929 goto err_free; 12930 } 12931 12932 if ((fkeys->basic.n_proto != htons(ETH_P_IP) && 12933 fkeys->basic.n_proto != htons(ETH_P_IPV6)) || 12934 ((fkeys->basic.ip_proto != IPPROTO_TCP) && 12935 (fkeys->basic.ip_proto != IPPROTO_UDP))) { 12936 rc = -EPROTONOSUPPORT; 12937 goto err_free; 12938 } 12939 if (fkeys->basic.n_proto == htons(ETH_P_IPV6) && 12940 bp->hwrm_spec_code < 0x10601) { 12941 rc = -EPROTONOSUPPORT; 12942 goto err_free; 12943 } 12944 flags = fkeys->control.flags; 12945 if (((flags & FLOW_DIS_ENCAPSULATION) && 12946 bp->hwrm_spec_code < 0x10601) || (flags & FLOW_DIS_IS_FRAGMENT)) { 12947 rc = -EPROTONOSUPPORT; 12948 goto err_free; 12949 } 12950 12951 memcpy(new_fltr->dst_mac_addr, eth->h_dest, ETH_ALEN); 12952 memcpy(new_fltr->src_mac_addr, eth->h_source, ETH_ALEN); 12953 12954 idx = skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK; 12955 head = &bp->ntp_fltr_hash_tbl[idx]; 12956 rcu_read_lock(); 12957 hlist_for_each_entry_rcu(fltr, head, hash) { 12958 if (bnxt_fltr_match(fltr, new_fltr)) { 12959 rc = fltr->sw_id; 12960 rcu_read_unlock(); 12961 goto err_free; 12962 } 12963 } 12964 rcu_read_unlock(); 12965 12966 spin_lock_bh(&bp->ntp_fltr_lock); 12967 bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap, 12968 BNXT_NTP_FLTR_MAX_FLTR, 0); 12969 if (bit_id < 0) { 12970 spin_unlock_bh(&bp->ntp_fltr_lock); 12971 rc = -ENOMEM; 12972 goto err_free; 12973 } 12974 12975 new_fltr->sw_id = (u16)bit_id; 12976 new_fltr->flow_id = flow_id; 12977 new_fltr->l2_fltr_idx = l2_idx; 12978 new_fltr->rxq = rxq_index; 12979 hlist_add_head_rcu(&new_fltr->hash, head); 12980 bp->ntp_fltr_count++; 12981 spin_unlock_bh(&bp->ntp_fltr_lock); 12982 12983 bnxt_queue_sp_work(bp, BNXT_RX_NTP_FLTR_SP_EVENT); 12984 12985 return new_fltr->sw_id; 12986 12987 err_free: 12988 kfree(new_fltr); 12989 return rc; 12990 } 12991 12992 static void bnxt_cfg_ntp_filters(struct bnxt *bp) 12993 { 12994 int i; 12995 12996 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) { 12997 struct hlist_head *head; 12998 struct hlist_node *tmp; 12999 struct bnxt_ntuple_filter *fltr; 13000 int rc; 13001 13002 head = &bp->ntp_fltr_hash_tbl[i]; 13003 hlist_for_each_entry_safe(fltr, tmp, head, hash) { 13004 bool del = false; 13005 13006 if (test_bit(BNXT_FLTR_VALID, &fltr->state)) { 13007 if (rps_may_expire_flow(bp->dev, fltr->rxq, 13008 fltr->flow_id, 13009 fltr->sw_id)) { 13010 bnxt_hwrm_cfa_ntuple_filter_free(bp, 13011 fltr); 13012 del = true; 13013 } 13014 } else { 13015 rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp, 13016 fltr); 13017 if (rc) 13018 del = true; 13019 else 13020 set_bit(BNXT_FLTR_VALID, &fltr->state); 13021 } 13022 13023 if (del) { 13024 spin_lock_bh(&bp->ntp_fltr_lock); 13025 hlist_del_rcu(&fltr->hash); 13026 bp->ntp_fltr_count--; 13027 spin_unlock_bh(&bp->ntp_fltr_lock); 13028 synchronize_rcu(); 13029 clear_bit(fltr->sw_id, bp->ntp_fltr_bmap); 13030 kfree(fltr); 13031 } 13032 } 13033 } 13034 if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event)) 13035 netdev_info(bp->dev, "Receive PF driver unload event!\n"); 13036 } 13037 13038 #else 13039 13040 static void bnxt_cfg_ntp_filters(struct bnxt *bp) 13041 { 13042 } 13043 13044 #endif /* CONFIG_RFS_ACCEL */ 13045 13046 static int bnxt_udp_tunnel_set_port(struct net_device *netdev, unsigned int table, 13047 unsigned int entry, struct udp_tunnel_info *ti) 13048 { 13049 struct bnxt *bp = netdev_priv(netdev); 13050 unsigned int cmd; 13051 13052 if (ti->type == UDP_TUNNEL_TYPE_VXLAN) 13053 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN; 13054 else 13055 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE; 13056 13057 return bnxt_hwrm_tunnel_dst_port_alloc(bp, ti->port, cmd); 13058 } 13059 13060 static int bnxt_udp_tunnel_unset_port(struct net_device *netdev, unsigned int table, 13061 unsigned int entry, struct udp_tunnel_info *ti) 13062 { 13063 struct bnxt *bp = netdev_priv(netdev); 13064 unsigned int cmd; 13065 13066 if (ti->type == UDP_TUNNEL_TYPE_VXLAN) 13067 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN; 13068 else 13069 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE; 13070 13071 return bnxt_hwrm_tunnel_dst_port_free(bp, cmd); 13072 } 13073 13074 static const struct udp_tunnel_nic_info bnxt_udp_tunnels = { 13075 .set_port = bnxt_udp_tunnel_set_port, 13076 .unset_port = bnxt_udp_tunnel_unset_port, 13077 .flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP | 13078 UDP_TUNNEL_NIC_INFO_OPEN_ONLY, 13079 .tables = { 13080 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN, }, 13081 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_GENEVE, }, 13082 }, 13083 }; 13084 13085 static int bnxt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 13086 struct net_device *dev, u32 filter_mask, 13087 int nlflags) 13088 { 13089 struct bnxt *bp = netdev_priv(dev); 13090 13091 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bp->br_mode, 0, 0, 13092 nlflags, filter_mask, NULL); 13093 } 13094 13095 static int bnxt_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh, 13096 u16 flags, struct netlink_ext_ack *extack) 13097 { 13098 struct bnxt *bp = netdev_priv(dev); 13099 struct nlattr *attr, *br_spec; 13100 int rem, rc = 0; 13101 13102 if (bp->hwrm_spec_code < 0x10708 || !BNXT_SINGLE_PF(bp)) 13103 return -EOPNOTSUPP; 13104 13105 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 13106 if (!br_spec) 13107 return -EINVAL; 13108 13109 nla_for_each_nested(attr, br_spec, rem) { 13110 u16 mode; 13111 13112 if (nla_type(attr) != IFLA_BRIDGE_MODE) 13113 continue; 13114 13115 mode = nla_get_u16(attr); 13116 if (mode == bp->br_mode) 13117 break; 13118 13119 rc = bnxt_hwrm_set_br_mode(bp, mode); 13120 if (!rc) 13121 bp->br_mode = mode; 13122 break; 13123 } 13124 return rc; 13125 } 13126 13127 int bnxt_get_port_parent_id(struct net_device *dev, 13128 struct netdev_phys_item_id *ppid) 13129 { 13130 struct bnxt *bp = netdev_priv(dev); 13131 13132 if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV) 13133 return -EOPNOTSUPP; 13134 13135 /* The PF and it's VF-reps only support the switchdev framework */ 13136 if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_DSN_VALID)) 13137 return -EOPNOTSUPP; 13138 13139 ppid->id_len = sizeof(bp->dsn); 13140 memcpy(ppid->id, bp->dsn, ppid->id_len); 13141 13142 return 0; 13143 } 13144 13145 static const struct net_device_ops bnxt_netdev_ops = { 13146 .ndo_open = bnxt_open, 13147 .ndo_start_xmit = bnxt_start_xmit, 13148 .ndo_stop = bnxt_close, 13149 .ndo_get_stats64 = bnxt_get_stats64, 13150 .ndo_set_rx_mode = bnxt_set_rx_mode, 13151 .ndo_eth_ioctl = bnxt_ioctl, 13152 .ndo_validate_addr = eth_validate_addr, 13153 .ndo_set_mac_address = bnxt_change_mac_addr, 13154 .ndo_change_mtu = bnxt_change_mtu, 13155 .ndo_fix_features = bnxt_fix_features, 13156 .ndo_set_features = bnxt_set_features, 13157 .ndo_features_check = bnxt_features_check, 13158 .ndo_tx_timeout = bnxt_tx_timeout, 13159 #ifdef CONFIG_BNXT_SRIOV 13160 .ndo_get_vf_config = bnxt_get_vf_config, 13161 .ndo_set_vf_mac = bnxt_set_vf_mac, 13162 .ndo_set_vf_vlan = bnxt_set_vf_vlan, 13163 .ndo_set_vf_rate = bnxt_set_vf_bw, 13164 .ndo_set_vf_link_state = bnxt_set_vf_link_state, 13165 .ndo_set_vf_spoofchk = bnxt_set_vf_spoofchk, 13166 .ndo_set_vf_trust = bnxt_set_vf_trust, 13167 #endif 13168 .ndo_setup_tc = bnxt_setup_tc, 13169 #ifdef CONFIG_RFS_ACCEL 13170 .ndo_rx_flow_steer = bnxt_rx_flow_steer, 13171 #endif 13172 .ndo_bpf = bnxt_xdp, 13173 .ndo_xdp_xmit = bnxt_xdp_xmit, 13174 .ndo_bridge_getlink = bnxt_bridge_getlink, 13175 .ndo_bridge_setlink = bnxt_bridge_setlink, 13176 }; 13177 13178 static void bnxt_remove_one(struct pci_dev *pdev) 13179 { 13180 struct net_device *dev = pci_get_drvdata(pdev); 13181 struct bnxt *bp = netdev_priv(dev); 13182 13183 if (BNXT_PF(bp)) 13184 bnxt_sriov_disable(bp); 13185 13186 bnxt_rdma_aux_device_uninit(bp); 13187 13188 bnxt_ptp_clear(bp); 13189 unregister_netdev(dev); 13190 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 13191 /* Flush any pending tasks */ 13192 cancel_work_sync(&bp->sp_task); 13193 cancel_delayed_work_sync(&bp->fw_reset_task); 13194 bp->sp_event = 0; 13195 13196 bnxt_dl_fw_reporters_destroy(bp); 13197 bnxt_dl_unregister(bp); 13198 bnxt_shutdown_tc(bp); 13199 13200 bnxt_clear_int_mode(bp); 13201 bnxt_hwrm_func_drv_unrgtr(bp); 13202 bnxt_free_hwrm_resources(bp); 13203 bnxt_ethtool_free(bp); 13204 bnxt_dcb_free(bp); 13205 kfree(bp->ptp_cfg); 13206 bp->ptp_cfg = NULL; 13207 kfree(bp->fw_health); 13208 bp->fw_health = NULL; 13209 bnxt_cleanup_pci(bp); 13210 bnxt_free_ctx_mem(bp); 13211 kfree(bp->ctx); 13212 bp->ctx = NULL; 13213 kfree(bp->rss_indir_tbl); 13214 bp->rss_indir_tbl = NULL; 13215 bnxt_free_port_stats(bp); 13216 free_netdev(dev); 13217 } 13218 13219 static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt) 13220 { 13221 int rc = 0; 13222 struct bnxt_link_info *link_info = &bp->link_info; 13223 13224 bp->phy_flags = 0; 13225 rc = bnxt_hwrm_phy_qcaps(bp); 13226 if (rc) { 13227 netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n", 13228 rc); 13229 return rc; 13230 } 13231 if (bp->phy_flags & BNXT_PHY_FL_NO_FCS) 13232 bp->dev->priv_flags |= IFF_SUPP_NOFCS; 13233 else 13234 bp->dev->priv_flags &= ~IFF_SUPP_NOFCS; 13235 if (!fw_dflt) 13236 return 0; 13237 13238 mutex_lock(&bp->link_lock); 13239 rc = bnxt_update_link(bp, false); 13240 if (rc) { 13241 mutex_unlock(&bp->link_lock); 13242 netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n", 13243 rc); 13244 return rc; 13245 } 13246 13247 /* Older firmware does not have supported_auto_speeds, so assume 13248 * that all supported speeds can be autonegotiated. 13249 */ 13250 if (link_info->auto_link_speeds && !link_info->support_auto_speeds) 13251 link_info->support_auto_speeds = link_info->support_speeds; 13252 13253 bnxt_init_ethtool_link_settings(bp); 13254 mutex_unlock(&bp->link_lock); 13255 return 0; 13256 } 13257 13258 static int bnxt_get_max_irq(struct pci_dev *pdev) 13259 { 13260 u16 ctrl; 13261 13262 if (!pdev->msix_cap) 13263 return 1; 13264 13265 pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl); 13266 return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1; 13267 } 13268 13269 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, 13270 int *max_cp) 13271 { 13272 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 13273 int max_ring_grps = 0, max_irq; 13274 13275 *max_tx = hw_resc->max_tx_rings; 13276 *max_rx = hw_resc->max_rx_rings; 13277 *max_cp = bnxt_get_max_func_cp_rings_for_en(bp); 13278 max_irq = min_t(int, bnxt_get_max_func_irqs(bp) - 13279 bnxt_get_ulp_msix_num(bp), 13280 hw_resc->max_stat_ctxs - bnxt_get_ulp_stat_ctxs(bp)); 13281 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 13282 *max_cp = min_t(int, *max_cp, max_irq); 13283 max_ring_grps = hw_resc->max_hw_ring_grps; 13284 if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) { 13285 *max_cp -= 1; 13286 *max_rx -= 2; 13287 } 13288 if (bp->flags & BNXT_FLAG_AGG_RINGS) 13289 *max_rx >>= 1; 13290 if (bp->flags & BNXT_FLAG_CHIP_P5) { 13291 bnxt_trim_rings(bp, max_rx, max_tx, *max_cp, false); 13292 /* On P5 chips, max_cp output param should be available NQs */ 13293 *max_cp = max_irq; 13294 } 13295 *max_rx = min_t(int, *max_rx, max_ring_grps); 13296 } 13297 13298 int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared) 13299 { 13300 int rx, tx, cp; 13301 13302 _bnxt_get_max_rings(bp, &rx, &tx, &cp); 13303 *max_rx = rx; 13304 *max_tx = tx; 13305 if (!rx || !tx || !cp) 13306 return -ENOMEM; 13307 13308 return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared); 13309 } 13310 13311 static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx, 13312 bool shared) 13313 { 13314 int rc; 13315 13316 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared); 13317 if (rc && (bp->flags & BNXT_FLAG_AGG_RINGS)) { 13318 /* Not enough rings, try disabling agg rings. */ 13319 bp->flags &= ~BNXT_FLAG_AGG_RINGS; 13320 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared); 13321 if (rc) { 13322 /* set BNXT_FLAG_AGG_RINGS back for consistency */ 13323 bp->flags |= BNXT_FLAG_AGG_RINGS; 13324 return rc; 13325 } 13326 bp->flags |= BNXT_FLAG_NO_AGG_RINGS; 13327 bp->dev->hw_features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); 13328 bp->dev->features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); 13329 bnxt_set_ring_params(bp); 13330 } 13331 13332 if (bp->flags & BNXT_FLAG_ROCE_CAP) { 13333 int max_cp, max_stat, max_irq; 13334 13335 /* Reserve minimum resources for RoCE */ 13336 max_cp = bnxt_get_max_func_cp_rings(bp); 13337 max_stat = bnxt_get_max_func_stat_ctxs(bp); 13338 max_irq = bnxt_get_max_func_irqs(bp); 13339 if (max_cp <= BNXT_MIN_ROCE_CP_RINGS || 13340 max_irq <= BNXT_MIN_ROCE_CP_RINGS || 13341 max_stat <= BNXT_MIN_ROCE_STAT_CTXS) 13342 return 0; 13343 13344 max_cp -= BNXT_MIN_ROCE_CP_RINGS; 13345 max_irq -= BNXT_MIN_ROCE_CP_RINGS; 13346 max_stat -= BNXT_MIN_ROCE_STAT_CTXS; 13347 max_cp = min_t(int, max_cp, max_irq); 13348 max_cp = min_t(int, max_cp, max_stat); 13349 rc = bnxt_trim_rings(bp, max_rx, max_tx, max_cp, shared); 13350 if (rc) 13351 rc = 0; 13352 } 13353 return rc; 13354 } 13355 13356 /* In initial default shared ring setting, each shared ring must have a 13357 * RX/TX ring pair. 13358 */ 13359 static void bnxt_trim_dflt_sh_rings(struct bnxt *bp) 13360 { 13361 bp->cp_nr_rings = min_t(int, bp->tx_nr_rings_per_tc, bp->rx_nr_rings); 13362 bp->rx_nr_rings = bp->cp_nr_rings; 13363 bp->tx_nr_rings_per_tc = bp->cp_nr_rings; 13364 bp->tx_nr_rings = bp->tx_nr_rings_per_tc; 13365 } 13366 13367 static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh) 13368 { 13369 int dflt_rings, max_rx_rings, max_tx_rings, rc; 13370 13371 if (!bnxt_can_reserve_rings(bp)) 13372 return 0; 13373 13374 if (sh) 13375 bp->flags |= BNXT_FLAG_SHARED_RINGS; 13376 dflt_rings = is_kdump_kernel() ? 1 : netif_get_num_default_rss_queues(); 13377 /* Reduce default rings on multi-port cards so that total default 13378 * rings do not exceed CPU count. 13379 */ 13380 if (bp->port_count > 1) { 13381 int max_rings = 13382 max_t(int, num_online_cpus() / bp->port_count, 1); 13383 13384 dflt_rings = min_t(int, dflt_rings, max_rings); 13385 } 13386 rc = bnxt_get_dflt_rings(bp, &max_rx_rings, &max_tx_rings, sh); 13387 if (rc) 13388 return rc; 13389 bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings); 13390 bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings); 13391 if (sh) 13392 bnxt_trim_dflt_sh_rings(bp); 13393 else 13394 bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings; 13395 bp->tx_nr_rings = bp->tx_nr_rings_per_tc; 13396 13397 rc = __bnxt_reserve_rings(bp); 13398 if (rc && rc != -ENODEV) 13399 netdev_warn(bp->dev, "Unable to reserve tx rings\n"); 13400 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 13401 if (sh) 13402 bnxt_trim_dflt_sh_rings(bp); 13403 13404 /* Rings may have been trimmed, re-reserve the trimmed rings. */ 13405 if (bnxt_need_reserve_rings(bp)) { 13406 rc = __bnxt_reserve_rings(bp); 13407 if (rc && rc != -ENODEV) 13408 netdev_warn(bp->dev, "2nd rings reservation failed.\n"); 13409 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 13410 } 13411 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 13412 bp->rx_nr_rings++; 13413 bp->cp_nr_rings++; 13414 } 13415 if (rc) { 13416 bp->tx_nr_rings = 0; 13417 bp->rx_nr_rings = 0; 13418 } 13419 return rc; 13420 } 13421 13422 static int bnxt_init_dflt_ring_mode(struct bnxt *bp) 13423 { 13424 int rc; 13425 13426 if (bp->tx_nr_rings) 13427 return 0; 13428 13429 bnxt_ulp_irq_stop(bp); 13430 bnxt_clear_int_mode(bp); 13431 rc = bnxt_set_dflt_rings(bp, true); 13432 if (rc) { 13433 if (BNXT_VF(bp) && rc == -ENODEV) 13434 netdev_err(bp->dev, "Cannot configure VF rings while PF is unavailable.\n"); 13435 else 13436 netdev_err(bp->dev, "Not enough rings available.\n"); 13437 goto init_dflt_ring_err; 13438 } 13439 rc = bnxt_init_int_mode(bp); 13440 if (rc) 13441 goto init_dflt_ring_err; 13442 13443 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 13444 13445 bnxt_set_dflt_rfs(bp); 13446 13447 init_dflt_ring_err: 13448 bnxt_ulp_irq_restart(bp, rc); 13449 return rc; 13450 } 13451 13452 int bnxt_restore_pf_fw_resources(struct bnxt *bp) 13453 { 13454 int rc; 13455 13456 ASSERT_RTNL(); 13457 bnxt_hwrm_func_qcaps(bp); 13458 13459 if (netif_running(bp->dev)) 13460 __bnxt_close_nic(bp, true, false); 13461 13462 bnxt_ulp_irq_stop(bp); 13463 bnxt_clear_int_mode(bp); 13464 rc = bnxt_init_int_mode(bp); 13465 bnxt_ulp_irq_restart(bp, rc); 13466 13467 if (netif_running(bp->dev)) { 13468 if (rc) 13469 dev_close(bp->dev); 13470 else 13471 rc = bnxt_open_nic(bp, true, false); 13472 } 13473 13474 return rc; 13475 } 13476 13477 static int bnxt_init_mac_addr(struct bnxt *bp) 13478 { 13479 int rc = 0; 13480 13481 if (BNXT_PF(bp)) { 13482 eth_hw_addr_set(bp->dev, bp->pf.mac_addr); 13483 } else { 13484 #ifdef CONFIG_BNXT_SRIOV 13485 struct bnxt_vf_info *vf = &bp->vf; 13486 bool strict_approval = true; 13487 13488 if (is_valid_ether_addr(vf->mac_addr)) { 13489 /* overwrite netdev dev_addr with admin VF MAC */ 13490 eth_hw_addr_set(bp->dev, vf->mac_addr); 13491 /* Older PF driver or firmware may not approve this 13492 * correctly. 13493 */ 13494 strict_approval = false; 13495 } else { 13496 eth_hw_addr_random(bp->dev); 13497 } 13498 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, strict_approval); 13499 #endif 13500 } 13501 return rc; 13502 } 13503 13504 static void bnxt_vpd_read_info(struct bnxt *bp) 13505 { 13506 struct pci_dev *pdev = bp->pdev; 13507 unsigned int vpd_size, kw_len; 13508 int pos, size; 13509 u8 *vpd_data; 13510 13511 vpd_data = pci_vpd_alloc(pdev, &vpd_size); 13512 if (IS_ERR(vpd_data)) { 13513 pci_warn(pdev, "Unable to read VPD\n"); 13514 return; 13515 } 13516 13517 pos = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size, 13518 PCI_VPD_RO_KEYWORD_PARTNO, &kw_len); 13519 if (pos < 0) 13520 goto read_sn; 13521 13522 size = min_t(int, kw_len, BNXT_VPD_FLD_LEN - 1); 13523 memcpy(bp->board_partno, &vpd_data[pos], size); 13524 13525 read_sn: 13526 pos = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size, 13527 PCI_VPD_RO_KEYWORD_SERIALNO, 13528 &kw_len); 13529 if (pos < 0) 13530 goto exit; 13531 13532 size = min_t(int, kw_len, BNXT_VPD_FLD_LEN - 1); 13533 memcpy(bp->board_serialno, &vpd_data[pos], size); 13534 exit: 13535 kfree(vpd_data); 13536 } 13537 13538 static int bnxt_pcie_dsn_get(struct bnxt *bp, u8 dsn[]) 13539 { 13540 struct pci_dev *pdev = bp->pdev; 13541 u64 qword; 13542 13543 qword = pci_get_dsn(pdev); 13544 if (!qword) { 13545 netdev_info(bp->dev, "Unable to read adapter's DSN\n"); 13546 return -EOPNOTSUPP; 13547 } 13548 13549 put_unaligned_le64(qword, dsn); 13550 13551 bp->flags |= BNXT_FLAG_DSN_VALID; 13552 return 0; 13553 } 13554 13555 static int bnxt_map_db_bar(struct bnxt *bp) 13556 { 13557 if (!bp->db_size) 13558 return -ENODEV; 13559 bp->bar1 = pci_iomap(bp->pdev, 2, bp->db_size); 13560 if (!bp->bar1) 13561 return -ENOMEM; 13562 return 0; 13563 } 13564 13565 void bnxt_print_device_info(struct bnxt *bp) 13566 { 13567 netdev_info(bp->dev, "%s found at mem %lx, node addr %pM\n", 13568 board_info[bp->board_idx].name, 13569 (long)pci_resource_start(bp->pdev, 0), bp->dev->dev_addr); 13570 13571 pcie_print_link_status(bp->pdev); 13572 } 13573 13574 static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) 13575 { 13576 struct net_device *dev; 13577 struct bnxt *bp; 13578 int rc, max_irqs; 13579 13580 if (pci_is_bridge(pdev)) 13581 return -ENODEV; 13582 13583 /* Clear any pending DMA transactions from crash kernel 13584 * while loading driver in capture kernel. 13585 */ 13586 if (is_kdump_kernel()) { 13587 pci_clear_master(pdev); 13588 pcie_flr(pdev); 13589 } 13590 13591 max_irqs = bnxt_get_max_irq(pdev); 13592 dev = alloc_etherdev_mq(sizeof(*bp), max_irqs); 13593 if (!dev) 13594 return -ENOMEM; 13595 13596 bp = netdev_priv(dev); 13597 bp->board_idx = ent->driver_data; 13598 bp->msg_enable = BNXT_DEF_MSG_ENABLE; 13599 bnxt_set_max_func_irqs(bp, max_irqs); 13600 13601 if (bnxt_vf_pciid(bp->board_idx)) 13602 bp->flags |= BNXT_FLAG_VF; 13603 13604 /* No devlink port registration in case of a VF */ 13605 if (BNXT_PF(bp)) 13606 SET_NETDEV_DEVLINK_PORT(dev, &bp->dl_port); 13607 13608 if (pdev->msix_cap) 13609 bp->flags |= BNXT_FLAG_MSIX_CAP; 13610 13611 rc = bnxt_init_board(pdev, dev); 13612 if (rc < 0) 13613 goto init_err_free; 13614 13615 dev->netdev_ops = &bnxt_netdev_ops; 13616 dev->watchdog_timeo = BNXT_TX_TIMEOUT; 13617 dev->ethtool_ops = &bnxt_ethtool_ops; 13618 pci_set_drvdata(pdev, dev); 13619 13620 rc = bnxt_alloc_hwrm_resources(bp); 13621 if (rc) 13622 goto init_err_pci_clean; 13623 13624 mutex_init(&bp->hwrm_cmd_lock); 13625 mutex_init(&bp->link_lock); 13626 13627 rc = bnxt_fw_init_one_p1(bp); 13628 if (rc) 13629 goto init_err_pci_clean; 13630 13631 if (BNXT_PF(bp)) 13632 bnxt_vpd_read_info(bp); 13633 13634 if (BNXT_CHIP_P5(bp)) { 13635 bp->flags |= BNXT_FLAG_CHIP_P5; 13636 if (BNXT_CHIP_SR2(bp)) 13637 bp->flags |= BNXT_FLAG_CHIP_SR2; 13638 } 13639 13640 rc = bnxt_alloc_rss_indir_tbl(bp); 13641 if (rc) 13642 goto init_err_pci_clean; 13643 13644 rc = bnxt_fw_init_one_p2(bp); 13645 if (rc) 13646 goto init_err_pci_clean; 13647 13648 rc = bnxt_map_db_bar(bp); 13649 if (rc) { 13650 dev_err(&pdev->dev, "Cannot map doorbell BAR rc = %d, aborting\n", 13651 rc); 13652 goto init_err_pci_clean; 13653 } 13654 13655 dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG | 13656 NETIF_F_TSO | NETIF_F_TSO6 | 13657 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE | 13658 NETIF_F_GSO_IPXIP4 | 13659 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM | 13660 NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH | 13661 NETIF_F_RXCSUM | NETIF_F_GRO; 13662 13663 if (BNXT_SUPPORTS_TPA(bp)) 13664 dev->hw_features |= NETIF_F_LRO; 13665 13666 dev->hw_enc_features = 13667 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG | 13668 NETIF_F_TSO | NETIF_F_TSO6 | 13669 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE | 13670 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM | 13671 NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL; 13672 dev->udp_tunnel_nic_info = &bnxt_udp_tunnels; 13673 13674 dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM | 13675 NETIF_F_GSO_GRE_CSUM; 13676 dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA; 13677 if (bp->fw_cap & BNXT_FW_CAP_VLAN_RX_STRIP) 13678 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_RX; 13679 if (bp->fw_cap & BNXT_FW_CAP_VLAN_TX_INSERT) 13680 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_TX; 13681 if (BNXT_SUPPORTS_TPA(bp)) 13682 dev->hw_features |= NETIF_F_GRO_HW; 13683 dev->features |= dev->hw_features | NETIF_F_HIGHDMA; 13684 if (dev->features & NETIF_F_GRO_HW) 13685 dev->features &= ~NETIF_F_LRO; 13686 dev->priv_flags |= IFF_UNICAST_FLT; 13687 13688 netif_set_tso_max_size(dev, GSO_MAX_SIZE); 13689 13690 dev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | 13691 NETDEV_XDP_ACT_RX_SG; 13692 13693 #ifdef CONFIG_BNXT_SRIOV 13694 init_waitqueue_head(&bp->sriov_cfg_wait); 13695 #endif 13696 if (BNXT_SUPPORTS_TPA(bp)) { 13697 bp->gro_func = bnxt_gro_func_5730x; 13698 if (BNXT_CHIP_P4(bp)) 13699 bp->gro_func = bnxt_gro_func_5731x; 13700 else if (BNXT_CHIP_P5(bp)) 13701 bp->gro_func = bnxt_gro_func_5750x; 13702 } 13703 if (!BNXT_CHIP_P4_PLUS(bp)) 13704 bp->flags |= BNXT_FLAG_DOUBLE_DB; 13705 13706 rc = bnxt_init_mac_addr(bp); 13707 if (rc) { 13708 dev_err(&pdev->dev, "Unable to initialize mac address.\n"); 13709 rc = -EADDRNOTAVAIL; 13710 goto init_err_pci_clean; 13711 } 13712 13713 if (BNXT_PF(bp)) { 13714 /* Read the adapter's DSN to use as the eswitch switch_id */ 13715 rc = bnxt_pcie_dsn_get(bp, bp->dsn); 13716 } 13717 13718 /* MTU range: 60 - FW defined max */ 13719 dev->min_mtu = ETH_ZLEN; 13720 dev->max_mtu = bp->max_mtu; 13721 13722 rc = bnxt_probe_phy(bp, true); 13723 if (rc) 13724 goto init_err_pci_clean; 13725 13726 bnxt_set_rx_skb_mode(bp, false); 13727 bnxt_set_tpa_flags(bp); 13728 bnxt_set_ring_params(bp); 13729 rc = bnxt_set_dflt_rings(bp, true); 13730 if (rc) { 13731 if (BNXT_VF(bp) && rc == -ENODEV) { 13732 netdev_err(bp->dev, "Cannot configure VF rings while PF is unavailable.\n"); 13733 } else { 13734 netdev_err(bp->dev, "Not enough rings available.\n"); 13735 rc = -ENOMEM; 13736 } 13737 goto init_err_pci_clean; 13738 } 13739 13740 bnxt_fw_init_one_p3(bp); 13741 13742 bnxt_init_dflt_coal(bp); 13743 13744 if (dev->hw_features & BNXT_HW_FEATURE_VLAN_ALL_RX) 13745 bp->flags |= BNXT_FLAG_STRIP_VLAN; 13746 13747 rc = bnxt_init_int_mode(bp); 13748 if (rc) 13749 goto init_err_pci_clean; 13750 13751 /* No TC has been set yet and rings may have been trimmed due to 13752 * limited MSIX, so we re-initialize the TX rings per TC. 13753 */ 13754 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 13755 13756 if (BNXT_PF(bp)) { 13757 if (!bnxt_pf_wq) { 13758 bnxt_pf_wq = 13759 create_singlethread_workqueue("bnxt_pf_wq"); 13760 if (!bnxt_pf_wq) { 13761 dev_err(&pdev->dev, "Unable to create workqueue.\n"); 13762 rc = -ENOMEM; 13763 goto init_err_pci_clean; 13764 } 13765 } 13766 rc = bnxt_init_tc(bp); 13767 if (rc) 13768 netdev_err(dev, "Failed to initialize TC flower offload, err = %d.\n", 13769 rc); 13770 } 13771 13772 bnxt_inv_fw_health_reg(bp); 13773 rc = bnxt_dl_register(bp); 13774 if (rc) 13775 goto init_err_dl; 13776 13777 rc = register_netdev(dev); 13778 if (rc) 13779 goto init_err_cleanup; 13780 13781 bnxt_dl_fw_reporters_create(bp); 13782 13783 bnxt_rdma_aux_device_init(bp); 13784 13785 bnxt_print_device_info(bp); 13786 13787 pci_save_state(pdev); 13788 13789 return 0; 13790 init_err_cleanup: 13791 bnxt_dl_unregister(bp); 13792 init_err_dl: 13793 bnxt_shutdown_tc(bp); 13794 bnxt_clear_int_mode(bp); 13795 13796 init_err_pci_clean: 13797 bnxt_hwrm_func_drv_unrgtr(bp); 13798 bnxt_free_hwrm_resources(bp); 13799 bnxt_ethtool_free(bp); 13800 bnxt_ptp_clear(bp); 13801 kfree(bp->ptp_cfg); 13802 bp->ptp_cfg = NULL; 13803 kfree(bp->fw_health); 13804 bp->fw_health = NULL; 13805 bnxt_cleanup_pci(bp); 13806 bnxt_free_ctx_mem(bp); 13807 kfree(bp->ctx); 13808 bp->ctx = NULL; 13809 kfree(bp->rss_indir_tbl); 13810 bp->rss_indir_tbl = NULL; 13811 13812 init_err_free: 13813 free_netdev(dev); 13814 return rc; 13815 } 13816 13817 static void bnxt_shutdown(struct pci_dev *pdev) 13818 { 13819 struct net_device *dev = pci_get_drvdata(pdev); 13820 struct bnxt *bp; 13821 13822 if (!dev) 13823 return; 13824 13825 rtnl_lock(); 13826 bp = netdev_priv(dev); 13827 if (!bp) 13828 goto shutdown_exit; 13829 13830 if (netif_running(dev)) 13831 dev_close(dev); 13832 13833 bnxt_clear_int_mode(bp); 13834 pci_disable_device(pdev); 13835 13836 if (system_state == SYSTEM_POWER_OFF) { 13837 pci_wake_from_d3(pdev, bp->wol); 13838 pci_set_power_state(pdev, PCI_D3hot); 13839 } 13840 13841 shutdown_exit: 13842 rtnl_unlock(); 13843 } 13844 13845 #ifdef CONFIG_PM_SLEEP 13846 static int bnxt_suspend(struct device *device) 13847 { 13848 struct net_device *dev = dev_get_drvdata(device); 13849 struct bnxt *bp = netdev_priv(dev); 13850 int rc = 0; 13851 13852 rtnl_lock(); 13853 bnxt_ulp_stop(bp); 13854 if (netif_running(dev)) { 13855 netif_device_detach(dev); 13856 rc = bnxt_close(dev); 13857 } 13858 bnxt_hwrm_func_drv_unrgtr(bp); 13859 pci_disable_device(bp->pdev); 13860 bnxt_free_ctx_mem(bp); 13861 kfree(bp->ctx); 13862 bp->ctx = NULL; 13863 rtnl_unlock(); 13864 return rc; 13865 } 13866 13867 static int bnxt_resume(struct device *device) 13868 { 13869 struct net_device *dev = dev_get_drvdata(device); 13870 struct bnxt *bp = netdev_priv(dev); 13871 int rc = 0; 13872 13873 rtnl_lock(); 13874 rc = pci_enable_device(bp->pdev); 13875 if (rc) { 13876 netdev_err(dev, "Cannot re-enable PCI device during resume, err = %d\n", 13877 rc); 13878 goto resume_exit; 13879 } 13880 pci_set_master(bp->pdev); 13881 if (bnxt_hwrm_ver_get(bp)) { 13882 rc = -ENODEV; 13883 goto resume_exit; 13884 } 13885 rc = bnxt_hwrm_func_reset(bp); 13886 if (rc) { 13887 rc = -EBUSY; 13888 goto resume_exit; 13889 } 13890 13891 rc = bnxt_hwrm_func_qcaps(bp); 13892 if (rc) 13893 goto resume_exit; 13894 13895 if (bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false)) { 13896 rc = -ENODEV; 13897 goto resume_exit; 13898 } 13899 13900 bnxt_get_wol_settings(bp); 13901 if (netif_running(dev)) { 13902 rc = bnxt_open(dev); 13903 if (!rc) 13904 netif_device_attach(dev); 13905 } 13906 13907 resume_exit: 13908 bnxt_ulp_start(bp, rc); 13909 if (!rc) 13910 bnxt_reenable_sriov(bp); 13911 rtnl_unlock(); 13912 return rc; 13913 } 13914 13915 static SIMPLE_DEV_PM_OPS(bnxt_pm_ops, bnxt_suspend, bnxt_resume); 13916 #define BNXT_PM_OPS (&bnxt_pm_ops) 13917 13918 #else 13919 13920 #define BNXT_PM_OPS NULL 13921 13922 #endif /* CONFIG_PM_SLEEP */ 13923 13924 /** 13925 * bnxt_io_error_detected - called when PCI error is detected 13926 * @pdev: Pointer to PCI device 13927 * @state: The current pci connection state 13928 * 13929 * This function is called after a PCI bus error affecting 13930 * this device has been detected. 13931 */ 13932 static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev, 13933 pci_channel_state_t state) 13934 { 13935 struct net_device *netdev = pci_get_drvdata(pdev); 13936 struct bnxt *bp = netdev_priv(netdev); 13937 13938 netdev_info(netdev, "PCI I/O error detected\n"); 13939 13940 rtnl_lock(); 13941 netif_device_detach(netdev); 13942 13943 bnxt_ulp_stop(bp); 13944 13945 if (state == pci_channel_io_perm_failure) { 13946 rtnl_unlock(); 13947 return PCI_ERS_RESULT_DISCONNECT; 13948 } 13949 13950 if (state == pci_channel_io_frozen) 13951 set_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, &bp->state); 13952 13953 if (netif_running(netdev)) 13954 bnxt_close(netdev); 13955 13956 if (pci_is_enabled(pdev)) 13957 pci_disable_device(pdev); 13958 bnxt_free_ctx_mem(bp); 13959 kfree(bp->ctx); 13960 bp->ctx = NULL; 13961 rtnl_unlock(); 13962 13963 /* Request a slot slot reset. */ 13964 return PCI_ERS_RESULT_NEED_RESET; 13965 } 13966 13967 /** 13968 * bnxt_io_slot_reset - called after the pci bus has been reset. 13969 * @pdev: Pointer to PCI device 13970 * 13971 * Restart the card from scratch, as if from a cold-boot. 13972 * At this point, the card has exprienced a hard reset, 13973 * followed by fixups by BIOS, and has its config space 13974 * set up identically to what it was at cold boot. 13975 */ 13976 static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev) 13977 { 13978 pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT; 13979 struct net_device *netdev = pci_get_drvdata(pdev); 13980 struct bnxt *bp = netdev_priv(netdev); 13981 int retry = 0; 13982 int err = 0; 13983 int off; 13984 13985 netdev_info(bp->dev, "PCI Slot Reset\n"); 13986 13987 rtnl_lock(); 13988 13989 if (pci_enable_device(pdev)) { 13990 dev_err(&pdev->dev, 13991 "Cannot re-enable PCI device after reset.\n"); 13992 } else { 13993 pci_set_master(pdev); 13994 /* Upon fatal error, our device internal logic that latches to 13995 * BAR value is getting reset and will restore only upon 13996 * rewritting the BARs. 13997 * 13998 * As pci_restore_state() does not re-write the BARs if the 13999 * value is same as saved value earlier, driver needs to 14000 * write the BARs to 0 to force restore, in case of fatal error. 14001 */ 14002 if (test_and_clear_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, 14003 &bp->state)) { 14004 for (off = PCI_BASE_ADDRESS_0; 14005 off <= PCI_BASE_ADDRESS_5; off += 4) 14006 pci_write_config_dword(bp->pdev, off, 0); 14007 } 14008 pci_restore_state(pdev); 14009 pci_save_state(pdev); 14010 14011 bnxt_inv_fw_health_reg(bp); 14012 bnxt_try_map_fw_health_reg(bp); 14013 14014 /* In some PCIe AER scenarios, firmware may take up to 14015 * 10 seconds to become ready in the worst case. 14016 */ 14017 do { 14018 err = bnxt_try_recover_fw(bp); 14019 if (!err) 14020 break; 14021 retry++; 14022 } while (retry < BNXT_FW_SLOT_RESET_RETRY); 14023 14024 if (err) { 14025 dev_err(&pdev->dev, "Firmware not ready\n"); 14026 goto reset_exit; 14027 } 14028 14029 err = bnxt_hwrm_func_reset(bp); 14030 if (!err) 14031 result = PCI_ERS_RESULT_RECOVERED; 14032 14033 bnxt_ulp_irq_stop(bp); 14034 bnxt_clear_int_mode(bp); 14035 err = bnxt_init_int_mode(bp); 14036 bnxt_ulp_irq_restart(bp, err); 14037 } 14038 14039 reset_exit: 14040 bnxt_clear_reservations(bp, true); 14041 rtnl_unlock(); 14042 14043 return result; 14044 } 14045 14046 /** 14047 * bnxt_io_resume - called when traffic can start flowing again. 14048 * @pdev: Pointer to PCI device 14049 * 14050 * This callback is called when the error recovery driver tells 14051 * us that its OK to resume normal operation. 14052 */ 14053 static void bnxt_io_resume(struct pci_dev *pdev) 14054 { 14055 struct net_device *netdev = pci_get_drvdata(pdev); 14056 struct bnxt *bp = netdev_priv(netdev); 14057 int err; 14058 14059 netdev_info(bp->dev, "PCI Slot Resume\n"); 14060 rtnl_lock(); 14061 14062 err = bnxt_hwrm_func_qcaps(bp); 14063 if (!err && netif_running(netdev)) 14064 err = bnxt_open(netdev); 14065 14066 bnxt_ulp_start(bp, err); 14067 if (!err) { 14068 bnxt_reenable_sriov(bp); 14069 netif_device_attach(netdev); 14070 } 14071 14072 rtnl_unlock(); 14073 } 14074 14075 static const struct pci_error_handlers bnxt_err_handler = { 14076 .error_detected = bnxt_io_error_detected, 14077 .slot_reset = bnxt_io_slot_reset, 14078 .resume = bnxt_io_resume 14079 }; 14080 14081 static struct pci_driver bnxt_pci_driver = { 14082 .name = DRV_MODULE_NAME, 14083 .id_table = bnxt_pci_tbl, 14084 .probe = bnxt_init_one, 14085 .remove = bnxt_remove_one, 14086 .shutdown = bnxt_shutdown, 14087 .driver.pm = BNXT_PM_OPS, 14088 .err_handler = &bnxt_err_handler, 14089 #if defined(CONFIG_BNXT_SRIOV) 14090 .sriov_configure = bnxt_sriov_configure, 14091 #endif 14092 }; 14093 14094 static int __init bnxt_init(void) 14095 { 14096 int err; 14097 14098 bnxt_debug_init(); 14099 err = pci_register_driver(&bnxt_pci_driver); 14100 if (err) { 14101 bnxt_debug_exit(); 14102 return err; 14103 } 14104 14105 return 0; 14106 } 14107 14108 static void __exit bnxt_exit(void) 14109 { 14110 pci_unregister_driver(&bnxt_pci_driver); 14111 if (bnxt_pf_wq) 14112 destroy_workqueue(bnxt_pf_wq); 14113 bnxt_debug_exit(); 14114 } 14115 14116 module_init(bnxt_init); 14117 module_exit(bnxt_exit); 14118