1 /* Broadcom NetXtreme-C/E network driver. 2 * 3 * Copyright (c) 2014-2016 Broadcom Corporation 4 * Copyright (c) 2016-2019 Broadcom Limited 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License as published by 8 * the Free Software Foundation. 9 */ 10 11 #include <linux/module.h> 12 13 #include <linux/stringify.h> 14 #include <linux/kernel.h> 15 #include <linux/timer.h> 16 #include <linux/errno.h> 17 #include <linux/ioport.h> 18 #include <linux/slab.h> 19 #include <linux/vmalloc.h> 20 #include <linux/interrupt.h> 21 #include <linux/pci.h> 22 #include <linux/netdevice.h> 23 #include <linux/etherdevice.h> 24 #include <linux/skbuff.h> 25 #include <linux/dma-mapping.h> 26 #include <linux/bitops.h> 27 #include <linux/io.h> 28 #include <linux/irq.h> 29 #include <linux/delay.h> 30 #include <asm/byteorder.h> 31 #include <asm/page.h> 32 #include <linux/time.h> 33 #include <linux/mii.h> 34 #include <linux/mdio.h> 35 #include <linux/if.h> 36 #include <linux/if_vlan.h> 37 #include <linux/if_bridge.h> 38 #include <linux/rtc.h> 39 #include <linux/bpf.h> 40 #include <net/ip.h> 41 #include <net/tcp.h> 42 #include <net/udp.h> 43 #include <net/checksum.h> 44 #include <net/ip6_checksum.h> 45 #include <net/udp_tunnel.h> 46 #include <linux/workqueue.h> 47 #include <linux/prefetch.h> 48 #include <linux/cache.h> 49 #include <linux/log2.h> 50 #include <linux/aer.h> 51 #include <linux/bitmap.h> 52 #include <linux/cpu_rmap.h> 53 #include <linux/cpumask.h> 54 #include <net/pkt_cls.h> 55 #include <linux/hwmon.h> 56 #include <linux/hwmon-sysfs.h> 57 #include <net/page_pool.h> 58 59 #include "bnxt_hsi.h" 60 #include "bnxt.h" 61 #include "bnxt_ulp.h" 62 #include "bnxt_sriov.h" 63 #include "bnxt_ethtool.h" 64 #include "bnxt_dcb.h" 65 #include "bnxt_xdp.h" 66 #include "bnxt_vfr.h" 67 #include "bnxt_tc.h" 68 #include "bnxt_devlink.h" 69 #include "bnxt_debugfs.h" 70 71 #define BNXT_TX_TIMEOUT (5 * HZ) 72 #define BNXT_DEF_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_HW) 73 74 MODULE_LICENSE("GPL"); 75 MODULE_DESCRIPTION("Broadcom BCM573xx network driver"); 76 77 #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN) 78 #define BNXT_RX_DMA_OFFSET NET_SKB_PAD 79 #define BNXT_RX_COPY_THRESH 256 80 81 #define BNXT_TX_PUSH_THRESH 164 82 83 enum board_idx { 84 BCM57301, 85 BCM57302, 86 BCM57304, 87 BCM57417_NPAR, 88 BCM58700, 89 BCM57311, 90 BCM57312, 91 BCM57402, 92 BCM57404, 93 BCM57406, 94 BCM57402_NPAR, 95 BCM57407, 96 BCM57412, 97 BCM57414, 98 BCM57416, 99 BCM57417, 100 BCM57412_NPAR, 101 BCM57314, 102 BCM57417_SFP, 103 BCM57416_SFP, 104 BCM57404_NPAR, 105 BCM57406_NPAR, 106 BCM57407_SFP, 107 BCM57407_NPAR, 108 BCM57414_NPAR, 109 BCM57416_NPAR, 110 BCM57452, 111 BCM57454, 112 BCM5745x_NPAR, 113 BCM57508, 114 BCM57504, 115 BCM57502, 116 BCM57508_NPAR, 117 BCM57504_NPAR, 118 BCM57502_NPAR, 119 BCM58802, 120 BCM58804, 121 BCM58808, 122 NETXTREME_E_VF, 123 NETXTREME_C_VF, 124 NETXTREME_S_VF, 125 NETXTREME_E_P5_VF, 126 }; 127 128 /* indexed by enum above */ 129 static const struct { 130 char *name; 131 } board_info[] = { 132 [BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" }, 133 [BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" }, 134 [BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" }, 135 [BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" }, 136 [BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" }, 137 [BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" }, 138 [BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" }, 139 [BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" }, 140 [BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" }, 141 [BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" }, 142 [BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" }, 143 [BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" }, 144 [BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" }, 145 [BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" }, 146 [BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" }, 147 [BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" }, 148 [BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" }, 149 [BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" }, 150 [BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" }, 151 [BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" }, 152 [BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" }, 153 [BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" }, 154 [BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" }, 155 [BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" }, 156 [BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" }, 157 [BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" }, 158 [BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" }, 159 [BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" }, 160 [BCM5745x_NPAR] = { "Broadcom BCM5745x NetXtreme-E Ethernet Partition" }, 161 [BCM57508] = { "Broadcom BCM57508 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" }, 162 [BCM57504] = { "Broadcom BCM57504 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" }, 163 [BCM57502] = { "Broadcom BCM57502 NetXtreme-E 10Gb/25Gb/50Gb Ethernet" }, 164 [BCM57508_NPAR] = { "Broadcom BCM57508 NetXtreme-E Ethernet Partition" }, 165 [BCM57504_NPAR] = { "Broadcom BCM57504 NetXtreme-E Ethernet Partition" }, 166 [BCM57502_NPAR] = { "Broadcom BCM57502 NetXtreme-E Ethernet Partition" }, 167 [BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" }, 168 [BCM58804] = { "Broadcom BCM58804 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" }, 169 [BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" }, 170 [NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" }, 171 [NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" }, 172 [NETXTREME_S_VF] = { "Broadcom NetXtreme-S Ethernet Virtual Function" }, 173 [NETXTREME_E_P5_VF] = { "Broadcom BCM5750X NetXtreme-E Ethernet Virtual Function" }, 174 }; 175 176 static const struct pci_device_id bnxt_pci_tbl[] = { 177 { PCI_VDEVICE(BROADCOM, 0x1604), .driver_data = BCM5745x_NPAR }, 178 { PCI_VDEVICE(BROADCOM, 0x1605), .driver_data = BCM5745x_NPAR }, 179 { PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 }, 180 { PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR }, 181 { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 }, 182 { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 }, 183 { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 }, 184 { PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR }, 185 { PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 }, 186 { PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 }, 187 { PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 }, 188 { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 }, 189 { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 }, 190 { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 }, 191 { PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR }, 192 { PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 }, 193 { PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 }, 194 { PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 }, 195 { PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 }, 196 { PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 }, 197 { PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR }, 198 { PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 }, 199 { PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP }, 200 { PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP }, 201 { PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR }, 202 { PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR }, 203 { PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP }, 204 { PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR }, 205 { PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR }, 206 { PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR }, 207 { PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR }, 208 { PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR }, 209 { PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR }, 210 { PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 }, 211 { PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 }, 212 { PCI_VDEVICE(BROADCOM, 0x1750), .driver_data = BCM57508 }, 213 { PCI_VDEVICE(BROADCOM, 0x1751), .driver_data = BCM57504 }, 214 { PCI_VDEVICE(BROADCOM, 0x1752), .driver_data = BCM57502 }, 215 { PCI_VDEVICE(BROADCOM, 0x1800), .driver_data = BCM57508_NPAR }, 216 { PCI_VDEVICE(BROADCOM, 0x1801), .driver_data = BCM57504_NPAR }, 217 { PCI_VDEVICE(BROADCOM, 0x1802), .driver_data = BCM57502_NPAR }, 218 { PCI_VDEVICE(BROADCOM, 0x1803), .driver_data = BCM57508_NPAR }, 219 { PCI_VDEVICE(BROADCOM, 0x1804), .driver_data = BCM57504_NPAR }, 220 { PCI_VDEVICE(BROADCOM, 0x1805), .driver_data = BCM57502_NPAR }, 221 { PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 }, 222 { PCI_VDEVICE(BROADCOM, 0xd804), .driver_data = BCM58804 }, 223 #ifdef CONFIG_BNXT_SRIOV 224 { PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF }, 225 { PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF }, 226 { PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF }, 227 { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF }, 228 { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF }, 229 { PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF }, 230 { PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF }, 231 { PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF }, 232 { PCI_VDEVICE(BROADCOM, 0x1806), .driver_data = NETXTREME_E_P5_VF }, 233 { PCI_VDEVICE(BROADCOM, 0x1807), .driver_data = NETXTREME_E_P5_VF }, 234 { PCI_VDEVICE(BROADCOM, 0xd800), .driver_data = NETXTREME_S_VF }, 235 #endif 236 { 0 } 237 }; 238 239 MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl); 240 241 static const u16 bnxt_vf_req_snif[] = { 242 HWRM_FUNC_CFG, 243 HWRM_FUNC_VF_CFG, 244 HWRM_PORT_PHY_QCFG, 245 HWRM_CFA_L2_FILTER_ALLOC, 246 }; 247 248 static const u16 bnxt_async_events_arr[] = { 249 ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE, 250 ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE, 251 ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD, 252 ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED, 253 ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE, 254 ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE, 255 ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE, 256 ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY, 257 ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY, 258 ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION, 259 ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG, 260 ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST, 261 }; 262 263 static struct workqueue_struct *bnxt_pf_wq; 264 265 static bool bnxt_vf_pciid(enum board_idx idx) 266 { 267 return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF || 268 idx == NETXTREME_S_VF || idx == NETXTREME_E_P5_VF); 269 } 270 271 #define DB_CP_REARM_FLAGS (DB_KEY_CP | DB_IDX_VALID) 272 #define DB_CP_FLAGS (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS) 273 #define DB_CP_IRQ_DIS_FLAGS (DB_KEY_CP | DB_IRQ_DIS) 274 275 #define BNXT_CP_DB_IRQ_DIS(db) \ 276 writel(DB_CP_IRQ_DIS_FLAGS, db) 277 278 #define BNXT_DB_CQ(db, idx) \ 279 writel(DB_CP_FLAGS | RING_CMP(idx), (db)->doorbell) 280 281 #define BNXT_DB_NQ_P5(db, idx) \ 282 writeq((db)->db_key64 | DBR_TYPE_NQ | RING_CMP(idx), (db)->doorbell) 283 284 #define BNXT_DB_CQ_ARM(db, idx) \ 285 writel(DB_CP_REARM_FLAGS | RING_CMP(idx), (db)->doorbell) 286 287 #define BNXT_DB_NQ_ARM_P5(db, idx) \ 288 writeq((db)->db_key64 | DBR_TYPE_NQ_ARM | RING_CMP(idx), (db)->doorbell) 289 290 static void bnxt_db_nq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx) 291 { 292 if (bp->flags & BNXT_FLAG_CHIP_P5) 293 BNXT_DB_NQ_P5(db, idx); 294 else 295 BNXT_DB_CQ(db, idx); 296 } 297 298 static void bnxt_db_nq_arm(struct bnxt *bp, struct bnxt_db_info *db, u32 idx) 299 { 300 if (bp->flags & BNXT_FLAG_CHIP_P5) 301 BNXT_DB_NQ_ARM_P5(db, idx); 302 else 303 BNXT_DB_CQ_ARM(db, idx); 304 } 305 306 static void bnxt_db_cq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx) 307 { 308 if (bp->flags & BNXT_FLAG_CHIP_P5) 309 writeq(db->db_key64 | DBR_TYPE_CQ_ARMALL | RING_CMP(idx), 310 db->doorbell); 311 else 312 BNXT_DB_CQ(db, idx); 313 } 314 315 const u16 bnxt_lhint_arr[] = { 316 TX_BD_FLAGS_LHINT_512_AND_SMALLER, 317 TX_BD_FLAGS_LHINT_512_TO_1023, 318 TX_BD_FLAGS_LHINT_1024_TO_2047, 319 TX_BD_FLAGS_LHINT_1024_TO_2047, 320 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 321 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 322 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 323 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 324 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 325 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 326 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 327 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 328 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 329 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 330 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 331 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 332 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 333 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 334 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 335 }; 336 337 static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb) 338 { 339 struct metadata_dst *md_dst = skb_metadata_dst(skb); 340 341 if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX) 342 return 0; 343 344 return md_dst->u.port_info.port_id; 345 } 346 347 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev) 348 { 349 struct bnxt *bp = netdev_priv(dev); 350 struct tx_bd *txbd; 351 struct tx_bd_ext *txbd1; 352 struct netdev_queue *txq; 353 int i; 354 dma_addr_t mapping; 355 unsigned int length, pad = 0; 356 u32 len, free_size, vlan_tag_flags, cfa_action, flags; 357 u16 prod, last_frag; 358 struct pci_dev *pdev = bp->pdev; 359 struct bnxt_tx_ring_info *txr; 360 struct bnxt_sw_tx_bd *tx_buf; 361 362 i = skb_get_queue_mapping(skb); 363 if (unlikely(i >= bp->tx_nr_rings)) { 364 dev_kfree_skb_any(skb); 365 return NETDEV_TX_OK; 366 } 367 368 txq = netdev_get_tx_queue(dev, i); 369 txr = &bp->tx_ring[bp->tx_ring_map[i]]; 370 prod = txr->tx_prod; 371 372 free_size = bnxt_tx_avail(bp, txr); 373 if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) { 374 netif_tx_stop_queue(txq); 375 return NETDEV_TX_BUSY; 376 } 377 378 length = skb->len; 379 len = skb_headlen(skb); 380 last_frag = skb_shinfo(skb)->nr_frags; 381 382 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 383 384 txbd->tx_bd_opaque = prod; 385 386 tx_buf = &txr->tx_buf_ring[prod]; 387 tx_buf->skb = skb; 388 tx_buf->nr_frags = last_frag; 389 390 vlan_tag_flags = 0; 391 cfa_action = bnxt_xmit_get_cfa_action(skb); 392 if (skb_vlan_tag_present(skb)) { 393 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN | 394 skb_vlan_tag_get(skb); 395 /* Currently supports 8021Q, 8021AD vlan offloads 396 * QINQ1, QINQ2, QINQ3 vlan headers are deprecated 397 */ 398 if (skb->vlan_proto == htons(ETH_P_8021Q)) 399 vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT; 400 } 401 402 if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh) { 403 struct tx_push_buffer *tx_push_buf = txr->tx_push; 404 struct tx_push_bd *tx_push = &tx_push_buf->push_bd; 405 struct tx_bd_ext *tx_push1 = &tx_push->txbd2; 406 void __iomem *db = txr->tx_db.doorbell; 407 void *pdata = tx_push_buf->data; 408 u64 *end; 409 int j, push_len; 410 411 /* Set COAL_NOW to be ready quickly for the next push */ 412 tx_push->tx_bd_len_flags_type = 413 cpu_to_le32((length << TX_BD_LEN_SHIFT) | 414 TX_BD_TYPE_LONG_TX_BD | 415 TX_BD_FLAGS_LHINT_512_AND_SMALLER | 416 TX_BD_FLAGS_COAL_NOW | 417 TX_BD_FLAGS_PACKET_END | 418 (2 << TX_BD_FLAGS_BD_CNT_SHIFT)); 419 420 if (skb->ip_summed == CHECKSUM_PARTIAL) 421 tx_push1->tx_bd_hsize_lflags = 422 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM); 423 else 424 tx_push1->tx_bd_hsize_lflags = 0; 425 426 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags); 427 tx_push1->tx_bd_cfa_action = 428 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT); 429 430 end = pdata + length; 431 end = PTR_ALIGN(end, 8) - 1; 432 *end = 0; 433 434 skb_copy_from_linear_data(skb, pdata, len); 435 pdata += len; 436 for (j = 0; j < last_frag; j++) { 437 skb_frag_t *frag = &skb_shinfo(skb)->frags[j]; 438 void *fptr; 439 440 fptr = skb_frag_address_safe(frag); 441 if (!fptr) 442 goto normal_tx; 443 444 memcpy(pdata, fptr, skb_frag_size(frag)); 445 pdata += skb_frag_size(frag); 446 } 447 448 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type; 449 txbd->tx_bd_haddr = txr->data_mapping; 450 prod = NEXT_TX(prod); 451 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 452 memcpy(txbd, tx_push1, sizeof(*txbd)); 453 prod = NEXT_TX(prod); 454 tx_push->doorbell = 455 cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | prod); 456 txr->tx_prod = prod; 457 458 tx_buf->is_push = 1; 459 netdev_tx_sent_queue(txq, skb->len); 460 wmb(); /* Sync is_push and byte queue before pushing data */ 461 462 push_len = (length + sizeof(*tx_push) + 7) / 8; 463 if (push_len > 16) { 464 __iowrite64_copy(db, tx_push_buf, 16); 465 __iowrite32_copy(db + 4, tx_push_buf + 1, 466 (push_len - 16) << 1); 467 } else { 468 __iowrite64_copy(db, tx_push_buf, push_len); 469 } 470 471 goto tx_done; 472 } 473 474 normal_tx: 475 if (length < BNXT_MIN_PKT_SIZE) { 476 pad = BNXT_MIN_PKT_SIZE - length; 477 if (skb_pad(skb, pad)) { 478 /* SKB already freed. */ 479 tx_buf->skb = NULL; 480 return NETDEV_TX_OK; 481 } 482 length = BNXT_MIN_PKT_SIZE; 483 } 484 485 mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE); 486 487 if (unlikely(dma_mapping_error(&pdev->dev, mapping))) { 488 dev_kfree_skb_any(skb); 489 tx_buf->skb = NULL; 490 return NETDEV_TX_OK; 491 } 492 493 dma_unmap_addr_set(tx_buf, mapping, mapping); 494 flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD | 495 ((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT); 496 497 txbd->tx_bd_haddr = cpu_to_le64(mapping); 498 499 prod = NEXT_TX(prod); 500 txbd1 = (struct tx_bd_ext *) 501 &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 502 503 txbd1->tx_bd_hsize_lflags = 0; 504 if (skb_is_gso(skb)) { 505 u32 hdr_len; 506 507 if (skb->encapsulation) 508 hdr_len = skb_inner_network_offset(skb) + 509 skb_inner_network_header_len(skb) + 510 inner_tcp_hdrlen(skb); 511 else 512 hdr_len = skb_transport_offset(skb) + 513 tcp_hdrlen(skb); 514 515 txbd1->tx_bd_hsize_lflags = cpu_to_le32(TX_BD_FLAGS_LSO | 516 TX_BD_FLAGS_T_IPID | 517 (hdr_len << (TX_BD_HSIZE_SHIFT - 1))); 518 length = skb_shinfo(skb)->gso_size; 519 txbd1->tx_bd_mss = cpu_to_le32(length); 520 length += hdr_len; 521 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { 522 txbd1->tx_bd_hsize_lflags = 523 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM); 524 txbd1->tx_bd_mss = 0; 525 } 526 527 length >>= 9; 528 if (unlikely(length >= ARRAY_SIZE(bnxt_lhint_arr))) { 529 dev_warn_ratelimited(&pdev->dev, "Dropped oversize %d bytes TX packet.\n", 530 skb->len); 531 i = 0; 532 goto tx_dma_error; 533 } 534 flags |= bnxt_lhint_arr[length]; 535 txbd->tx_bd_len_flags_type = cpu_to_le32(flags); 536 537 txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags); 538 txbd1->tx_bd_cfa_action = 539 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT); 540 for (i = 0; i < last_frag; i++) { 541 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 542 543 prod = NEXT_TX(prod); 544 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 545 546 len = skb_frag_size(frag); 547 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len, 548 DMA_TO_DEVICE); 549 550 if (unlikely(dma_mapping_error(&pdev->dev, mapping))) 551 goto tx_dma_error; 552 553 tx_buf = &txr->tx_buf_ring[prod]; 554 dma_unmap_addr_set(tx_buf, mapping, mapping); 555 556 txbd->tx_bd_haddr = cpu_to_le64(mapping); 557 558 flags = len << TX_BD_LEN_SHIFT; 559 txbd->tx_bd_len_flags_type = cpu_to_le32(flags); 560 } 561 562 flags &= ~TX_BD_LEN; 563 txbd->tx_bd_len_flags_type = 564 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags | 565 TX_BD_FLAGS_PACKET_END); 566 567 netdev_tx_sent_queue(txq, skb->len); 568 569 /* Sync BD data before updating doorbell */ 570 wmb(); 571 572 prod = NEXT_TX(prod); 573 txr->tx_prod = prod; 574 575 if (!netdev_xmit_more() || netif_xmit_stopped(txq)) 576 bnxt_db_write(bp, &txr->tx_db, prod); 577 578 tx_done: 579 580 if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) { 581 if (netdev_xmit_more() && !tx_buf->is_push) 582 bnxt_db_write(bp, &txr->tx_db, prod); 583 584 netif_tx_stop_queue(txq); 585 586 /* netif_tx_stop_queue() must be done before checking 587 * tx index in bnxt_tx_avail() below, because in 588 * bnxt_tx_int(), we update tx index before checking for 589 * netif_tx_queue_stopped(). 590 */ 591 smp_mb(); 592 if (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh) 593 netif_tx_wake_queue(txq); 594 } 595 return NETDEV_TX_OK; 596 597 tx_dma_error: 598 last_frag = i; 599 600 /* start back at beginning and unmap skb */ 601 prod = txr->tx_prod; 602 tx_buf = &txr->tx_buf_ring[prod]; 603 tx_buf->skb = NULL; 604 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping), 605 skb_headlen(skb), PCI_DMA_TODEVICE); 606 prod = NEXT_TX(prod); 607 608 /* unmap remaining mapped pages */ 609 for (i = 0; i < last_frag; i++) { 610 prod = NEXT_TX(prod); 611 tx_buf = &txr->tx_buf_ring[prod]; 612 dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping), 613 skb_frag_size(&skb_shinfo(skb)->frags[i]), 614 PCI_DMA_TODEVICE); 615 } 616 617 dev_kfree_skb_any(skb); 618 return NETDEV_TX_OK; 619 } 620 621 static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts) 622 { 623 struct bnxt_tx_ring_info *txr = bnapi->tx_ring; 624 struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index); 625 u16 cons = txr->tx_cons; 626 struct pci_dev *pdev = bp->pdev; 627 int i; 628 unsigned int tx_bytes = 0; 629 630 for (i = 0; i < nr_pkts; i++) { 631 struct bnxt_sw_tx_bd *tx_buf; 632 struct sk_buff *skb; 633 int j, last; 634 635 tx_buf = &txr->tx_buf_ring[cons]; 636 cons = NEXT_TX(cons); 637 skb = tx_buf->skb; 638 tx_buf->skb = NULL; 639 640 if (tx_buf->is_push) { 641 tx_buf->is_push = 0; 642 goto next_tx_int; 643 } 644 645 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping), 646 skb_headlen(skb), PCI_DMA_TODEVICE); 647 last = tx_buf->nr_frags; 648 649 for (j = 0; j < last; j++) { 650 cons = NEXT_TX(cons); 651 tx_buf = &txr->tx_buf_ring[cons]; 652 dma_unmap_page( 653 &pdev->dev, 654 dma_unmap_addr(tx_buf, mapping), 655 skb_frag_size(&skb_shinfo(skb)->frags[j]), 656 PCI_DMA_TODEVICE); 657 } 658 659 next_tx_int: 660 cons = NEXT_TX(cons); 661 662 tx_bytes += skb->len; 663 dev_kfree_skb_any(skb); 664 } 665 666 netdev_tx_completed_queue(txq, nr_pkts, tx_bytes); 667 txr->tx_cons = cons; 668 669 /* Need to make the tx_cons update visible to bnxt_start_xmit() 670 * before checking for netif_tx_queue_stopped(). Without the 671 * memory barrier, there is a small possibility that bnxt_start_xmit() 672 * will miss it and cause the queue to be stopped forever. 673 */ 674 smp_mb(); 675 676 if (unlikely(netif_tx_queue_stopped(txq)) && 677 (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)) { 678 __netif_tx_lock(txq, smp_processor_id()); 679 if (netif_tx_queue_stopped(txq) && 680 bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh && 681 txr->dev_state != BNXT_DEV_STATE_CLOSING) 682 netif_tx_wake_queue(txq); 683 __netif_tx_unlock(txq); 684 } 685 } 686 687 static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping, 688 struct bnxt_rx_ring_info *rxr, 689 gfp_t gfp) 690 { 691 struct device *dev = &bp->pdev->dev; 692 struct page *page; 693 694 page = page_pool_dev_alloc_pages(rxr->page_pool); 695 if (!page) 696 return NULL; 697 698 *mapping = dma_map_page_attrs(dev, page, 0, PAGE_SIZE, bp->rx_dir, 699 DMA_ATTR_WEAK_ORDERING); 700 if (dma_mapping_error(dev, *mapping)) { 701 page_pool_recycle_direct(rxr->page_pool, page); 702 return NULL; 703 } 704 *mapping += bp->rx_dma_offset; 705 return page; 706 } 707 708 static inline u8 *__bnxt_alloc_rx_data(struct bnxt *bp, dma_addr_t *mapping, 709 gfp_t gfp) 710 { 711 u8 *data; 712 struct pci_dev *pdev = bp->pdev; 713 714 data = kmalloc(bp->rx_buf_size, gfp); 715 if (!data) 716 return NULL; 717 718 *mapping = dma_map_single_attrs(&pdev->dev, data + bp->rx_dma_offset, 719 bp->rx_buf_use_size, bp->rx_dir, 720 DMA_ATTR_WEAK_ORDERING); 721 722 if (dma_mapping_error(&pdev->dev, *mapping)) { 723 kfree(data); 724 data = NULL; 725 } 726 return data; 727 } 728 729 int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, 730 u16 prod, gfp_t gfp) 731 { 732 struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 733 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod]; 734 dma_addr_t mapping; 735 736 if (BNXT_RX_PAGE_MODE(bp)) { 737 struct page *page = 738 __bnxt_alloc_rx_page(bp, &mapping, rxr, gfp); 739 740 if (!page) 741 return -ENOMEM; 742 743 rx_buf->data = page; 744 rx_buf->data_ptr = page_address(page) + bp->rx_offset; 745 } else { 746 u8 *data = __bnxt_alloc_rx_data(bp, &mapping, gfp); 747 748 if (!data) 749 return -ENOMEM; 750 751 rx_buf->data = data; 752 rx_buf->data_ptr = data + bp->rx_offset; 753 } 754 rx_buf->mapping = mapping; 755 756 rxbd->rx_bd_haddr = cpu_to_le64(mapping); 757 return 0; 758 } 759 760 void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data) 761 { 762 u16 prod = rxr->rx_prod; 763 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf; 764 struct rx_bd *cons_bd, *prod_bd; 765 766 prod_rx_buf = &rxr->rx_buf_ring[prod]; 767 cons_rx_buf = &rxr->rx_buf_ring[cons]; 768 769 prod_rx_buf->data = data; 770 prod_rx_buf->data_ptr = cons_rx_buf->data_ptr; 771 772 prod_rx_buf->mapping = cons_rx_buf->mapping; 773 774 prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 775 cons_bd = &rxr->rx_desc_ring[RX_RING(cons)][RX_IDX(cons)]; 776 777 prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr; 778 } 779 780 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx) 781 { 782 u16 next, max = rxr->rx_agg_bmap_size; 783 784 next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx); 785 if (next >= max) 786 next = find_first_zero_bit(rxr->rx_agg_bmap, max); 787 return next; 788 } 789 790 static inline int bnxt_alloc_rx_page(struct bnxt *bp, 791 struct bnxt_rx_ring_info *rxr, 792 u16 prod, gfp_t gfp) 793 { 794 struct rx_bd *rxbd = 795 &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 796 struct bnxt_sw_rx_agg_bd *rx_agg_buf; 797 struct pci_dev *pdev = bp->pdev; 798 struct page *page; 799 dma_addr_t mapping; 800 u16 sw_prod = rxr->rx_sw_agg_prod; 801 unsigned int offset = 0; 802 803 if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) { 804 page = rxr->rx_page; 805 if (!page) { 806 page = alloc_page(gfp); 807 if (!page) 808 return -ENOMEM; 809 rxr->rx_page = page; 810 rxr->rx_page_offset = 0; 811 } 812 offset = rxr->rx_page_offset; 813 rxr->rx_page_offset += BNXT_RX_PAGE_SIZE; 814 if (rxr->rx_page_offset == PAGE_SIZE) 815 rxr->rx_page = NULL; 816 else 817 get_page(page); 818 } else { 819 page = alloc_page(gfp); 820 if (!page) 821 return -ENOMEM; 822 } 823 824 mapping = dma_map_page_attrs(&pdev->dev, page, offset, 825 BNXT_RX_PAGE_SIZE, PCI_DMA_FROMDEVICE, 826 DMA_ATTR_WEAK_ORDERING); 827 if (dma_mapping_error(&pdev->dev, mapping)) { 828 __free_page(page); 829 return -EIO; 830 } 831 832 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap))) 833 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod); 834 835 __set_bit(sw_prod, rxr->rx_agg_bmap); 836 rx_agg_buf = &rxr->rx_agg_ring[sw_prod]; 837 rxr->rx_sw_agg_prod = NEXT_RX_AGG(sw_prod); 838 839 rx_agg_buf->page = page; 840 rx_agg_buf->offset = offset; 841 rx_agg_buf->mapping = mapping; 842 rxbd->rx_bd_haddr = cpu_to_le64(mapping); 843 rxbd->rx_bd_opaque = sw_prod; 844 return 0; 845 } 846 847 static struct rx_agg_cmp *bnxt_get_agg(struct bnxt *bp, 848 struct bnxt_cp_ring_info *cpr, 849 u16 cp_cons, u16 curr) 850 { 851 struct rx_agg_cmp *agg; 852 853 cp_cons = RING_CMP(ADV_RAW_CMP(cp_cons, curr)); 854 agg = (struct rx_agg_cmp *) 855 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 856 return agg; 857 } 858 859 static struct rx_agg_cmp *bnxt_get_tpa_agg_p5(struct bnxt *bp, 860 struct bnxt_rx_ring_info *rxr, 861 u16 agg_id, u16 curr) 862 { 863 struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[agg_id]; 864 865 return &tpa_info->agg_arr[curr]; 866 } 867 868 static void bnxt_reuse_rx_agg_bufs(struct bnxt_cp_ring_info *cpr, u16 idx, 869 u16 start, u32 agg_bufs, bool tpa) 870 { 871 struct bnxt_napi *bnapi = cpr->bnapi; 872 struct bnxt *bp = bnapi->bp; 873 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 874 u16 prod = rxr->rx_agg_prod; 875 u16 sw_prod = rxr->rx_sw_agg_prod; 876 bool p5_tpa = false; 877 u32 i; 878 879 if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa) 880 p5_tpa = true; 881 882 for (i = 0; i < agg_bufs; i++) { 883 u16 cons; 884 struct rx_agg_cmp *agg; 885 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf; 886 struct rx_bd *prod_bd; 887 struct page *page; 888 889 if (p5_tpa) 890 agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, start + i); 891 else 892 agg = bnxt_get_agg(bp, cpr, idx, start + i); 893 cons = agg->rx_agg_cmp_opaque; 894 __clear_bit(cons, rxr->rx_agg_bmap); 895 896 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap))) 897 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod); 898 899 __set_bit(sw_prod, rxr->rx_agg_bmap); 900 prod_rx_buf = &rxr->rx_agg_ring[sw_prod]; 901 cons_rx_buf = &rxr->rx_agg_ring[cons]; 902 903 /* It is possible for sw_prod to be equal to cons, so 904 * set cons_rx_buf->page to NULL first. 905 */ 906 page = cons_rx_buf->page; 907 cons_rx_buf->page = NULL; 908 prod_rx_buf->page = page; 909 prod_rx_buf->offset = cons_rx_buf->offset; 910 911 prod_rx_buf->mapping = cons_rx_buf->mapping; 912 913 prod_bd = &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 914 915 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping); 916 prod_bd->rx_bd_opaque = sw_prod; 917 918 prod = NEXT_RX_AGG(prod); 919 sw_prod = NEXT_RX_AGG(sw_prod); 920 } 921 rxr->rx_agg_prod = prod; 922 rxr->rx_sw_agg_prod = sw_prod; 923 } 924 925 static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp, 926 struct bnxt_rx_ring_info *rxr, 927 u16 cons, void *data, u8 *data_ptr, 928 dma_addr_t dma_addr, 929 unsigned int offset_and_len) 930 { 931 unsigned int payload = offset_and_len >> 16; 932 unsigned int len = offset_and_len & 0xffff; 933 skb_frag_t *frag; 934 struct page *page = data; 935 u16 prod = rxr->rx_prod; 936 struct sk_buff *skb; 937 int off, err; 938 939 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC); 940 if (unlikely(err)) { 941 bnxt_reuse_rx_data(rxr, cons, data); 942 return NULL; 943 } 944 dma_addr -= bp->rx_dma_offset; 945 dma_unmap_page_attrs(&bp->pdev->dev, dma_addr, PAGE_SIZE, bp->rx_dir, 946 DMA_ATTR_WEAK_ORDERING); 947 page_pool_release_page(rxr->page_pool, page); 948 949 if (unlikely(!payload)) 950 payload = eth_get_headlen(bp->dev, data_ptr, len); 951 952 skb = napi_alloc_skb(&rxr->bnapi->napi, payload); 953 if (!skb) { 954 __free_page(page); 955 return NULL; 956 } 957 958 off = (void *)data_ptr - page_address(page); 959 skb_add_rx_frag(skb, 0, page, off, len, PAGE_SIZE); 960 memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN, 961 payload + NET_IP_ALIGN); 962 963 frag = &skb_shinfo(skb)->frags[0]; 964 skb_frag_size_sub(frag, payload); 965 skb_frag_off_add(frag, payload); 966 skb->data_len -= payload; 967 skb->tail += payload; 968 969 return skb; 970 } 971 972 static struct sk_buff *bnxt_rx_skb(struct bnxt *bp, 973 struct bnxt_rx_ring_info *rxr, u16 cons, 974 void *data, u8 *data_ptr, 975 dma_addr_t dma_addr, 976 unsigned int offset_and_len) 977 { 978 u16 prod = rxr->rx_prod; 979 struct sk_buff *skb; 980 int err; 981 982 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC); 983 if (unlikely(err)) { 984 bnxt_reuse_rx_data(rxr, cons, data); 985 return NULL; 986 } 987 988 skb = build_skb(data, 0); 989 dma_unmap_single_attrs(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size, 990 bp->rx_dir, DMA_ATTR_WEAK_ORDERING); 991 if (!skb) { 992 kfree(data); 993 return NULL; 994 } 995 996 skb_reserve(skb, bp->rx_offset); 997 skb_put(skb, offset_and_len & 0xffff); 998 return skb; 999 } 1000 1001 static struct sk_buff *bnxt_rx_pages(struct bnxt *bp, 1002 struct bnxt_cp_ring_info *cpr, 1003 struct sk_buff *skb, u16 idx, 1004 u32 agg_bufs, bool tpa) 1005 { 1006 struct bnxt_napi *bnapi = cpr->bnapi; 1007 struct pci_dev *pdev = bp->pdev; 1008 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1009 u16 prod = rxr->rx_agg_prod; 1010 bool p5_tpa = false; 1011 u32 i; 1012 1013 if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa) 1014 p5_tpa = true; 1015 1016 for (i = 0; i < agg_bufs; i++) { 1017 u16 cons, frag_len; 1018 struct rx_agg_cmp *agg; 1019 struct bnxt_sw_rx_agg_bd *cons_rx_buf; 1020 struct page *page; 1021 dma_addr_t mapping; 1022 1023 if (p5_tpa) 1024 agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, i); 1025 else 1026 agg = bnxt_get_agg(bp, cpr, idx, i); 1027 cons = agg->rx_agg_cmp_opaque; 1028 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) & 1029 RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT; 1030 1031 cons_rx_buf = &rxr->rx_agg_ring[cons]; 1032 skb_fill_page_desc(skb, i, cons_rx_buf->page, 1033 cons_rx_buf->offset, frag_len); 1034 __clear_bit(cons, rxr->rx_agg_bmap); 1035 1036 /* It is possible for bnxt_alloc_rx_page() to allocate 1037 * a sw_prod index that equals the cons index, so we 1038 * need to clear the cons entry now. 1039 */ 1040 mapping = cons_rx_buf->mapping; 1041 page = cons_rx_buf->page; 1042 cons_rx_buf->page = NULL; 1043 1044 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) { 1045 struct skb_shared_info *shinfo; 1046 unsigned int nr_frags; 1047 1048 shinfo = skb_shinfo(skb); 1049 nr_frags = --shinfo->nr_frags; 1050 __skb_frag_set_page(&shinfo->frags[nr_frags], NULL); 1051 1052 dev_kfree_skb(skb); 1053 1054 cons_rx_buf->page = page; 1055 1056 /* Update prod since possibly some pages have been 1057 * allocated already. 1058 */ 1059 rxr->rx_agg_prod = prod; 1060 bnxt_reuse_rx_agg_bufs(cpr, idx, i, agg_bufs - i, tpa); 1061 return NULL; 1062 } 1063 1064 dma_unmap_page_attrs(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE, 1065 PCI_DMA_FROMDEVICE, 1066 DMA_ATTR_WEAK_ORDERING); 1067 1068 skb->data_len += frag_len; 1069 skb->len += frag_len; 1070 skb->truesize += PAGE_SIZE; 1071 1072 prod = NEXT_RX_AGG(prod); 1073 } 1074 rxr->rx_agg_prod = prod; 1075 return skb; 1076 } 1077 1078 static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 1079 u8 agg_bufs, u32 *raw_cons) 1080 { 1081 u16 last; 1082 struct rx_agg_cmp *agg; 1083 1084 *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs); 1085 last = RING_CMP(*raw_cons); 1086 agg = (struct rx_agg_cmp *) 1087 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)]; 1088 return RX_AGG_CMP_VALID(agg, *raw_cons); 1089 } 1090 1091 static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data, 1092 unsigned int len, 1093 dma_addr_t mapping) 1094 { 1095 struct bnxt *bp = bnapi->bp; 1096 struct pci_dev *pdev = bp->pdev; 1097 struct sk_buff *skb; 1098 1099 skb = napi_alloc_skb(&bnapi->napi, len); 1100 if (!skb) 1101 return NULL; 1102 1103 dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copy_thresh, 1104 bp->rx_dir); 1105 1106 memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN, 1107 len + NET_IP_ALIGN); 1108 1109 dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copy_thresh, 1110 bp->rx_dir); 1111 1112 skb_put(skb, len); 1113 return skb; 1114 } 1115 1116 static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 1117 u32 *raw_cons, void *cmp) 1118 { 1119 struct rx_cmp *rxcmp = cmp; 1120 u32 tmp_raw_cons = *raw_cons; 1121 u8 cmp_type, agg_bufs = 0; 1122 1123 cmp_type = RX_CMP_TYPE(rxcmp); 1124 1125 if (cmp_type == CMP_TYPE_RX_L2_CMP) { 1126 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) & 1127 RX_CMP_AGG_BUFS) >> 1128 RX_CMP_AGG_BUFS_SHIFT; 1129 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) { 1130 struct rx_tpa_end_cmp *tpa_end = cmp; 1131 1132 if (bp->flags & BNXT_FLAG_CHIP_P5) 1133 return 0; 1134 1135 agg_bufs = TPA_END_AGG_BUFS(tpa_end); 1136 } 1137 1138 if (agg_bufs) { 1139 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons)) 1140 return -EBUSY; 1141 } 1142 *raw_cons = tmp_raw_cons; 1143 return 0; 1144 } 1145 1146 static void bnxt_queue_fw_reset_work(struct bnxt *bp, unsigned long delay) 1147 { 1148 if (!(test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))) 1149 return; 1150 1151 if (BNXT_PF(bp)) 1152 queue_delayed_work(bnxt_pf_wq, &bp->fw_reset_task, delay); 1153 else 1154 schedule_delayed_work(&bp->fw_reset_task, delay); 1155 } 1156 1157 static void bnxt_queue_sp_work(struct bnxt *bp) 1158 { 1159 if (BNXT_PF(bp)) 1160 queue_work(bnxt_pf_wq, &bp->sp_task); 1161 else 1162 schedule_work(&bp->sp_task); 1163 } 1164 1165 static void bnxt_sched_reset(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) 1166 { 1167 if (!rxr->bnapi->in_reset) { 1168 rxr->bnapi->in_reset = true; 1169 if (bp->flags & BNXT_FLAG_CHIP_P5) 1170 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event); 1171 else 1172 set_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event); 1173 bnxt_queue_sp_work(bp); 1174 } 1175 rxr->rx_next_cons = 0xffff; 1176 } 1177 1178 static u16 bnxt_alloc_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id) 1179 { 1180 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map; 1181 u16 idx = agg_id & MAX_TPA_P5_MASK; 1182 1183 if (test_bit(idx, map->agg_idx_bmap)) 1184 idx = find_first_zero_bit(map->agg_idx_bmap, 1185 BNXT_AGG_IDX_BMAP_SIZE); 1186 __set_bit(idx, map->agg_idx_bmap); 1187 map->agg_id_tbl[agg_id] = idx; 1188 return idx; 1189 } 1190 1191 static void bnxt_free_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx) 1192 { 1193 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map; 1194 1195 __clear_bit(idx, map->agg_idx_bmap); 1196 } 1197 1198 static u16 bnxt_lookup_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id) 1199 { 1200 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map; 1201 1202 return map->agg_id_tbl[agg_id]; 1203 } 1204 1205 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, 1206 struct rx_tpa_start_cmp *tpa_start, 1207 struct rx_tpa_start_cmp_ext *tpa_start1) 1208 { 1209 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf; 1210 struct bnxt_tpa_info *tpa_info; 1211 u16 cons, prod, agg_id; 1212 struct rx_bd *prod_bd; 1213 dma_addr_t mapping; 1214 1215 if (bp->flags & BNXT_FLAG_CHIP_P5) { 1216 agg_id = TPA_START_AGG_ID_P5(tpa_start); 1217 agg_id = bnxt_alloc_agg_idx(rxr, agg_id); 1218 } else { 1219 agg_id = TPA_START_AGG_ID(tpa_start); 1220 } 1221 cons = tpa_start->rx_tpa_start_cmp_opaque; 1222 prod = rxr->rx_prod; 1223 cons_rx_buf = &rxr->rx_buf_ring[cons]; 1224 prod_rx_buf = &rxr->rx_buf_ring[prod]; 1225 tpa_info = &rxr->rx_tpa[agg_id]; 1226 1227 if (unlikely(cons != rxr->rx_next_cons || 1228 TPA_START_ERROR(tpa_start))) { 1229 netdev_warn(bp->dev, "TPA cons %x, expected cons %x, error code %x\n", 1230 cons, rxr->rx_next_cons, 1231 TPA_START_ERROR_CODE(tpa_start1)); 1232 bnxt_sched_reset(bp, rxr); 1233 return; 1234 } 1235 /* Store cfa_code in tpa_info to use in tpa_end 1236 * completion processing. 1237 */ 1238 tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1); 1239 prod_rx_buf->data = tpa_info->data; 1240 prod_rx_buf->data_ptr = tpa_info->data_ptr; 1241 1242 mapping = tpa_info->mapping; 1243 prod_rx_buf->mapping = mapping; 1244 1245 prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 1246 1247 prod_bd->rx_bd_haddr = cpu_to_le64(mapping); 1248 1249 tpa_info->data = cons_rx_buf->data; 1250 tpa_info->data_ptr = cons_rx_buf->data_ptr; 1251 cons_rx_buf->data = NULL; 1252 tpa_info->mapping = cons_rx_buf->mapping; 1253 1254 tpa_info->len = 1255 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >> 1256 RX_TPA_START_CMP_LEN_SHIFT; 1257 if (likely(TPA_START_HASH_VALID(tpa_start))) { 1258 u32 hash_type = TPA_START_HASH_TYPE(tpa_start); 1259 1260 tpa_info->hash_type = PKT_HASH_TYPE_L4; 1261 tpa_info->gso_type = SKB_GSO_TCPV4; 1262 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */ 1263 if (hash_type == 3 || TPA_START_IS_IPV6(tpa_start1)) 1264 tpa_info->gso_type = SKB_GSO_TCPV6; 1265 tpa_info->rss_hash = 1266 le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash); 1267 } else { 1268 tpa_info->hash_type = PKT_HASH_TYPE_NONE; 1269 tpa_info->gso_type = 0; 1270 netif_warn(bp, rx_err, bp->dev, "TPA packet without valid hash\n"); 1271 } 1272 tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2); 1273 tpa_info->metadata = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata); 1274 tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info); 1275 tpa_info->agg_count = 0; 1276 1277 rxr->rx_prod = NEXT_RX(prod); 1278 cons = NEXT_RX(cons); 1279 rxr->rx_next_cons = NEXT_RX(cons); 1280 cons_rx_buf = &rxr->rx_buf_ring[cons]; 1281 1282 bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data); 1283 rxr->rx_prod = NEXT_RX(rxr->rx_prod); 1284 cons_rx_buf->data = NULL; 1285 } 1286 1287 static void bnxt_abort_tpa(struct bnxt_cp_ring_info *cpr, u16 idx, u32 agg_bufs) 1288 { 1289 if (agg_bufs) 1290 bnxt_reuse_rx_agg_bufs(cpr, idx, 0, agg_bufs, true); 1291 } 1292 1293 #ifdef CONFIG_INET 1294 static void bnxt_gro_tunnel(struct sk_buff *skb, __be16 ip_proto) 1295 { 1296 struct udphdr *uh = NULL; 1297 1298 if (ip_proto == htons(ETH_P_IP)) { 1299 struct iphdr *iph = (struct iphdr *)skb->data; 1300 1301 if (iph->protocol == IPPROTO_UDP) 1302 uh = (struct udphdr *)(iph + 1); 1303 } else { 1304 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data; 1305 1306 if (iph->nexthdr == IPPROTO_UDP) 1307 uh = (struct udphdr *)(iph + 1); 1308 } 1309 if (uh) { 1310 if (uh->check) 1311 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM; 1312 else 1313 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL; 1314 } 1315 } 1316 #endif 1317 1318 static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info, 1319 int payload_off, int tcp_ts, 1320 struct sk_buff *skb) 1321 { 1322 #ifdef CONFIG_INET 1323 struct tcphdr *th; 1324 int len, nw_off; 1325 u16 outer_ip_off, inner_ip_off, inner_mac_off; 1326 u32 hdr_info = tpa_info->hdr_info; 1327 bool loopback = false; 1328 1329 inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info); 1330 inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info); 1331 outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info); 1332 1333 /* If the packet is an internal loopback packet, the offsets will 1334 * have an extra 4 bytes. 1335 */ 1336 if (inner_mac_off == 4) { 1337 loopback = true; 1338 } else if (inner_mac_off > 4) { 1339 __be16 proto = *((__be16 *)(skb->data + inner_ip_off - 1340 ETH_HLEN - 2)); 1341 1342 /* We only support inner iPv4/ipv6. If we don't see the 1343 * correct protocol ID, it must be a loopback packet where 1344 * the offsets are off by 4. 1345 */ 1346 if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6)) 1347 loopback = true; 1348 } 1349 if (loopback) { 1350 /* internal loopback packet, subtract all offsets by 4 */ 1351 inner_ip_off -= 4; 1352 inner_mac_off -= 4; 1353 outer_ip_off -= 4; 1354 } 1355 1356 nw_off = inner_ip_off - ETH_HLEN; 1357 skb_set_network_header(skb, nw_off); 1358 if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) { 1359 struct ipv6hdr *iph = ipv6_hdr(skb); 1360 1361 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr)); 1362 len = skb->len - skb_transport_offset(skb); 1363 th = tcp_hdr(skb); 1364 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0); 1365 } else { 1366 struct iphdr *iph = ip_hdr(skb); 1367 1368 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr)); 1369 len = skb->len - skb_transport_offset(skb); 1370 th = tcp_hdr(skb); 1371 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0); 1372 } 1373 1374 if (inner_mac_off) { /* tunnel */ 1375 __be16 proto = *((__be16 *)(skb->data + outer_ip_off - 1376 ETH_HLEN - 2)); 1377 1378 bnxt_gro_tunnel(skb, proto); 1379 } 1380 #endif 1381 return skb; 1382 } 1383 1384 static struct sk_buff *bnxt_gro_func_5750x(struct bnxt_tpa_info *tpa_info, 1385 int payload_off, int tcp_ts, 1386 struct sk_buff *skb) 1387 { 1388 #ifdef CONFIG_INET 1389 u16 outer_ip_off, inner_ip_off, inner_mac_off; 1390 u32 hdr_info = tpa_info->hdr_info; 1391 int iphdr_len, nw_off; 1392 1393 inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info); 1394 inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info); 1395 outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info); 1396 1397 nw_off = inner_ip_off - ETH_HLEN; 1398 skb_set_network_header(skb, nw_off); 1399 iphdr_len = (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) ? 1400 sizeof(struct ipv6hdr) : sizeof(struct iphdr); 1401 skb_set_transport_header(skb, nw_off + iphdr_len); 1402 1403 if (inner_mac_off) { /* tunnel */ 1404 __be16 proto = *((__be16 *)(skb->data + outer_ip_off - 1405 ETH_HLEN - 2)); 1406 1407 bnxt_gro_tunnel(skb, proto); 1408 } 1409 #endif 1410 return skb; 1411 } 1412 1413 #define BNXT_IPV4_HDR_SIZE (sizeof(struct iphdr) + sizeof(struct tcphdr)) 1414 #define BNXT_IPV6_HDR_SIZE (sizeof(struct ipv6hdr) + sizeof(struct tcphdr)) 1415 1416 static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info, 1417 int payload_off, int tcp_ts, 1418 struct sk_buff *skb) 1419 { 1420 #ifdef CONFIG_INET 1421 struct tcphdr *th; 1422 int len, nw_off, tcp_opt_len = 0; 1423 1424 if (tcp_ts) 1425 tcp_opt_len = 12; 1426 1427 if (tpa_info->gso_type == SKB_GSO_TCPV4) { 1428 struct iphdr *iph; 1429 1430 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len - 1431 ETH_HLEN; 1432 skb_set_network_header(skb, nw_off); 1433 iph = ip_hdr(skb); 1434 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr)); 1435 len = skb->len - skb_transport_offset(skb); 1436 th = tcp_hdr(skb); 1437 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0); 1438 } else if (tpa_info->gso_type == SKB_GSO_TCPV6) { 1439 struct ipv6hdr *iph; 1440 1441 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len - 1442 ETH_HLEN; 1443 skb_set_network_header(skb, nw_off); 1444 iph = ipv6_hdr(skb); 1445 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr)); 1446 len = skb->len - skb_transport_offset(skb); 1447 th = tcp_hdr(skb); 1448 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0); 1449 } else { 1450 dev_kfree_skb_any(skb); 1451 return NULL; 1452 } 1453 1454 if (nw_off) /* tunnel */ 1455 bnxt_gro_tunnel(skb, skb->protocol); 1456 #endif 1457 return skb; 1458 } 1459 1460 static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp, 1461 struct bnxt_tpa_info *tpa_info, 1462 struct rx_tpa_end_cmp *tpa_end, 1463 struct rx_tpa_end_cmp_ext *tpa_end1, 1464 struct sk_buff *skb) 1465 { 1466 #ifdef CONFIG_INET 1467 int payload_off; 1468 u16 segs; 1469 1470 segs = TPA_END_TPA_SEGS(tpa_end); 1471 if (segs == 1) 1472 return skb; 1473 1474 NAPI_GRO_CB(skb)->count = segs; 1475 skb_shinfo(skb)->gso_size = 1476 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len); 1477 skb_shinfo(skb)->gso_type = tpa_info->gso_type; 1478 if (bp->flags & BNXT_FLAG_CHIP_P5) 1479 payload_off = TPA_END_PAYLOAD_OFF_P5(tpa_end1); 1480 else 1481 payload_off = TPA_END_PAYLOAD_OFF(tpa_end); 1482 skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb); 1483 if (likely(skb)) 1484 tcp_gro_complete(skb); 1485 #endif 1486 return skb; 1487 } 1488 1489 /* Given the cfa_code of a received packet determine which 1490 * netdev (vf-rep or PF) the packet is destined to. 1491 */ 1492 static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code) 1493 { 1494 struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code); 1495 1496 /* if vf-rep dev is NULL, the must belongs to the PF */ 1497 return dev ? dev : bp->dev; 1498 } 1499 1500 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp, 1501 struct bnxt_cp_ring_info *cpr, 1502 u32 *raw_cons, 1503 struct rx_tpa_end_cmp *tpa_end, 1504 struct rx_tpa_end_cmp_ext *tpa_end1, 1505 u8 *event) 1506 { 1507 struct bnxt_napi *bnapi = cpr->bnapi; 1508 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1509 u8 *data_ptr, agg_bufs; 1510 unsigned int len; 1511 struct bnxt_tpa_info *tpa_info; 1512 dma_addr_t mapping; 1513 struct sk_buff *skb; 1514 u16 idx = 0, agg_id; 1515 void *data; 1516 bool gro; 1517 1518 if (unlikely(bnapi->in_reset)) { 1519 int rc = bnxt_discard_rx(bp, cpr, raw_cons, tpa_end); 1520 1521 if (rc < 0) 1522 return ERR_PTR(-EBUSY); 1523 return NULL; 1524 } 1525 1526 if (bp->flags & BNXT_FLAG_CHIP_P5) { 1527 agg_id = TPA_END_AGG_ID_P5(tpa_end); 1528 agg_id = bnxt_lookup_agg_idx(rxr, agg_id); 1529 agg_bufs = TPA_END_AGG_BUFS_P5(tpa_end1); 1530 tpa_info = &rxr->rx_tpa[agg_id]; 1531 if (unlikely(agg_bufs != tpa_info->agg_count)) { 1532 netdev_warn(bp->dev, "TPA end agg_buf %d != expected agg_bufs %d\n", 1533 agg_bufs, tpa_info->agg_count); 1534 agg_bufs = tpa_info->agg_count; 1535 } 1536 tpa_info->agg_count = 0; 1537 *event |= BNXT_AGG_EVENT; 1538 bnxt_free_agg_idx(rxr, agg_id); 1539 idx = agg_id; 1540 gro = !!(bp->flags & BNXT_FLAG_GRO); 1541 } else { 1542 agg_id = TPA_END_AGG_ID(tpa_end); 1543 agg_bufs = TPA_END_AGG_BUFS(tpa_end); 1544 tpa_info = &rxr->rx_tpa[agg_id]; 1545 idx = RING_CMP(*raw_cons); 1546 if (agg_bufs) { 1547 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons)) 1548 return ERR_PTR(-EBUSY); 1549 1550 *event |= BNXT_AGG_EVENT; 1551 idx = NEXT_CMP(idx); 1552 } 1553 gro = !!TPA_END_GRO(tpa_end); 1554 } 1555 data = tpa_info->data; 1556 data_ptr = tpa_info->data_ptr; 1557 prefetch(data_ptr); 1558 len = tpa_info->len; 1559 mapping = tpa_info->mapping; 1560 1561 if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) { 1562 bnxt_abort_tpa(cpr, idx, agg_bufs); 1563 if (agg_bufs > MAX_SKB_FRAGS) 1564 netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n", 1565 agg_bufs, (int)MAX_SKB_FRAGS); 1566 return NULL; 1567 } 1568 1569 if (len <= bp->rx_copy_thresh) { 1570 skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping); 1571 if (!skb) { 1572 bnxt_abort_tpa(cpr, idx, agg_bufs); 1573 return NULL; 1574 } 1575 } else { 1576 u8 *new_data; 1577 dma_addr_t new_mapping; 1578 1579 new_data = __bnxt_alloc_rx_data(bp, &new_mapping, GFP_ATOMIC); 1580 if (!new_data) { 1581 bnxt_abort_tpa(cpr, idx, agg_bufs); 1582 return NULL; 1583 } 1584 1585 tpa_info->data = new_data; 1586 tpa_info->data_ptr = new_data + bp->rx_offset; 1587 tpa_info->mapping = new_mapping; 1588 1589 skb = build_skb(data, 0); 1590 dma_unmap_single_attrs(&bp->pdev->dev, mapping, 1591 bp->rx_buf_use_size, bp->rx_dir, 1592 DMA_ATTR_WEAK_ORDERING); 1593 1594 if (!skb) { 1595 kfree(data); 1596 bnxt_abort_tpa(cpr, idx, agg_bufs); 1597 return NULL; 1598 } 1599 skb_reserve(skb, bp->rx_offset); 1600 skb_put(skb, len); 1601 } 1602 1603 if (agg_bufs) { 1604 skb = bnxt_rx_pages(bp, cpr, skb, idx, agg_bufs, true); 1605 if (!skb) { 1606 /* Page reuse already handled by bnxt_rx_pages(). */ 1607 return NULL; 1608 } 1609 } 1610 1611 skb->protocol = 1612 eth_type_trans(skb, bnxt_get_pkt_dev(bp, tpa_info->cfa_code)); 1613 1614 if (tpa_info->hash_type != PKT_HASH_TYPE_NONE) 1615 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type); 1616 1617 if ((tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) && 1618 (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) { 1619 u16 vlan_proto = tpa_info->metadata >> 1620 RX_CMP_FLAGS2_METADATA_TPID_SFT; 1621 u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_TCI_MASK; 1622 1623 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag); 1624 } 1625 1626 skb_checksum_none_assert(skb); 1627 if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) { 1628 skb->ip_summed = CHECKSUM_UNNECESSARY; 1629 skb->csum_level = 1630 (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3; 1631 } 1632 1633 if (gro) 1634 skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb); 1635 1636 return skb; 1637 } 1638 1639 static void bnxt_tpa_agg(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, 1640 struct rx_agg_cmp *rx_agg) 1641 { 1642 u16 agg_id = TPA_AGG_AGG_ID(rx_agg); 1643 struct bnxt_tpa_info *tpa_info; 1644 1645 agg_id = bnxt_lookup_agg_idx(rxr, agg_id); 1646 tpa_info = &rxr->rx_tpa[agg_id]; 1647 BUG_ON(tpa_info->agg_count >= MAX_SKB_FRAGS); 1648 tpa_info->agg_arr[tpa_info->agg_count++] = *rx_agg; 1649 } 1650 1651 static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi, 1652 struct sk_buff *skb) 1653 { 1654 if (skb->dev != bp->dev) { 1655 /* this packet belongs to a vf-rep */ 1656 bnxt_vf_rep_rx(bp, skb); 1657 return; 1658 } 1659 skb_record_rx_queue(skb, bnapi->index); 1660 napi_gro_receive(&bnapi->napi, skb); 1661 } 1662 1663 /* returns the following: 1664 * 1 - 1 packet successfully received 1665 * 0 - successful TPA_START, packet not completed yet 1666 * -EBUSY - completion ring does not have all the agg buffers yet 1667 * -ENOMEM - packet aborted due to out of memory 1668 * -EIO - packet aborted due to hw error indicated in BD 1669 */ 1670 static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 1671 u32 *raw_cons, u8 *event) 1672 { 1673 struct bnxt_napi *bnapi = cpr->bnapi; 1674 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1675 struct net_device *dev = bp->dev; 1676 struct rx_cmp *rxcmp; 1677 struct rx_cmp_ext *rxcmp1; 1678 u32 tmp_raw_cons = *raw_cons; 1679 u16 cfa_code, cons, prod, cp_cons = RING_CMP(tmp_raw_cons); 1680 struct bnxt_sw_rx_bd *rx_buf; 1681 unsigned int len; 1682 u8 *data_ptr, agg_bufs, cmp_type; 1683 dma_addr_t dma_addr; 1684 struct sk_buff *skb; 1685 void *data; 1686 int rc = 0; 1687 u32 misc; 1688 1689 rxcmp = (struct rx_cmp *) 1690 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1691 1692 cmp_type = RX_CMP_TYPE(rxcmp); 1693 1694 if (cmp_type == CMP_TYPE_RX_TPA_AGG_CMP) { 1695 bnxt_tpa_agg(bp, rxr, (struct rx_agg_cmp *)rxcmp); 1696 goto next_rx_no_prod_no_len; 1697 } 1698 1699 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons); 1700 cp_cons = RING_CMP(tmp_raw_cons); 1701 rxcmp1 = (struct rx_cmp_ext *) 1702 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1703 1704 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons)) 1705 return -EBUSY; 1706 1707 prod = rxr->rx_prod; 1708 1709 if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) { 1710 bnxt_tpa_start(bp, rxr, (struct rx_tpa_start_cmp *)rxcmp, 1711 (struct rx_tpa_start_cmp_ext *)rxcmp1); 1712 1713 *event |= BNXT_RX_EVENT; 1714 goto next_rx_no_prod_no_len; 1715 1716 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) { 1717 skb = bnxt_tpa_end(bp, cpr, &tmp_raw_cons, 1718 (struct rx_tpa_end_cmp *)rxcmp, 1719 (struct rx_tpa_end_cmp_ext *)rxcmp1, event); 1720 1721 if (IS_ERR(skb)) 1722 return -EBUSY; 1723 1724 rc = -ENOMEM; 1725 if (likely(skb)) { 1726 bnxt_deliver_skb(bp, bnapi, skb); 1727 rc = 1; 1728 } 1729 *event |= BNXT_RX_EVENT; 1730 goto next_rx_no_prod_no_len; 1731 } 1732 1733 cons = rxcmp->rx_cmp_opaque; 1734 if (unlikely(cons != rxr->rx_next_cons)) { 1735 int rc1 = bnxt_discard_rx(bp, cpr, raw_cons, rxcmp); 1736 1737 /* 0xffff is forced error, don't print it */ 1738 if (rxr->rx_next_cons != 0xffff) 1739 netdev_warn(bp->dev, "RX cons %x != expected cons %x\n", 1740 cons, rxr->rx_next_cons); 1741 bnxt_sched_reset(bp, rxr); 1742 return rc1; 1743 } 1744 rx_buf = &rxr->rx_buf_ring[cons]; 1745 data = rx_buf->data; 1746 data_ptr = rx_buf->data_ptr; 1747 prefetch(data_ptr); 1748 1749 misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1); 1750 agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT; 1751 1752 if (agg_bufs) { 1753 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons)) 1754 return -EBUSY; 1755 1756 cp_cons = NEXT_CMP(cp_cons); 1757 *event |= BNXT_AGG_EVENT; 1758 } 1759 *event |= BNXT_RX_EVENT; 1760 1761 rx_buf->data = NULL; 1762 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) { 1763 u32 rx_err = le32_to_cpu(rxcmp1->rx_cmp_cfa_code_errors_v2); 1764 1765 bnxt_reuse_rx_data(rxr, cons, data); 1766 if (agg_bufs) 1767 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, agg_bufs, 1768 false); 1769 1770 rc = -EIO; 1771 if (rx_err & RX_CMPL_ERRORS_BUFFER_ERROR_MASK) { 1772 bnapi->cp_ring.sw_stats.rx.rx_buf_errors++; 1773 if (!(bp->flags & BNXT_FLAG_CHIP_P5) && 1774 !(bp->fw_cap & BNXT_FW_CAP_RING_MONITOR)) { 1775 netdev_warn_once(bp->dev, "RX buffer error %x\n", 1776 rx_err); 1777 bnxt_sched_reset(bp, rxr); 1778 } 1779 } 1780 goto next_rx_no_len; 1781 } 1782 1783 len = le32_to_cpu(rxcmp->rx_cmp_len_flags_type) >> RX_CMP_LEN_SHIFT; 1784 dma_addr = rx_buf->mapping; 1785 1786 if (bnxt_rx_xdp(bp, rxr, cons, data, &data_ptr, &len, event)) { 1787 rc = 1; 1788 goto next_rx; 1789 } 1790 1791 if (len <= bp->rx_copy_thresh) { 1792 skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr); 1793 bnxt_reuse_rx_data(rxr, cons, data); 1794 if (!skb) { 1795 if (agg_bufs) 1796 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, 1797 agg_bufs, false); 1798 rc = -ENOMEM; 1799 goto next_rx; 1800 } 1801 } else { 1802 u32 payload; 1803 1804 if (rx_buf->data_ptr == data_ptr) 1805 payload = misc & RX_CMP_PAYLOAD_OFFSET; 1806 else 1807 payload = 0; 1808 skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr, 1809 payload | len); 1810 if (!skb) { 1811 rc = -ENOMEM; 1812 goto next_rx; 1813 } 1814 } 1815 1816 if (agg_bufs) { 1817 skb = bnxt_rx_pages(bp, cpr, skb, cp_cons, agg_bufs, false); 1818 if (!skb) { 1819 rc = -ENOMEM; 1820 goto next_rx; 1821 } 1822 } 1823 1824 if (RX_CMP_HASH_VALID(rxcmp)) { 1825 u32 hash_type = RX_CMP_HASH_TYPE(rxcmp); 1826 enum pkt_hash_types type = PKT_HASH_TYPE_L4; 1827 1828 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */ 1829 if (hash_type != 1 && hash_type != 3) 1830 type = PKT_HASH_TYPE_L3; 1831 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type); 1832 } 1833 1834 cfa_code = RX_CMP_CFA_CODE(rxcmp1); 1835 skb->protocol = eth_type_trans(skb, bnxt_get_pkt_dev(bp, cfa_code)); 1836 1837 if ((rxcmp1->rx_cmp_flags2 & 1838 cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) && 1839 (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) { 1840 u32 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data); 1841 u16 vtag = meta_data & RX_CMP_FLAGS2_METADATA_TCI_MASK; 1842 u16 vlan_proto = meta_data >> RX_CMP_FLAGS2_METADATA_TPID_SFT; 1843 1844 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag); 1845 } 1846 1847 skb_checksum_none_assert(skb); 1848 if (RX_CMP_L4_CS_OK(rxcmp1)) { 1849 if (dev->features & NETIF_F_RXCSUM) { 1850 skb->ip_summed = CHECKSUM_UNNECESSARY; 1851 skb->csum_level = RX_CMP_ENCAP(rxcmp1); 1852 } 1853 } else { 1854 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) { 1855 if (dev->features & NETIF_F_RXCSUM) 1856 bnapi->cp_ring.sw_stats.rx.rx_l4_csum_errors++; 1857 } 1858 } 1859 1860 bnxt_deliver_skb(bp, bnapi, skb); 1861 rc = 1; 1862 1863 next_rx: 1864 cpr->rx_packets += 1; 1865 cpr->rx_bytes += len; 1866 1867 next_rx_no_len: 1868 rxr->rx_prod = NEXT_RX(prod); 1869 rxr->rx_next_cons = NEXT_RX(cons); 1870 1871 next_rx_no_prod_no_len: 1872 *raw_cons = tmp_raw_cons; 1873 1874 return rc; 1875 } 1876 1877 /* In netpoll mode, if we are using a combined completion ring, we need to 1878 * discard the rx packets and recycle the buffers. 1879 */ 1880 static int bnxt_force_rx_discard(struct bnxt *bp, 1881 struct bnxt_cp_ring_info *cpr, 1882 u32 *raw_cons, u8 *event) 1883 { 1884 u32 tmp_raw_cons = *raw_cons; 1885 struct rx_cmp_ext *rxcmp1; 1886 struct rx_cmp *rxcmp; 1887 u16 cp_cons; 1888 u8 cmp_type; 1889 1890 cp_cons = RING_CMP(tmp_raw_cons); 1891 rxcmp = (struct rx_cmp *) 1892 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1893 1894 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons); 1895 cp_cons = RING_CMP(tmp_raw_cons); 1896 rxcmp1 = (struct rx_cmp_ext *) 1897 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1898 1899 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons)) 1900 return -EBUSY; 1901 1902 cmp_type = RX_CMP_TYPE(rxcmp); 1903 if (cmp_type == CMP_TYPE_RX_L2_CMP) { 1904 rxcmp1->rx_cmp_cfa_code_errors_v2 |= 1905 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR); 1906 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) { 1907 struct rx_tpa_end_cmp_ext *tpa_end1; 1908 1909 tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1; 1910 tpa_end1->rx_tpa_end_cmp_errors_v2 |= 1911 cpu_to_le32(RX_TPA_END_CMP_ERRORS); 1912 } 1913 return bnxt_rx_pkt(bp, cpr, raw_cons, event); 1914 } 1915 1916 u32 bnxt_fw_health_readl(struct bnxt *bp, int reg_idx) 1917 { 1918 struct bnxt_fw_health *fw_health = bp->fw_health; 1919 u32 reg = fw_health->regs[reg_idx]; 1920 u32 reg_type, reg_off, val = 0; 1921 1922 reg_type = BNXT_FW_HEALTH_REG_TYPE(reg); 1923 reg_off = BNXT_FW_HEALTH_REG_OFF(reg); 1924 switch (reg_type) { 1925 case BNXT_FW_HEALTH_REG_TYPE_CFG: 1926 pci_read_config_dword(bp->pdev, reg_off, &val); 1927 break; 1928 case BNXT_FW_HEALTH_REG_TYPE_GRC: 1929 reg_off = fw_health->mapped_regs[reg_idx]; 1930 fallthrough; 1931 case BNXT_FW_HEALTH_REG_TYPE_BAR0: 1932 val = readl(bp->bar0 + reg_off); 1933 break; 1934 case BNXT_FW_HEALTH_REG_TYPE_BAR1: 1935 val = readl(bp->bar1 + reg_off); 1936 break; 1937 } 1938 if (reg_idx == BNXT_FW_RESET_INPROG_REG) 1939 val &= fw_health->fw_reset_inprog_reg_mask; 1940 return val; 1941 } 1942 1943 static u16 bnxt_agg_ring_id_to_grp_idx(struct bnxt *bp, u16 ring_id) 1944 { 1945 int i; 1946 1947 for (i = 0; i < bp->rx_nr_rings; i++) { 1948 u16 grp_idx = bp->rx_ring[i].bnapi->index; 1949 struct bnxt_ring_grp_info *grp_info; 1950 1951 grp_info = &bp->grp_info[grp_idx]; 1952 if (grp_info->agg_fw_ring_id == ring_id) 1953 return grp_idx; 1954 } 1955 return INVALID_HW_RING_ID; 1956 } 1957 1958 #define BNXT_GET_EVENT_PORT(data) \ 1959 ((data) & \ 1960 ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK) 1961 1962 #define BNXT_EVENT_RING_TYPE(data2) \ 1963 ((data2) & \ 1964 ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_MASK) 1965 1966 #define BNXT_EVENT_RING_TYPE_RX(data2) \ 1967 (BNXT_EVENT_RING_TYPE(data2) == \ 1968 ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_RX) 1969 1970 static int bnxt_async_event_process(struct bnxt *bp, 1971 struct hwrm_async_event_cmpl *cmpl) 1972 { 1973 u16 event_id = le16_to_cpu(cmpl->event_id); 1974 u32 data1 = le32_to_cpu(cmpl->event_data1); 1975 u32 data2 = le32_to_cpu(cmpl->event_data2); 1976 1977 /* TODO CHIMP_FW: Define event id's for link change, error etc */ 1978 switch (event_id) { 1979 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: { 1980 struct bnxt_link_info *link_info = &bp->link_info; 1981 1982 if (BNXT_VF(bp)) 1983 goto async_event_process_exit; 1984 1985 /* print unsupported speed warning in forced speed mode only */ 1986 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED) && 1987 (data1 & 0x20000)) { 1988 u16 fw_speed = link_info->force_link_speed; 1989 u32 speed = bnxt_fw_to_ethtool_speed(fw_speed); 1990 1991 if (speed != SPEED_UNKNOWN) 1992 netdev_warn(bp->dev, "Link speed %d no longer supported\n", 1993 speed); 1994 } 1995 set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event); 1996 } 1997 fallthrough; 1998 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE: 1999 case ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE: 2000 set_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT, &bp->sp_event); 2001 fallthrough; 2002 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE: 2003 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event); 2004 break; 2005 case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD: 2006 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event); 2007 break; 2008 case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: { 2009 u16 port_id = BNXT_GET_EVENT_PORT(data1); 2010 2011 if (BNXT_VF(bp)) 2012 break; 2013 2014 if (bp->pf.port_id != port_id) 2015 break; 2016 2017 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event); 2018 break; 2019 } 2020 case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE: 2021 if (BNXT_PF(bp)) 2022 goto async_event_process_exit; 2023 set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event); 2024 break; 2025 case ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY: { 2026 char *fatal_str = "non-fatal"; 2027 2028 if (!bp->fw_health) 2029 goto async_event_process_exit; 2030 2031 bp->fw_reset_timestamp = jiffies; 2032 bp->fw_reset_min_dsecs = cmpl->timestamp_lo; 2033 if (!bp->fw_reset_min_dsecs) 2034 bp->fw_reset_min_dsecs = BNXT_DFLT_FW_RST_MIN_DSECS; 2035 bp->fw_reset_max_dsecs = le16_to_cpu(cmpl->timestamp_hi); 2036 if (!bp->fw_reset_max_dsecs) 2037 bp->fw_reset_max_dsecs = BNXT_DFLT_FW_RST_MAX_DSECS; 2038 if (EVENT_DATA1_RESET_NOTIFY_FATAL(data1)) { 2039 fatal_str = "fatal"; 2040 set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state); 2041 } 2042 netif_warn(bp, hw, bp->dev, 2043 "Firmware %s reset event, data1: 0x%x, data2: 0x%x, min wait %u ms, max wait %u ms\n", 2044 fatal_str, data1, data2, 2045 bp->fw_reset_min_dsecs * 100, 2046 bp->fw_reset_max_dsecs * 100); 2047 set_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event); 2048 break; 2049 } 2050 case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY: { 2051 struct bnxt_fw_health *fw_health = bp->fw_health; 2052 2053 if (!fw_health) 2054 goto async_event_process_exit; 2055 2056 fw_health->enabled = EVENT_DATA1_RECOVERY_ENABLED(data1); 2057 fw_health->master = EVENT_DATA1_RECOVERY_MASTER_FUNC(data1); 2058 if (!fw_health->enabled) { 2059 netif_info(bp, drv, bp->dev, 2060 "Error recovery info: error recovery[0]\n"); 2061 break; 2062 } 2063 fw_health->tmr_multiplier = 2064 DIV_ROUND_UP(fw_health->polling_dsecs * HZ, 2065 bp->current_interval * 10); 2066 fw_health->tmr_counter = fw_health->tmr_multiplier; 2067 fw_health->last_fw_heartbeat = 2068 bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG); 2069 fw_health->last_fw_reset_cnt = 2070 bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG); 2071 netif_info(bp, drv, bp->dev, 2072 "Error recovery info: error recovery[1], master[%d], reset count[%u], health status: 0x%x\n", 2073 fw_health->master, fw_health->last_fw_reset_cnt, 2074 bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG)); 2075 goto async_event_process_exit; 2076 } 2077 case ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION: 2078 netif_notice(bp, hw, bp->dev, 2079 "Received firmware debug notification, data1: 0x%x, data2: 0x%x\n", 2080 data1, data2); 2081 goto async_event_process_exit; 2082 case ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG: { 2083 struct bnxt_rx_ring_info *rxr; 2084 u16 grp_idx; 2085 2086 if (bp->flags & BNXT_FLAG_CHIP_P5) 2087 goto async_event_process_exit; 2088 2089 netdev_warn(bp->dev, "Ring monitor event, ring type %lu id 0x%x\n", 2090 BNXT_EVENT_RING_TYPE(data2), data1); 2091 if (!BNXT_EVENT_RING_TYPE_RX(data2)) 2092 goto async_event_process_exit; 2093 2094 grp_idx = bnxt_agg_ring_id_to_grp_idx(bp, data1); 2095 if (grp_idx == INVALID_HW_RING_ID) { 2096 netdev_warn(bp->dev, "Unknown RX agg ring id 0x%x\n", 2097 data1); 2098 goto async_event_process_exit; 2099 } 2100 rxr = bp->bnapi[grp_idx]->rx_ring; 2101 bnxt_sched_reset(bp, rxr); 2102 goto async_event_process_exit; 2103 } 2104 case ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST: { 2105 struct bnxt_fw_health *fw_health = bp->fw_health; 2106 2107 netif_notice(bp, hw, bp->dev, 2108 "Received firmware echo request, data1: 0x%x, data2: 0x%x\n", 2109 data1, data2); 2110 if (fw_health) { 2111 fw_health->echo_req_data1 = data1; 2112 fw_health->echo_req_data2 = data2; 2113 set_bit(BNXT_FW_ECHO_REQUEST_SP_EVENT, &bp->sp_event); 2114 break; 2115 } 2116 goto async_event_process_exit; 2117 } 2118 default: 2119 goto async_event_process_exit; 2120 } 2121 bnxt_queue_sp_work(bp); 2122 async_event_process_exit: 2123 bnxt_ulp_async_events(bp, cmpl); 2124 return 0; 2125 } 2126 2127 static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp) 2128 { 2129 u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id; 2130 struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp; 2131 struct hwrm_fwd_req_cmpl *fwd_req_cmpl = 2132 (struct hwrm_fwd_req_cmpl *)txcmp; 2133 2134 switch (cmpl_type) { 2135 case CMPL_BASE_TYPE_HWRM_DONE: 2136 seq_id = le16_to_cpu(h_cmpl->sequence_id); 2137 if (seq_id == bp->hwrm_intr_seq_id) 2138 bp->hwrm_intr_seq_id = (u16)~bp->hwrm_intr_seq_id; 2139 else 2140 netdev_err(bp->dev, "Invalid hwrm seq id %d\n", seq_id); 2141 break; 2142 2143 case CMPL_BASE_TYPE_HWRM_FWD_REQ: 2144 vf_id = le16_to_cpu(fwd_req_cmpl->source_id); 2145 2146 if ((vf_id < bp->pf.first_vf_id) || 2147 (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) { 2148 netdev_err(bp->dev, "Msg contains invalid VF id %x\n", 2149 vf_id); 2150 return -EINVAL; 2151 } 2152 2153 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap); 2154 set_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event); 2155 bnxt_queue_sp_work(bp); 2156 break; 2157 2158 case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT: 2159 bnxt_async_event_process(bp, 2160 (struct hwrm_async_event_cmpl *)txcmp); 2161 2162 default: 2163 break; 2164 } 2165 2166 return 0; 2167 } 2168 2169 static irqreturn_t bnxt_msix(int irq, void *dev_instance) 2170 { 2171 struct bnxt_napi *bnapi = dev_instance; 2172 struct bnxt *bp = bnapi->bp; 2173 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2174 u32 cons = RING_CMP(cpr->cp_raw_cons); 2175 2176 cpr->event_ctr++; 2177 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]); 2178 napi_schedule(&bnapi->napi); 2179 return IRQ_HANDLED; 2180 } 2181 2182 static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr) 2183 { 2184 u32 raw_cons = cpr->cp_raw_cons; 2185 u16 cons = RING_CMP(raw_cons); 2186 struct tx_cmp *txcmp; 2187 2188 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]; 2189 2190 return TX_CMP_VALID(txcmp, raw_cons); 2191 } 2192 2193 static irqreturn_t bnxt_inta(int irq, void *dev_instance) 2194 { 2195 struct bnxt_napi *bnapi = dev_instance; 2196 struct bnxt *bp = bnapi->bp; 2197 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2198 u32 cons = RING_CMP(cpr->cp_raw_cons); 2199 u32 int_status; 2200 2201 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]); 2202 2203 if (!bnxt_has_work(bp, cpr)) { 2204 int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS); 2205 /* return if erroneous interrupt */ 2206 if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id))) 2207 return IRQ_NONE; 2208 } 2209 2210 /* disable ring IRQ */ 2211 BNXT_CP_DB_IRQ_DIS(cpr->cp_db.doorbell); 2212 2213 /* Return here if interrupt is shared and is disabled. */ 2214 if (unlikely(atomic_read(&bp->intr_sem) != 0)) 2215 return IRQ_HANDLED; 2216 2217 napi_schedule(&bnapi->napi); 2218 return IRQ_HANDLED; 2219 } 2220 2221 static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 2222 int budget) 2223 { 2224 struct bnxt_napi *bnapi = cpr->bnapi; 2225 u32 raw_cons = cpr->cp_raw_cons; 2226 u32 cons; 2227 int tx_pkts = 0; 2228 int rx_pkts = 0; 2229 u8 event = 0; 2230 struct tx_cmp *txcmp; 2231 2232 cpr->has_more_work = 0; 2233 cpr->had_work_done = 1; 2234 while (1) { 2235 int rc; 2236 2237 cons = RING_CMP(raw_cons); 2238 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]; 2239 2240 if (!TX_CMP_VALID(txcmp, raw_cons)) 2241 break; 2242 2243 /* The valid test of the entry must be done first before 2244 * reading any further. 2245 */ 2246 dma_rmb(); 2247 if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) { 2248 tx_pkts++; 2249 /* return full budget so NAPI will complete. */ 2250 if (unlikely(tx_pkts > bp->tx_wake_thresh)) { 2251 rx_pkts = budget; 2252 raw_cons = NEXT_RAW_CMP(raw_cons); 2253 if (budget) 2254 cpr->has_more_work = 1; 2255 break; 2256 } 2257 } else if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) { 2258 if (likely(budget)) 2259 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event); 2260 else 2261 rc = bnxt_force_rx_discard(bp, cpr, &raw_cons, 2262 &event); 2263 if (likely(rc >= 0)) 2264 rx_pkts += rc; 2265 /* Increment rx_pkts when rc is -ENOMEM to count towards 2266 * the NAPI budget. Otherwise, we may potentially loop 2267 * here forever if we consistently cannot allocate 2268 * buffers. 2269 */ 2270 else if (rc == -ENOMEM && budget) 2271 rx_pkts++; 2272 else if (rc == -EBUSY) /* partial completion */ 2273 break; 2274 } else if (unlikely((TX_CMP_TYPE(txcmp) == 2275 CMPL_BASE_TYPE_HWRM_DONE) || 2276 (TX_CMP_TYPE(txcmp) == 2277 CMPL_BASE_TYPE_HWRM_FWD_REQ) || 2278 (TX_CMP_TYPE(txcmp) == 2279 CMPL_BASE_TYPE_HWRM_ASYNC_EVENT))) { 2280 bnxt_hwrm_handler(bp, txcmp); 2281 } 2282 raw_cons = NEXT_RAW_CMP(raw_cons); 2283 2284 if (rx_pkts && rx_pkts == budget) { 2285 cpr->has_more_work = 1; 2286 break; 2287 } 2288 } 2289 2290 if (event & BNXT_REDIRECT_EVENT) 2291 xdp_do_flush_map(); 2292 2293 if (event & BNXT_TX_EVENT) { 2294 struct bnxt_tx_ring_info *txr = bnapi->tx_ring; 2295 u16 prod = txr->tx_prod; 2296 2297 /* Sync BD data before updating doorbell */ 2298 wmb(); 2299 2300 bnxt_db_write_relaxed(bp, &txr->tx_db, prod); 2301 } 2302 2303 cpr->cp_raw_cons = raw_cons; 2304 bnapi->tx_pkts += tx_pkts; 2305 bnapi->events |= event; 2306 return rx_pkts; 2307 } 2308 2309 static void __bnxt_poll_work_done(struct bnxt *bp, struct bnxt_napi *bnapi) 2310 { 2311 if (bnapi->tx_pkts) { 2312 bnapi->tx_int(bp, bnapi, bnapi->tx_pkts); 2313 bnapi->tx_pkts = 0; 2314 } 2315 2316 if ((bnapi->events & BNXT_RX_EVENT) && !(bnapi->in_reset)) { 2317 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 2318 2319 if (bnapi->events & BNXT_AGG_EVENT) 2320 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); 2321 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 2322 } 2323 bnapi->events = 0; 2324 } 2325 2326 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 2327 int budget) 2328 { 2329 struct bnxt_napi *bnapi = cpr->bnapi; 2330 int rx_pkts; 2331 2332 rx_pkts = __bnxt_poll_work(bp, cpr, budget); 2333 2334 /* ACK completion ring before freeing tx ring and producing new 2335 * buffers in rx/agg rings to prevent overflowing the completion 2336 * ring. 2337 */ 2338 bnxt_db_cq(bp, &cpr->cp_db, cpr->cp_raw_cons); 2339 2340 __bnxt_poll_work_done(bp, bnapi); 2341 return rx_pkts; 2342 } 2343 2344 static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget) 2345 { 2346 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi); 2347 struct bnxt *bp = bnapi->bp; 2348 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2349 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 2350 struct tx_cmp *txcmp; 2351 struct rx_cmp_ext *rxcmp1; 2352 u32 cp_cons, tmp_raw_cons; 2353 u32 raw_cons = cpr->cp_raw_cons; 2354 u32 rx_pkts = 0; 2355 u8 event = 0; 2356 2357 while (1) { 2358 int rc; 2359 2360 cp_cons = RING_CMP(raw_cons); 2361 txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 2362 2363 if (!TX_CMP_VALID(txcmp, raw_cons)) 2364 break; 2365 2366 if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) { 2367 tmp_raw_cons = NEXT_RAW_CMP(raw_cons); 2368 cp_cons = RING_CMP(tmp_raw_cons); 2369 rxcmp1 = (struct rx_cmp_ext *) 2370 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 2371 2372 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons)) 2373 break; 2374 2375 /* force an error to recycle the buffer */ 2376 rxcmp1->rx_cmp_cfa_code_errors_v2 |= 2377 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR); 2378 2379 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event); 2380 if (likely(rc == -EIO) && budget) 2381 rx_pkts++; 2382 else if (rc == -EBUSY) /* partial completion */ 2383 break; 2384 } else if (unlikely(TX_CMP_TYPE(txcmp) == 2385 CMPL_BASE_TYPE_HWRM_DONE)) { 2386 bnxt_hwrm_handler(bp, txcmp); 2387 } else { 2388 netdev_err(bp->dev, 2389 "Invalid completion received on special ring\n"); 2390 } 2391 raw_cons = NEXT_RAW_CMP(raw_cons); 2392 2393 if (rx_pkts == budget) 2394 break; 2395 } 2396 2397 cpr->cp_raw_cons = raw_cons; 2398 BNXT_DB_CQ(&cpr->cp_db, cpr->cp_raw_cons); 2399 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 2400 2401 if (event & BNXT_AGG_EVENT) 2402 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); 2403 2404 if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) { 2405 napi_complete_done(napi, rx_pkts); 2406 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons); 2407 } 2408 return rx_pkts; 2409 } 2410 2411 static int bnxt_poll(struct napi_struct *napi, int budget) 2412 { 2413 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi); 2414 struct bnxt *bp = bnapi->bp; 2415 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2416 int work_done = 0; 2417 2418 if (unlikely(test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))) { 2419 napi_complete(napi); 2420 return 0; 2421 } 2422 while (1) { 2423 work_done += bnxt_poll_work(bp, cpr, budget - work_done); 2424 2425 if (work_done >= budget) { 2426 if (!budget) 2427 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons); 2428 break; 2429 } 2430 2431 if (!bnxt_has_work(bp, cpr)) { 2432 if (napi_complete_done(napi, work_done)) 2433 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons); 2434 break; 2435 } 2436 } 2437 if (bp->flags & BNXT_FLAG_DIM) { 2438 struct dim_sample dim_sample = {}; 2439 2440 dim_update_sample(cpr->event_ctr, 2441 cpr->rx_packets, 2442 cpr->rx_bytes, 2443 &dim_sample); 2444 net_dim(&cpr->dim, dim_sample); 2445 } 2446 return work_done; 2447 } 2448 2449 static int __bnxt_poll_cqs(struct bnxt *bp, struct bnxt_napi *bnapi, int budget) 2450 { 2451 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2452 int i, work_done = 0; 2453 2454 for (i = 0; i < 2; i++) { 2455 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i]; 2456 2457 if (cpr2) { 2458 work_done += __bnxt_poll_work(bp, cpr2, 2459 budget - work_done); 2460 cpr->has_more_work |= cpr2->has_more_work; 2461 } 2462 } 2463 return work_done; 2464 } 2465 2466 static void __bnxt_poll_cqs_done(struct bnxt *bp, struct bnxt_napi *bnapi, 2467 u64 dbr_type) 2468 { 2469 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2470 int i; 2471 2472 for (i = 0; i < 2; i++) { 2473 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i]; 2474 struct bnxt_db_info *db; 2475 2476 if (cpr2 && cpr2->had_work_done) { 2477 db = &cpr2->cp_db; 2478 writeq(db->db_key64 | dbr_type | 2479 RING_CMP(cpr2->cp_raw_cons), db->doorbell); 2480 cpr2->had_work_done = 0; 2481 } 2482 } 2483 __bnxt_poll_work_done(bp, bnapi); 2484 } 2485 2486 static int bnxt_poll_p5(struct napi_struct *napi, int budget) 2487 { 2488 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi); 2489 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2490 u32 raw_cons = cpr->cp_raw_cons; 2491 struct bnxt *bp = bnapi->bp; 2492 struct nqe_cn *nqcmp; 2493 int work_done = 0; 2494 u32 cons; 2495 2496 if (unlikely(test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))) { 2497 napi_complete(napi); 2498 return 0; 2499 } 2500 if (cpr->has_more_work) { 2501 cpr->has_more_work = 0; 2502 work_done = __bnxt_poll_cqs(bp, bnapi, budget); 2503 } 2504 while (1) { 2505 cons = RING_CMP(raw_cons); 2506 nqcmp = &cpr->nq_desc_ring[CP_RING(cons)][CP_IDX(cons)]; 2507 2508 if (!NQ_CMP_VALID(nqcmp, raw_cons)) { 2509 if (cpr->has_more_work) 2510 break; 2511 2512 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ_ARMALL); 2513 cpr->cp_raw_cons = raw_cons; 2514 if (napi_complete_done(napi, work_done)) 2515 BNXT_DB_NQ_ARM_P5(&cpr->cp_db, 2516 cpr->cp_raw_cons); 2517 return work_done; 2518 } 2519 2520 /* The valid test of the entry must be done first before 2521 * reading any further. 2522 */ 2523 dma_rmb(); 2524 2525 if (nqcmp->type == cpu_to_le16(NQ_CN_TYPE_CQ_NOTIFICATION)) { 2526 u32 idx = le32_to_cpu(nqcmp->cq_handle_low); 2527 struct bnxt_cp_ring_info *cpr2; 2528 2529 cpr2 = cpr->cp_ring_arr[idx]; 2530 work_done += __bnxt_poll_work(bp, cpr2, 2531 budget - work_done); 2532 cpr->has_more_work |= cpr2->has_more_work; 2533 } else { 2534 bnxt_hwrm_handler(bp, (struct tx_cmp *)nqcmp); 2535 } 2536 raw_cons = NEXT_RAW_CMP(raw_cons); 2537 } 2538 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ); 2539 if (raw_cons != cpr->cp_raw_cons) { 2540 cpr->cp_raw_cons = raw_cons; 2541 BNXT_DB_NQ_P5(&cpr->cp_db, raw_cons); 2542 } 2543 return work_done; 2544 } 2545 2546 static void bnxt_free_tx_skbs(struct bnxt *bp) 2547 { 2548 int i, max_idx; 2549 struct pci_dev *pdev = bp->pdev; 2550 2551 if (!bp->tx_ring) 2552 return; 2553 2554 max_idx = bp->tx_nr_pages * TX_DESC_CNT; 2555 for (i = 0; i < bp->tx_nr_rings; i++) { 2556 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 2557 int j; 2558 2559 for (j = 0; j < max_idx;) { 2560 struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j]; 2561 struct sk_buff *skb; 2562 int k, last; 2563 2564 if (i < bp->tx_nr_rings_xdp && 2565 tx_buf->action == XDP_REDIRECT) { 2566 dma_unmap_single(&pdev->dev, 2567 dma_unmap_addr(tx_buf, mapping), 2568 dma_unmap_len(tx_buf, len), 2569 PCI_DMA_TODEVICE); 2570 xdp_return_frame(tx_buf->xdpf); 2571 tx_buf->action = 0; 2572 tx_buf->xdpf = NULL; 2573 j++; 2574 continue; 2575 } 2576 2577 skb = tx_buf->skb; 2578 if (!skb) { 2579 j++; 2580 continue; 2581 } 2582 2583 tx_buf->skb = NULL; 2584 2585 if (tx_buf->is_push) { 2586 dev_kfree_skb(skb); 2587 j += 2; 2588 continue; 2589 } 2590 2591 dma_unmap_single(&pdev->dev, 2592 dma_unmap_addr(tx_buf, mapping), 2593 skb_headlen(skb), 2594 PCI_DMA_TODEVICE); 2595 2596 last = tx_buf->nr_frags; 2597 j += 2; 2598 for (k = 0; k < last; k++, j++) { 2599 int ring_idx = j & bp->tx_ring_mask; 2600 skb_frag_t *frag = &skb_shinfo(skb)->frags[k]; 2601 2602 tx_buf = &txr->tx_buf_ring[ring_idx]; 2603 dma_unmap_page( 2604 &pdev->dev, 2605 dma_unmap_addr(tx_buf, mapping), 2606 skb_frag_size(frag), PCI_DMA_TODEVICE); 2607 } 2608 dev_kfree_skb(skb); 2609 } 2610 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i)); 2611 } 2612 } 2613 2614 static void bnxt_free_one_rx_ring_skbs(struct bnxt *bp, int ring_nr) 2615 { 2616 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr]; 2617 struct pci_dev *pdev = bp->pdev; 2618 struct bnxt_tpa_idx_map *map; 2619 int i, max_idx, max_agg_idx; 2620 2621 max_idx = bp->rx_nr_pages * RX_DESC_CNT; 2622 max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT; 2623 if (!rxr->rx_tpa) 2624 goto skip_rx_tpa_free; 2625 2626 for (i = 0; i < bp->max_tpa; i++) { 2627 struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[i]; 2628 u8 *data = tpa_info->data; 2629 2630 if (!data) 2631 continue; 2632 2633 dma_unmap_single_attrs(&pdev->dev, tpa_info->mapping, 2634 bp->rx_buf_use_size, bp->rx_dir, 2635 DMA_ATTR_WEAK_ORDERING); 2636 2637 tpa_info->data = NULL; 2638 2639 kfree(data); 2640 } 2641 2642 skip_rx_tpa_free: 2643 for (i = 0; i < max_idx; i++) { 2644 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[i]; 2645 dma_addr_t mapping = rx_buf->mapping; 2646 void *data = rx_buf->data; 2647 2648 if (!data) 2649 continue; 2650 2651 rx_buf->data = NULL; 2652 if (BNXT_RX_PAGE_MODE(bp)) { 2653 mapping -= bp->rx_dma_offset; 2654 dma_unmap_page_attrs(&pdev->dev, mapping, PAGE_SIZE, 2655 bp->rx_dir, 2656 DMA_ATTR_WEAK_ORDERING); 2657 page_pool_recycle_direct(rxr->page_pool, data); 2658 } else { 2659 dma_unmap_single_attrs(&pdev->dev, mapping, 2660 bp->rx_buf_use_size, bp->rx_dir, 2661 DMA_ATTR_WEAK_ORDERING); 2662 kfree(data); 2663 } 2664 } 2665 for (i = 0; i < max_agg_idx; i++) { 2666 struct bnxt_sw_rx_agg_bd *rx_agg_buf = &rxr->rx_agg_ring[i]; 2667 struct page *page = rx_agg_buf->page; 2668 2669 if (!page) 2670 continue; 2671 2672 dma_unmap_page_attrs(&pdev->dev, rx_agg_buf->mapping, 2673 BNXT_RX_PAGE_SIZE, PCI_DMA_FROMDEVICE, 2674 DMA_ATTR_WEAK_ORDERING); 2675 2676 rx_agg_buf->page = NULL; 2677 __clear_bit(i, rxr->rx_agg_bmap); 2678 2679 __free_page(page); 2680 } 2681 if (rxr->rx_page) { 2682 __free_page(rxr->rx_page); 2683 rxr->rx_page = NULL; 2684 } 2685 map = rxr->rx_tpa_idx_map; 2686 if (map) 2687 memset(map->agg_idx_bmap, 0, sizeof(map->agg_idx_bmap)); 2688 } 2689 2690 static void bnxt_free_rx_skbs(struct bnxt *bp) 2691 { 2692 int i; 2693 2694 if (!bp->rx_ring) 2695 return; 2696 2697 for (i = 0; i < bp->rx_nr_rings; i++) 2698 bnxt_free_one_rx_ring_skbs(bp, i); 2699 } 2700 2701 static void bnxt_free_skbs(struct bnxt *bp) 2702 { 2703 bnxt_free_tx_skbs(bp); 2704 bnxt_free_rx_skbs(bp); 2705 } 2706 2707 static void bnxt_init_ctx_mem(struct bnxt_mem_init *mem_init, void *p, int len) 2708 { 2709 u8 init_val = mem_init->init_val; 2710 u16 offset = mem_init->offset; 2711 u8 *p2 = p; 2712 int i; 2713 2714 if (!init_val) 2715 return; 2716 if (offset == BNXT_MEM_INVALID_OFFSET) { 2717 memset(p, init_val, len); 2718 return; 2719 } 2720 for (i = 0; i < len; i += mem_init->size) 2721 *(p2 + i + offset) = init_val; 2722 } 2723 2724 static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem) 2725 { 2726 struct pci_dev *pdev = bp->pdev; 2727 int i; 2728 2729 for (i = 0; i < rmem->nr_pages; i++) { 2730 if (!rmem->pg_arr[i]) 2731 continue; 2732 2733 dma_free_coherent(&pdev->dev, rmem->page_size, 2734 rmem->pg_arr[i], rmem->dma_arr[i]); 2735 2736 rmem->pg_arr[i] = NULL; 2737 } 2738 if (rmem->pg_tbl) { 2739 size_t pg_tbl_size = rmem->nr_pages * 8; 2740 2741 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG) 2742 pg_tbl_size = rmem->page_size; 2743 dma_free_coherent(&pdev->dev, pg_tbl_size, 2744 rmem->pg_tbl, rmem->pg_tbl_map); 2745 rmem->pg_tbl = NULL; 2746 } 2747 if (rmem->vmem_size && *rmem->vmem) { 2748 vfree(*rmem->vmem); 2749 *rmem->vmem = NULL; 2750 } 2751 } 2752 2753 static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem) 2754 { 2755 struct pci_dev *pdev = bp->pdev; 2756 u64 valid_bit = 0; 2757 int i; 2758 2759 if (rmem->flags & (BNXT_RMEM_VALID_PTE_FLAG | BNXT_RMEM_RING_PTE_FLAG)) 2760 valid_bit = PTU_PTE_VALID; 2761 if ((rmem->nr_pages > 1 || rmem->depth > 0) && !rmem->pg_tbl) { 2762 size_t pg_tbl_size = rmem->nr_pages * 8; 2763 2764 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG) 2765 pg_tbl_size = rmem->page_size; 2766 rmem->pg_tbl = dma_alloc_coherent(&pdev->dev, pg_tbl_size, 2767 &rmem->pg_tbl_map, 2768 GFP_KERNEL); 2769 if (!rmem->pg_tbl) 2770 return -ENOMEM; 2771 } 2772 2773 for (i = 0; i < rmem->nr_pages; i++) { 2774 u64 extra_bits = valid_bit; 2775 2776 rmem->pg_arr[i] = dma_alloc_coherent(&pdev->dev, 2777 rmem->page_size, 2778 &rmem->dma_arr[i], 2779 GFP_KERNEL); 2780 if (!rmem->pg_arr[i]) 2781 return -ENOMEM; 2782 2783 if (rmem->mem_init) 2784 bnxt_init_ctx_mem(rmem->mem_init, rmem->pg_arr[i], 2785 rmem->page_size); 2786 if (rmem->nr_pages > 1 || rmem->depth > 0) { 2787 if (i == rmem->nr_pages - 2 && 2788 (rmem->flags & BNXT_RMEM_RING_PTE_FLAG)) 2789 extra_bits |= PTU_PTE_NEXT_TO_LAST; 2790 else if (i == rmem->nr_pages - 1 && 2791 (rmem->flags & BNXT_RMEM_RING_PTE_FLAG)) 2792 extra_bits |= PTU_PTE_LAST; 2793 rmem->pg_tbl[i] = 2794 cpu_to_le64(rmem->dma_arr[i] | extra_bits); 2795 } 2796 } 2797 2798 if (rmem->vmem_size) { 2799 *rmem->vmem = vzalloc(rmem->vmem_size); 2800 if (!(*rmem->vmem)) 2801 return -ENOMEM; 2802 } 2803 return 0; 2804 } 2805 2806 static void bnxt_free_tpa_info(struct bnxt *bp) 2807 { 2808 int i; 2809 2810 for (i = 0; i < bp->rx_nr_rings; i++) { 2811 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 2812 2813 kfree(rxr->rx_tpa_idx_map); 2814 rxr->rx_tpa_idx_map = NULL; 2815 if (rxr->rx_tpa) { 2816 kfree(rxr->rx_tpa[0].agg_arr); 2817 rxr->rx_tpa[0].agg_arr = NULL; 2818 } 2819 kfree(rxr->rx_tpa); 2820 rxr->rx_tpa = NULL; 2821 } 2822 } 2823 2824 static int bnxt_alloc_tpa_info(struct bnxt *bp) 2825 { 2826 int i, j, total_aggs = 0; 2827 2828 bp->max_tpa = MAX_TPA; 2829 if (bp->flags & BNXT_FLAG_CHIP_P5) { 2830 if (!bp->max_tpa_v2) 2831 return 0; 2832 bp->max_tpa = max_t(u16, bp->max_tpa_v2, MAX_TPA_P5); 2833 total_aggs = bp->max_tpa * MAX_SKB_FRAGS; 2834 } 2835 2836 for (i = 0; i < bp->rx_nr_rings; i++) { 2837 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 2838 struct rx_agg_cmp *agg; 2839 2840 rxr->rx_tpa = kcalloc(bp->max_tpa, sizeof(struct bnxt_tpa_info), 2841 GFP_KERNEL); 2842 if (!rxr->rx_tpa) 2843 return -ENOMEM; 2844 2845 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 2846 continue; 2847 agg = kcalloc(total_aggs, sizeof(*agg), GFP_KERNEL); 2848 rxr->rx_tpa[0].agg_arr = agg; 2849 if (!agg) 2850 return -ENOMEM; 2851 for (j = 1; j < bp->max_tpa; j++) 2852 rxr->rx_tpa[j].agg_arr = agg + j * MAX_SKB_FRAGS; 2853 rxr->rx_tpa_idx_map = kzalloc(sizeof(*rxr->rx_tpa_idx_map), 2854 GFP_KERNEL); 2855 if (!rxr->rx_tpa_idx_map) 2856 return -ENOMEM; 2857 } 2858 return 0; 2859 } 2860 2861 static void bnxt_free_rx_rings(struct bnxt *bp) 2862 { 2863 int i; 2864 2865 if (!bp->rx_ring) 2866 return; 2867 2868 bnxt_free_tpa_info(bp); 2869 for (i = 0; i < bp->rx_nr_rings; i++) { 2870 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 2871 struct bnxt_ring_struct *ring; 2872 2873 if (rxr->xdp_prog) 2874 bpf_prog_put(rxr->xdp_prog); 2875 2876 if (xdp_rxq_info_is_reg(&rxr->xdp_rxq)) 2877 xdp_rxq_info_unreg(&rxr->xdp_rxq); 2878 2879 page_pool_destroy(rxr->page_pool); 2880 rxr->page_pool = NULL; 2881 2882 kfree(rxr->rx_agg_bmap); 2883 rxr->rx_agg_bmap = NULL; 2884 2885 ring = &rxr->rx_ring_struct; 2886 bnxt_free_ring(bp, &ring->ring_mem); 2887 2888 ring = &rxr->rx_agg_ring_struct; 2889 bnxt_free_ring(bp, &ring->ring_mem); 2890 } 2891 } 2892 2893 static int bnxt_alloc_rx_page_pool(struct bnxt *bp, 2894 struct bnxt_rx_ring_info *rxr) 2895 { 2896 struct page_pool_params pp = { 0 }; 2897 2898 pp.pool_size = bp->rx_ring_size; 2899 pp.nid = dev_to_node(&bp->pdev->dev); 2900 pp.dev = &bp->pdev->dev; 2901 pp.dma_dir = DMA_BIDIRECTIONAL; 2902 2903 rxr->page_pool = page_pool_create(&pp); 2904 if (IS_ERR(rxr->page_pool)) { 2905 int err = PTR_ERR(rxr->page_pool); 2906 2907 rxr->page_pool = NULL; 2908 return err; 2909 } 2910 return 0; 2911 } 2912 2913 static int bnxt_alloc_rx_rings(struct bnxt *bp) 2914 { 2915 int i, rc = 0, agg_rings = 0; 2916 2917 if (!bp->rx_ring) 2918 return -ENOMEM; 2919 2920 if (bp->flags & BNXT_FLAG_AGG_RINGS) 2921 agg_rings = 1; 2922 2923 for (i = 0; i < bp->rx_nr_rings; i++) { 2924 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 2925 struct bnxt_ring_struct *ring; 2926 2927 ring = &rxr->rx_ring_struct; 2928 2929 rc = bnxt_alloc_rx_page_pool(bp, rxr); 2930 if (rc) 2931 return rc; 2932 2933 rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i, 0); 2934 if (rc < 0) 2935 return rc; 2936 2937 rc = xdp_rxq_info_reg_mem_model(&rxr->xdp_rxq, 2938 MEM_TYPE_PAGE_POOL, 2939 rxr->page_pool); 2940 if (rc) { 2941 xdp_rxq_info_unreg(&rxr->xdp_rxq); 2942 return rc; 2943 } 2944 2945 rc = bnxt_alloc_ring(bp, &ring->ring_mem); 2946 if (rc) 2947 return rc; 2948 2949 ring->grp_idx = i; 2950 if (agg_rings) { 2951 u16 mem_size; 2952 2953 ring = &rxr->rx_agg_ring_struct; 2954 rc = bnxt_alloc_ring(bp, &ring->ring_mem); 2955 if (rc) 2956 return rc; 2957 2958 ring->grp_idx = i; 2959 rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1; 2960 mem_size = rxr->rx_agg_bmap_size / 8; 2961 rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL); 2962 if (!rxr->rx_agg_bmap) 2963 return -ENOMEM; 2964 } 2965 } 2966 if (bp->flags & BNXT_FLAG_TPA) 2967 rc = bnxt_alloc_tpa_info(bp); 2968 return rc; 2969 } 2970 2971 static void bnxt_free_tx_rings(struct bnxt *bp) 2972 { 2973 int i; 2974 struct pci_dev *pdev = bp->pdev; 2975 2976 if (!bp->tx_ring) 2977 return; 2978 2979 for (i = 0; i < bp->tx_nr_rings; i++) { 2980 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 2981 struct bnxt_ring_struct *ring; 2982 2983 if (txr->tx_push) { 2984 dma_free_coherent(&pdev->dev, bp->tx_push_size, 2985 txr->tx_push, txr->tx_push_mapping); 2986 txr->tx_push = NULL; 2987 } 2988 2989 ring = &txr->tx_ring_struct; 2990 2991 bnxt_free_ring(bp, &ring->ring_mem); 2992 } 2993 } 2994 2995 static int bnxt_alloc_tx_rings(struct bnxt *bp) 2996 { 2997 int i, j, rc; 2998 struct pci_dev *pdev = bp->pdev; 2999 3000 bp->tx_push_size = 0; 3001 if (bp->tx_push_thresh) { 3002 int push_size; 3003 3004 push_size = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) + 3005 bp->tx_push_thresh); 3006 3007 if (push_size > 256) { 3008 push_size = 0; 3009 bp->tx_push_thresh = 0; 3010 } 3011 3012 bp->tx_push_size = push_size; 3013 } 3014 3015 for (i = 0, j = 0; i < bp->tx_nr_rings; i++) { 3016 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 3017 struct bnxt_ring_struct *ring; 3018 u8 qidx; 3019 3020 ring = &txr->tx_ring_struct; 3021 3022 rc = bnxt_alloc_ring(bp, &ring->ring_mem); 3023 if (rc) 3024 return rc; 3025 3026 ring->grp_idx = txr->bnapi->index; 3027 if (bp->tx_push_size) { 3028 dma_addr_t mapping; 3029 3030 /* One pre-allocated DMA buffer to backup 3031 * TX push operation 3032 */ 3033 txr->tx_push = dma_alloc_coherent(&pdev->dev, 3034 bp->tx_push_size, 3035 &txr->tx_push_mapping, 3036 GFP_KERNEL); 3037 3038 if (!txr->tx_push) 3039 return -ENOMEM; 3040 3041 mapping = txr->tx_push_mapping + 3042 sizeof(struct tx_push_bd); 3043 txr->data_mapping = cpu_to_le64(mapping); 3044 } 3045 qidx = bp->tc_to_qidx[j]; 3046 ring->queue_id = bp->q_info[qidx].queue_id; 3047 if (i < bp->tx_nr_rings_xdp) 3048 continue; 3049 if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1)) 3050 j++; 3051 } 3052 return 0; 3053 } 3054 3055 static void bnxt_free_cp_rings(struct bnxt *bp) 3056 { 3057 int i; 3058 3059 if (!bp->bnapi) 3060 return; 3061 3062 for (i = 0; i < bp->cp_nr_rings; i++) { 3063 struct bnxt_napi *bnapi = bp->bnapi[i]; 3064 struct bnxt_cp_ring_info *cpr; 3065 struct bnxt_ring_struct *ring; 3066 int j; 3067 3068 if (!bnapi) 3069 continue; 3070 3071 cpr = &bnapi->cp_ring; 3072 ring = &cpr->cp_ring_struct; 3073 3074 bnxt_free_ring(bp, &ring->ring_mem); 3075 3076 for (j = 0; j < 2; j++) { 3077 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j]; 3078 3079 if (cpr2) { 3080 ring = &cpr2->cp_ring_struct; 3081 bnxt_free_ring(bp, &ring->ring_mem); 3082 kfree(cpr2); 3083 cpr->cp_ring_arr[j] = NULL; 3084 } 3085 } 3086 } 3087 } 3088 3089 static struct bnxt_cp_ring_info *bnxt_alloc_cp_sub_ring(struct bnxt *bp) 3090 { 3091 struct bnxt_ring_mem_info *rmem; 3092 struct bnxt_ring_struct *ring; 3093 struct bnxt_cp_ring_info *cpr; 3094 int rc; 3095 3096 cpr = kzalloc(sizeof(*cpr), GFP_KERNEL); 3097 if (!cpr) 3098 return NULL; 3099 3100 ring = &cpr->cp_ring_struct; 3101 rmem = &ring->ring_mem; 3102 rmem->nr_pages = bp->cp_nr_pages; 3103 rmem->page_size = HW_CMPD_RING_SIZE; 3104 rmem->pg_arr = (void **)cpr->cp_desc_ring; 3105 rmem->dma_arr = cpr->cp_desc_mapping; 3106 rmem->flags = BNXT_RMEM_RING_PTE_FLAG; 3107 rc = bnxt_alloc_ring(bp, rmem); 3108 if (rc) { 3109 bnxt_free_ring(bp, rmem); 3110 kfree(cpr); 3111 cpr = NULL; 3112 } 3113 return cpr; 3114 } 3115 3116 static int bnxt_alloc_cp_rings(struct bnxt *bp) 3117 { 3118 bool sh = !!(bp->flags & BNXT_FLAG_SHARED_RINGS); 3119 int i, rc, ulp_base_vec, ulp_msix; 3120 3121 ulp_msix = bnxt_get_ulp_msix_num(bp); 3122 ulp_base_vec = bnxt_get_ulp_msix_base(bp); 3123 for (i = 0; i < bp->cp_nr_rings; i++) { 3124 struct bnxt_napi *bnapi = bp->bnapi[i]; 3125 struct bnxt_cp_ring_info *cpr; 3126 struct bnxt_ring_struct *ring; 3127 3128 if (!bnapi) 3129 continue; 3130 3131 cpr = &bnapi->cp_ring; 3132 cpr->bnapi = bnapi; 3133 ring = &cpr->cp_ring_struct; 3134 3135 rc = bnxt_alloc_ring(bp, &ring->ring_mem); 3136 if (rc) 3137 return rc; 3138 3139 if (ulp_msix && i >= ulp_base_vec) 3140 ring->map_idx = i + ulp_msix; 3141 else 3142 ring->map_idx = i; 3143 3144 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 3145 continue; 3146 3147 if (i < bp->rx_nr_rings) { 3148 struct bnxt_cp_ring_info *cpr2 = 3149 bnxt_alloc_cp_sub_ring(bp); 3150 3151 cpr->cp_ring_arr[BNXT_RX_HDL] = cpr2; 3152 if (!cpr2) 3153 return -ENOMEM; 3154 cpr2->bnapi = bnapi; 3155 } 3156 if ((sh && i < bp->tx_nr_rings) || 3157 (!sh && i >= bp->rx_nr_rings)) { 3158 struct bnxt_cp_ring_info *cpr2 = 3159 bnxt_alloc_cp_sub_ring(bp); 3160 3161 cpr->cp_ring_arr[BNXT_TX_HDL] = cpr2; 3162 if (!cpr2) 3163 return -ENOMEM; 3164 cpr2->bnapi = bnapi; 3165 } 3166 } 3167 return 0; 3168 } 3169 3170 static void bnxt_init_ring_struct(struct bnxt *bp) 3171 { 3172 int i; 3173 3174 for (i = 0; i < bp->cp_nr_rings; i++) { 3175 struct bnxt_napi *bnapi = bp->bnapi[i]; 3176 struct bnxt_ring_mem_info *rmem; 3177 struct bnxt_cp_ring_info *cpr; 3178 struct bnxt_rx_ring_info *rxr; 3179 struct bnxt_tx_ring_info *txr; 3180 struct bnxt_ring_struct *ring; 3181 3182 if (!bnapi) 3183 continue; 3184 3185 cpr = &bnapi->cp_ring; 3186 ring = &cpr->cp_ring_struct; 3187 rmem = &ring->ring_mem; 3188 rmem->nr_pages = bp->cp_nr_pages; 3189 rmem->page_size = HW_CMPD_RING_SIZE; 3190 rmem->pg_arr = (void **)cpr->cp_desc_ring; 3191 rmem->dma_arr = cpr->cp_desc_mapping; 3192 rmem->vmem_size = 0; 3193 3194 rxr = bnapi->rx_ring; 3195 if (!rxr) 3196 goto skip_rx; 3197 3198 ring = &rxr->rx_ring_struct; 3199 rmem = &ring->ring_mem; 3200 rmem->nr_pages = bp->rx_nr_pages; 3201 rmem->page_size = HW_RXBD_RING_SIZE; 3202 rmem->pg_arr = (void **)rxr->rx_desc_ring; 3203 rmem->dma_arr = rxr->rx_desc_mapping; 3204 rmem->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages; 3205 rmem->vmem = (void **)&rxr->rx_buf_ring; 3206 3207 ring = &rxr->rx_agg_ring_struct; 3208 rmem = &ring->ring_mem; 3209 rmem->nr_pages = bp->rx_agg_nr_pages; 3210 rmem->page_size = HW_RXBD_RING_SIZE; 3211 rmem->pg_arr = (void **)rxr->rx_agg_desc_ring; 3212 rmem->dma_arr = rxr->rx_agg_desc_mapping; 3213 rmem->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages; 3214 rmem->vmem = (void **)&rxr->rx_agg_ring; 3215 3216 skip_rx: 3217 txr = bnapi->tx_ring; 3218 if (!txr) 3219 continue; 3220 3221 ring = &txr->tx_ring_struct; 3222 rmem = &ring->ring_mem; 3223 rmem->nr_pages = bp->tx_nr_pages; 3224 rmem->page_size = HW_RXBD_RING_SIZE; 3225 rmem->pg_arr = (void **)txr->tx_desc_ring; 3226 rmem->dma_arr = txr->tx_desc_mapping; 3227 rmem->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages; 3228 rmem->vmem = (void **)&txr->tx_buf_ring; 3229 } 3230 } 3231 3232 static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type) 3233 { 3234 int i; 3235 u32 prod; 3236 struct rx_bd **rx_buf_ring; 3237 3238 rx_buf_ring = (struct rx_bd **)ring->ring_mem.pg_arr; 3239 for (i = 0, prod = 0; i < ring->ring_mem.nr_pages; i++) { 3240 int j; 3241 struct rx_bd *rxbd; 3242 3243 rxbd = rx_buf_ring[i]; 3244 if (!rxbd) 3245 continue; 3246 3247 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) { 3248 rxbd->rx_bd_len_flags_type = cpu_to_le32(type); 3249 rxbd->rx_bd_opaque = prod; 3250 } 3251 } 3252 } 3253 3254 static int bnxt_alloc_one_rx_ring(struct bnxt *bp, int ring_nr) 3255 { 3256 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr]; 3257 struct net_device *dev = bp->dev; 3258 u32 prod; 3259 int i; 3260 3261 prod = rxr->rx_prod; 3262 for (i = 0; i < bp->rx_ring_size; i++) { 3263 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL)) { 3264 netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n", 3265 ring_nr, i, bp->rx_ring_size); 3266 break; 3267 } 3268 prod = NEXT_RX(prod); 3269 } 3270 rxr->rx_prod = prod; 3271 3272 if (!(bp->flags & BNXT_FLAG_AGG_RINGS)) 3273 return 0; 3274 3275 prod = rxr->rx_agg_prod; 3276 for (i = 0; i < bp->rx_agg_ring_size; i++) { 3277 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL)) { 3278 netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n", 3279 ring_nr, i, bp->rx_ring_size); 3280 break; 3281 } 3282 prod = NEXT_RX_AGG(prod); 3283 } 3284 rxr->rx_agg_prod = prod; 3285 3286 if (rxr->rx_tpa) { 3287 dma_addr_t mapping; 3288 u8 *data; 3289 3290 for (i = 0; i < bp->max_tpa; i++) { 3291 data = __bnxt_alloc_rx_data(bp, &mapping, GFP_KERNEL); 3292 if (!data) 3293 return -ENOMEM; 3294 3295 rxr->rx_tpa[i].data = data; 3296 rxr->rx_tpa[i].data_ptr = data + bp->rx_offset; 3297 rxr->rx_tpa[i].mapping = mapping; 3298 } 3299 } 3300 return 0; 3301 } 3302 3303 static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr) 3304 { 3305 struct bnxt_rx_ring_info *rxr; 3306 struct bnxt_ring_struct *ring; 3307 u32 type; 3308 3309 type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) | 3310 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP; 3311 3312 if (NET_IP_ALIGN == 2) 3313 type |= RX_BD_FLAGS_SOP; 3314 3315 rxr = &bp->rx_ring[ring_nr]; 3316 ring = &rxr->rx_ring_struct; 3317 bnxt_init_rxbd_pages(ring, type); 3318 3319 if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) { 3320 bpf_prog_add(bp->xdp_prog, 1); 3321 rxr->xdp_prog = bp->xdp_prog; 3322 } 3323 ring->fw_ring_id = INVALID_HW_RING_ID; 3324 3325 ring = &rxr->rx_agg_ring_struct; 3326 ring->fw_ring_id = INVALID_HW_RING_ID; 3327 3328 if ((bp->flags & BNXT_FLAG_AGG_RINGS)) { 3329 type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) | 3330 RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP; 3331 3332 bnxt_init_rxbd_pages(ring, type); 3333 } 3334 3335 return bnxt_alloc_one_rx_ring(bp, ring_nr); 3336 } 3337 3338 static void bnxt_init_cp_rings(struct bnxt *bp) 3339 { 3340 int i, j; 3341 3342 for (i = 0; i < bp->cp_nr_rings; i++) { 3343 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring; 3344 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 3345 3346 ring->fw_ring_id = INVALID_HW_RING_ID; 3347 cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks; 3348 cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs; 3349 for (j = 0; j < 2; j++) { 3350 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j]; 3351 3352 if (!cpr2) 3353 continue; 3354 3355 ring = &cpr2->cp_ring_struct; 3356 ring->fw_ring_id = INVALID_HW_RING_ID; 3357 cpr2->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks; 3358 cpr2->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs; 3359 } 3360 } 3361 } 3362 3363 static int bnxt_init_rx_rings(struct bnxt *bp) 3364 { 3365 int i, rc = 0; 3366 3367 if (BNXT_RX_PAGE_MODE(bp)) { 3368 bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM; 3369 bp->rx_dma_offset = XDP_PACKET_HEADROOM; 3370 } else { 3371 bp->rx_offset = BNXT_RX_OFFSET; 3372 bp->rx_dma_offset = BNXT_RX_DMA_OFFSET; 3373 } 3374 3375 for (i = 0; i < bp->rx_nr_rings; i++) { 3376 rc = bnxt_init_one_rx_ring(bp, i); 3377 if (rc) 3378 break; 3379 } 3380 3381 return rc; 3382 } 3383 3384 static int bnxt_init_tx_rings(struct bnxt *bp) 3385 { 3386 u16 i; 3387 3388 bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2, 3389 MAX_SKB_FRAGS + 1); 3390 3391 for (i = 0; i < bp->tx_nr_rings; i++) { 3392 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 3393 struct bnxt_ring_struct *ring = &txr->tx_ring_struct; 3394 3395 ring->fw_ring_id = INVALID_HW_RING_ID; 3396 } 3397 3398 return 0; 3399 } 3400 3401 static void bnxt_free_ring_grps(struct bnxt *bp) 3402 { 3403 kfree(bp->grp_info); 3404 bp->grp_info = NULL; 3405 } 3406 3407 static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init) 3408 { 3409 int i; 3410 3411 if (irq_re_init) { 3412 bp->grp_info = kcalloc(bp->cp_nr_rings, 3413 sizeof(struct bnxt_ring_grp_info), 3414 GFP_KERNEL); 3415 if (!bp->grp_info) 3416 return -ENOMEM; 3417 } 3418 for (i = 0; i < bp->cp_nr_rings; i++) { 3419 if (irq_re_init) 3420 bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID; 3421 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID; 3422 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID; 3423 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID; 3424 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID; 3425 } 3426 return 0; 3427 } 3428 3429 static void bnxt_free_vnics(struct bnxt *bp) 3430 { 3431 kfree(bp->vnic_info); 3432 bp->vnic_info = NULL; 3433 bp->nr_vnics = 0; 3434 } 3435 3436 static int bnxt_alloc_vnics(struct bnxt *bp) 3437 { 3438 int num_vnics = 1; 3439 3440 #ifdef CONFIG_RFS_ACCEL 3441 if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS) 3442 num_vnics += bp->rx_nr_rings; 3443 #endif 3444 3445 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 3446 num_vnics++; 3447 3448 bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info), 3449 GFP_KERNEL); 3450 if (!bp->vnic_info) 3451 return -ENOMEM; 3452 3453 bp->nr_vnics = num_vnics; 3454 return 0; 3455 } 3456 3457 static void bnxt_init_vnics(struct bnxt *bp) 3458 { 3459 int i; 3460 3461 for (i = 0; i < bp->nr_vnics; i++) { 3462 struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; 3463 int j; 3464 3465 vnic->fw_vnic_id = INVALID_HW_RING_ID; 3466 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) 3467 vnic->fw_rss_cos_lb_ctx[j] = INVALID_HW_RING_ID; 3468 3469 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID; 3470 3471 if (bp->vnic_info[i].rss_hash_key) { 3472 if (i == 0) 3473 prandom_bytes(vnic->rss_hash_key, 3474 HW_HASH_KEY_SIZE); 3475 else 3476 memcpy(vnic->rss_hash_key, 3477 bp->vnic_info[0].rss_hash_key, 3478 HW_HASH_KEY_SIZE); 3479 } 3480 } 3481 } 3482 3483 static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg) 3484 { 3485 int pages; 3486 3487 pages = ring_size / desc_per_pg; 3488 3489 if (!pages) 3490 return 1; 3491 3492 pages++; 3493 3494 while (pages & (pages - 1)) 3495 pages++; 3496 3497 return pages; 3498 } 3499 3500 void bnxt_set_tpa_flags(struct bnxt *bp) 3501 { 3502 bp->flags &= ~BNXT_FLAG_TPA; 3503 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS) 3504 return; 3505 if (bp->dev->features & NETIF_F_LRO) 3506 bp->flags |= BNXT_FLAG_LRO; 3507 else if (bp->dev->features & NETIF_F_GRO_HW) 3508 bp->flags |= BNXT_FLAG_GRO; 3509 } 3510 3511 /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must 3512 * be set on entry. 3513 */ 3514 void bnxt_set_ring_params(struct bnxt *bp) 3515 { 3516 u32 ring_size, rx_size, rx_space, max_rx_cmpl; 3517 u32 agg_factor = 0, agg_ring_size = 0; 3518 3519 /* 8 for CRC and VLAN */ 3520 rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8); 3521 3522 rx_space = rx_size + NET_SKB_PAD + 3523 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 3524 3525 bp->rx_copy_thresh = BNXT_RX_COPY_THRESH; 3526 ring_size = bp->rx_ring_size; 3527 bp->rx_agg_ring_size = 0; 3528 bp->rx_agg_nr_pages = 0; 3529 3530 if (bp->flags & BNXT_FLAG_TPA) 3531 agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE); 3532 3533 bp->flags &= ~BNXT_FLAG_JUMBO; 3534 if (rx_space > PAGE_SIZE && !(bp->flags & BNXT_FLAG_NO_AGG_RINGS)) { 3535 u32 jumbo_factor; 3536 3537 bp->flags |= BNXT_FLAG_JUMBO; 3538 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT; 3539 if (jumbo_factor > agg_factor) 3540 agg_factor = jumbo_factor; 3541 } 3542 agg_ring_size = ring_size * agg_factor; 3543 3544 if (agg_ring_size) { 3545 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size, 3546 RX_DESC_CNT); 3547 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) { 3548 u32 tmp = agg_ring_size; 3549 3550 bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES; 3551 agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1; 3552 netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n", 3553 tmp, agg_ring_size); 3554 } 3555 bp->rx_agg_ring_size = agg_ring_size; 3556 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1; 3557 rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN); 3558 rx_space = rx_size + NET_SKB_PAD + 3559 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 3560 } 3561 3562 bp->rx_buf_use_size = rx_size; 3563 bp->rx_buf_size = rx_space; 3564 3565 bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT); 3566 bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1; 3567 3568 ring_size = bp->tx_ring_size; 3569 bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT); 3570 bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1; 3571 3572 max_rx_cmpl = bp->rx_ring_size; 3573 /* MAX TPA needs to be added because TPA_START completions are 3574 * immediately recycled, so the TPA completions are not bound by 3575 * the RX ring size. 3576 */ 3577 if (bp->flags & BNXT_FLAG_TPA) 3578 max_rx_cmpl += bp->max_tpa; 3579 /* RX and TPA completions are 32-byte, all others are 16-byte */ 3580 ring_size = max_rx_cmpl * 2 + agg_ring_size + bp->tx_ring_size; 3581 bp->cp_ring_size = ring_size; 3582 3583 bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT); 3584 if (bp->cp_nr_pages > MAX_CP_PAGES) { 3585 bp->cp_nr_pages = MAX_CP_PAGES; 3586 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1; 3587 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n", 3588 ring_size, bp->cp_ring_size); 3589 } 3590 bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT; 3591 bp->cp_ring_mask = bp->cp_bit - 1; 3592 } 3593 3594 /* Changing allocation mode of RX rings. 3595 * TODO: Update when extending xdp_rxq_info to support allocation modes. 3596 */ 3597 int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode) 3598 { 3599 if (page_mode) { 3600 if (bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU) 3601 return -EOPNOTSUPP; 3602 bp->dev->max_mtu = 3603 min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU); 3604 bp->flags &= ~BNXT_FLAG_AGG_RINGS; 3605 bp->flags |= BNXT_FLAG_NO_AGG_RINGS | BNXT_FLAG_RX_PAGE_MODE; 3606 bp->rx_dir = DMA_BIDIRECTIONAL; 3607 bp->rx_skb_func = bnxt_rx_page_skb; 3608 /* Disable LRO or GRO_HW */ 3609 netdev_update_features(bp->dev); 3610 } else { 3611 bp->dev->max_mtu = bp->max_mtu; 3612 bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE; 3613 bp->rx_dir = DMA_FROM_DEVICE; 3614 bp->rx_skb_func = bnxt_rx_skb; 3615 } 3616 return 0; 3617 } 3618 3619 static void bnxt_free_vnic_attributes(struct bnxt *bp) 3620 { 3621 int i; 3622 struct bnxt_vnic_info *vnic; 3623 struct pci_dev *pdev = bp->pdev; 3624 3625 if (!bp->vnic_info) 3626 return; 3627 3628 for (i = 0; i < bp->nr_vnics; i++) { 3629 vnic = &bp->vnic_info[i]; 3630 3631 kfree(vnic->fw_grp_ids); 3632 vnic->fw_grp_ids = NULL; 3633 3634 kfree(vnic->uc_list); 3635 vnic->uc_list = NULL; 3636 3637 if (vnic->mc_list) { 3638 dma_free_coherent(&pdev->dev, vnic->mc_list_size, 3639 vnic->mc_list, vnic->mc_list_mapping); 3640 vnic->mc_list = NULL; 3641 } 3642 3643 if (vnic->rss_table) { 3644 dma_free_coherent(&pdev->dev, vnic->rss_table_size, 3645 vnic->rss_table, 3646 vnic->rss_table_dma_addr); 3647 vnic->rss_table = NULL; 3648 } 3649 3650 vnic->rss_hash_key = NULL; 3651 vnic->flags = 0; 3652 } 3653 } 3654 3655 static int bnxt_alloc_vnic_attributes(struct bnxt *bp) 3656 { 3657 int i, rc = 0, size; 3658 struct bnxt_vnic_info *vnic; 3659 struct pci_dev *pdev = bp->pdev; 3660 int max_rings; 3661 3662 for (i = 0; i < bp->nr_vnics; i++) { 3663 vnic = &bp->vnic_info[i]; 3664 3665 if (vnic->flags & BNXT_VNIC_UCAST_FLAG) { 3666 int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN; 3667 3668 if (mem_size > 0) { 3669 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL); 3670 if (!vnic->uc_list) { 3671 rc = -ENOMEM; 3672 goto out; 3673 } 3674 } 3675 } 3676 3677 if (vnic->flags & BNXT_VNIC_MCAST_FLAG) { 3678 vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN; 3679 vnic->mc_list = 3680 dma_alloc_coherent(&pdev->dev, 3681 vnic->mc_list_size, 3682 &vnic->mc_list_mapping, 3683 GFP_KERNEL); 3684 if (!vnic->mc_list) { 3685 rc = -ENOMEM; 3686 goto out; 3687 } 3688 } 3689 3690 if (bp->flags & BNXT_FLAG_CHIP_P5) 3691 goto vnic_skip_grps; 3692 3693 if (vnic->flags & BNXT_VNIC_RSS_FLAG) 3694 max_rings = bp->rx_nr_rings; 3695 else 3696 max_rings = 1; 3697 3698 vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL); 3699 if (!vnic->fw_grp_ids) { 3700 rc = -ENOMEM; 3701 goto out; 3702 } 3703 vnic_skip_grps: 3704 if ((bp->flags & BNXT_FLAG_NEW_RSS_CAP) && 3705 !(vnic->flags & BNXT_VNIC_RSS_FLAG)) 3706 continue; 3707 3708 /* Allocate rss table and hash key */ 3709 size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16)); 3710 if (bp->flags & BNXT_FLAG_CHIP_P5) 3711 size = L1_CACHE_ALIGN(BNXT_MAX_RSS_TABLE_SIZE_P5); 3712 3713 vnic->rss_table_size = size + HW_HASH_KEY_SIZE; 3714 vnic->rss_table = dma_alloc_coherent(&pdev->dev, 3715 vnic->rss_table_size, 3716 &vnic->rss_table_dma_addr, 3717 GFP_KERNEL); 3718 if (!vnic->rss_table) { 3719 rc = -ENOMEM; 3720 goto out; 3721 } 3722 3723 vnic->rss_hash_key = ((void *)vnic->rss_table) + size; 3724 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size; 3725 } 3726 return 0; 3727 3728 out: 3729 return rc; 3730 } 3731 3732 static void bnxt_free_hwrm_resources(struct bnxt *bp) 3733 { 3734 struct pci_dev *pdev = bp->pdev; 3735 3736 if (bp->hwrm_cmd_resp_addr) { 3737 dma_free_coherent(&pdev->dev, PAGE_SIZE, bp->hwrm_cmd_resp_addr, 3738 bp->hwrm_cmd_resp_dma_addr); 3739 bp->hwrm_cmd_resp_addr = NULL; 3740 } 3741 3742 if (bp->hwrm_cmd_kong_resp_addr) { 3743 dma_free_coherent(&pdev->dev, PAGE_SIZE, 3744 bp->hwrm_cmd_kong_resp_addr, 3745 bp->hwrm_cmd_kong_resp_dma_addr); 3746 bp->hwrm_cmd_kong_resp_addr = NULL; 3747 } 3748 } 3749 3750 static int bnxt_alloc_kong_hwrm_resources(struct bnxt *bp) 3751 { 3752 struct pci_dev *pdev = bp->pdev; 3753 3754 if (bp->hwrm_cmd_kong_resp_addr) 3755 return 0; 3756 3757 bp->hwrm_cmd_kong_resp_addr = 3758 dma_alloc_coherent(&pdev->dev, PAGE_SIZE, 3759 &bp->hwrm_cmd_kong_resp_dma_addr, 3760 GFP_KERNEL); 3761 if (!bp->hwrm_cmd_kong_resp_addr) 3762 return -ENOMEM; 3763 3764 return 0; 3765 } 3766 3767 static int bnxt_alloc_hwrm_resources(struct bnxt *bp) 3768 { 3769 struct pci_dev *pdev = bp->pdev; 3770 3771 bp->hwrm_cmd_resp_addr = dma_alloc_coherent(&pdev->dev, PAGE_SIZE, 3772 &bp->hwrm_cmd_resp_dma_addr, 3773 GFP_KERNEL); 3774 if (!bp->hwrm_cmd_resp_addr) 3775 return -ENOMEM; 3776 3777 return 0; 3778 } 3779 3780 static void bnxt_free_hwrm_short_cmd_req(struct bnxt *bp) 3781 { 3782 if (bp->hwrm_short_cmd_req_addr) { 3783 struct pci_dev *pdev = bp->pdev; 3784 3785 dma_free_coherent(&pdev->dev, bp->hwrm_max_ext_req_len, 3786 bp->hwrm_short_cmd_req_addr, 3787 bp->hwrm_short_cmd_req_dma_addr); 3788 bp->hwrm_short_cmd_req_addr = NULL; 3789 } 3790 } 3791 3792 static int bnxt_alloc_hwrm_short_cmd_req(struct bnxt *bp) 3793 { 3794 struct pci_dev *pdev = bp->pdev; 3795 3796 if (bp->hwrm_short_cmd_req_addr) 3797 return 0; 3798 3799 bp->hwrm_short_cmd_req_addr = 3800 dma_alloc_coherent(&pdev->dev, bp->hwrm_max_ext_req_len, 3801 &bp->hwrm_short_cmd_req_dma_addr, 3802 GFP_KERNEL); 3803 if (!bp->hwrm_short_cmd_req_addr) 3804 return -ENOMEM; 3805 3806 return 0; 3807 } 3808 3809 static void bnxt_free_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats) 3810 { 3811 kfree(stats->hw_masks); 3812 stats->hw_masks = NULL; 3813 kfree(stats->sw_stats); 3814 stats->sw_stats = NULL; 3815 if (stats->hw_stats) { 3816 dma_free_coherent(&bp->pdev->dev, stats->len, stats->hw_stats, 3817 stats->hw_stats_map); 3818 stats->hw_stats = NULL; 3819 } 3820 } 3821 3822 static int bnxt_alloc_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats, 3823 bool alloc_masks) 3824 { 3825 stats->hw_stats = dma_alloc_coherent(&bp->pdev->dev, stats->len, 3826 &stats->hw_stats_map, GFP_KERNEL); 3827 if (!stats->hw_stats) 3828 return -ENOMEM; 3829 3830 stats->sw_stats = kzalloc(stats->len, GFP_KERNEL); 3831 if (!stats->sw_stats) 3832 goto stats_mem_err; 3833 3834 if (alloc_masks) { 3835 stats->hw_masks = kzalloc(stats->len, GFP_KERNEL); 3836 if (!stats->hw_masks) 3837 goto stats_mem_err; 3838 } 3839 return 0; 3840 3841 stats_mem_err: 3842 bnxt_free_stats_mem(bp, stats); 3843 return -ENOMEM; 3844 } 3845 3846 static void bnxt_fill_masks(u64 *mask_arr, u64 mask, int count) 3847 { 3848 int i; 3849 3850 for (i = 0; i < count; i++) 3851 mask_arr[i] = mask; 3852 } 3853 3854 static void bnxt_copy_hw_masks(u64 *mask_arr, __le64 *hw_mask_arr, int count) 3855 { 3856 int i; 3857 3858 for (i = 0; i < count; i++) 3859 mask_arr[i] = le64_to_cpu(hw_mask_arr[i]); 3860 } 3861 3862 static int bnxt_hwrm_func_qstat_ext(struct bnxt *bp, 3863 struct bnxt_stats_mem *stats) 3864 { 3865 struct hwrm_func_qstats_ext_output *resp = bp->hwrm_cmd_resp_addr; 3866 struct hwrm_func_qstats_ext_input req = {0}; 3867 __le64 *hw_masks; 3868 int rc; 3869 3870 if (!(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED) || 3871 !(bp->flags & BNXT_FLAG_CHIP_P5)) 3872 return -EOPNOTSUPP; 3873 3874 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QSTATS_EXT, -1, -1); 3875 req.fid = cpu_to_le16(0xffff); 3876 req.flags = FUNC_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK; 3877 mutex_lock(&bp->hwrm_cmd_lock); 3878 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3879 if (rc) 3880 goto qstat_exit; 3881 3882 hw_masks = &resp->rx_ucast_pkts; 3883 bnxt_copy_hw_masks(stats->hw_masks, hw_masks, stats->len / 8); 3884 3885 qstat_exit: 3886 mutex_unlock(&bp->hwrm_cmd_lock); 3887 return rc; 3888 } 3889 3890 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags); 3891 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags); 3892 3893 static void bnxt_init_stats(struct bnxt *bp) 3894 { 3895 struct bnxt_napi *bnapi = bp->bnapi[0]; 3896 struct bnxt_cp_ring_info *cpr; 3897 struct bnxt_stats_mem *stats; 3898 __le64 *rx_stats, *tx_stats; 3899 int rc, rx_count, tx_count; 3900 u64 *rx_masks, *tx_masks; 3901 u64 mask; 3902 u8 flags; 3903 3904 cpr = &bnapi->cp_ring; 3905 stats = &cpr->stats; 3906 rc = bnxt_hwrm_func_qstat_ext(bp, stats); 3907 if (rc) { 3908 if (bp->flags & BNXT_FLAG_CHIP_P5) 3909 mask = (1ULL << 48) - 1; 3910 else 3911 mask = -1ULL; 3912 bnxt_fill_masks(stats->hw_masks, mask, stats->len / 8); 3913 } 3914 if (bp->flags & BNXT_FLAG_PORT_STATS) { 3915 stats = &bp->port_stats; 3916 rx_stats = stats->hw_stats; 3917 rx_masks = stats->hw_masks; 3918 rx_count = sizeof(struct rx_port_stats) / 8; 3919 tx_stats = rx_stats + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 3920 tx_masks = rx_masks + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 3921 tx_count = sizeof(struct tx_port_stats) / 8; 3922 3923 flags = PORT_QSTATS_REQ_FLAGS_COUNTER_MASK; 3924 rc = bnxt_hwrm_port_qstats(bp, flags); 3925 if (rc) { 3926 mask = (1ULL << 40) - 1; 3927 3928 bnxt_fill_masks(rx_masks, mask, rx_count); 3929 bnxt_fill_masks(tx_masks, mask, tx_count); 3930 } else { 3931 bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count); 3932 bnxt_copy_hw_masks(tx_masks, tx_stats, tx_count); 3933 bnxt_hwrm_port_qstats(bp, 0); 3934 } 3935 } 3936 if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) { 3937 stats = &bp->rx_port_stats_ext; 3938 rx_stats = stats->hw_stats; 3939 rx_masks = stats->hw_masks; 3940 rx_count = sizeof(struct rx_port_stats_ext) / 8; 3941 stats = &bp->tx_port_stats_ext; 3942 tx_stats = stats->hw_stats; 3943 tx_masks = stats->hw_masks; 3944 tx_count = sizeof(struct tx_port_stats_ext) / 8; 3945 3946 flags = PORT_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK; 3947 rc = bnxt_hwrm_port_qstats_ext(bp, flags); 3948 if (rc) { 3949 mask = (1ULL << 40) - 1; 3950 3951 bnxt_fill_masks(rx_masks, mask, rx_count); 3952 if (tx_stats) 3953 bnxt_fill_masks(tx_masks, mask, tx_count); 3954 } else { 3955 bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count); 3956 if (tx_stats) 3957 bnxt_copy_hw_masks(tx_masks, tx_stats, 3958 tx_count); 3959 bnxt_hwrm_port_qstats_ext(bp, 0); 3960 } 3961 } 3962 } 3963 3964 static void bnxt_free_port_stats(struct bnxt *bp) 3965 { 3966 bp->flags &= ~BNXT_FLAG_PORT_STATS; 3967 bp->flags &= ~BNXT_FLAG_PORT_STATS_EXT; 3968 3969 bnxt_free_stats_mem(bp, &bp->port_stats); 3970 bnxt_free_stats_mem(bp, &bp->rx_port_stats_ext); 3971 bnxt_free_stats_mem(bp, &bp->tx_port_stats_ext); 3972 } 3973 3974 static void bnxt_free_ring_stats(struct bnxt *bp) 3975 { 3976 int i; 3977 3978 if (!bp->bnapi) 3979 return; 3980 3981 for (i = 0; i < bp->cp_nr_rings; i++) { 3982 struct bnxt_napi *bnapi = bp->bnapi[i]; 3983 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 3984 3985 bnxt_free_stats_mem(bp, &cpr->stats); 3986 } 3987 } 3988 3989 static int bnxt_alloc_stats(struct bnxt *bp) 3990 { 3991 u32 size, i; 3992 int rc; 3993 3994 size = bp->hw_ring_stats_size; 3995 3996 for (i = 0; i < bp->cp_nr_rings; i++) { 3997 struct bnxt_napi *bnapi = bp->bnapi[i]; 3998 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 3999 4000 cpr->stats.len = size; 4001 rc = bnxt_alloc_stats_mem(bp, &cpr->stats, !i); 4002 if (rc) 4003 return rc; 4004 4005 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID; 4006 } 4007 4008 if (BNXT_VF(bp) || bp->chip_num == CHIP_NUM_58700) 4009 return 0; 4010 4011 if (bp->port_stats.hw_stats) 4012 goto alloc_ext_stats; 4013 4014 bp->port_stats.len = BNXT_PORT_STATS_SIZE; 4015 rc = bnxt_alloc_stats_mem(bp, &bp->port_stats, true); 4016 if (rc) 4017 return rc; 4018 4019 bp->flags |= BNXT_FLAG_PORT_STATS; 4020 4021 alloc_ext_stats: 4022 /* Display extended statistics only if FW supports it */ 4023 if (bp->hwrm_spec_code < 0x10804 || bp->hwrm_spec_code == 0x10900) 4024 if (!(bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED)) 4025 return 0; 4026 4027 if (bp->rx_port_stats_ext.hw_stats) 4028 goto alloc_tx_ext_stats; 4029 4030 bp->rx_port_stats_ext.len = sizeof(struct rx_port_stats_ext); 4031 rc = bnxt_alloc_stats_mem(bp, &bp->rx_port_stats_ext, true); 4032 /* Extended stats are optional */ 4033 if (rc) 4034 return 0; 4035 4036 alloc_tx_ext_stats: 4037 if (bp->tx_port_stats_ext.hw_stats) 4038 return 0; 4039 4040 if (bp->hwrm_spec_code >= 0x10902 || 4041 (bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED)) { 4042 bp->tx_port_stats_ext.len = sizeof(struct tx_port_stats_ext); 4043 rc = bnxt_alloc_stats_mem(bp, &bp->tx_port_stats_ext, true); 4044 /* Extended stats are optional */ 4045 if (rc) 4046 return 0; 4047 } 4048 bp->flags |= BNXT_FLAG_PORT_STATS_EXT; 4049 return 0; 4050 } 4051 4052 static void bnxt_clear_ring_indices(struct bnxt *bp) 4053 { 4054 int i; 4055 4056 if (!bp->bnapi) 4057 return; 4058 4059 for (i = 0; i < bp->cp_nr_rings; i++) { 4060 struct bnxt_napi *bnapi = bp->bnapi[i]; 4061 struct bnxt_cp_ring_info *cpr; 4062 struct bnxt_rx_ring_info *rxr; 4063 struct bnxt_tx_ring_info *txr; 4064 4065 if (!bnapi) 4066 continue; 4067 4068 cpr = &bnapi->cp_ring; 4069 cpr->cp_raw_cons = 0; 4070 4071 txr = bnapi->tx_ring; 4072 if (txr) { 4073 txr->tx_prod = 0; 4074 txr->tx_cons = 0; 4075 } 4076 4077 rxr = bnapi->rx_ring; 4078 if (rxr) { 4079 rxr->rx_prod = 0; 4080 rxr->rx_agg_prod = 0; 4081 rxr->rx_sw_agg_prod = 0; 4082 rxr->rx_next_cons = 0; 4083 } 4084 } 4085 } 4086 4087 static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool irq_reinit) 4088 { 4089 #ifdef CONFIG_RFS_ACCEL 4090 int i; 4091 4092 /* Under rtnl_lock and all our NAPIs have been disabled. It's 4093 * safe to delete the hash table. 4094 */ 4095 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) { 4096 struct hlist_head *head; 4097 struct hlist_node *tmp; 4098 struct bnxt_ntuple_filter *fltr; 4099 4100 head = &bp->ntp_fltr_hash_tbl[i]; 4101 hlist_for_each_entry_safe(fltr, tmp, head, hash) { 4102 hlist_del(&fltr->hash); 4103 kfree(fltr); 4104 } 4105 } 4106 if (irq_reinit) { 4107 kfree(bp->ntp_fltr_bmap); 4108 bp->ntp_fltr_bmap = NULL; 4109 } 4110 bp->ntp_fltr_count = 0; 4111 #endif 4112 } 4113 4114 static int bnxt_alloc_ntp_fltrs(struct bnxt *bp) 4115 { 4116 #ifdef CONFIG_RFS_ACCEL 4117 int i, rc = 0; 4118 4119 if (!(bp->flags & BNXT_FLAG_RFS)) 4120 return 0; 4121 4122 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) 4123 INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]); 4124 4125 bp->ntp_fltr_count = 0; 4126 bp->ntp_fltr_bmap = kcalloc(BITS_TO_LONGS(BNXT_NTP_FLTR_MAX_FLTR), 4127 sizeof(long), 4128 GFP_KERNEL); 4129 4130 if (!bp->ntp_fltr_bmap) 4131 rc = -ENOMEM; 4132 4133 return rc; 4134 #else 4135 return 0; 4136 #endif 4137 } 4138 4139 static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init) 4140 { 4141 bnxt_free_vnic_attributes(bp); 4142 bnxt_free_tx_rings(bp); 4143 bnxt_free_rx_rings(bp); 4144 bnxt_free_cp_rings(bp); 4145 bnxt_free_ntp_fltrs(bp, irq_re_init); 4146 if (irq_re_init) { 4147 bnxt_free_ring_stats(bp); 4148 if (!(bp->fw_cap & BNXT_FW_CAP_PORT_STATS_NO_RESET) || 4149 test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 4150 bnxt_free_port_stats(bp); 4151 bnxt_free_ring_grps(bp); 4152 bnxt_free_vnics(bp); 4153 kfree(bp->tx_ring_map); 4154 bp->tx_ring_map = NULL; 4155 kfree(bp->tx_ring); 4156 bp->tx_ring = NULL; 4157 kfree(bp->rx_ring); 4158 bp->rx_ring = NULL; 4159 kfree(bp->bnapi); 4160 bp->bnapi = NULL; 4161 } else { 4162 bnxt_clear_ring_indices(bp); 4163 } 4164 } 4165 4166 static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init) 4167 { 4168 int i, j, rc, size, arr_size; 4169 void *bnapi; 4170 4171 if (irq_re_init) { 4172 /* Allocate bnapi mem pointer array and mem block for 4173 * all queues 4174 */ 4175 arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) * 4176 bp->cp_nr_rings); 4177 size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi)); 4178 bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL); 4179 if (!bnapi) 4180 return -ENOMEM; 4181 4182 bp->bnapi = bnapi; 4183 bnapi += arr_size; 4184 for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) { 4185 bp->bnapi[i] = bnapi; 4186 bp->bnapi[i]->index = i; 4187 bp->bnapi[i]->bp = bp; 4188 if (bp->flags & BNXT_FLAG_CHIP_P5) { 4189 struct bnxt_cp_ring_info *cpr = 4190 &bp->bnapi[i]->cp_ring; 4191 4192 cpr->cp_ring_struct.ring_mem.flags = 4193 BNXT_RMEM_RING_PTE_FLAG; 4194 } 4195 } 4196 4197 bp->rx_ring = kcalloc(bp->rx_nr_rings, 4198 sizeof(struct bnxt_rx_ring_info), 4199 GFP_KERNEL); 4200 if (!bp->rx_ring) 4201 return -ENOMEM; 4202 4203 for (i = 0; i < bp->rx_nr_rings; i++) { 4204 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 4205 4206 if (bp->flags & BNXT_FLAG_CHIP_P5) { 4207 rxr->rx_ring_struct.ring_mem.flags = 4208 BNXT_RMEM_RING_PTE_FLAG; 4209 rxr->rx_agg_ring_struct.ring_mem.flags = 4210 BNXT_RMEM_RING_PTE_FLAG; 4211 } 4212 rxr->bnapi = bp->bnapi[i]; 4213 bp->bnapi[i]->rx_ring = &bp->rx_ring[i]; 4214 } 4215 4216 bp->tx_ring = kcalloc(bp->tx_nr_rings, 4217 sizeof(struct bnxt_tx_ring_info), 4218 GFP_KERNEL); 4219 if (!bp->tx_ring) 4220 return -ENOMEM; 4221 4222 bp->tx_ring_map = kcalloc(bp->tx_nr_rings, sizeof(u16), 4223 GFP_KERNEL); 4224 4225 if (!bp->tx_ring_map) 4226 return -ENOMEM; 4227 4228 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 4229 j = 0; 4230 else 4231 j = bp->rx_nr_rings; 4232 4233 for (i = 0; i < bp->tx_nr_rings; i++, j++) { 4234 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 4235 4236 if (bp->flags & BNXT_FLAG_CHIP_P5) 4237 txr->tx_ring_struct.ring_mem.flags = 4238 BNXT_RMEM_RING_PTE_FLAG; 4239 txr->bnapi = bp->bnapi[j]; 4240 bp->bnapi[j]->tx_ring = txr; 4241 bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i; 4242 if (i >= bp->tx_nr_rings_xdp) { 4243 txr->txq_index = i - bp->tx_nr_rings_xdp; 4244 bp->bnapi[j]->tx_int = bnxt_tx_int; 4245 } else { 4246 bp->bnapi[j]->flags |= BNXT_NAPI_FLAG_XDP; 4247 bp->bnapi[j]->tx_int = bnxt_tx_int_xdp; 4248 } 4249 } 4250 4251 rc = bnxt_alloc_stats(bp); 4252 if (rc) 4253 goto alloc_mem_err; 4254 bnxt_init_stats(bp); 4255 4256 rc = bnxt_alloc_ntp_fltrs(bp); 4257 if (rc) 4258 goto alloc_mem_err; 4259 4260 rc = bnxt_alloc_vnics(bp); 4261 if (rc) 4262 goto alloc_mem_err; 4263 } 4264 4265 bnxt_init_ring_struct(bp); 4266 4267 rc = bnxt_alloc_rx_rings(bp); 4268 if (rc) 4269 goto alloc_mem_err; 4270 4271 rc = bnxt_alloc_tx_rings(bp); 4272 if (rc) 4273 goto alloc_mem_err; 4274 4275 rc = bnxt_alloc_cp_rings(bp); 4276 if (rc) 4277 goto alloc_mem_err; 4278 4279 bp->vnic_info[0].flags |= BNXT_VNIC_RSS_FLAG | BNXT_VNIC_MCAST_FLAG | 4280 BNXT_VNIC_UCAST_FLAG; 4281 rc = bnxt_alloc_vnic_attributes(bp); 4282 if (rc) 4283 goto alloc_mem_err; 4284 return 0; 4285 4286 alloc_mem_err: 4287 bnxt_free_mem(bp, true); 4288 return rc; 4289 } 4290 4291 static void bnxt_disable_int(struct bnxt *bp) 4292 { 4293 int i; 4294 4295 if (!bp->bnapi) 4296 return; 4297 4298 for (i = 0; i < bp->cp_nr_rings; i++) { 4299 struct bnxt_napi *bnapi = bp->bnapi[i]; 4300 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 4301 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 4302 4303 if (ring->fw_ring_id != INVALID_HW_RING_ID) 4304 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons); 4305 } 4306 } 4307 4308 static int bnxt_cp_num_to_irq_num(struct bnxt *bp, int n) 4309 { 4310 struct bnxt_napi *bnapi = bp->bnapi[n]; 4311 struct bnxt_cp_ring_info *cpr; 4312 4313 cpr = &bnapi->cp_ring; 4314 return cpr->cp_ring_struct.map_idx; 4315 } 4316 4317 static void bnxt_disable_int_sync(struct bnxt *bp) 4318 { 4319 int i; 4320 4321 if (!bp->irq_tbl) 4322 return; 4323 4324 atomic_inc(&bp->intr_sem); 4325 4326 bnxt_disable_int(bp); 4327 for (i = 0; i < bp->cp_nr_rings; i++) { 4328 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 4329 4330 synchronize_irq(bp->irq_tbl[map_idx].vector); 4331 } 4332 } 4333 4334 static void bnxt_enable_int(struct bnxt *bp) 4335 { 4336 int i; 4337 4338 atomic_set(&bp->intr_sem, 0); 4339 for (i = 0; i < bp->cp_nr_rings; i++) { 4340 struct bnxt_napi *bnapi = bp->bnapi[i]; 4341 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 4342 4343 bnxt_db_nq_arm(bp, &cpr->cp_db, cpr->cp_raw_cons); 4344 } 4345 } 4346 4347 void bnxt_hwrm_cmd_hdr_init(struct bnxt *bp, void *request, u16 req_type, 4348 u16 cmpl_ring, u16 target_id) 4349 { 4350 struct input *req = request; 4351 4352 req->req_type = cpu_to_le16(req_type); 4353 req->cmpl_ring = cpu_to_le16(cmpl_ring); 4354 req->target_id = cpu_to_le16(target_id); 4355 if (bnxt_kong_hwrm_message(bp, req)) 4356 req->resp_addr = cpu_to_le64(bp->hwrm_cmd_kong_resp_dma_addr); 4357 else 4358 req->resp_addr = cpu_to_le64(bp->hwrm_cmd_resp_dma_addr); 4359 } 4360 4361 static int bnxt_hwrm_to_stderr(u32 hwrm_err) 4362 { 4363 switch (hwrm_err) { 4364 case HWRM_ERR_CODE_SUCCESS: 4365 return 0; 4366 case HWRM_ERR_CODE_RESOURCE_LOCKED: 4367 return -EROFS; 4368 case HWRM_ERR_CODE_RESOURCE_ACCESS_DENIED: 4369 return -EACCES; 4370 case HWRM_ERR_CODE_RESOURCE_ALLOC_ERROR: 4371 return -ENOSPC; 4372 case HWRM_ERR_CODE_INVALID_PARAMS: 4373 case HWRM_ERR_CODE_INVALID_FLAGS: 4374 case HWRM_ERR_CODE_INVALID_ENABLES: 4375 case HWRM_ERR_CODE_UNSUPPORTED_TLV: 4376 case HWRM_ERR_CODE_UNSUPPORTED_OPTION_ERR: 4377 return -EINVAL; 4378 case HWRM_ERR_CODE_NO_BUFFER: 4379 return -ENOMEM; 4380 case HWRM_ERR_CODE_HOT_RESET_PROGRESS: 4381 case HWRM_ERR_CODE_BUSY: 4382 return -EAGAIN; 4383 case HWRM_ERR_CODE_CMD_NOT_SUPPORTED: 4384 return -EOPNOTSUPP; 4385 default: 4386 return -EIO; 4387 } 4388 } 4389 4390 static int bnxt_hwrm_do_send_msg(struct bnxt *bp, void *msg, u32 msg_len, 4391 int timeout, bool silent) 4392 { 4393 int i, intr_process, rc, tmo_count; 4394 struct input *req = msg; 4395 u32 *data = msg; 4396 u8 *valid; 4397 u16 cp_ring_id, len = 0; 4398 struct hwrm_err_output *resp = bp->hwrm_cmd_resp_addr; 4399 u16 max_req_len = BNXT_HWRM_MAX_REQ_LEN; 4400 struct hwrm_short_input short_input = {0}; 4401 u32 doorbell_offset = BNXT_GRCPF_REG_CHIMP_COMM_TRIGGER; 4402 u32 bar_offset = BNXT_GRCPF_REG_CHIMP_COMM; 4403 u16 dst = BNXT_HWRM_CHNL_CHIMP; 4404 4405 if (BNXT_NO_FW_ACCESS(bp) && 4406 le16_to_cpu(req->req_type) != HWRM_FUNC_RESET) 4407 return -EBUSY; 4408 4409 if (msg_len > BNXT_HWRM_MAX_REQ_LEN) { 4410 if (msg_len > bp->hwrm_max_ext_req_len || 4411 !bp->hwrm_short_cmd_req_addr) 4412 return -EINVAL; 4413 } 4414 4415 if (bnxt_hwrm_kong_chnl(bp, req)) { 4416 dst = BNXT_HWRM_CHNL_KONG; 4417 bar_offset = BNXT_GRCPF_REG_KONG_COMM; 4418 doorbell_offset = BNXT_GRCPF_REG_KONG_COMM_TRIGGER; 4419 resp = bp->hwrm_cmd_kong_resp_addr; 4420 } 4421 4422 memset(resp, 0, PAGE_SIZE); 4423 cp_ring_id = le16_to_cpu(req->cmpl_ring); 4424 intr_process = (cp_ring_id == INVALID_HW_RING_ID) ? 0 : 1; 4425 4426 req->seq_id = cpu_to_le16(bnxt_get_hwrm_seq_id(bp, dst)); 4427 /* currently supports only one outstanding message */ 4428 if (intr_process) 4429 bp->hwrm_intr_seq_id = le16_to_cpu(req->seq_id); 4430 4431 if ((bp->fw_cap & BNXT_FW_CAP_SHORT_CMD) || 4432 msg_len > BNXT_HWRM_MAX_REQ_LEN) { 4433 void *short_cmd_req = bp->hwrm_short_cmd_req_addr; 4434 u16 max_msg_len; 4435 4436 /* Set boundary for maximum extended request length for short 4437 * cmd format. If passed up from device use the max supported 4438 * internal req length. 4439 */ 4440 max_msg_len = bp->hwrm_max_ext_req_len; 4441 4442 memcpy(short_cmd_req, req, msg_len); 4443 if (msg_len < max_msg_len) 4444 memset(short_cmd_req + msg_len, 0, 4445 max_msg_len - msg_len); 4446 4447 short_input.req_type = req->req_type; 4448 short_input.signature = 4449 cpu_to_le16(SHORT_REQ_SIGNATURE_SHORT_CMD); 4450 short_input.size = cpu_to_le16(msg_len); 4451 short_input.req_addr = 4452 cpu_to_le64(bp->hwrm_short_cmd_req_dma_addr); 4453 4454 data = (u32 *)&short_input; 4455 msg_len = sizeof(short_input); 4456 4457 /* Sync memory write before updating doorbell */ 4458 wmb(); 4459 4460 max_req_len = BNXT_HWRM_SHORT_REQ_LEN; 4461 } 4462 4463 /* Write request msg to hwrm channel */ 4464 __iowrite32_copy(bp->bar0 + bar_offset, data, msg_len / 4); 4465 4466 for (i = msg_len; i < max_req_len; i += 4) 4467 writel(0, bp->bar0 + bar_offset + i); 4468 4469 /* Ring channel doorbell */ 4470 writel(1, bp->bar0 + doorbell_offset); 4471 4472 if (!pci_is_enabled(bp->pdev)) 4473 return 0; 4474 4475 if (!timeout) 4476 timeout = DFLT_HWRM_CMD_TIMEOUT; 4477 /* Limit timeout to an upper limit */ 4478 timeout = min(timeout, HWRM_CMD_MAX_TIMEOUT); 4479 /* convert timeout to usec */ 4480 timeout *= 1000; 4481 4482 i = 0; 4483 /* Short timeout for the first few iterations: 4484 * number of loops = number of loops for short timeout + 4485 * number of loops for standard timeout. 4486 */ 4487 tmo_count = HWRM_SHORT_TIMEOUT_COUNTER; 4488 timeout = timeout - HWRM_SHORT_MIN_TIMEOUT * HWRM_SHORT_TIMEOUT_COUNTER; 4489 tmo_count += DIV_ROUND_UP(timeout, HWRM_MIN_TIMEOUT); 4490 4491 if (intr_process) { 4492 u16 seq_id = bp->hwrm_intr_seq_id; 4493 4494 /* Wait until hwrm response cmpl interrupt is processed */ 4495 while (bp->hwrm_intr_seq_id != (u16)~seq_id && 4496 i++ < tmo_count) { 4497 /* Abort the wait for completion if the FW health 4498 * check has failed. 4499 */ 4500 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) 4501 return -EBUSY; 4502 /* on first few passes, just barely sleep */ 4503 if (i < HWRM_SHORT_TIMEOUT_COUNTER) 4504 usleep_range(HWRM_SHORT_MIN_TIMEOUT, 4505 HWRM_SHORT_MAX_TIMEOUT); 4506 else 4507 usleep_range(HWRM_MIN_TIMEOUT, 4508 HWRM_MAX_TIMEOUT); 4509 } 4510 4511 if (bp->hwrm_intr_seq_id != (u16)~seq_id) { 4512 if (!silent) 4513 netdev_err(bp->dev, "Resp cmpl intr err msg: 0x%x\n", 4514 le16_to_cpu(req->req_type)); 4515 return -EBUSY; 4516 } 4517 len = le16_to_cpu(resp->resp_len); 4518 valid = ((u8 *)resp) + len - 1; 4519 } else { 4520 int j; 4521 4522 /* Check if response len is updated */ 4523 for (i = 0; i < tmo_count; i++) { 4524 /* Abort the wait for completion if the FW health 4525 * check has failed. 4526 */ 4527 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) 4528 return -EBUSY; 4529 len = le16_to_cpu(resp->resp_len); 4530 if (len) 4531 break; 4532 /* on first few passes, just barely sleep */ 4533 if (i < HWRM_SHORT_TIMEOUT_COUNTER) 4534 usleep_range(HWRM_SHORT_MIN_TIMEOUT, 4535 HWRM_SHORT_MAX_TIMEOUT); 4536 else 4537 usleep_range(HWRM_MIN_TIMEOUT, 4538 HWRM_MAX_TIMEOUT); 4539 } 4540 4541 if (i >= tmo_count) { 4542 if (!silent) 4543 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d\n", 4544 HWRM_TOTAL_TIMEOUT(i), 4545 le16_to_cpu(req->req_type), 4546 le16_to_cpu(req->seq_id), len); 4547 return -EBUSY; 4548 } 4549 4550 /* Last byte of resp contains valid bit */ 4551 valid = ((u8 *)resp) + len - 1; 4552 for (j = 0; j < HWRM_VALID_BIT_DELAY_USEC; j++) { 4553 /* make sure we read from updated DMA memory */ 4554 dma_rmb(); 4555 if (*valid) 4556 break; 4557 usleep_range(1, 5); 4558 } 4559 4560 if (j >= HWRM_VALID_BIT_DELAY_USEC) { 4561 if (!silent) 4562 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d v:%d\n", 4563 HWRM_TOTAL_TIMEOUT(i), 4564 le16_to_cpu(req->req_type), 4565 le16_to_cpu(req->seq_id), len, 4566 *valid); 4567 return -EBUSY; 4568 } 4569 } 4570 4571 /* Zero valid bit for compatibility. Valid bit in an older spec 4572 * may become a new field in a newer spec. We must make sure that 4573 * a new field not implemented by old spec will read zero. 4574 */ 4575 *valid = 0; 4576 rc = le16_to_cpu(resp->error_code); 4577 if (rc && !silent) 4578 netdev_err(bp->dev, "hwrm req_type 0x%x seq id 0x%x error 0x%x\n", 4579 le16_to_cpu(resp->req_type), 4580 le16_to_cpu(resp->seq_id), rc); 4581 return bnxt_hwrm_to_stderr(rc); 4582 } 4583 4584 int _hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout) 4585 { 4586 return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, false); 4587 } 4588 4589 int _hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len, 4590 int timeout) 4591 { 4592 return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true); 4593 } 4594 4595 int hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout) 4596 { 4597 int rc; 4598 4599 mutex_lock(&bp->hwrm_cmd_lock); 4600 rc = _hwrm_send_message(bp, msg, msg_len, timeout); 4601 mutex_unlock(&bp->hwrm_cmd_lock); 4602 return rc; 4603 } 4604 4605 int hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len, 4606 int timeout) 4607 { 4608 int rc; 4609 4610 mutex_lock(&bp->hwrm_cmd_lock); 4611 rc = bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true); 4612 mutex_unlock(&bp->hwrm_cmd_lock); 4613 return rc; 4614 } 4615 4616 int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp, unsigned long *bmap, int bmap_size, 4617 bool async_only) 4618 { 4619 struct hwrm_func_drv_rgtr_output *resp = bp->hwrm_cmd_resp_addr; 4620 struct hwrm_func_drv_rgtr_input req = {0}; 4621 DECLARE_BITMAP(async_events_bmap, 256); 4622 u32 *events = (u32 *)async_events_bmap; 4623 u32 flags; 4624 int rc, i; 4625 4626 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1); 4627 4628 req.enables = 4629 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE | 4630 FUNC_DRV_RGTR_REQ_ENABLES_VER | 4631 FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD); 4632 4633 req.os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX); 4634 flags = FUNC_DRV_RGTR_REQ_FLAGS_16BIT_VER_MODE; 4635 if (bp->fw_cap & BNXT_FW_CAP_HOT_RESET) 4636 flags |= FUNC_DRV_RGTR_REQ_FLAGS_HOT_RESET_SUPPORT; 4637 if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) 4638 flags |= FUNC_DRV_RGTR_REQ_FLAGS_ERROR_RECOVERY_SUPPORT | 4639 FUNC_DRV_RGTR_REQ_FLAGS_MASTER_SUPPORT; 4640 req.flags = cpu_to_le32(flags); 4641 req.ver_maj_8b = DRV_VER_MAJ; 4642 req.ver_min_8b = DRV_VER_MIN; 4643 req.ver_upd_8b = DRV_VER_UPD; 4644 req.ver_maj = cpu_to_le16(DRV_VER_MAJ); 4645 req.ver_min = cpu_to_le16(DRV_VER_MIN); 4646 req.ver_upd = cpu_to_le16(DRV_VER_UPD); 4647 4648 if (BNXT_PF(bp)) { 4649 u32 data[8]; 4650 int i; 4651 4652 memset(data, 0, sizeof(data)); 4653 for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) { 4654 u16 cmd = bnxt_vf_req_snif[i]; 4655 unsigned int bit, idx; 4656 4657 idx = cmd / 32; 4658 bit = cmd % 32; 4659 data[idx] |= 1 << bit; 4660 } 4661 4662 for (i = 0; i < 8; i++) 4663 req.vf_req_fwd[i] = cpu_to_le32(data[i]); 4664 4665 req.enables |= 4666 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD); 4667 } 4668 4669 if (bp->fw_cap & BNXT_FW_CAP_OVS_64BIT_HANDLE) 4670 req.flags |= cpu_to_le32( 4671 FUNC_DRV_RGTR_REQ_FLAGS_FLOW_HANDLE_64BIT_MODE); 4672 4673 memset(async_events_bmap, 0, sizeof(async_events_bmap)); 4674 for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++) { 4675 u16 event_id = bnxt_async_events_arr[i]; 4676 4677 if (event_id == ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY && 4678 !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) 4679 continue; 4680 __set_bit(bnxt_async_events_arr[i], async_events_bmap); 4681 } 4682 if (bmap && bmap_size) { 4683 for (i = 0; i < bmap_size; i++) { 4684 if (test_bit(i, bmap)) 4685 __set_bit(i, async_events_bmap); 4686 } 4687 } 4688 for (i = 0; i < 8; i++) 4689 req.async_event_fwd[i] |= cpu_to_le32(events[i]); 4690 4691 if (async_only) 4692 req.enables = 4693 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD); 4694 4695 mutex_lock(&bp->hwrm_cmd_lock); 4696 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4697 if (!rc) { 4698 set_bit(BNXT_STATE_DRV_REGISTERED, &bp->state); 4699 if (resp->flags & 4700 cpu_to_le32(FUNC_DRV_RGTR_RESP_FLAGS_IF_CHANGE_SUPPORTED)) 4701 bp->fw_cap |= BNXT_FW_CAP_IF_CHANGE; 4702 } 4703 mutex_unlock(&bp->hwrm_cmd_lock); 4704 return rc; 4705 } 4706 4707 static int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp) 4708 { 4709 struct hwrm_func_drv_unrgtr_input req = {0}; 4710 4711 if (!test_and_clear_bit(BNXT_STATE_DRV_REGISTERED, &bp->state)) 4712 return 0; 4713 4714 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_UNRGTR, -1, -1); 4715 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4716 } 4717 4718 static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type) 4719 { 4720 u32 rc = 0; 4721 struct hwrm_tunnel_dst_port_free_input req = {0}; 4722 4723 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_FREE, -1, -1); 4724 req.tunnel_type = tunnel_type; 4725 4726 switch (tunnel_type) { 4727 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN: 4728 req.tunnel_dst_port_id = cpu_to_le16(bp->vxlan_fw_dst_port_id); 4729 bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID; 4730 break; 4731 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE: 4732 req.tunnel_dst_port_id = cpu_to_le16(bp->nge_fw_dst_port_id); 4733 bp->nge_fw_dst_port_id = INVALID_HW_RING_ID; 4734 break; 4735 default: 4736 break; 4737 } 4738 4739 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4740 if (rc) 4741 netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n", 4742 rc); 4743 return rc; 4744 } 4745 4746 static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port, 4747 u8 tunnel_type) 4748 { 4749 u32 rc = 0; 4750 struct hwrm_tunnel_dst_port_alloc_input req = {0}; 4751 struct hwrm_tunnel_dst_port_alloc_output *resp = bp->hwrm_cmd_resp_addr; 4752 4753 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_ALLOC, -1, -1); 4754 4755 req.tunnel_type = tunnel_type; 4756 req.tunnel_dst_port_val = port; 4757 4758 mutex_lock(&bp->hwrm_cmd_lock); 4759 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4760 if (rc) { 4761 netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n", 4762 rc); 4763 goto err_out; 4764 } 4765 4766 switch (tunnel_type) { 4767 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN: 4768 bp->vxlan_fw_dst_port_id = 4769 le16_to_cpu(resp->tunnel_dst_port_id); 4770 break; 4771 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE: 4772 bp->nge_fw_dst_port_id = le16_to_cpu(resp->tunnel_dst_port_id); 4773 break; 4774 default: 4775 break; 4776 } 4777 4778 err_out: 4779 mutex_unlock(&bp->hwrm_cmd_lock); 4780 return rc; 4781 } 4782 4783 static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id) 4784 { 4785 struct hwrm_cfa_l2_set_rx_mask_input req = {0}; 4786 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 4787 4788 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_SET_RX_MASK, -1, -1); 4789 req.vnic_id = cpu_to_le32(vnic->fw_vnic_id); 4790 4791 req.num_mc_entries = cpu_to_le32(vnic->mc_list_count); 4792 req.mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping); 4793 req.mask = cpu_to_le32(vnic->rx_mask); 4794 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4795 } 4796 4797 #ifdef CONFIG_RFS_ACCEL 4798 static int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp, 4799 struct bnxt_ntuple_filter *fltr) 4800 { 4801 struct hwrm_cfa_ntuple_filter_free_input req = {0}; 4802 4803 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_FREE, -1, -1); 4804 req.ntuple_filter_id = fltr->filter_id; 4805 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4806 } 4807 4808 #define BNXT_NTP_FLTR_FLAGS \ 4809 (CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID | \ 4810 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE | \ 4811 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR | \ 4812 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE | \ 4813 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR | \ 4814 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK | \ 4815 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR | \ 4816 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK | \ 4817 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL | \ 4818 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT | \ 4819 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK | \ 4820 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT | \ 4821 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK | \ 4822 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID) 4823 4824 #define BNXT_NTP_TUNNEL_FLTR_FLAG \ 4825 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE 4826 4827 static int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp, 4828 struct bnxt_ntuple_filter *fltr) 4829 { 4830 struct hwrm_cfa_ntuple_filter_alloc_input req = {0}; 4831 struct hwrm_cfa_ntuple_filter_alloc_output *resp; 4832 struct flow_keys *keys = &fltr->fkeys; 4833 struct bnxt_vnic_info *vnic; 4834 u32 flags = 0; 4835 int rc = 0; 4836 4837 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_ALLOC, -1, -1); 4838 req.l2_filter_id = bp->vnic_info[0].fw_l2_filter_id[fltr->l2_fltr_idx]; 4839 4840 if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2) { 4841 flags = CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DEST_RFS_RING_IDX; 4842 req.dst_id = cpu_to_le16(fltr->rxq); 4843 } else { 4844 vnic = &bp->vnic_info[fltr->rxq + 1]; 4845 req.dst_id = cpu_to_le16(vnic->fw_vnic_id); 4846 } 4847 req.flags = cpu_to_le32(flags); 4848 req.enables = cpu_to_le32(BNXT_NTP_FLTR_FLAGS); 4849 4850 req.ethertype = htons(ETH_P_IP); 4851 memcpy(req.src_macaddr, fltr->src_mac_addr, ETH_ALEN); 4852 req.ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4; 4853 req.ip_protocol = keys->basic.ip_proto; 4854 4855 if (keys->basic.n_proto == htons(ETH_P_IPV6)) { 4856 int i; 4857 4858 req.ethertype = htons(ETH_P_IPV6); 4859 req.ip_addr_type = 4860 CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6; 4861 *(struct in6_addr *)&req.src_ipaddr[0] = 4862 keys->addrs.v6addrs.src; 4863 *(struct in6_addr *)&req.dst_ipaddr[0] = 4864 keys->addrs.v6addrs.dst; 4865 for (i = 0; i < 4; i++) { 4866 req.src_ipaddr_mask[i] = cpu_to_be32(0xffffffff); 4867 req.dst_ipaddr_mask[i] = cpu_to_be32(0xffffffff); 4868 } 4869 } else { 4870 req.src_ipaddr[0] = keys->addrs.v4addrs.src; 4871 req.src_ipaddr_mask[0] = cpu_to_be32(0xffffffff); 4872 req.dst_ipaddr[0] = keys->addrs.v4addrs.dst; 4873 req.dst_ipaddr_mask[0] = cpu_to_be32(0xffffffff); 4874 } 4875 if (keys->control.flags & FLOW_DIS_ENCAPSULATION) { 4876 req.enables |= cpu_to_le32(BNXT_NTP_TUNNEL_FLTR_FLAG); 4877 req.tunnel_type = 4878 CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL; 4879 } 4880 4881 req.src_port = keys->ports.src; 4882 req.src_port_mask = cpu_to_be16(0xffff); 4883 req.dst_port = keys->ports.dst; 4884 req.dst_port_mask = cpu_to_be16(0xffff); 4885 4886 mutex_lock(&bp->hwrm_cmd_lock); 4887 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4888 if (!rc) { 4889 resp = bnxt_get_hwrm_resp_addr(bp, &req); 4890 fltr->filter_id = resp->ntuple_filter_id; 4891 } 4892 mutex_unlock(&bp->hwrm_cmd_lock); 4893 return rc; 4894 } 4895 #endif 4896 4897 static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx, 4898 u8 *mac_addr) 4899 { 4900 u32 rc = 0; 4901 struct hwrm_cfa_l2_filter_alloc_input req = {0}; 4902 struct hwrm_cfa_l2_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr; 4903 4904 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_ALLOC, -1, -1); 4905 req.flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX); 4906 if (!BNXT_CHIP_TYPE_NITRO_A0(bp)) 4907 req.flags |= 4908 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST); 4909 req.dst_id = cpu_to_le16(bp->vnic_info[vnic_id].fw_vnic_id); 4910 req.enables = 4911 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR | 4912 CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID | 4913 CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK); 4914 memcpy(req.l2_addr, mac_addr, ETH_ALEN); 4915 req.l2_addr_mask[0] = 0xff; 4916 req.l2_addr_mask[1] = 0xff; 4917 req.l2_addr_mask[2] = 0xff; 4918 req.l2_addr_mask[3] = 0xff; 4919 req.l2_addr_mask[4] = 0xff; 4920 req.l2_addr_mask[5] = 0xff; 4921 4922 mutex_lock(&bp->hwrm_cmd_lock); 4923 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4924 if (!rc) 4925 bp->vnic_info[vnic_id].fw_l2_filter_id[idx] = 4926 resp->l2_filter_id; 4927 mutex_unlock(&bp->hwrm_cmd_lock); 4928 return rc; 4929 } 4930 4931 static int bnxt_hwrm_clear_vnic_filter(struct bnxt *bp) 4932 { 4933 u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */ 4934 int rc = 0; 4935 4936 /* Any associated ntuple filters will also be cleared by firmware. */ 4937 mutex_lock(&bp->hwrm_cmd_lock); 4938 for (i = 0; i < num_of_vnics; i++) { 4939 struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; 4940 4941 for (j = 0; j < vnic->uc_filter_count; j++) { 4942 struct hwrm_cfa_l2_filter_free_input req = {0}; 4943 4944 bnxt_hwrm_cmd_hdr_init(bp, &req, 4945 HWRM_CFA_L2_FILTER_FREE, -1, -1); 4946 4947 req.l2_filter_id = vnic->fw_l2_filter_id[j]; 4948 4949 rc = _hwrm_send_message(bp, &req, sizeof(req), 4950 HWRM_CMD_TIMEOUT); 4951 } 4952 vnic->uc_filter_count = 0; 4953 } 4954 mutex_unlock(&bp->hwrm_cmd_lock); 4955 4956 return rc; 4957 } 4958 4959 static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags) 4960 { 4961 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 4962 u16 max_aggs = VNIC_TPA_CFG_REQ_MAX_AGGS_MAX; 4963 struct hwrm_vnic_tpa_cfg_input req = {0}; 4964 4965 if (vnic->fw_vnic_id == INVALID_HW_RING_ID) 4966 return 0; 4967 4968 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_TPA_CFG, -1, -1); 4969 4970 if (tpa_flags) { 4971 u16 mss = bp->dev->mtu - 40; 4972 u32 nsegs, n, segs = 0, flags; 4973 4974 flags = VNIC_TPA_CFG_REQ_FLAGS_TPA | 4975 VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA | 4976 VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE | 4977 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN | 4978 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ; 4979 if (tpa_flags & BNXT_FLAG_GRO) 4980 flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO; 4981 4982 req.flags = cpu_to_le32(flags); 4983 4984 req.enables = 4985 cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS | 4986 VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS | 4987 VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN); 4988 4989 /* Number of segs are log2 units, and first packet is not 4990 * included as part of this units. 4991 */ 4992 if (mss <= BNXT_RX_PAGE_SIZE) { 4993 n = BNXT_RX_PAGE_SIZE / mss; 4994 nsegs = (MAX_SKB_FRAGS - 1) * n; 4995 } else { 4996 n = mss / BNXT_RX_PAGE_SIZE; 4997 if (mss & (BNXT_RX_PAGE_SIZE - 1)) 4998 n++; 4999 nsegs = (MAX_SKB_FRAGS - n) / n; 5000 } 5001 5002 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5003 segs = MAX_TPA_SEGS_P5; 5004 max_aggs = bp->max_tpa; 5005 } else { 5006 segs = ilog2(nsegs); 5007 } 5008 req.max_agg_segs = cpu_to_le16(segs); 5009 req.max_aggs = cpu_to_le16(max_aggs); 5010 5011 req.min_agg_len = cpu_to_le32(512); 5012 } 5013 req.vnic_id = cpu_to_le16(vnic->fw_vnic_id); 5014 5015 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5016 } 5017 5018 static u16 bnxt_cp_ring_from_grp(struct bnxt *bp, struct bnxt_ring_struct *ring) 5019 { 5020 struct bnxt_ring_grp_info *grp_info; 5021 5022 grp_info = &bp->grp_info[ring->grp_idx]; 5023 return grp_info->cp_fw_ring_id; 5024 } 5025 5026 static u16 bnxt_cp_ring_for_rx(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) 5027 { 5028 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5029 struct bnxt_napi *bnapi = rxr->bnapi; 5030 struct bnxt_cp_ring_info *cpr; 5031 5032 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_RX_HDL]; 5033 return cpr->cp_ring_struct.fw_ring_id; 5034 } else { 5035 return bnxt_cp_ring_from_grp(bp, &rxr->rx_ring_struct); 5036 } 5037 } 5038 5039 static u16 bnxt_cp_ring_for_tx(struct bnxt *bp, struct bnxt_tx_ring_info *txr) 5040 { 5041 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5042 struct bnxt_napi *bnapi = txr->bnapi; 5043 struct bnxt_cp_ring_info *cpr; 5044 5045 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_TX_HDL]; 5046 return cpr->cp_ring_struct.fw_ring_id; 5047 } else { 5048 return bnxt_cp_ring_from_grp(bp, &txr->tx_ring_struct); 5049 } 5050 } 5051 5052 static int bnxt_alloc_rss_indir_tbl(struct bnxt *bp) 5053 { 5054 int entries; 5055 5056 if (bp->flags & BNXT_FLAG_CHIP_P5) 5057 entries = BNXT_MAX_RSS_TABLE_ENTRIES_P5; 5058 else 5059 entries = HW_HASH_INDEX_SIZE; 5060 5061 bp->rss_indir_tbl_entries = entries; 5062 bp->rss_indir_tbl = kmalloc_array(entries, sizeof(*bp->rss_indir_tbl), 5063 GFP_KERNEL); 5064 if (!bp->rss_indir_tbl) 5065 return -ENOMEM; 5066 return 0; 5067 } 5068 5069 static void bnxt_set_dflt_rss_indir_tbl(struct bnxt *bp) 5070 { 5071 u16 max_rings, max_entries, pad, i; 5072 5073 if (!bp->rx_nr_rings) 5074 return; 5075 5076 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 5077 max_rings = bp->rx_nr_rings - 1; 5078 else 5079 max_rings = bp->rx_nr_rings; 5080 5081 max_entries = bnxt_get_rxfh_indir_size(bp->dev); 5082 5083 for (i = 0; i < max_entries; i++) 5084 bp->rss_indir_tbl[i] = ethtool_rxfh_indir_default(i, max_rings); 5085 5086 pad = bp->rss_indir_tbl_entries - max_entries; 5087 if (pad) 5088 memset(&bp->rss_indir_tbl[i], 0, pad * sizeof(u16)); 5089 } 5090 5091 static u16 bnxt_get_max_rss_ring(struct bnxt *bp) 5092 { 5093 u16 i, tbl_size, max_ring = 0; 5094 5095 if (!bp->rss_indir_tbl) 5096 return 0; 5097 5098 tbl_size = bnxt_get_rxfh_indir_size(bp->dev); 5099 for (i = 0; i < tbl_size; i++) 5100 max_ring = max(max_ring, bp->rss_indir_tbl[i]); 5101 return max_ring; 5102 } 5103 5104 int bnxt_get_nr_rss_ctxs(struct bnxt *bp, int rx_rings) 5105 { 5106 if (bp->flags & BNXT_FLAG_CHIP_P5) 5107 return DIV_ROUND_UP(rx_rings, BNXT_RSS_TABLE_ENTRIES_P5); 5108 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 5109 return 2; 5110 return 1; 5111 } 5112 5113 static void __bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic) 5114 { 5115 bool no_rss = !(vnic->flags & BNXT_VNIC_RSS_FLAG); 5116 u16 i, j; 5117 5118 /* Fill the RSS indirection table with ring group ids */ 5119 for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++) { 5120 if (!no_rss) 5121 j = bp->rss_indir_tbl[i]; 5122 vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]); 5123 } 5124 } 5125 5126 static void __bnxt_fill_hw_rss_tbl_p5(struct bnxt *bp, 5127 struct bnxt_vnic_info *vnic) 5128 { 5129 __le16 *ring_tbl = vnic->rss_table; 5130 struct bnxt_rx_ring_info *rxr; 5131 u16 tbl_size, i; 5132 5133 tbl_size = bnxt_get_rxfh_indir_size(bp->dev); 5134 5135 for (i = 0; i < tbl_size; i++) { 5136 u16 ring_id, j; 5137 5138 j = bp->rss_indir_tbl[i]; 5139 rxr = &bp->rx_ring[j]; 5140 5141 ring_id = rxr->rx_ring_struct.fw_ring_id; 5142 *ring_tbl++ = cpu_to_le16(ring_id); 5143 ring_id = bnxt_cp_ring_for_rx(bp, rxr); 5144 *ring_tbl++ = cpu_to_le16(ring_id); 5145 } 5146 } 5147 5148 static void bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic) 5149 { 5150 if (bp->flags & BNXT_FLAG_CHIP_P5) 5151 __bnxt_fill_hw_rss_tbl_p5(bp, vnic); 5152 else 5153 __bnxt_fill_hw_rss_tbl(bp, vnic); 5154 } 5155 5156 static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss) 5157 { 5158 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5159 struct hwrm_vnic_rss_cfg_input req = {0}; 5160 5161 if ((bp->flags & BNXT_FLAG_CHIP_P5) || 5162 vnic->fw_rss_cos_lb_ctx[0] == INVALID_HW_RING_ID) 5163 return 0; 5164 5165 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1); 5166 if (set_rss) { 5167 bnxt_fill_hw_rss_tbl(bp, vnic); 5168 req.hash_type = cpu_to_le32(bp->rss_hash_cfg); 5169 req.hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT; 5170 req.ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr); 5171 req.hash_key_tbl_addr = 5172 cpu_to_le64(vnic->rss_hash_key_dma_addr); 5173 } 5174 req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]); 5175 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5176 } 5177 5178 static int bnxt_hwrm_vnic_set_rss_p5(struct bnxt *bp, u16 vnic_id, bool set_rss) 5179 { 5180 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5181 struct hwrm_vnic_rss_cfg_input req = {0}; 5182 dma_addr_t ring_tbl_map; 5183 u32 i, nr_ctxs; 5184 5185 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1); 5186 req.vnic_id = cpu_to_le16(vnic->fw_vnic_id); 5187 if (!set_rss) { 5188 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5189 return 0; 5190 } 5191 bnxt_fill_hw_rss_tbl(bp, vnic); 5192 req.hash_type = cpu_to_le32(bp->rss_hash_cfg); 5193 req.hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT; 5194 req.hash_key_tbl_addr = cpu_to_le64(vnic->rss_hash_key_dma_addr); 5195 ring_tbl_map = vnic->rss_table_dma_addr; 5196 nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings); 5197 for (i = 0; i < nr_ctxs; ring_tbl_map += BNXT_RSS_TABLE_SIZE_P5, i++) { 5198 int rc; 5199 5200 req.ring_grp_tbl_addr = cpu_to_le64(ring_tbl_map); 5201 req.ring_table_pair_index = i; 5202 req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[i]); 5203 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5204 if (rc) 5205 return rc; 5206 } 5207 return 0; 5208 } 5209 5210 static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id) 5211 { 5212 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5213 struct hwrm_vnic_plcmodes_cfg_input req = {0}; 5214 5215 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_PLCMODES_CFG, -1, -1); 5216 req.flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT | 5217 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 | 5218 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6); 5219 req.enables = 5220 cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID | 5221 VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID); 5222 /* thresholds not implemented in firmware yet */ 5223 req.jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh); 5224 req.hds_threshold = cpu_to_le16(bp->rx_copy_thresh); 5225 req.vnic_id = cpu_to_le32(vnic->fw_vnic_id); 5226 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5227 } 5228 5229 static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id, 5230 u16 ctx_idx) 5231 { 5232 struct hwrm_vnic_rss_cos_lb_ctx_free_input req = {0}; 5233 5234 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_FREE, -1, -1); 5235 req.rss_cos_lb_ctx_id = 5236 cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx]); 5237 5238 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5239 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID; 5240 } 5241 5242 static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp) 5243 { 5244 int i, j; 5245 5246 for (i = 0; i < bp->nr_vnics; i++) { 5247 struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; 5248 5249 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) { 5250 if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID) 5251 bnxt_hwrm_vnic_ctx_free_one(bp, i, j); 5252 } 5253 } 5254 bp->rsscos_nr_ctxs = 0; 5255 } 5256 5257 static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id, u16 ctx_idx) 5258 { 5259 int rc; 5260 struct hwrm_vnic_rss_cos_lb_ctx_alloc_input req = {0}; 5261 struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp = 5262 bp->hwrm_cmd_resp_addr; 5263 5264 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC, -1, 5265 -1); 5266 5267 mutex_lock(&bp->hwrm_cmd_lock); 5268 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5269 if (!rc) 5270 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = 5271 le16_to_cpu(resp->rss_cos_lb_ctx_id); 5272 mutex_unlock(&bp->hwrm_cmd_lock); 5273 5274 return rc; 5275 } 5276 5277 static u32 bnxt_get_roce_vnic_mode(struct bnxt *bp) 5278 { 5279 if (bp->flags & BNXT_FLAG_ROCE_MIRROR_CAP) 5280 return VNIC_CFG_REQ_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE; 5281 return VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE; 5282 } 5283 5284 int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id) 5285 { 5286 unsigned int ring = 0, grp_idx; 5287 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5288 struct hwrm_vnic_cfg_input req = {0}; 5289 u16 def_vlan = 0; 5290 5291 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_CFG, -1, -1); 5292 5293 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5294 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0]; 5295 5296 req.default_rx_ring_id = 5297 cpu_to_le16(rxr->rx_ring_struct.fw_ring_id); 5298 req.default_cmpl_ring_id = 5299 cpu_to_le16(bnxt_cp_ring_for_rx(bp, rxr)); 5300 req.enables = 5301 cpu_to_le32(VNIC_CFG_REQ_ENABLES_DEFAULT_RX_RING_ID | 5302 VNIC_CFG_REQ_ENABLES_DEFAULT_CMPL_RING_ID); 5303 goto vnic_mru; 5304 } 5305 req.enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP); 5306 /* Only RSS support for now TBD: COS & LB */ 5307 if (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID) { 5308 req.rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]); 5309 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE | 5310 VNIC_CFG_REQ_ENABLES_MRU); 5311 } else if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) { 5312 req.rss_rule = 5313 cpu_to_le16(bp->vnic_info[0].fw_rss_cos_lb_ctx[0]); 5314 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE | 5315 VNIC_CFG_REQ_ENABLES_MRU); 5316 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE); 5317 } else { 5318 req.rss_rule = cpu_to_le16(0xffff); 5319 } 5320 5321 if (BNXT_CHIP_TYPE_NITRO_A0(bp) && 5322 (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID)) { 5323 req.cos_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[1]); 5324 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_COS_RULE); 5325 } else { 5326 req.cos_rule = cpu_to_le16(0xffff); 5327 } 5328 5329 if (vnic->flags & BNXT_VNIC_RSS_FLAG) 5330 ring = 0; 5331 else if (vnic->flags & BNXT_VNIC_RFS_FLAG) 5332 ring = vnic_id - 1; 5333 else if ((vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp)) 5334 ring = bp->rx_nr_rings - 1; 5335 5336 grp_idx = bp->rx_ring[ring].bnapi->index; 5337 req.dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id); 5338 req.lb_rule = cpu_to_le16(0xffff); 5339 vnic_mru: 5340 req.mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + VLAN_HLEN); 5341 5342 req.vnic_id = cpu_to_le16(vnic->fw_vnic_id); 5343 #ifdef CONFIG_BNXT_SRIOV 5344 if (BNXT_VF(bp)) 5345 def_vlan = bp->vf.vlan; 5346 #endif 5347 if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan) 5348 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE); 5349 if (!vnic_id && bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP)) 5350 req.flags |= cpu_to_le32(bnxt_get_roce_vnic_mode(bp)); 5351 5352 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5353 } 5354 5355 static void bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id) 5356 { 5357 if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) { 5358 struct hwrm_vnic_free_input req = {0}; 5359 5360 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_FREE, -1, -1); 5361 req.vnic_id = 5362 cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id); 5363 5364 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5365 bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID; 5366 } 5367 } 5368 5369 static void bnxt_hwrm_vnic_free(struct bnxt *bp) 5370 { 5371 u16 i; 5372 5373 for (i = 0; i < bp->nr_vnics; i++) 5374 bnxt_hwrm_vnic_free_one(bp, i); 5375 } 5376 5377 static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id, 5378 unsigned int start_rx_ring_idx, 5379 unsigned int nr_rings) 5380 { 5381 int rc = 0; 5382 unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings; 5383 struct hwrm_vnic_alloc_input req = {0}; 5384 struct hwrm_vnic_alloc_output *resp = bp->hwrm_cmd_resp_addr; 5385 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5386 5387 if (bp->flags & BNXT_FLAG_CHIP_P5) 5388 goto vnic_no_ring_grps; 5389 5390 /* map ring groups to this vnic */ 5391 for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) { 5392 grp_idx = bp->rx_ring[i].bnapi->index; 5393 if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) { 5394 netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n", 5395 j, nr_rings); 5396 break; 5397 } 5398 vnic->fw_grp_ids[j] = bp->grp_info[grp_idx].fw_grp_id; 5399 } 5400 5401 vnic_no_ring_grps: 5402 for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++) 5403 vnic->fw_rss_cos_lb_ctx[i] = INVALID_HW_RING_ID; 5404 if (vnic_id == 0) 5405 req.flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT); 5406 5407 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_ALLOC, -1, -1); 5408 5409 mutex_lock(&bp->hwrm_cmd_lock); 5410 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5411 if (!rc) 5412 vnic->fw_vnic_id = le32_to_cpu(resp->vnic_id); 5413 mutex_unlock(&bp->hwrm_cmd_lock); 5414 return rc; 5415 } 5416 5417 static int bnxt_hwrm_vnic_qcaps(struct bnxt *bp) 5418 { 5419 struct hwrm_vnic_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 5420 struct hwrm_vnic_qcaps_input req = {0}; 5421 int rc; 5422 5423 bp->hw_ring_stats_size = sizeof(struct ctx_hw_stats); 5424 bp->flags &= ~(BNXT_FLAG_NEW_RSS_CAP | BNXT_FLAG_ROCE_MIRROR_CAP); 5425 if (bp->hwrm_spec_code < 0x10600) 5426 return 0; 5427 5428 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_QCAPS, -1, -1); 5429 mutex_lock(&bp->hwrm_cmd_lock); 5430 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5431 if (!rc) { 5432 u32 flags = le32_to_cpu(resp->flags); 5433 5434 if (!(bp->flags & BNXT_FLAG_CHIP_P5) && 5435 (flags & VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP)) 5436 bp->flags |= BNXT_FLAG_NEW_RSS_CAP; 5437 if (flags & 5438 VNIC_QCAPS_RESP_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_CAP) 5439 bp->flags |= BNXT_FLAG_ROCE_MIRROR_CAP; 5440 5441 /* Older P5 fw before EXT_HW_STATS support did not set 5442 * VLAN_STRIP_CAP properly. 5443 */ 5444 if ((flags & VNIC_QCAPS_RESP_FLAGS_VLAN_STRIP_CAP) || 5445 (BNXT_CHIP_P5_THOR(bp) && 5446 !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))) 5447 bp->fw_cap |= BNXT_FW_CAP_VLAN_RX_STRIP; 5448 bp->max_tpa_v2 = le16_to_cpu(resp->max_aggs_supported); 5449 if (bp->max_tpa_v2) { 5450 if (BNXT_CHIP_P5_THOR(bp)) 5451 bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P5; 5452 else 5453 bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P5_SR2; 5454 } 5455 } 5456 mutex_unlock(&bp->hwrm_cmd_lock); 5457 return rc; 5458 } 5459 5460 static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp) 5461 { 5462 u16 i; 5463 u32 rc = 0; 5464 5465 if (bp->flags & BNXT_FLAG_CHIP_P5) 5466 return 0; 5467 5468 mutex_lock(&bp->hwrm_cmd_lock); 5469 for (i = 0; i < bp->rx_nr_rings; i++) { 5470 struct hwrm_ring_grp_alloc_input req = {0}; 5471 struct hwrm_ring_grp_alloc_output *resp = 5472 bp->hwrm_cmd_resp_addr; 5473 unsigned int grp_idx = bp->rx_ring[i].bnapi->index; 5474 5475 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_ALLOC, -1, -1); 5476 5477 req.cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id); 5478 req.rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id); 5479 req.ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id); 5480 req.sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx); 5481 5482 rc = _hwrm_send_message(bp, &req, sizeof(req), 5483 HWRM_CMD_TIMEOUT); 5484 if (rc) 5485 break; 5486 5487 bp->grp_info[grp_idx].fw_grp_id = 5488 le32_to_cpu(resp->ring_group_id); 5489 } 5490 mutex_unlock(&bp->hwrm_cmd_lock); 5491 return rc; 5492 } 5493 5494 static void bnxt_hwrm_ring_grp_free(struct bnxt *bp) 5495 { 5496 u16 i; 5497 struct hwrm_ring_grp_free_input req = {0}; 5498 5499 if (!bp->grp_info || (bp->flags & BNXT_FLAG_CHIP_P5)) 5500 return; 5501 5502 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_FREE, -1, -1); 5503 5504 mutex_lock(&bp->hwrm_cmd_lock); 5505 for (i = 0; i < bp->cp_nr_rings; i++) { 5506 if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID) 5507 continue; 5508 req.ring_group_id = 5509 cpu_to_le32(bp->grp_info[i].fw_grp_id); 5510 5511 _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5512 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID; 5513 } 5514 mutex_unlock(&bp->hwrm_cmd_lock); 5515 } 5516 5517 static int hwrm_ring_alloc_send_msg(struct bnxt *bp, 5518 struct bnxt_ring_struct *ring, 5519 u32 ring_type, u32 map_index) 5520 { 5521 int rc = 0, err = 0; 5522 struct hwrm_ring_alloc_input req = {0}; 5523 struct hwrm_ring_alloc_output *resp = bp->hwrm_cmd_resp_addr; 5524 struct bnxt_ring_mem_info *rmem = &ring->ring_mem; 5525 struct bnxt_ring_grp_info *grp_info; 5526 u16 ring_id; 5527 5528 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_ALLOC, -1, -1); 5529 5530 req.enables = 0; 5531 if (rmem->nr_pages > 1) { 5532 req.page_tbl_addr = cpu_to_le64(rmem->pg_tbl_map); 5533 /* Page size is in log2 units */ 5534 req.page_size = BNXT_PAGE_SHIFT; 5535 req.page_tbl_depth = 1; 5536 } else { 5537 req.page_tbl_addr = cpu_to_le64(rmem->dma_arr[0]); 5538 } 5539 req.fbo = 0; 5540 /* Association of ring index with doorbell index and MSIX number */ 5541 req.logical_id = cpu_to_le16(map_index); 5542 5543 switch (ring_type) { 5544 case HWRM_RING_ALLOC_TX: { 5545 struct bnxt_tx_ring_info *txr; 5546 5547 txr = container_of(ring, struct bnxt_tx_ring_info, 5548 tx_ring_struct); 5549 req.ring_type = RING_ALLOC_REQ_RING_TYPE_TX; 5550 /* Association of transmit ring with completion ring */ 5551 grp_info = &bp->grp_info[ring->grp_idx]; 5552 req.cmpl_ring_id = cpu_to_le16(bnxt_cp_ring_for_tx(bp, txr)); 5553 req.length = cpu_to_le32(bp->tx_ring_mask + 1); 5554 req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx); 5555 req.queue_id = cpu_to_le16(ring->queue_id); 5556 break; 5557 } 5558 case HWRM_RING_ALLOC_RX: 5559 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX; 5560 req.length = cpu_to_le32(bp->rx_ring_mask + 1); 5561 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5562 u16 flags = 0; 5563 5564 /* Association of rx ring with stats context */ 5565 grp_info = &bp->grp_info[ring->grp_idx]; 5566 req.rx_buf_size = cpu_to_le16(bp->rx_buf_use_size); 5567 req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx); 5568 req.enables |= cpu_to_le32( 5569 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID); 5570 if (NET_IP_ALIGN == 2) 5571 flags = RING_ALLOC_REQ_FLAGS_RX_SOP_PAD; 5572 req.flags = cpu_to_le16(flags); 5573 } 5574 break; 5575 case HWRM_RING_ALLOC_AGG: 5576 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5577 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX_AGG; 5578 /* Association of agg ring with rx ring */ 5579 grp_info = &bp->grp_info[ring->grp_idx]; 5580 req.rx_ring_id = cpu_to_le16(grp_info->rx_fw_ring_id); 5581 req.rx_buf_size = cpu_to_le16(BNXT_RX_PAGE_SIZE); 5582 req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx); 5583 req.enables |= cpu_to_le32( 5584 RING_ALLOC_REQ_ENABLES_RX_RING_ID_VALID | 5585 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID); 5586 } else { 5587 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX; 5588 } 5589 req.length = cpu_to_le32(bp->rx_agg_ring_mask + 1); 5590 break; 5591 case HWRM_RING_ALLOC_CMPL: 5592 req.ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL; 5593 req.length = cpu_to_le32(bp->cp_ring_mask + 1); 5594 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5595 /* Association of cp ring with nq */ 5596 grp_info = &bp->grp_info[map_index]; 5597 req.nq_ring_id = cpu_to_le16(grp_info->cp_fw_ring_id); 5598 req.cq_handle = cpu_to_le64(ring->handle); 5599 req.enables |= cpu_to_le32( 5600 RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID); 5601 } else if (bp->flags & BNXT_FLAG_USING_MSIX) { 5602 req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX; 5603 } 5604 break; 5605 case HWRM_RING_ALLOC_NQ: 5606 req.ring_type = RING_ALLOC_REQ_RING_TYPE_NQ; 5607 req.length = cpu_to_le32(bp->cp_ring_mask + 1); 5608 if (bp->flags & BNXT_FLAG_USING_MSIX) 5609 req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX; 5610 break; 5611 default: 5612 netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n", 5613 ring_type); 5614 return -1; 5615 } 5616 5617 mutex_lock(&bp->hwrm_cmd_lock); 5618 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5619 err = le16_to_cpu(resp->error_code); 5620 ring_id = le16_to_cpu(resp->ring_id); 5621 mutex_unlock(&bp->hwrm_cmd_lock); 5622 5623 if (rc || err) { 5624 netdev_err(bp->dev, "hwrm_ring_alloc type %d failed. rc:%x err:%x\n", 5625 ring_type, rc, err); 5626 return -EIO; 5627 } 5628 ring->fw_ring_id = ring_id; 5629 return rc; 5630 } 5631 5632 static int bnxt_hwrm_set_async_event_cr(struct bnxt *bp, int idx) 5633 { 5634 int rc; 5635 5636 if (BNXT_PF(bp)) { 5637 struct hwrm_func_cfg_input req = {0}; 5638 5639 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1); 5640 req.fid = cpu_to_le16(0xffff); 5641 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR); 5642 req.async_event_cr = cpu_to_le16(idx); 5643 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5644 } else { 5645 struct hwrm_func_vf_cfg_input req = {0}; 5646 5647 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_VF_CFG, -1, -1); 5648 req.enables = 5649 cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR); 5650 req.async_event_cr = cpu_to_le16(idx); 5651 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5652 } 5653 return rc; 5654 } 5655 5656 static void bnxt_set_db(struct bnxt *bp, struct bnxt_db_info *db, u32 ring_type, 5657 u32 map_idx, u32 xid) 5658 { 5659 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5660 if (BNXT_PF(bp)) 5661 db->doorbell = bp->bar1 + DB_PF_OFFSET_P5; 5662 else 5663 db->doorbell = bp->bar1 + DB_VF_OFFSET_P5; 5664 switch (ring_type) { 5665 case HWRM_RING_ALLOC_TX: 5666 db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SQ; 5667 break; 5668 case HWRM_RING_ALLOC_RX: 5669 case HWRM_RING_ALLOC_AGG: 5670 db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SRQ; 5671 break; 5672 case HWRM_RING_ALLOC_CMPL: 5673 db->db_key64 = DBR_PATH_L2; 5674 break; 5675 case HWRM_RING_ALLOC_NQ: 5676 db->db_key64 = DBR_PATH_L2; 5677 break; 5678 } 5679 db->db_key64 |= (u64)xid << DBR_XID_SFT; 5680 } else { 5681 db->doorbell = bp->bar1 + map_idx * 0x80; 5682 switch (ring_type) { 5683 case HWRM_RING_ALLOC_TX: 5684 db->db_key32 = DB_KEY_TX; 5685 break; 5686 case HWRM_RING_ALLOC_RX: 5687 case HWRM_RING_ALLOC_AGG: 5688 db->db_key32 = DB_KEY_RX; 5689 break; 5690 case HWRM_RING_ALLOC_CMPL: 5691 db->db_key32 = DB_KEY_CP; 5692 break; 5693 } 5694 } 5695 } 5696 5697 static int bnxt_hwrm_ring_alloc(struct bnxt *bp) 5698 { 5699 bool agg_rings = !!(bp->flags & BNXT_FLAG_AGG_RINGS); 5700 int i, rc = 0; 5701 u32 type; 5702 5703 if (bp->flags & BNXT_FLAG_CHIP_P5) 5704 type = HWRM_RING_ALLOC_NQ; 5705 else 5706 type = HWRM_RING_ALLOC_CMPL; 5707 for (i = 0; i < bp->cp_nr_rings; i++) { 5708 struct bnxt_napi *bnapi = bp->bnapi[i]; 5709 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 5710 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 5711 u32 map_idx = ring->map_idx; 5712 unsigned int vector; 5713 5714 vector = bp->irq_tbl[map_idx].vector; 5715 disable_irq_nosync(vector); 5716 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 5717 if (rc) { 5718 enable_irq(vector); 5719 goto err_out; 5720 } 5721 bnxt_set_db(bp, &cpr->cp_db, type, map_idx, ring->fw_ring_id); 5722 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons); 5723 enable_irq(vector); 5724 bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id; 5725 5726 if (!i) { 5727 rc = bnxt_hwrm_set_async_event_cr(bp, ring->fw_ring_id); 5728 if (rc) 5729 netdev_warn(bp->dev, "Failed to set async event completion ring.\n"); 5730 } 5731 } 5732 5733 type = HWRM_RING_ALLOC_TX; 5734 for (i = 0; i < bp->tx_nr_rings; i++) { 5735 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 5736 struct bnxt_ring_struct *ring; 5737 u32 map_idx; 5738 5739 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5740 struct bnxt_napi *bnapi = txr->bnapi; 5741 struct bnxt_cp_ring_info *cpr, *cpr2; 5742 u32 type2 = HWRM_RING_ALLOC_CMPL; 5743 5744 cpr = &bnapi->cp_ring; 5745 cpr2 = cpr->cp_ring_arr[BNXT_TX_HDL]; 5746 ring = &cpr2->cp_ring_struct; 5747 ring->handle = BNXT_TX_HDL; 5748 map_idx = bnapi->index; 5749 rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx); 5750 if (rc) 5751 goto err_out; 5752 bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx, 5753 ring->fw_ring_id); 5754 bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons); 5755 } 5756 ring = &txr->tx_ring_struct; 5757 map_idx = i; 5758 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 5759 if (rc) 5760 goto err_out; 5761 bnxt_set_db(bp, &txr->tx_db, type, map_idx, ring->fw_ring_id); 5762 } 5763 5764 type = HWRM_RING_ALLOC_RX; 5765 for (i = 0; i < bp->rx_nr_rings; i++) { 5766 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 5767 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct; 5768 struct bnxt_napi *bnapi = rxr->bnapi; 5769 u32 map_idx = bnapi->index; 5770 5771 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 5772 if (rc) 5773 goto err_out; 5774 bnxt_set_db(bp, &rxr->rx_db, type, map_idx, ring->fw_ring_id); 5775 /* If we have agg rings, post agg buffers first. */ 5776 if (!agg_rings) 5777 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 5778 bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id; 5779 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5780 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 5781 u32 type2 = HWRM_RING_ALLOC_CMPL; 5782 struct bnxt_cp_ring_info *cpr2; 5783 5784 cpr2 = cpr->cp_ring_arr[BNXT_RX_HDL]; 5785 ring = &cpr2->cp_ring_struct; 5786 ring->handle = BNXT_RX_HDL; 5787 rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx); 5788 if (rc) 5789 goto err_out; 5790 bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx, 5791 ring->fw_ring_id); 5792 bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons); 5793 } 5794 } 5795 5796 if (agg_rings) { 5797 type = HWRM_RING_ALLOC_AGG; 5798 for (i = 0; i < bp->rx_nr_rings; i++) { 5799 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 5800 struct bnxt_ring_struct *ring = 5801 &rxr->rx_agg_ring_struct; 5802 u32 grp_idx = ring->grp_idx; 5803 u32 map_idx = grp_idx + bp->rx_nr_rings; 5804 5805 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 5806 if (rc) 5807 goto err_out; 5808 5809 bnxt_set_db(bp, &rxr->rx_agg_db, type, map_idx, 5810 ring->fw_ring_id); 5811 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); 5812 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 5813 bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id; 5814 } 5815 } 5816 err_out: 5817 return rc; 5818 } 5819 5820 static int hwrm_ring_free_send_msg(struct bnxt *bp, 5821 struct bnxt_ring_struct *ring, 5822 u32 ring_type, int cmpl_ring_id) 5823 { 5824 int rc; 5825 struct hwrm_ring_free_input req = {0}; 5826 struct hwrm_ring_free_output *resp = bp->hwrm_cmd_resp_addr; 5827 u16 error_code; 5828 5829 if (BNXT_NO_FW_ACCESS(bp)) 5830 return 0; 5831 5832 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_FREE, cmpl_ring_id, -1); 5833 req.ring_type = ring_type; 5834 req.ring_id = cpu_to_le16(ring->fw_ring_id); 5835 5836 mutex_lock(&bp->hwrm_cmd_lock); 5837 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5838 error_code = le16_to_cpu(resp->error_code); 5839 mutex_unlock(&bp->hwrm_cmd_lock); 5840 5841 if (rc || error_code) { 5842 netdev_err(bp->dev, "hwrm_ring_free type %d failed. rc:%x err:%x\n", 5843 ring_type, rc, error_code); 5844 return -EIO; 5845 } 5846 return 0; 5847 } 5848 5849 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path) 5850 { 5851 u32 type; 5852 int i; 5853 5854 if (!bp->bnapi) 5855 return; 5856 5857 for (i = 0; i < bp->tx_nr_rings; i++) { 5858 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 5859 struct bnxt_ring_struct *ring = &txr->tx_ring_struct; 5860 5861 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 5862 u32 cmpl_ring_id = bnxt_cp_ring_for_tx(bp, txr); 5863 5864 hwrm_ring_free_send_msg(bp, ring, 5865 RING_FREE_REQ_RING_TYPE_TX, 5866 close_path ? cmpl_ring_id : 5867 INVALID_HW_RING_ID); 5868 ring->fw_ring_id = INVALID_HW_RING_ID; 5869 } 5870 } 5871 5872 for (i = 0; i < bp->rx_nr_rings; i++) { 5873 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 5874 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct; 5875 u32 grp_idx = rxr->bnapi->index; 5876 5877 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 5878 u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr); 5879 5880 hwrm_ring_free_send_msg(bp, ring, 5881 RING_FREE_REQ_RING_TYPE_RX, 5882 close_path ? cmpl_ring_id : 5883 INVALID_HW_RING_ID); 5884 ring->fw_ring_id = INVALID_HW_RING_ID; 5885 bp->grp_info[grp_idx].rx_fw_ring_id = 5886 INVALID_HW_RING_ID; 5887 } 5888 } 5889 5890 if (bp->flags & BNXT_FLAG_CHIP_P5) 5891 type = RING_FREE_REQ_RING_TYPE_RX_AGG; 5892 else 5893 type = RING_FREE_REQ_RING_TYPE_RX; 5894 for (i = 0; i < bp->rx_nr_rings; i++) { 5895 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 5896 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct; 5897 u32 grp_idx = rxr->bnapi->index; 5898 5899 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 5900 u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr); 5901 5902 hwrm_ring_free_send_msg(bp, ring, type, 5903 close_path ? cmpl_ring_id : 5904 INVALID_HW_RING_ID); 5905 ring->fw_ring_id = INVALID_HW_RING_ID; 5906 bp->grp_info[grp_idx].agg_fw_ring_id = 5907 INVALID_HW_RING_ID; 5908 } 5909 } 5910 5911 /* The completion rings are about to be freed. After that the 5912 * IRQ doorbell will not work anymore. So we need to disable 5913 * IRQ here. 5914 */ 5915 bnxt_disable_int_sync(bp); 5916 5917 if (bp->flags & BNXT_FLAG_CHIP_P5) 5918 type = RING_FREE_REQ_RING_TYPE_NQ; 5919 else 5920 type = RING_FREE_REQ_RING_TYPE_L2_CMPL; 5921 for (i = 0; i < bp->cp_nr_rings; i++) { 5922 struct bnxt_napi *bnapi = bp->bnapi[i]; 5923 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 5924 struct bnxt_ring_struct *ring; 5925 int j; 5926 5927 for (j = 0; j < 2; j++) { 5928 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j]; 5929 5930 if (cpr2) { 5931 ring = &cpr2->cp_ring_struct; 5932 if (ring->fw_ring_id == INVALID_HW_RING_ID) 5933 continue; 5934 hwrm_ring_free_send_msg(bp, ring, 5935 RING_FREE_REQ_RING_TYPE_L2_CMPL, 5936 INVALID_HW_RING_ID); 5937 ring->fw_ring_id = INVALID_HW_RING_ID; 5938 } 5939 } 5940 ring = &cpr->cp_ring_struct; 5941 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 5942 hwrm_ring_free_send_msg(bp, ring, type, 5943 INVALID_HW_RING_ID); 5944 ring->fw_ring_id = INVALID_HW_RING_ID; 5945 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID; 5946 } 5947 } 5948 } 5949 5950 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max, 5951 bool shared); 5952 5953 static int bnxt_hwrm_get_rings(struct bnxt *bp) 5954 { 5955 struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 5956 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 5957 struct hwrm_func_qcfg_input req = {0}; 5958 int rc; 5959 5960 if (bp->hwrm_spec_code < 0x10601) 5961 return 0; 5962 5963 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1); 5964 req.fid = cpu_to_le16(0xffff); 5965 mutex_lock(&bp->hwrm_cmd_lock); 5966 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5967 if (rc) { 5968 mutex_unlock(&bp->hwrm_cmd_lock); 5969 return rc; 5970 } 5971 5972 hw_resc->resv_tx_rings = le16_to_cpu(resp->alloc_tx_rings); 5973 if (BNXT_NEW_RM(bp)) { 5974 u16 cp, stats; 5975 5976 hw_resc->resv_rx_rings = le16_to_cpu(resp->alloc_rx_rings); 5977 hw_resc->resv_hw_ring_grps = 5978 le32_to_cpu(resp->alloc_hw_ring_grps); 5979 hw_resc->resv_vnics = le16_to_cpu(resp->alloc_vnics); 5980 cp = le16_to_cpu(resp->alloc_cmpl_rings); 5981 stats = le16_to_cpu(resp->alloc_stat_ctx); 5982 hw_resc->resv_irqs = cp; 5983 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5984 int rx = hw_resc->resv_rx_rings; 5985 int tx = hw_resc->resv_tx_rings; 5986 5987 if (bp->flags & BNXT_FLAG_AGG_RINGS) 5988 rx >>= 1; 5989 if (cp < (rx + tx)) { 5990 bnxt_trim_rings(bp, &rx, &tx, cp, false); 5991 if (bp->flags & BNXT_FLAG_AGG_RINGS) 5992 rx <<= 1; 5993 hw_resc->resv_rx_rings = rx; 5994 hw_resc->resv_tx_rings = tx; 5995 } 5996 hw_resc->resv_irqs = le16_to_cpu(resp->alloc_msix); 5997 hw_resc->resv_hw_ring_grps = rx; 5998 } 5999 hw_resc->resv_cp_rings = cp; 6000 hw_resc->resv_stat_ctxs = stats; 6001 } 6002 mutex_unlock(&bp->hwrm_cmd_lock); 6003 return 0; 6004 } 6005 6006 /* Caller must hold bp->hwrm_cmd_lock */ 6007 int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings) 6008 { 6009 struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 6010 struct hwrm_func_qcfg_input req = {0}; 6011 int rc; 6012 6013 if (bp->hwrm_spec_code < 0x10601) 6014 return 0; 6015 6016 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1); 6017 req.fid = cpu_to_le16(fid); 6018 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6019 if (!rc) 6020 *tx_rings = le16_to_cpu(resp->alloc_tx_rings); 6021 6022 return rc; 6023 } 6024 6025 static bool bnxt_rfs_supported(struct bnxt *bp); 6026 6027 static void 6028 __bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, struct hwrm_func_cfg_input *req, 6029 int tx_rings, int rx_rings, int ring_grps, 6030 int cp_rings, int stats, int vnics) 6031 { 6032 u32 enables = 0; 6033 6034 bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_CFG, -1, -1); 6035 req->fid = cpu_to_le16(0xffff); 6036 enables |= tx_rings ? FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS : 0; 6037 req->num_tx_rings = cpu_to_le16(tx_rings); 6038 if (BNXT_NEW_RM(bp)) { 6039 enables |= rx_rings ? FUNC_CFG_REQ_ENABLES_NUM_RX_RINGS : 0; 6040 enables |= stats ? FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0; 6041 if (bp->flags & BNXT_FLAG_CHIP_P5) { 6042 enables |= cp_rings ? FUNC_CFG_REQ_ENABLES_NUM_MSIX : 0; 6043 enables |= tx_rings + ring_grps ? 6044 FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0; 6045 enables |= rx_rings ? 6046 FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0; 6047 } else { 6048 enables |= cp_rings ? 6049 FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0; 6050 enables |= ring_grps ? 6051 FUNC_CFG_REQ_ENABLES_NUM_HW_RING_GRPS | 6052 FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0; 6053 } 6054 enables |= vnics ? FUNC_CFG_REQ_ENABLES_NUM_VNICS : 0; 6055 6056 req->num_rx_rings = cpu_to_le16(rx_rings); 6057 if (bp->flags & BNXT_FLAG_CHIP_P5) { 6058 req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps); 6059 req->num_msix = cpu_to_le16(cp_rings); 6060 req->num_rsscos_ctxs = 6061 cpu_to_le16(DIV_ROUND_UP(ring_grps, 64)); 6062 } else { 6063 req->num_cmpl_rings = cpu_to_le16(cp_rings); 6064 req->num_hw_ring_grps = cpu_to_le16(ring_grps); 6065 req->num_rsscos_ctxs = cpu_to_le16(1); 6066 if (!(bp->flags & BNXT_FLAG_NEW_RSS_CAP) && 6067 bnxt_rfs_supported(bp)) 6068 req->num_rsscos_ctxs = 6069 cpu_to_le16(ring_grps + 1); 6070 } 6071 req->num_stat_ctxs = cpu_to_le16(stats); 6072 req->num_vnics = cpu_to_le16(vnics); 6073 } 6074 req->enables = cpu_to_le32(enables); 6075 } 6076 6077 static void 6078 __bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, 6079 struct hwrm_func_vf_cfg_input *req, int tx_rings, 6080 int rx_rings, int ring_grps, int cp_rings, 6081 int stats, int vnics) 6082 { 6083 u32 enables = 0; 6084 6085 bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_VF_CFG, -1, -1); 6086 enables |= tx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_TX_RINGS : 0; 6087 enables |= rx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_RX_RINGS | 6088 FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0; 6089 enables |= stats ? FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0; 6090 if (bp->flags & BNXT_FLAG_CHIP_P5) { 6091 enables |= tx_rings + ring_grps ? 6092 FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0; 6093 } else { 6094 enables |= cp_rings ? 6095 FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0; 6096 enables |= ring_grps ? 6097 FUNC_VF_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0; 6098 } 6099 enables |= vnics ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0; 6100 enables |= FUNC_VF_CFG_REQ_ENABLES_NUM_L2_CTXS; 6101 6102 req->num_l2_ctxs = cpu_to_le16(BNXT_VF_MAX_L2_CTX); 6103 req->num_tx_rings = cpu_to_le16(tx_rings); 6104 req->num_rx_rings = cpu_to_le16(rx_rings); 6105 if (bp->flags & BNXT_FLAG_CHIP_P5) { 6106 req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps); 6107 req->num_rsscos_ctxs = cpu_to_le16(DIV_ROUND_UP(ring_grps, 64)); 6108 } else { 6109 req->num_cmpl_rings = cpu_to_le16(cp_rings); 6110 req->num_hw_ring_grps = cpu_to_le16(ring_grps); 6111 req->num_rsscos_ctxs = cpu_to_le16(BNXT_VF_MAX_RSS_CTX); 6112 } 6113 req->num_stat_ctxs = cpu_to_le16(stats); 6114 req->num_vnics = cpu_to_le16(vnics); 6115 6116 req->enables = cpu_to_le32(enables); 6117 } 6118 6119 static int 6120 bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6121 int ring_grps, int cp_rings, int stats, int vnics) 6122 { 6123 struct hwrm_func_cfg_input req = {0}; 6124 int rc; 6125 6126 __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps, 6127 cp_rings, stats, vnics); 6128 if (!req.enables) 6129 return 0; 6130 6131 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6132 if (rc) 6133 return rc; 6134 6135 if (bp->hwrm_spec_code < 0x10601) 6136 bp->hw_resc.resv_tx_rings = tx_rings; 6137 6138 return bnxt_hwrm_get_rings(bp); 6139 } 6140 6141 static int 6142 bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6143 int ring_grps, int cp_rings, int stats, int vnics) 6144 { 6145 struct hwrm_func_vf_cfg_input req = {0}; 6146 int rc; 6147 6148 if (!BNXT_NEW_RM(bp)) { 6149 bp->hw_resc.resv_tx_rings = tx_rings; 6150 return 0; 6151 } 6152 6153 __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps, 6154 cp_rings, stats, vnics); 6155 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6156 if (rc) 6157 return rc; 6158 6159 return bnxt_hwrm_get_rings(bp); 6160 } 6161 6162 static int bnxt_hwrm_reserve_rings(struct bnxt *bp, int tx, int rx, int grp, 6163 int cp, int stat, int vnic) 6164 { 6165 if (BNXT_PF(bp)) 6166 return bnxt_hwrm_reserve_pf_rings(bp, tx, rx, grp, cp, stat, 6167 vnic); 6168 else 6169 return bnxt_hwrm_reserve_vf_rings(bp, tx, rx, grp, cp, stat, 6170 vnic); 6171 } 6172 6173 int bnxt_nq_rings_in_use(struct bnxt *bp) 6174 { 6175 int cp = bp->cp_nr_rings; 6176 int ulp_msix, ulp_base; 6177 6178 ulp_msix = bnxt_get_ulp_msix_num(bp); 6179 if (ulp_msix) { 6180 ulp_base = bnxt_get_ulp_msix_base(bp); 6181 cp += ulp_msix; 6182 if ((ulp_base + ulp_msix) > cp) 6183 cp = ulp_base + ulp_msix; 6184 } 6185 return cp; 6186 } 6187 6188 static int bnxt_cp_rings_in_use(struct bnxt *bp) 6189 { 6190 int cp; 6191 6192 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 6193 return bnxt_nq_rings_in_use(bp); 6194 6195 cp = bp->tx_nr_rings + bp->rx_nr_rings; 6196 return cp; 6197 } 6198 6199 static int bnxt_get_func_stat_ctxs(struct bnxt *bp) 6200 { 6201 int ulp_stat = bnxt_get_ulp_stat_ctxs(bp); 6202 int cp = bp->cp_nr_rings; 6203 6204 if (!ulp_stat) 6205 return cp; 6206 6207 if (bnxt_nq_rings_in_use(bp) > cp + bnxt_get_ulp_msix_num(bp)) 6208 return bnxt_get_ulp_msix_base(bp) + ulp_stat; 6209 6210 return cp + ulp_stat; 6211 } 6212 6213 /* Check if a default RSS map needs to be setup. This function is only 6214 * used on older firmware that does not require reserving RX rings. 6215 */ 6216 static void bnxt_check_rss_tbl_no_rmgr(struct bnxt *bp) 6217 { 6218 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 6219 6220 /* The RSS map is valid for RX rings set to resv_rx_rings */ 6221 if (hw_resc->resv_rx_rings != bp->rx_nr_rings) { 6222 hw_resc->resv_rx_rings = bp->rx_nr_rings; 6223 if (!netif_is_rxfh_configured(bp->dev)) 6224 bnxt_set_dflt_rss_indir_tbl(bp); 6225 } 6226 } 6227 6228 static bool bnxt_need_reserve_rings(struct bnxt *bp) 6229 { 6230 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 6231 int cp = bnxt_cp_rings_in_use(bp); 6232 int nq = bnxt_nq_rings_in_use(bp); 6233 int rx = bp->rx_nr_rings, stat; 6234 int vnic = 1, grp = rx; 6235 6236 if (hw_resc->resv_tx_rings != bp->tx_nr_rings && 6237 bp->hwrm_spec_code >= 0x10601) 6238 return true; 6239 6240 /* Old firmware does not need RX ring reservations but we still 6241 * need to setup a default RSS map when needed. With new firmware 6242 * we go through RX ring reservations first and then set up the 6243 * RSS map for the successfully reserved RX rings when needed. 6244 */ 6245 if (!BNXT_NEW_RM(bp)) { 6246 bnxt_check_rss_tbl_no_rmgr(bp); 6247 return false; 6248 } 6249 if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5)) 6250 vnic = rx + 1; 6251 if (bp->flags & BNXT_FLAG_AGG_RINGS) 6252 rx <<= 1; 6253 stat = bnxt_get_func_stat_ctxs(bp); 6254 if (hw_resc->resv_rx_rings != rx || hw_resc->resv_cp_rings != cp || 6255 hw_resc->resv_vnics != vnic || hw_resc->resv_stat_ctxs != stat || 6256 (hw_resc->resv_hw_ring_grps != grp && 6257 !(bp->flags & BNXT_FLAG_CHIP_P5))) 6258 return true; 6259 if ((bp->flags & BNXT_FLAG_CHIP_P5) && BNXT_PF(bp) && 6260 hw_resc->resv_irqs != nq) 6261 return true; 6262 return false; 6263 } 6264 6265 static int __bnxt_reserve_rings(struct bnxt *bp) 6266 { 6267 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 6268 int cp = bnxt_nq_rings_in_use(bp); 6269 int tx = bp->tx_nr_rings; 6270 int rx = bp->rx_nr_rings; 6271 int grp, rx_rings, rc; 6272 int vnic = 1, stat; 6273 bool sh = false; 6274 6275 if (!bnxt_need_reserve_rings(bp)) 6276 return 0; 6277 6278 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 6279 sh = true; 6280 if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5)) 6281 vnic = rx + 1; 6282 if (bp->flags & BNXT_FLAG_AGG_RINGS) 6283 rx <<= 1; 6284 grp = bp->rx_nr_rings; 6285 stat = bnxt_get_func_stat_ctxs(bp); 6286 6287 rc = bnxt_hwrm_reserve_rings(bp, tx, rx, grp, cp, stat, vnic); 6288 if (rc) 6289 return rc; 6290 6291 tx = hw_resc->resv_tx_rings; 6292 if (BNXT_NEW_RM(bp)) { 6293 rx = hw_resc->resv_rx_rings; 6294 cp = hw_resc->resv_irqs; 6295 grp = hw_resc->resv_hw_ring_grps; 6296 vnic = hw_resc->resv_vnics; 6297 stat = hw_resc->resv_stat_ctxs; 6298 } 6299 6300 rx_rings = rx; 6301 if (bp->flags & BNXT_FLAG_AGG_RINGS) { 6302 if (rx >= 2) { 6303 rx_rings = rx >> 1; 6304 } else { 6305 if (netif_running(bp->dev)) 6306 return -ENOMEM; 6307 6308 bp->flags &= ~BNXT_FLAG_AGG_RINGS; 6309 bp->flags |= BNXT_FLAG_NO_AGG_RINGS; 6310 bp->dev->hw_features &= ~NETIF_F_LRO; 6311 bp->dev->features &= ~NETIF_F_LRO; 6312 bnxt_set_ring_params(bp); 6313 } 6314 } 6315 rx_rings = min_t(int, rx_rings, grp); 6316 cp = min_t(int, cp, bp->cp_nr_rings); 6317 if (stat > bnxt_get_ulp_stat_ctxs(bp)) 6318 stat -= bnxt_get_ulp_stat_ctxs(bp); 6319 cp = min_t(int, cp, stat); 6320 rc = bnxt_trim_rings(bp, &rx_rings, &tx, cp, sh); 6321 if (bp->flags & BNXT_FLAG_AGG_RINGS) 6322 rx = rx_rings << 1; 6323 cp = sh ? max_t(int, tx, rx_rings) : tx + rx_rings; 6324 bp->tx_nr_rings = tx; 6325 6326 /* If we cannot reserve all the RX rings, reset the RSS map only 6327 * if absolutely necessary 6328 */ 6329 if (rx_rings != bp->rx_nr_rings) { 6330 netdev_warn(bp->dev, "Able to reserve only %d out of %d requested RX rings\n", 6331 rx_rings, bp->rx_nr_rings); 6332 if ((bp->dev->priv_flags & IFF_RXFH_CONFIGURED) && 6333 (bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings) != 6334 bnxt_get_nr_rss_ctxs(bp, rx_rings) || 6335 bnxt_get_max_rss_ring(bp) >= rx_rings)) { 6336 netdev_warn(bp->dev, "RSS table entries reverting to default\n"); 6337 bp->dev->priv_flags &= ~IFF_RXFH_CONFIGURED; 6338 } 6339 } 6340 bp->rx_nr_rings = rx_rings; 6341 bp->cp_nr_rings = cp; 6342 6343 if (!tx || !rx || !cp || !grp || !vnic || !stat) 6344 return -ENOMEM; 6345 6346 if (!netif_is_rxfh_configured(bp->dev)) 6347 bnxt_set_dflt_rss_indir_tbl(bp); 6348 6349 return rc; 6350 } 6351 6352 static int bnxt_hwrm_check_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6353 int ring_grps, int cp_rings, int stats, 6354 int vnics) 6355 { 6356 struct hwrm_func_vf_cfg_input req = {0}; 6357 u32 flags; 6358 6359 if (!BNXT_NEW_RM(bp)) 6360 return 0; 6361 6362 __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps, 6363 cp_rings, stats, vnics); 6364 flags = FUNC_VF_CFG_REQ_FLAGS_TX_ASSETS_TEST | 6365 FUNC_VF_CFG_REQ_FLAGS_RX_ASSETS_TEST | 6366 FUNC_VF_CFG_REQ_FLAGS_CMPL_ASSETS_TEST | 6367 FUNC_VF_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST | 6368 FUNC_VF_CFG_REQ_FLAGS_VNIC_ASSETS_TEST | 6369 FUNC_VF_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST; 6370 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 6371 flags |= FUNC_VF_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST; 6372 6373 req.flags = cpu_to_le32(flags); 6374 return hwrm_send_message_silent(bp, &req, sizeof(req), 6375 HWRM_CMD_TIMEOUT); 6376 } 6377 6378 static int bnxt_hwrm_check_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6379 int ring_grps, int cp_rings, int stats, 6380 int vnics) 6381 { 6382 struct hwrm_func_cfg_input req = {0}; 6383 u32 flags; 6384 6385 __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps, 6386 cp_rings, stats, vnics); 6387 flags = FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST; 6388 if (BNXT_NEW_RM(bp)) { 6389 flags |= FUNC_CFG_REQ_FLAGS_RX_ASSETS_TEST | 6390 FUNC_CFG_REQ_FLAGS_CMPL_ASSETS_TEST | 6391 FUNC_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST | 6392 FUNC_CFG_REQ_FLAGS_VNIC_ASSETS_TEST; 6393 if (bp->flags & BNXT_FLAG_CHIP_P5) 6394 flags |= FUNC_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST | 6395 FUNC_CFG_REQ_FLAGS_NQ_ASSETS_TEST; 6396 else 6397 flags |= FUNC_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST; 6398 } 6399 6400 req.flags = cpu_to_le32(flags); 6401 return hwrm_send_message_silent(bp, &req, sizeof(req), 6402 HWRM_CMD_TIMEOUT); 6403 } 6404 6405 static int bnxt_hwrm_check_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6406 int ring_grps, int cp_rings, int stats, 6407 int vnics) 6408 { 6409 if (bp->hwrm_spec_code < 0x10801) 6410 return 0; 6411 6412 if (BNXT_PF(bp)) 6413 return bnxt_hwrm_check_pf_rings(bp, tx_rings, rx_rings, 6414 ring_grps, cp_rings, stats, 6415 vnics); 6416 6417 return bnxt_hwrm_check_vf_rings(bp, tx_rings, rx_rings, ring_grps, 6418 cp_rings, stats, vnics); 6419 } 6420 6421 static void bnxt_hwrm_coal_params_qcaps(struct bnxt *bp) 6422 { 6423 struct hwrm_ring_aggint_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 6424 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 6425 struct hwrm_ring_aggint_qcaps_input req = {0}; 6426 int rc; 6427 6428 coal_cap->cmpl_params = BNXT_LEGACY_COAL_CMPL_PARAMS; 6429 coal_cap->num_cmpl_dma_aggr_max = 63; 6430 coal_cap->num_cmpl_dma_aggr_during_int_max = 63; 6431 coal_cap->cmpl_aggr_dma_tmr_max = 65535; 6432 coal_cap->cmpl_aggr_dma_tmr_during_int_max = 65535; 6433 coal_cap->int_lat_tmr_min_max = 65535; 6434 coal_cap->int_lat_tmr_max_max = 65535; 6435 coal_cap->num_cmpl_aggr_int_max = 65535; 6436 coal_cap->timer_units = 80; 6437 6438 if (bp->hwrm_spec_code < 0x10902) 6439 return; 6440 6441 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_AGGINT_QCAPS, -1, -1); 6442 mutex_lock(&bp->hwrm_cmd_lock); 6443 rc = _hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6444 if (!rc) { 6445 coal_cap->cmpl_params = le32_to_cpu(resp->cmpl_params); 6446 coal_cap->nq_params = le32_to_cpu(resp->nq_params); 6447 coal_cap->num_cmpl_dma_aggr_max = 6448 le16_to_cpu(resp->num_cmpl_dma_aggr_max); 6449 coal_cap->num_cmpl_dma_aggr_during_int_max = 6450 le16_to_cpu(resp->num_cmpl_dma_aggr_during_int_max); 6451 coal_cap->cmpl_aggr_dma_tmr_max = 6452 le16_to_cpu(resp->cmpl_aggr_dma_tmr_max); 6453 coal_cap->cmpl_aggr_dma_tmr_during_int_max = 6454 le16_to_cpu(resp->cmpl_aggr_dma_tmr_during_int_max); 6455 coal_cap->int_lat_tmr_min_max = 6456 le16_to_cpu(resp->int_lat_tmr_min_max); 6457 coal_cap->int_lat_tmr_max_max = 6458 le16_to_cpu(resp->int_lat_tmr_max_max); 6459 coal_cap->num_cmpl_aggr_int_max = 6460 le16_to_cpu(resp->num_cmpl_aggr_int_max); 6461 coal_cap->timer_units = le16_to_cpu(resp->timer_units); 6462 } 6463 mutex_unlock(&bp->hwrm_cmd_lock); 6464 } 6465 6466 static u16 bnxt_usec_to_coal_tmr(struct bnxt *bp, u16 usec) 6467 { 6468 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 6469 6470 return usec * 1000 / coal_cap->timer_units; 6471 } 6472 6473 static void bnxt_hwrm_set_coal_params(struct bnxt *bp, 6474 struct bnxt_coal *hw_coal, 6475 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req) 6476 { 6477 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 6478 u32 cmpl_params = coal_cap->cmpl_params; 6479 u16 val, tmr, max, flags = 0; 6480 6481 max = hw_coal->bufs_per_record * 128; 6482 if (hw_coal->budget) 6483 max = hw_coal->bufs_per_record * hw_coal->budget; 6484 max = min_t(u16, max, coal_cap->num_cmpl_aggr_int_max); 6485 6486 val = clamp_t(u16, hw_coal->coal_bufs, 1, max); 6487 req->num_cmpl_aggr_int = cpu_to_le16(val); 6488 6489 val = min_t(u16, val, coal_cap->num_cmpl_dma_aggr_max); 6490 req->num_cmpl_dma_aggr = cpu_to_le16(val); 6491 6492 val = clamp_t(u16, hw_coal->coal_bufs_irq, 1, 6493 coal_cap->num_cmpl_dma_aggr_during_int_max); 6494 req->num_cmpl_dma_aggr_during_int = cpu_to_le16(val); 6495 6496 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks); 6497 tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_max_max); 6498 req->int_lat_tmr_max = cpu_to_le16(tmr); 6499 6500 /* min timer set to 1/2 of interrupt timer */ 6501 if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_INT_LAT_TMR_MIN) { 6502 val = tmr / 2; 6503 val = clamp_t(u16, val, 1, coal_cap->int_lat_tmr_min_max); 6504 req->int_lat_tmr_min = cpu_to_le16(val); 6505 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE); 6506 } 6507 6508 /* buf timer set to 1/4 of interrupt timer */ 6509 val = clamp_t(u16, tmr / 4, 1, coal_cap->cmpl_aggr_dma_tmr_max); 6510 req->cmpl_aggr_dma_tmr = cpu_to_le16(val); 6511 6512 if (cmpl_params & 6513 RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_NUM_CMPL_DMA_AGGR_DURING_INT) { 6514 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks_irq); 6515 val = clamp_t(u16, tmr, 1, 6516 coal_cap->cmpl_aggr_dma_tmr_during_int_max); 6517 req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(val); 6518 req->enables |= 6519 cpu_to_le16(BNXT_COAL_CMPL_AGGR_TMR_DURING_INT_ENABLE); 6520 } 6521 6522 if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_TIMER_RESET) 6523 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET; 6524 if ((cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_RING_IDLE) && 6525 hw_coal->idle_thresh && hw_coal->coal_ticks < hw_coal->idle_thresh) 6526 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE; 6527 req->flags = cpu_to_le16(flags); 6528 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_ENABLES); 6529 } 6530 6531 /* Caller holds bp->hwrm_cmd_lock */ 6532 static int __bnxt_hwrm_set_coal_nq(struct bnxt *bp, struct bnxt_napi *bnapi, 6533 struct bnxt_coal *hw_coal) 6534 { 6535 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req = {0}; 6536 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 6537 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 6538 u32 nq_params = coal_cap->nq_params; 6539 u16 tmr; 6540 6541 if (!(nq_params & RING_AGGINT_QCAPS_RESP_NQ_PARAMS_INT_LAT_TMR_MIN)) 6542 return 0; 6543 6544 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, 6545 -1, -1); 6546 req.ring_id = cpu_to_le16(cpr->cp_ring_struct.fw_ring_id); 6547 req.flags = 6548 cpu_to_le16(RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_IS_NQ); 6549 6550 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks) / 2; 6551 tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_min_max); 6552 req.int_lat_tmr_min = cpu_to_le16(tmr); 6553 req.enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE); 6554 return _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6555 } 6556 6557 int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi) 6558 { 6559 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0}; 6560 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 6561 struct bnxt_coal coal; 6562 6563 /* Tick values in micro seconds. 6564 * 1 coal_buf x bufs_per_record = 1 completion record. 6565 */ 6566 memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal)); 6567 6568 coal.coal_ticks = cpr->rx_ring_coal.coal_ticks; 6569 coal.coal_bufs = cpr->rx_ring_coal.coal_bufs; 6570 6571 if (!bnapi->rx_ring) 6572 return -ENODEV; 6573 6574 bnxt_hwrm_cmd_hdr_init(bp, &req_rx, 6575 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1); 6576 6577 bnxt_hwrm_set_coal_params(bp, &coal, &req_rx); 6578 6579 req_rx.ring_id = cpu_to_le16(bnxt_cp_ring_for_rx(bp, bnapi->rx_ring)); 6580 6581 return hwrm_send_message(bp, &req_rx, sizeof(req_rx), 6582 HWRM_CMD_TIMEOUT); 6583 } 6584 6585 int bnxt_hwrm_set_coal(struct bnxt *bp) 6586 { 6587 int i, rc = 0; 6588 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0}, 6589 req_tx = {0}, *req; 6590 6591 bnxt_hwrm_cmd_hdr_init(bp, &req_rx, 6592 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1); 6593 bnxt_hwrm_cmd_hdr_init(bp, &req_tx, 6594 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1); 6595 6596 bnxt_hwrm_set_coal_params(bp, &bp->rx_coal, &req_rx); 6597 bnxt_hwrm_set_coal_params(bp, &bp->tx_coal, &req_tx); 6598 6599 mutex_lock(&bp->hwrm_cmd_lock); 6600 for (i = 0; i < bp->cp_nr_rings; i++) { 6601 struct bnxt_napi *bnapi = bp->bnapi[i]; 6602 struct bnxt_coal *hw_coal; 6603 u16 ring_id; 6604 6605 req = &req_rx; 6606 if (!bnapi->rx_ring) { 6607 ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring); 6608 req = &req_tx; 6609 } else { 6610 ring_id = bnxt_cp_ring_for_rx(bp, bnapi->rx_ring); 6611 } 6612 req->ring_id = cpu_to_le16(ring_id); 6613 6614 rc = _hwrm_send_message(bp, req, sizeof(*req), 6615 HWRM_CMD_TIMEOUT); 6616 if (rc) 6617 break; 6618 6619 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 6620 continue; 6621 6622 if (bnapi->rx_ring && bnapi->tx_ring) { 6623 req = &req_tx; 6624 ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring); 6625 req->ring_id = cpu_to_le16(ring_id); 6626 rc = _hwrm_send_message(bp, req, sizeof(*req), 6627 HWRM_CMD_TIMEOUT); 6628 if (rc) 6629 break; 6630 } 6631 if (bnapi->rx_ring) 6632 hw_coal = &bp->rx_coal; 6633 else 6634 hw_coal = &bp->tx_coal; 6635 __bnxt_hwrm_set_coal_nq(bp, bnapi, hw_coal); 6636 } 6637 mutex_unlock(&bp->hwrm_cmd_lock); 6638 return rc; 6639 } 6640 6641 static void bnxt_hwrm_stat_ctx_free(struct bnxt *bp) 6642 { 6643 struct hwrm_stat_ctx_clr_stats_input req0 = {0}; 6644 struct hwrm_stat_ctx_free_input req = {0}; 6645 int i; 6646 6647 if (!bp->bnapi) 6648 return; 6649 6650 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 6651 return; 6652 6653 bnxt_hwrm_cmd_hdr_init(bp, &req0, HWRM_STAT_CTX_CLR_STATS, -1, -1); 6654 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_FREE, -1, -1); 6655 6656 mutex_lock(&bp->hwrm_cmd_lock); 6657 for (i = 0; i < bp->cp_nr_rings; i++) { 6658 struct bnxt_napi *bnapi = bp->bnapi[i]; 6659 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 6660 6661 if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) { 6662 req.stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id); 6663 if (BNXT_FW_MAJ(bp) <= 20) { 6664 req0.stat_ctx_id = req.stat_ctx_id; 6665 _hwrm_send_message(bp, &req0, sizeof(req0), 6666 HWRM_CMD_TIMEOUT); 6667 } 6668 _hwrm_send_message(bp, &req, sizeof(req), 6669 HWRM_CMD_TIMEOUT); 6670 6671 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID; 6672 } 6673 } 6674 mutex_unlock(&bp->hwrm_cmd_lock); 6675 } 6676 6677 static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp) 6678 { 6679 int rc = 0, i; 6680 struct hwrm_stat_ctx_alloc_input req = {0}; 6681 struct hwrm_stat_ctx_alloc_output *resp = bp->hwrm_cmd_resp_addr; 6682 6683 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 6684 return 0; 6685 6686 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_ALLOC, -1, -1); 6687 6688 req.stats_dma_length = cpu_to_le16(bp->hw_ring_stats_size); 6689 req.update_period_ms = cpu_to_le32(bp->stats_coal_ticks / 1000); 6690 6691 mutex_lock(&bp->hwrm_cmd_lock); 6692 for (i = 0; i < bp->cp_nr_rings; i++) { 6693 struct bnxt_napi *bnapi = bp->bnapi[i]; 6694 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 6695 6696 req.stats_dma_addr = cpu_to_le64(cpr->stats.hw_stats_map); 6697 6698 rc = _hwrm_send_message(bp, &req, sizeof(req), 6699 HWRM_CMD_TIMEOUT); 6700 if (rc) 6701 break; 6702 6703 cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id); 6704 6705 bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id; 6706 } 6707 mutex_unlock(&bp->hwrm_cmd_lock); 6708 return rc; 6709 } 6710 6711 static int bnxt_hwrm_func_qcfg(struct bnxt *bp) 6712 { 6713 struct hwrm_func_qcfg_input req = {0}; 6714 struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 6715 u32 min_db_offset = 0; 6716 u16 flags; 6717 int rc; 6718 6719 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1); 6720 req.fid = cpu_to_le16(0xffff); 6721 mutex_lock(&bp->hwrm_cmd_lock); 6722 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6723 if (rc) 6724 goto func_qcfg_exit; 6725 6726 #ifdef CONFIG_BNXT_SRIOV 6727 if (BNXT_VF(bp)) { 6728 struct bnxt_vf_info *vf = &bp->vf; 6729 6730 vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK; 6731 } else { 6732 bp->pf.registered_vfs = le16_to_cpu(resp->registered_vfs); 6733 } 6734 #endif 6735 flags = le16_to_cpu(resp->flags); 6736 if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED | 6737 FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED)) { 6738 bp->fw_cap |= BNXT_FW_CAP_LLDP_AGENT; 6739 if (flags & FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED) 6740 bp->fw_cap |= BNXT_FW_CAP_DCBX_AGENT; 6741 } 6742 if (BNXT_PF(bp) && (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST)) 6743 bp->flags |= BNXT_FLAG_MULTI_HOST; 6744 if (flags & FUNC_QCFG_RESP_FLAGS_RING_MONITOR_ENABLED) 6745 bp->fw_cap |= BNXT_FW_CAP_RING_MONITOR; 6746 6747 switch (resp->port_partition_type) { 6748 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0: 6749 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5: 6750 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0: 6751 bp->port_partition_type = resp->port_partition_type; 6752 break; 6753 } 6754 if (bp->hwrm_spec_code < 0x10707 || 6755 resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEB) 6756 bp->br_mode = BRIDGE_MODE_VEB; 6757 else if (resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEPA) 6758 bp->br_mode = BRIDGE_MODE_VEPA; 6759 else 6760 bp->br_mode = BRIDGE_MODE_UNDEF; 6761 6762 bp->max_mtu = le16_to_cpu(resp->max_mtu_configured); 6763 if (!bp->max_mtu) 6764 bp->max_mtu = BNXT_MAX_MTU; 6765 6766 if (bp->db_size) 6767 goto func_qcfg_exit; 6768 6769 if (bp->flags & BNXT_FLAG_CHIP_P5) { 6770 if (BNXT_PF(bp)) 6771 min_db_offset = DB_PF_OFFSET_P5; 6772 else 6773 min_db_offset = DB_VF_OFFSET_P5; 6774 } 6775 bp->db_size = PAGE_ALIGN(le16_to_cpu(resp->l2_doorbell_bar_size_kb) * 6776 1024); 6777 if (!bp->db_size || bp->db_size > pci_resource_len(bp->pdev, 2) || 6778 bp->db_size <= min_db_offset) 6779 bp->db_size = pci_resource_len(bp->pdev, 2); 6780 6781 func_qcfg_exit: 6782 mutex_unlock(&bp->hwrm_cmd_lock); 6783 return rc; 6784 } 6785 6786 static void bnxt_init_ctx_initializer(struct bnxt_ctx_mem_info *ctx, 6787 struct hwrm_func_backing_store_qcaps_output *resp) 6788 { 6789 struct bnxt_mem_init *mem_init; 6790 u16 init_mask; 6791 u8 init_val; 6792 u8 *offset; 6793 int i; 6794 6795 init_val = resp->ctx_kind_initializer; 6796 init_mask = le16_to_cpu(resp->ctx_init_mask); 6797 offset = &resp->qp_init_offset; 6798 mem_init = &ctx->mem_init[BNXT_CTX_MEM_INIT_QP]; 6799 for (i = 0; i < BNXT_CTX_MEM_INIT_MAX; i++, mem_init++, offset++) { 6800 mem_init->init_val = init_val; 6801 mem_init->offset = BNXT_MEM_INVALID_OFFSET; 6802 if (!init_mask) 6803 continue; 6804 if (i == BNXT_CTX_MEM_INIT_STAT) 6805 offset = &resp->stat_init_offset; 6806 if (init_mask & (1 << i)) 6807 mem_init->offset = *offset * 4; 6808 else 6809 mem_init->init_val = 0; 6810 } 6811 ctx->mem_init[BNXT_CTX_MEM_INIT_QP].size = ctx->qp_entry_size; 6812 ctx->mem_init[BNXT_CTX_MEM_INIT_SRQ].size = ctx->srq_entry_size; 6813 ctx->mem_init[BNXT_CTX_MEM_INIT_CQ].size = ctx->cq_entry_size; 6814 ctx->mem_init[BNXT_CTX_MEM_INIT_VNIC].size = ctx->vnic_entry_size; 6815 ctx->mem_init[BNXT_CTX_MEM_INIT_STAT].size = ctx->stat_entry_size; 6816 ctx->mem_init[BNXT_CTX_MEM_INIT_MRAV].size = ctx->mrav_entry_size; 6817 } 6818 6819 static int bnxt_hwrm_func_backing_store_qcaps(struct bnxt *bp) 6820 { 6821 struct hwrm_func_backing_store_qcaps_input req = {0}; 6822 struct hwrm_func_backing_store_qcaps_output *resp = 6823 bp->hwrm_cmd_resp_addr; 6824 int rc; 6825 6826 if (bp->hwrm_spec_code < 0x10902 || BNXT_VF(bp) || bp->ctx) 6827 return 0; 6828 6829 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_BACKING_STORE_QCAPS, -1, -1); 6830 mutex_lock(&bp->hwrm_cmd_lock); 6831 rc = _hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6832 if (!rc) { 6833 struct bnxt_ctx_pg_info *ctx_pg; 6834 struct bnxt_ctx_mem_info *ctx; 6835 int i, tqm_rings; 6836 6837 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 6838 if (!ctx) { 6839 rc = -ENOMEM; 6840 goto ctx_err; 6841 } 6842 ctx->qp_max_entries = le32_to_cpu(resp->qp_max_entries); 6843 ctx->qp_min_qp1_entries = le16_to_cpu(resp->qp_min_qp1_entries); 6844 ctx->qp_max_l2_entries = le16_to_cpu(resp->qp_max_l2_entries); 6845 ctx->qp_entry_size = le16_to_cpu(resp->qp_entry_size); 6846 ctx->srq_max_l2_entries = le16_to_cpu(resp->srq_max_l2_entries); 6847 ctx->srq_max_entries = le32_to_cpu(resp->srq_max_entries); 6848 ctx->srq_entry_size = le16_to_cpu(resp->srq_entry_size); 6849 ctx->cq_max_l2_entries = le16_to_cpu(resp->cq_max_l2_entries); 6850 ctx->cq_max_entries = le32_to_cpu(resp->cq_max_entries); 6851 ctx->cq_entry_size = le16_to_cpu(resp->cq_entry_size); 6852 ctx->vnic_max_vnic_entries = 6853 le16_to_cpu(resp->vnic_max_vnic_entries); 6854 ctx->vnic_max_ring_table_entries = 6855 le16_to_cpu(resp->vnic_max_ring_table_entries); 6856 ctx->vnic_entry_size = le16_to_cpu(resp->vnic_entry_size); 6857 ctx->stat_max_entries = le32_to_cpu(resp->stat_max_entries); 6858 ctx->stat_entry_size = le16_to_cpu(resp->stat_entry_size); 6859 ctx->tqm_entry_size = le16_to_cpu(resp->tqm_entry_size); 6860 ctx->tqm_min_entries_per_ring = 6861 le32_to_cpu(resp->tqm_min_entries_per_ring); 6862 ctx->tqm_max_entries_per_ring = 6863 le32_to_cpu(resp->tqm_max_entries_per_ring); 6864 ctx->tqm_entries_multiple = resp->tqm_entries_multiple; 6865 if (!ctx->tqm_entries_multiple) 6866 ctx->tqm_entries_multiple = 1; 6867 ctx->mrav_max_entries = le32_to_cpu(resp->mrav_max_entries); 6868 ctx->mrav_entry_size = le16_to_cpu(resp->mrav_entry_size); 6869 ctx->mrav_num_entries_units = 6870 le16_to_cpu(resp->mrav_num_entries_units); 6871 ctx->tim_entry_size = le16_to_cpu(resp->tim_entry_size); 6872 ctx->tim_max_entries = le32_to_cpu(resp->tim_max_entries); 6873 6874 bnxt_init_ctx_initializer(ctx, resp); 6875 6876 ctx->tqm_fp_rings_count = resp->tqm_fp_rings_count; 6877 if (!ctx->tqm_fp_rings_count) 6878 ctx->tqm_fp_rings_count = bp->max_q; 6879 else if (ctx->tqm_fp_rings_count > BNXT_MAX_TQM_FP_RINGS) 6880 ctx->tqm_fp_rings_count = BNXT_MAX_TQM_FP_RINGS; 6881 6882 tqm_rings = ctx->tqm_fp_rings_count + BNXT_MAX_TQM_SP_RINGS; 6883 ctx_pg = kcalloc(tqm_rings, sizeof(*ctx_pg), GFP_KERNEL); 6884 if (!ctx_pg) { 6885 kfree(ctx); 6886 rc = -ENOMEM; 6887 goto ctx_err; 6888 } 6889 for (i = 0; i < tqm_rings; i++, ctx_pg++) 6890 ctx->tqm_mem[i] = ctx_pg; 6891 bp->ctx = ctx; 6892 } else { 6893 rc = 0; 6894 } 6895 ctx_err: 6896 mutex_unlock(&bp->hwrm_cmd_lock); 6897 return rc; 6898 } 6899 6900 static void bnxt_hwrm_set_pg_attr(struct bnxt_ring_mem_info *rmem, u8 *pg_attr, 6901 __le64 *pg_dir) 6902 { 6903 u8 pg_size = 0; 6904 6905 if (!rmem->nr_pages) 6906 return; 6907 6908 if (BNXT_PAGE_SHIFT == 13) 6909 pg_size = 1 << 4; 6910 else if (BNXT_PAGE_SIZE == 16) 6911 pg_size = 2 << 4; 6912 6913 *pg_attr = pg_size; 6914 if (rmem->depth >= 1) { 6915 if (rmem->depth == 2) 6916 *pg_attr |= 2; 6917 else 6918 *pg_attr |= 1; 6919 *pg_dir = cpu_to_le64(rmem->pg_tbl_map); 6920 } else { 6921 *pg_dir = cpu_to_le64(rmem->dma_arr[0]); 6922 } 6923 } 6924 6925 #define FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES \ 6926 (FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP | \ 6927 FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ | \ 6928 FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ | \ 6929 FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC | \ 6930 FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) 6931 6932 static int bnxt_hwrm_func_backing_store_cfg(struct bnxt *bp, u32 enables) 6933 { 6934 struct hwrm_func_backing_store_cfg_input req = {0}; 6935 struct bnxt_ctx_mem_info *ctx = bp->ctx; 6936 struct bnxt_ctx_pg_info *ctx_pg; 6937 u32 req_len = sizeof(req); 6938 __le32 *num_entries; 6939 __le64 *pg_dir; 6940 u32 flags = 0; 6941 u8 *pg_attr; 6942 u32 ena; 6943 int i; 6944 6945 if (!ctx) 6946 return 0; 6947 6948 if (req_len > bp->hwrm_max_ext_req_len) 6949 req_len = BNXT_BACKING_STORE_CFG_LEGACY_LEN; 6950 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_BACKING_STORE_CFG, -1, -1); 6951 req.enables = cpu_to_le32(enables); 6952 6953 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP) { 6954 ctx_pg = &ctx->qp_mem; 6955 req.qp_num_entries = cpu_to_le32(ctx_pg->entries); 6956 req.qp_num_qp1_entries = cpu_to_le16(ctx->qp_min_qp1_entries); 6957 req.qp_num_l2_entries = cpu_to_le16(ctx->qp_max_l2_entries); 6958 req.qp_entry_size = cpu_to_le16(ctx->qp_entry_size); 6959 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 6960 &req.qpc_pg_size_qpc_lvl, 6961 &req.qpc_page_dir); 6962 } 6963 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ) { 6964 ctx_pg = &ctx->srq_mem; 6965 req.srq_num_entries = cpu_to_le32(ctx_pg->entries); 6966 req.srq_num_l2_entries = cpu_to_le16(ctx->srq_max_l2_entries); 6967 req.srq_entry_size = cpu_to_le16(ctx->srq_entry_size); 6968 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 6969 &req.srq_pg_size_srq_lvl, 6970 &req.srq_page_dir); 6971 } 6972 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ) { 6973 ctx_pg = &ctx->cq_mem; 6974 req.cq_num_entries = cpu_to_le32(ctx_pg->entries); 6975 req.cq_num_l2_entries = cpu_to_le16(ctx->cq_max_l2_entries); 6976 req.cq_entry_size = cpu_to_le16(ctx->cq_entry_size); 6977 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, &req.cq_pg_size_cq_lvl, 6978 &req.cq_page_dir); 6979 } 6980 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC) { 6981 ctx_pg = &ctx->vnic_mem; 6982 req.vnic_num_vnic_entries = 6983 cpu_to_le16(ctx->vnic_max_vnic_entries); 6984 req.vnic_num_ring_table_entries = 6985 cpu_to_le16(ctx->vnic_max_ring_table_entries); 6986 req.vnic_entry_size = cpu_to_le16(ctx->vnic_entry_size); 6987 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 6988 &req.vnic_pg_size_vnic_lvl, 6989 &req.vnic_page_dir); 6990 } 6991 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) { 6992 ctx_pg = &ctx->stat_mem; 6993 req.stat_num_entries = cpu_to_le32(ctx->stat_max_entries); 6994 req.stat_entry_size = cpu_to_le16(ctx->stat_entry_size); 6995 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 6996 &req.stat_pg_size_stat_lvl, 6997 &req.stat_page_dir); 6998 } 6999 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV) { 7000 ctx_pg = &ctx->mrav_mem; 7001 req.mrav_num_entries = cpu_to_le32(ctx_pg->entries); 7002 if (ctx->mrav_num_entries_units) 7003 flags |= 7004 FUNC_BACKING_STORE_CFG_REQ_FLAGS_MRAV_RESERVATION_SPLIT; 7005 req.mrav_entry_size = cpu_to_le16(ctx->mrav_entry_size); 7006 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 7007 &req.mrav_pg_size_mrav_lvl, 7008 &req.mrav_page_dir); 7009 } 7010 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM) { 7011 ctx_pg = &ctx->tim_mem; 7012 req.tim_num_entries = cpu_to_le32(ctx_pg->entries); 7013 req.tim_entry_size = cpu_to_le16(ctx->tim_entry_size); 7014 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 7015 &req.tim_pg_size_tim_lvl, 7016 &req.tim_page_dir); 7017 } 7018 for (i = 0, num_entries = &req.tqm_sp_num_entries, 7019 pg_attr = &req.tqm_sp_pg_size_tqm_sp_lvl, 7020 pg_dir = &req.tqm_sp_page_dir, 7021 ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP; 7022 i < BNXT_MAX_TQM_RINGS; 7023 i++, num_entries++, pg_attr++, pg_dir++, ena <<= 1) { 7024 if (!(enables & ena)) 7025 continue; 7026 7027 req.tqm_entry_size = cpu_to_le16(ctx->tqm_entry_size); 7028 ctx_pg = ctx->tqm_mem[i]; 7029 *num_entries = cpu_to_le32(ctx_pg->entries); 7030 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, pg_attr, pg_dir); 7031 } 7032 req.flags = cpu_to_le32(flags); 7033 return hwrm_send_message(bp, &req, req_len, HWRM_CMD_TIMEOUT); 7034 } 7035 7036 static int bnxt_alloc_ctx_mem_blk(struct bnxt *bp, 7037 struct bnxt_ctx_pg_info *ctx_pg) 7038 { 7039 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem; 7040 7041 rmem->page_size = BNXT_PAGE_SIZE; 7042 rmem->pg_arr = ctx_pg->ctx_pg_arr; 7043 rmem->dma_arr = ctx_pg->ctx_dma_arr; 7044 rmem->flags = BNXT_RMEM_VALID_PTE_FLAG; 7045 if (rmem->depth >= 1) 7046 rmem->flags |= BNXT_RMEM_USE_FULL_PAGE_FLAG; 7047 return bnxt_alloc_ring(bp, rmem); 7048 } 7049 7050 static int bnxt_alloc_ctx_pg_tbls(struct bnxt *bp, 7051 struct bnxt_ctx_pg_info *ctx_pg, u32 mem_size, 7052 u8 depth, struct bnxt_mem_init *mem_init) 7053 { 7054 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem; 7055 int rc; 7056 7057 if (!mem_size) 7058 return -EINVAL; 7059 7060 ctx_pg->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE); 7061 if (ctx_pg->nr_pages > MAX_CTX_TOTAL_PAGES) { 7062 ctx_pg->nr_pages = 0; 7063 return -EINVAL; 7064 } 7065 if (ctx_pg->nr_pages > MAX_CTX_PAGES || depth > 1) { 7066 int nr_tbls, i; 7067 7068 rmem->depth = 2; 7069 ctx_pg->ctx_pg_tbl = kcalloc(MAX_CTX_PAGES, sizeof(ctx_pg), 7070 GFP_KERNEL); 7071 if (!ctx_pg->ctx_pg_tbl) 7072 return -ENOMEM; 7073 nr_tbls = DIV_ROUND_UP(ctx_pg->nr_pages, MAX_CTX_PAGES); 7074 rmem->nr_pages = nr_tbls; 7075 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg); 7076 if (rc) 7077 return rc; 7078 for (i = 0; i < nr_tbls; i++) { 7079 struct bnxt_ctx_pg_info *pg_tbl; 7080 7081 pg_tbl = kzalloc(sizeof(*pg_tbl), GFP_KERNEL); 7082 if (!pg_tbl) 7083 return -ENOMEM; 7084 ctx_pg->ctx_pg_tbl[i] = pg_tbl; 7085 rmem = &pg_tbl->ring_mem; 7086 rmem->pg_tbl = ctx_pg->ctx_pg_arr[i]; 7087 rmem->pg_tbl_map = ctx_pg->ctx_dma_arr[i]; 7088 rmem->depth = 1; 7089 rmem->nr_pages = MAX_CTX_PAGES; 7090 rmem->mem_init = mem_init; 7091 if (i == (nr_tbls - 1)) { 7092 int rem = ctx_pg->nr_pages % MAX_CTX_PAGES; 7093 7094 if (rem) 7095 rmem->nr_pages = rem; 7096 } 7097 rc = bnxt_alloc_ctx_mem_blk(bp, pg_tbl); 7098 if (rc) 7099 break; 7100 } 7101 } else { 7102 rmem->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE); 7103 if (rmem->nr_pages > 1 || depth) 7104 rmem->depth = 1; 7105 rmem->mem_init = mem_init; 7106 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg); 7107 } 7108 return rc; 7109 } 7110 7111 static void bnxt_free_ctx_pg_tbls(struct bnxt *bp, 7112 struct bnxt_ctx_pg_info *ctx_pg) 7113 { 7114 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem; 7115 7116 if (rmem->depth > 1 || ctx_pg->nr_pages > MAX_CTX_PAGES || 7117 ctx_pg->ctx_pg_tbl) { 7118 int i, nr_tbls = rmem->nr_pages; 7119 7120 for (i = 0; i < nr_tbls; i++) { 7121 struct bnxt_ctx_pg_info *pg_tbl; 7122 struct bnxt_ring_mem_info *rmem2; 7123 7124 pg_tbl = ctx_pg->ctx_pg_tbl[i]; 7125 if (!pg_tbl) 7126 continue; 7127 rmem2 = &pg_tbl->ring_mem; 7128 bnxt_free_ring(bp, rmem2); 7129 ctx_pg->ctx_pg_arr[i] = NULL; 7130 kfree(pg_tbl); 7131 ctx_pg->ctx_pg_tbl[i] = NULL; 7132 } 7133 kfree(ctx_pg->ctx_pg_tbl); 7134 ctx_pg->ctx_pg_tbl = NULL; 7135 } 7136 bnxt_free_ring(bp, rmem); 7137 ctx_pg->nr_pages = 0; 7138 } 7139 7140 static void bnxt_free_ctx_mem(struct bnxt *bp) 7141 { 7142 struct bnxt_ctx_mem_info *ctx = bp->ctx; 7143 int i; 7144 7145 if (!ctx) 7146 return; 7147 7148 if (ctx->tqm_mem[0]) { 7149 for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++) 7150 bnxt_free_ctx_pg_tbls(bp, ctx->tqm_mem[i]); 7151 kfree(ctx->tqm_mem[0]); 7152 ctx->tqm_mem[0] = NULL; 7153 } 7154 7155 bnxt_free_ctx_pg_tbls(bp, &ctx->tim_mem); 7156 bnxt_free_ctx_pg_tbls(bp, &ctx->mrav_mem); 7157 bnxt_free_ctx_pg_tbls(bp, &ctx->stat_mem); 7158 bnxt_free_ctx_pg_tbls(bp, &ctx->vnic_mem); 7159 bnxt_free_ctx_pg_tbls(bp, &ctx->cq_mem); 7160 bnxt_free_ctx_pg_tbls(bp, &ctx->srq_mem); 7161 bnxt_free_ctx_pg_tbls(bp, &ctx->qp_mem); 7162 ctx->flags &= ~BNXT_CTX_FLAG_INITED; 7163 } 7164 7165 static int bnxt_alloc_ctx_mem(struct bnxt *bp) 7166 { 7167 struct bnxt_ctx_pg_info *ctx_pg; 7168 struct bnxt_ctx_mem_info *ctx; 7169 struct bnxt_mem_init *init; 7170 u32 mem_size, ena, entries; 7171 u32 entries_sp, min; 7172 u32 num_mr, num_ah; 7173 u32 extra_srqs = 0; 7174 u32 extra_qps = 0; 7175 u8 pg_lvl = 1; 7176 int i, rc; 7177 7178 rc = bnxt_hwrm_func_backing_store_qcaps(bp); 7179 if (rc) { 7180 netdev_err(bp->dev, "Failed querying context mem capability, rc = %d.\n", 7181 rc); 7182 return rc; 7183 } 7184 ctx = bp->ctx; 7185 if (!ctx || (ctx->flags & BNXT_CTX_FLAG_INITED)) 7186 return 0; 7187 7188 if ((bp->flags & BNXT_FLAG_ROCE_CAP) && !is_kdump_kernel()) { 7189 pg_lvl = 2; 7190 extra_qps = 65536; 7191 extra_srqs = 8192; 7192 } 7193 7194 ctx_pg = &ctx->qp_mem; 7195 ctx_pg->entries = ctx->qp_min_qp1_entries + ctx->qp_max_l2_entries + 7196 extra_qps; 7197 if (ctx->qp_entry_size) { 7198 mem_size = ctx->qp_entry_size * ctx_pg->entries; 7199 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_QP]; 7200 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, init); 7201 if (rc) 7202 return rc; 7203 } 7204 7205 ctx_pg = &ctx->srq_mem; 7206 ctx_pg->entries = ctx->srq_max_l2_entries + extra_srqs; 7207 if (ctx->srq_entry_size) { 7208 mem_size = ctx->srq_entry_size * ctx_pg->entries; 7209 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_SRQ]; 7210 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, init); 7211 if (rc) 7212 return rc; 7213 } 7214 7215 ctx_pg = &ctx->cq_mem; 7216 ctx_pg->entries = ctx->cq_max_l2_entries + extra_qps * 2; 7217 if (ctx->cq_entry_size) { 7218 mem_size = ctx->cq_entry_size * ctx_pg->entries; 7219 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_CQ]; 7220 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, init); 7221 if (rc) 7222 return rc; 7223 } 7224 7225 ctx_pg = &ctx->vnic_mem; 7226 ctx_pg->entries = ctx->vnic_max_vnic_entries + 7227 ctx->vnic_max_ring_table_entries; 7228 if (ctx->vnic_entry_size) { 7229 mem_size = ctx->vnic_entry_size * ctx_pg->entries; 7230 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_VNIC]; 7231 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, init); 7232 if (rc) 7233 return rc; 7234 } 7235 7236 ctx_pg = &ctx->stat_mem; 7237 ctx_pg->entries = ctx->stat_max_entries; 7238 if (ctx->stat_entry_size) { 7239 mem_size = ctx->stat_entry_size * ctx_pg->entries; 7240 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_STAT]; 7241 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, init); 7242 if (rc) 7243 return rc; 7244 } 7245 7246 ena = 0; 7247 if (!(bp->flags & BNXT_FLAG_ROCE_CAP)) 7248 goto skip_rdma; 7249 7250 ctx_pg = &ctx->mrav_mem; 7251 /* 128K extra is needed to accommodate static AH context 7252 * allocation by f/w. 7253 */ 7254 num_mr = 1024 * 256; 7255 num_ah = 1024 * 128; 7256 ctx_pg->entries = num_mr + num_ah; 7257 if (ctx->mrav_entry_size) { 7258 mem_size = ctx->mrav_entry_size * ctx_pg->entries; 7259 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_MRAV]; 7260 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 2, init); 7261 if (rc) 7262 return rc; 7263 } 7264 ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV; 7265 if (ctx->mrav_num_entries_units) 7266 ctx_pg->entries = 7267 ((num_mr / ctx->mrav_num_entries_units) << 16) | 7268 (num_ah / ctx->mrav_num_entries_units); 7269 7270 ctx_pg = &ctx->tim_mem; 7271 ctx_pg->entries = ctx->qp_mem.entries; 7272 if (ctx->tim_entry_size) { 7273 mem_size = ctx->tim_entry_size * ctx_pg->entries; 7274 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, NULL); 7275 if (rc) 7276 return rc; 7277 } 7278 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM; 7279 7280 skip_rdma: 7281 min = ctx->tqm_min_entries_per_ring; 7282 entries_sp = ctx->vnic_max_vnic_entries + ctx->qp_max_l2_entries + 7283 2 * (extra_qps + ctx->qp_min_qp1_entries) + min; 7284 entries_sp = roundup(entries_sp, ctx->tqm_entries_multiple); 7285 entries = ctx->qp_max_l2_entries + extra_qps + ctx->qp_min_qp1_entries; 7286 entries = roundup(entries, ctx->tqm_entries_multiple); 7287 entries = clamp_t(u32, entries, min, ctx->tqm_max_entries_per_ring); 7288 for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++) { 7289 ctx_pg = ctx->tqm_mem[i]; 7290 ctx_pg->entries = i ? entries : entries_sp; 7291 if (ctx->tqm_entry_size) { 7292 mem_size = ctx->tqm_entry_size * ctx_pg->entries; 7293 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, 7294 NULL); 7295 if (rc) 7296 return rc; 7297 } 7298 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP << i; 7299 } 7300 ena |= FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES; 7301 rc = bnxt_hwrm_func_backing_store_cfg(bp, ena); 7302 if (rc) { 7303 netdev_err(bp->dev, "Failed configuring context mem, rc = %d.\n", 7304 rc); 7305 return rc; 7306 } 7307 ctx->flags |= BNXT_CTX_FLAG_INITED; 7308 return 0; 7309 } 7310 7311 int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp, bool all) 7312 { 7313 struct hwrm_func_resource_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 7314 struct hwrm_func_resource_qcaps_input req = {0}; 7315 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 7316 int rc; 7317 7318 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESOURCE_QCAPS, -1, -1); 7319 req.fid = cpu_to_le16(0xffff); 7320 7321 mutex_lock(&bp->hwrm_cmd_lock); 7322 rc = _hwrm_send_message_silent(bp, &req, sizeof(req), 7323 HWRM_CMD_TIMEOUT); 7324 if (rc) 7325 goto hwrm_func_resc_qcaps_exit; 7326 7327 hw_resc->max_tx_sch_inputs = le16_to_cpu(resp->max_tx_scheduler_inputs); 7328 if (!all) 7329 goto hwrm_func_resc_qcaps_exit; 7330 7331 hw_resc->min_rsscos_ctxs = le16_to_cpu(resp->min_rsscos_ctx); 7332 hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx); 7333 hw_resc->min_cp_rings = le16_to_cpu(resp->min_cmpl_rings); 7334 hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings); 7335 hw_resc->min_tx_rings = le16_to_cpu(resp->min_tx_rings); 7336 hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings); 7337 hw_resc->min_rx_rings = le16_to_cpu(resp->min_rx_rings); 7338 hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings); 7339 hw_resc->min_hw_ring_grps = le16_to_cpu(resp->min_hw_ring_grps); 7340 hw_resc->max_hw_ring_grps = le16_to_cpu(resp->max_hw_ring_grps); 7341 hw_resc->min_l2_ctxs = le16_to_cpu(resp->min_l2_ctxs); 7342 hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs); 7343 hw_resc->min_vnics = le16_to_cpu(resp->min_vnics); 7344 hw_resc->max_vnics = le16_to_cpu(resp->max_vnics); 7345 hw_resc->min_stat_ctxs = le16_to_cpu(resp->min_stat_ctx); 7346 hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx); 7347 7348 if (bp->flags & BNXT_FLAG_CHIP_P5) { 7349 u16 max_msix = le16_to_cpu(resp->max_msix); 7350 7351 hw_resc->max_nqs = max_msix; 7352 hw_resc->max_hw_ring_grps = hw_resc->max_rx_rings; 7353 } 7354 7355 if (BNXT_PF(bp)) { 7356 struct bnxt_pf_info *pf = &bp->pf; 7357 7358 pf->vf_resv_strategy = 7359 le16_to_cpu(resp->vf_reservation_strategy); 7360 if (pf->vf_resv_strategy > BNXT_VF_RESV_STRATEGY_MINIMAL_STATIC) 7361 pf->vf_resv_strategy = BNXT_VF_RESV_STRATEGY_MAXIMAL; 7362 } 7363 hwrm_func_resc_qcaps_exit: 7364 mutex_unlock(&bp->hwrm_cmd_lock); 7365 return rc; 7366 } 7367 7368 static int __bnxt_hwrm_func_qcaps(struct bnxt *bp) 7369 { 7370 int rc = 0; 7371 struct hwrm_func_qcaps_input req = {0}; 7372 struct hwrm_func_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 7373 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 7374 u32 flags, flags_ext; 7375 7376 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCAPS, -1, -1); 7377 req.fid = cpu_to_le16(0xffff); 7378 7379 mutex_lock(&bp->hwrm_cmd_lock); 7380 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 7381 if (rc) 7382 goto hwrm_func_qcaps_exit; 7383 7384 flags = le32_to_cpu(resp->flags); 7385 if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED) 7386 bp->flags |= BNXT_FLAG_ROCEV1_CAP; 7387 if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED) 7388 bp->flags |= BNXT_FLAG_ROCEV2_CAP; 7389 if (flags & FUNC_QCAPS_RESP_FLAGS_PCIE_STATS_SUPPORTED) 7390 bp->fw_cap |= BNXT_FW_CAP_PCIE_STATS_SUPPORTED; 7391 if (flags & FUNC_QCAPS_RESP_FLAGS_HOT_RESET_CAPABLE) 7392 bp->fw_cap |= BNXT_FW_CAP_HOT_RESET; 7393 if (flags & FUNC_QCAPS_RESP_FLAGS_EXT_STATS_SUPPORTED) 7394 bp->fw_cap |= BNXT_FW_CAP_EXT_STATS_SUPPORTED; 7395 if (flags & FUNC_QCAPS_RESP_FLAGS_ERROR_RECOVERY_CAPABLE) 7396 bp->fw_cap |= BNXT_FW_CAP_ERROR_RECOVERY; 7397 if (flags & FUNC_QCAPS_RESP_FLAGS_ERR_RECOVER_RELOAD) 7398 bp->fw_cap |= BNXT_FW_CAP_ERR_RECOVER_RELOAD; 7399 if (!(flags & FUNC_QCAPS_RESP_FLAGS_VLAN_ACCELERATION_TX_DISABLED)) 7400 bp->fw_cap |= BNXT_FW_CAP_VLAN_TX_INSERT; 7401 7402 flags_ext = le32_to_cpu(resp->flags_ext); 7403 if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_EXT_HW_STATS_SUPPORTED) 7404 bp->fw_cap |= BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED; 7405 7406 bp->tx_push_thresh = 0; 7407 if ((flags & FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED) && 7408 BNXT_FW_MAJ(bp) > 217) 7409 bp->tx_push_thresh = BNXT_TX_PUSH_THRESH; 7410 7411 hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx); 7412 hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings); 7413 hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings); 7414 hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings); 7415 hw_resc->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps); 7416 if (!hw_resc->max_hw_ring_grps) 7417 hw_resc->max_hw_ring_grps = hw_resc->max_tx_rings; 7418 hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs); 7419 hw_resc->max_vnics = le16_to_cpu(resp->max_vnics); 7420 hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx); 7421 7422 if (BNXT_PF(bp)) { 7423 struct bnxt_pf_info *pf = &bp->pf; 7424 7425 pf->fw_fid = le16_to_cpu(resp->fid); 7426 pf->port_id = le16_to_cpu(resp->port_id); 7427 memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN); 7428 pf->first_vf_id = le16_to_cpu(resp->first_vf_id); 7429 pf->max_vfs = le16_to_cpu(resp->max_vfs); 7430 pf->max_encap_records = le32_to_cpu(resp->max_encap_records); 7431 pf->max_decap_records = le32_to_cpu(resp->max_decap_records); 7432 pf->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows); 7433 pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows); 7434 pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows); 7435 pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows); 7436 bp->flags &= ~BNXT_FLAG_WOL_CAP; 7437 if (flags & FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED) 7438 bp->flags |= BNXT_FLAG_WOL_CAP; 7439 } else { 7440 #ifdef CONFIG_BNXT_SRIOV 7441 struct bnxt_vf_info *vf = &bp->vf; 7442 7443 vf->fw_fid = le16_to_cpu(resp->fid); 7444 memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN); 7445 #endif 7446 } 7447 7448 hwrm_func_qcaps_exit: 7449 mutex_unlock(&bp->hwrm_cmd_lock); 7450 return rc; 7451 } 7452 7453 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp); 7454 7455 static int bnxt_hwrm_func_qcaps(struct bnxt *bp) 7456 { 7457 int rc; 7458 7459 rc = __bnxt_hwrm_func_qcaps(bp); 7460 if (rc) 7461 return rc; 7462 rc = bnxt_hwrm_queue_qportcfg(bp); 7463 if (rc) { 7464 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %d\n", rc); 7465 return rc; 7466 } 7467 if (bp->hwrm_spec_code >= 0x10803) { 7468 rc = bnxt_alloc_ctx_mem(bp); 7469 if (rc) 7470 return rc; 7471 rc = bnxt_hwrm_func_resc_qcaps(bp, true); 7472 if (!rc) 7473 bp->fw_cap |= BNXT_FW_CAP_NEW_RM; 7474 } 7475 return 0; 7476 } 7477 7478 static int bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(struct bnxt *bp) 7479 { 7480 struct hwrm_cfa_adv_flow_mgnt_qcaps_input req = {0}; 7481 struct hwrm_cfa_adv_flow_mgnt_qcaps_output *resp; 7482 int rc = 0; 7483 u32 flags; 7484 7485 if (!(bp->fw_cap & BNXT_FW_CAP_CFA_ADV_FLOW)) 7486 return 0; 7487 7488 resp = bp->hwrm_cmd_resp_addr; 7489 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_ADV_FLOW_MGNT_QCAPS, -1, -1); 7490 7491 mutex_lock(&bp->hwrm_cmd_lock); 7492 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 7493 if (rc) 7494 goto hwrm_cfa_adv_qcaps_exit; 7495 7496 flags = le32_to_cpu(resp->flags); 7497 if (flags & 7498 CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_V2_SUPPORTED) 7499 bp->fw_cap |= BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2; 7500 7501 hwrm_cfa_adv_qcaps_exit: 7502 mutex_unlock(&bp->hwrm_cmd_lock); 7503 return rc; 7504 } 7505 7506 static int __bnxt_alloc_fw_health(struct bnxt *bp) 7507 { 7508 if (bp->fw_health) 7509 return 0; 7510 7511 bp->fw_health = kzalloc(sizeof(*bp->fw_health), GFP_KERNEL); 7512 if (!bp->fw_health) 7513 return -ENOMEM; 7514 7515 return 0; 7516 } 7517 7518 static int bnxt_alloc_fw_health(struct bnxt *bp) 7519 { 7520 int rc; 7521 7522 if (!(bp->fw_cap & BNXT_FW_CAP_HOT_RESET) && 7523 !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) 7524 return 0; 7525 7526 rc = __bnxt_alloc_fw_health(bp); 7527 if (rc) { 7528 bp->fw_cap &= ~BNXT_FW_CAP_HOT_RESET; 7529 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY; 7530 return rc; 7531 } 7532 7533 return 0; 7534 } 7535 7536 static void __bnxt_map_fw_health_reg(struct bnxt *bp, u32 reg) 7537 { 7538 writel(reg & BNXT_GRC_BASE_MASK, bp->bar0 + 7539 BNXT_GRCPF_REG_WINDOW_BASE_OUT + 7540 BNXT_FW_HEALTH_WIN_MAP_OFF); 7541 } 7542 7543 static void bnxt_try_map_fw_health_reg(struct bnxt *bp) 7544 { 7545 void __iomem *hs; 7546 u32 status_loc; 7547 u32 reg_type; 7548 u32 sig; 7549 7550 __bnxt_map_fw_health_reg(bp, HCOMM_STATUS_STRUCT_LOC); 7551 hs = bp->bar0 + BNXT_FW_HEALTH_WIN_OFF(HCOMM_STATUS_STRUCT_LOC); 7552 7553 sig = readl(hs + offsetof(struct hcomm_status, sig_ver)); 7554 if ((sig & HCOMM_STATUS_SIGNATURE_MASK) != HCOMM_STATUS_SIGNATURE_VAL) { 7555 if (!bp->chip_num) { 7556 __bnxt_map_fw_health_reg(bp, BNXT_GRC_REG_BASE); 7557 bp->chip_num = readl(bp->bar0 + 7558 BNXT_FW_HEALTH_WIN_BASE + 7559 BNXT_GRC_REG_CHIP_NUM); 7560 } 7561 if (!BNXT_CHIP_P5(bp)) { 7562 if (bp->fw_health) 7563 bp->fw_health->status_reliable = false; 7564 return; 7565 } 7566 status_loc = BNXT_GRC_REG_STATUS_P5 | 7567 BNXT_FW_HEALTH_REG_TYPE_BAR0; 7568 } else { 7569 status_loc = readl(hs + offsetof(struct hcomm_status, 7570 fw_status_loc)); 7571 } 7572 7573 if (__bnxt_alloc_fw_health(bp)) { 7574 netdev_warn(bp->dev, "no memory for firmware status checks\n"); 7575 return; 7576 } 7577 7578 bp->fw_health->regs[BNXT_FW_HEALTH_REG] = status_loc; 7579 reg_type = BNXT_FW_HEALTH_REG_TYPE(status_loc); 7580 if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC) { 7581 __bnxt_map_fw_health_reg(bp, status_loc); 7582 bp->fw_health->mapped_regs[BNXT_FW_HEALTH_REG] = 7583 BNXT_FW_HEALTH_WIN_OFF(status_loc); 7584 } 7585 7586 bp->fw_health->status_reliable = true; 7587 } 7588 7589 static int bnxt_map_fw_health_regs(struct bnxt *bp) 7590 { 7591 struct bnxt_fw_health *fw_health = bp->fw_health; 7592 u32 reg_base = 0xffffffff; 7593 int i; 7594 7595 /* Only pre-map the monitoring GRC registers using window 3 */ 7596 for (i = 0; i < 4; i++) { 7597 u32 reg = fw_health->regs[i]; 7598 7599 if (BNXT_FW_HEALTH_REG_TYPE(reg) != BNXT_FW_HEALTH_REG_TYPE_GRC) 7600 continue; 7601 if (reg_base == 0xffffffff) 7602 reg_base = reg & BNXT_GRC_BASE_MASK; 7603 if ((reg & BNXT_GRC_BASE_MASK) != reg_base) 7604 return -ERANGE; 7605 fw_health->mapped_regs[i] = BNXT_FW_HEALTH_WIN_OFF(reg); 7606 } 7607 if (reg_base == 0xffffffff) 7608 return 0; 7609 7610 __bnxt_map_fw_health_reg(bp, reg_base); 7611 return 0; 7612 } 7613 7614 static int bnxt_hwrm_error_recovery_qcfg(struct bnxt *bp) 7615 { 7616 struct hwrm_error_recovery_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 7617 struct bnxt_fw_health *fw_health = bp->fw_health; 7618 struct hwrm_error_recovery_qcfg_input req = {0}; 7619 int rc, i; 7620 7621 if (!(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) 7622 return 0; 7623 7624 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_ERROR_RECOVERY_QCFG, -1, -1); 7625 mutex_lock(&bp->hwrm_cmd_lock); 7626 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 7627 if (rc) 7628 goto err_recovery_out; 7629 fw_health->flags = le32_to_cpu(resp->flags); 7630 if ((fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) && 7631 !(bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL)) { 7632 rc = -EINVAL; 7633 goto err_recovery_out; 7634 } 7635 fw_health->polling_dsecs = le32_to_cpu(resp->driver_polling_freq); 7636 fw_health->master_func_wait_dsecs = 7637 le32_to_cpu(resp->master_func_wait_period); 7638 fw_health->normal_func_wait_dsecs = 7639 le32_to_cpu(resp->normal_func_wait_period); 7640 fw_health->post_reset_wait_dsecs = 7641 le32_to_cpu(resp->master_func_wait_period_after_reset); 7642 fw_health->post_reset_max_wait_dsecs = 7643 le32_to_cpu(resp->max_bailout_time_after_reset); 7644 fw_health->regs[BNXT_FW_HEALTH_REG] = 7645 le32_to_cpu(resp->fw_health_status_reg); 7646 fw_health->regs[BNXT_FW_HEARTBEAT_REG] = 7647 le32_to_cpu(resp->fw_heartbeat_reg); 7648 fw_health->regs[BNXT_FW_RESET_CNT_REG] = 7649 le32_to_cpu(resp->fw_reset_cnt_reg); 7650 fw_health->regs[BNXT_FW_RESET_INPROG_REG] = 7651 le32_to_cpu(resp->reset_inprogress_reg); 7652 fw_health->fw_reset_inprog_reg_mask = 7653 le32_to_cpu(resp->reset_inprogress_reg_mask); 7654 fw_health->fw_reset_seq_cnt = resp->reg_array_cnt; 7655 if (fw_health->fw_reset_seq_cnt >= 16) { 7656 rc = -EINVAL; 7657 goto err_recovery_out; 7658 } 7659 for (i = 0; i < fw_health->fw_reset_seq_cnt; i++) { 7660 fw_health->fw_reset_seq_regs[i] = 7661 le32_to_cpu(resp->reset_reg[i]); 7662 fw_health->fw_reset_seq_vals[i] = 7663 le32_to_cpu(resp->reset_reg_val[i]); 7664 fw_health->fw_reset_seq_delay_msec[i] = 7665 resp->delay_after_reset[i]; 7666 } 7667 err_recovery_out: 7668 mutex_unlock(&bp->hwrm_cmd_lock); 7669 if (!rc) 7670 rc = bnxt_map_fw_health_regs(bp); 7671 if (rc) 7672 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY; 7673 return rc; 7674 } 7675 7676 static int bnxt_hwrm_func_reset(struct bnxt *bp) 7677 { 7678 struct hwrm_func_reset_input req = {0}; 7679 7680 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESET, -1, -1); 7681 req.enables = 0; 7682 7683 return hwrm_send_message(bp, &req, sizeof(req), HWRM_RESET_TIMEOUT); 7684 } 7685 7686 static void bnxt_nvm_cfg_ver_get(struct bnxt *bp) 7687 { 7688 struct hwrm_nvm_get_dev_info_output nvm_info; 7689 7690 if (!bnxt_hwrm_nvm_get_dev_info(bp, &nvm_info)) 7691 snprintf(bp->nvm_cfg_ver, FW_VER_STR_LEN, "%d.%d.%d", 7692 nvm_info.nvm_cfg_ver_maj, nvm_info.nvm_cfg_ver_min, 7693 nvm_info.nvm_cfg_ver_upd); 7694 } 7695 7696 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp) 7697 { 7698 int rc = 0; 7699 struct hwrm_queue_qportcfg_input req = {0}; 7700 struct hwrm_queue_qportcfg_output *resp = bp->hwrm_cmd_resp_addr; 7701 u8 i, j, *qptr; 7702 bool no_rdma; 7703 7704 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_QUEUE_QPORTCFG, -1, -1); 7705 7706 mutex_lock(&bp->hwrm_cmd_lock); 7707 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 7708 if (rc) 7709 goto qportcfg_exit; 7710 7711 if (!resp->max_configurable_queues) { 7712 rc = -EINVAL; 7713 goto qportcfg_exit; 7714 } 7715 bp->max_tc = resp->max_configurable_queues; 7716 bp->max_lltc = resp->max_configurable_lossless_queues; 7717 if (bp->max_tc > BNXT_MAX_QUEUE) 7718 bp->max_tc = BNXT_MAX_QUEUE; 7719 7720 no_rdma = !(bp->flags & BNXT_FLAG_ROCE_CAP); 7721 qptr = &resp->queue_id0; 7722 for (i = 0, j = 0; i < bp->max_tc; i++) { 7723 bp->q_info[j].queue_id = *qptr; 7724 bp->q_ids[i] = *qptr++; 7725 bp->q_info[j].queue_profile = *qptr++; 7726 bp->tc_to_qidx[j] = j; 7727 if (!BNXT_CNPQ(bp->q_info[j].queue_profile) || 7728 (no_rdma && BNXT_PF(bp))) 7729 j++; 7730 } 7731 bp->max_q = bp->max_tc; 7732 bp->max_tc = max_t(u8, j, 1); 7733 7734 if (resp->queue_cfg_info & QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG) 7735 bp->max_tc = 1; 7736 7737 if (bp->max_lltc > bp->max_tc) 7738 bp->max_lltc = bp->max_tc; 7739 7740 qportcfg_exit: 7741 mutex_unlock(&bp->hwrm_cmd_lock); 7742 return rc; 7743 } 7744 7745 static int __bnxt_hwrm_ver_get(struct bnxt *bp, bool silent) 7746 { 7747 struct hwrm_ver_get_input req = {0}; 7748 int rc; 7749 7750 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VER_GET, -1, -1); 7751 req.hwrm_intf_maj = HWRM_VERSION_MAJOR; 7752 req.hwrm_intf_min = HWRM_VERSION_MINOR; 7753 req.hwrm_intf_upd = HWRM_VERSION_UPDATE; 7754 7755 rc = bnxt_hwrm_do_send_msg(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT, 7756 silent); 7757 return rc; 7758 } 7759 7760 static int bnxt_hwrm_ver_get(struct bnxt *bp) 7761 { 7762 struct hwrm_ver_get_output *resp = bp->hwrm_cmd_resp_addr; 7763 u16 fw_maj, fw_min, fw_bld, fw_rsv; 7764 u32 dev_caps_cfg, hwrm_ver; 7765 int rc, len; 7766 7767 bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN; 7768 mutex_lock(&bp->hwrm_cmd_lock); 7769 rc = __bnxt_hwrm_ver_get(bp, false); 7770 if (rc) 7771 goto hwrm_ver_get_exit; 7772 7773 memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output)); 7774 7775 bp->hwrm_spec_code = resp->hwrm_intf_maj_8b << 16 | 7776 resp->hwrm_intf_min_8b << 8 | 7777 resp->hwrm_intf_upd_8b; 7778 if (resp->hwrm_intf_maj_8b < 1) { 7779 netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n", 7780 resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b, 7781 resp->hwrm_intf_upd_8b); 7782 netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n"); 7783 } 7784 7785 hwrm_ver = HWRM_VERSION_MAJOR << 16 | HWRM_VERSION_MINOR << 8 | 7786 HWRM_VERSION_UPDATE; 7787 7788 if (bp->hwrm_spec_code > hwrm_ver) 7789 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d", 7790 HWRM_VERSION_MAJOR, HWRM_VERSION_MINOR, 7791 HWRM_VERSION_UPDATE); 7792 else 7793 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d", 7794 resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b, 7795 resp->hwrm_intf_upd_8b); 7796 7797 fw_maj = le16_to_cpu(resp->hwrm_fw_major); 7798 if (bp->hwrm_spec_code > 0x10803 && fw_maj) { 7799 fw_min = le16_to_cpu(resp->hwrm_fw_minor); 7800 fw_bld = le16_to_cpu(resp->hwrm_fw_build); 7801 fw_rsv = le16_to_cpu(resp->hwrm_fw_patch); 7802 len = FW_VER_STR_LEN; 7803 } else { 7804 fw_maj = resp->hwrm_fw_maj_8b; 7805 fw_min = resp->hwrm_fw_min_8b; 7806 fw_bld = resp->hwrm_fw_bld_8b; 7807 fw_rsv = resp->hwrm_fw_rsvd_8b; 7808 len = BC_HWRM_STR_LEN; 7809 } 7810 bp->fw_ver_code = BNXT_FW_VER_CODE(fw_maj, fw_min, fw_bld, fw_rsv); 7811 snprintf(bp->fw_ver_str, len, "%d.%d.%d.%d", fw_maj, fw_min, fw_bld, 7812 fw_rsv); 7813 7814 if (strlen(resp->active_pkg_name)) { 7815 int fw_ver_len = strlen(bp->fw_ver_str); 7816 7817 snprintf(bp->fw_ver_str + fw_ver_len, 7818 FW_VER_STR_LEN - fw_ver_len - 1, "/pkg %s", 7819 resp->active_pkg_name); 7820 bp->fw_cap |= BNXT_FW_CAP_PKG_VER; 7821 } 7822 7823 bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout); 7824 if (!bp->hwrm_cmd_timeout) 7825 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT; 7826 7827 if (resp->hwrm_intf_maj_8b >= 1) { 7828 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len); 7829 bp->hwrm_max_ext_req_len = le16_to_cpu(resp->max_ext_req_len); 7830 } 7831 if (bp->hwrm_max_ext_req_len < HWRM_MAX_REQ_LEN) 7832 bp->hwrm_max_ext_req_len = HWRM_MAX_REQ_LEN; 7833 7834 bp->chip_num = le16_to_cpu(resp->chip_num); 7835 bp->chip_rev = resp->chip_rev; 7836 if (bp->chip_num == CHIP_NUM_58700 && !resp->chip_rev && 7837 !resp->chip_metal) 7838 bp->flags |= BNXT_FLAG_CHIP_NITRO_A0; 7839 7840 dev_caps_cfg = le32_to_cpu(resp->dev_caps_cfg); 7841 if ((dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) && 7842 (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED)) 7843 bp->fw_cap |= BNXT_FW_CAP_SHORT_CMD; 7844 7845 if (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_KONG_MB_CHNL_SUPPORTED) 7846 bp->fw_cap |= BNXT_FW_CAP_KONG_MB_CHNL; 7847 7848 if (dev_caps_cfg & 7849 VER_GET_RESP_DEV_CAPS_CFG_FLOW_HANDLE_64BIT_SUPPORTED) 7850 bp->fw_cap |= BNXT_FW_CAP_OVS_64BIT_HANDLE; 7851 7852 if (dev_caps_cfg & 7853 VER_GET_RESP_DEV_CAPS_CFG_TRUSTED_VF_SUPPORTED) 7854 bp->fw_cap |= BNXT_FW_CAP_TRUSTED_VF; 7855 7856 if (dev_caps_cfg & 7857 VER_GET_RESP_DEV_CAPS_CFG_CFA_ADV_FLOW_MGNT_SUPPORTED) 7858 bp->fw_cap |= BNXT_FW_CAP_CFA_ADV_FLOW; 7859 7860 hwrm_ver_get_exit: 7861 mutex_unlock(&bp->hwrm_cmd_lock); 7862 return rc; 7863 } 7864 7865 int bnxt_hwrm_fw_set_time(struct bnxt *bp) 7866 { 7867 struct hwrm_fw_set_time_input req = {0}; 7868 struct tm tm; 7869 time64_t now = ktime_get_real_seconds(); 7870 7871 if ((BNXT_VF(bp) && bp->hwrm_spec_code < 0x10901) || 7872 bp->hwrm_spec_code < 0x10400) 7873 return -EOPNOTSUPP; 7874 7875 time64_to_tm(now, 0, &tm); 7876 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FW_SET_TIME, -1, -1); 7877 req.year = cpu_to_le16(1900 + tm.tm_year); 7878 req.month = 1 + tm.tm_mon; 7879 req.day = tm.tm_mday; 7880 req.hour = tm.tm_hour; 7881 req.minute = tm.tm_min; 7882 req.second = tm.tm_sec; 7883 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 7884 } 7885 7886 static void bnxt_add_one_ctr(u64 hw, u64 *sw, u64 mask) 7887 { 7888 u64 sw_tmp; 7889 7890 hw &= mask; 7891 sw_tmp = (*sw & ~mask) | hw; 7892 if (hw < (*sw & mask)) 7893 sw_tmp += mask + 1; 7894 WRITE_ONCE(*sw, sw_tmp); 7895 } 7896 7897 static void __bnxt_accumulate_stats(__le64 *hw_stats, u64 *sw_stats, u64 *masks, 7898 int count, bool ignore_zero) 7899 { 7900 int i; 7901 7902 for (i = 0; i < count; i++) { 7903 u64 hw = le64_to_cpu(READ_ONCE(hw_stats[i])); 7904 7905 if (ignore_zero && !hw) 7906 continue; 7907 7908 if (masks[i] == -1ULL) 7909 sw_stats[i] = hw; 7910 else 7911 bnxt_add_one_ctr(hw, &sw_stats[i], masks[i]); 7912 } 7913 } 7914 7915 static void bnxt_accumulate_stats(struct bnxt_stats_mem *stats) 7916 { 7917 if (!stats->hw_stats) 7918 return; 7919 7920 __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats, 7921 stats->hw_masks, stats->len / 8, false); 7922 } 7923 7924 static void bnxt_accumulate_all_stats(struct bnxt *bp) 7925 { 7926 struct bnxt_stats_mem *ring0_stats; 7927 bool ignore_zero = false; 7928 int i; 7929 7930 /* Chip bug. Counter intermittently becomes 0. */ 7931 if (bp->flags & BNXT_FLAG_CHIP_P5) 7932 ignore_zero = true; 7933 7934 for (i = 0; i < bp->cp_nr_rings; i++) { 7935 struct bnxt_napi *bnapi = bp->bnapi[i]; 7936 struct bnxt_cp_ring_info *cpr; 7937 struct bnxt_stats_mem *stats; 7938 7939 cpr = &bnapi->cp_ring; 7940 stats = &cpr->stats; 7941 if (!i) 7942 ring0_stats = stats; 7943 __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats, 7944 ring0_stats->hw_masks, 7945 ring0_stats->len / 8, ignore_zero); 7946 } 7947 if (bp->flags & BNXT_FLAG_PORT_STATS) { 7948 struct bnxt_stats_mem *stats = &bp->port_stats; 7949 __le64 *hw_stats = stats->hw_stats; 7950 u64 *sw_stats = stats->sw_stats; 7951 u64 *masks = stats->hw_masks; 7952 int cnt; 7953 7954 cnt = sizeof(struct rx_port_stats) / 8; 7955 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false); 7956 7957 hw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 7958 sw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 7959 masks += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 7960 cnt = sizeof(struct tx_port_stats) / 8; 7961 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false); 7962 } 7963 if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) { 7964 bnxt_accumulate_stats(&bp->rx_port_stats_ext); 7965 bnxt_accumulate_stats(&bp->tx_port_stats_ext); 7966 } 7967 } 7968 7969 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags) 7970 { 7971 struct bnxt_pf_info *pf = &bp->pf; 7972 struct hwrm_port_qstats_input req = {0}; 7973 7974 if (!(bp->flags & BNXT_FLAG_PORT_STATS)) 7975 return 0; 7976 7977 if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED)) 7978 return -EOPNOTSUPP; 7979 7980 req.flags = flags; 7981 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS, -1, -1); 7982 req.port_id = cpu_to_le16(pf->port_id); 7983 req.tx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map + 7984 BNXT_TX_PORT_STATS_BYTE_OFFSET); 7985 req.rx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map); 7986 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 7987 } 7988 7989 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags) 7990 { 7991 struct hwrm_port_qstats_ext_output *resp = bp->hwrm_cmd_resp_addr; 7992 struct hwrm_queue_pri2cos_qcfg_input req2 = {0}; 7993 struct hwrm_port_qstats_ext_input req = {0}; 7994 struct bnxt_pf_info *pf = &bp->pf; 7995 u32 tx_stat_size; 7996 int rc; 7997 7998 if (!(bp->flags & BNXT_FLAG_PORT_STATS_EXT)) 7999 return 0; 8000 8001 if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED)) 8002 return -EOPNOTSUPP; 8003 8004 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS_EXT, -1, -1); 8005 req.flags = flags; 8006 req.port_id = cpu_to_le16(pf->port_id); 8007 req.rx_stat_size = cpu_to_le16(sizeof(struct rx_port_stats_ext)); 8008 req.rx_stat_host_addr = cpu_to_le64(bp->rx_port_stats_ext.hw_stats_map); 8009 tx_stat_size = bp->tx_port_stats_ext.hw_stats ? 8010 sizeof(struct tx_port_stats_ext) : 0; 8011 req.tx_stat_size = cpu_to_le16(tx_stat_size); 8012 req.tx_stat_host_addr = cpu_to_le64(bp->tx_port_stats_ext.hw_stats_map); 8013 mutex_lock(&bp->hwrm_cmd_lock); 8014 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 8015 if (!rc) { 8016 bp->fw_rx_stats_ext_size = le16_to_cpu(resp->rx_stat_size) / 8; 8017 bp->fw_tx_stats_ext_size = tx_stat_size ? 8018 le16_to_cpu(resp->tx_stat_size) / 8 : 0; 8019 } else { 8020 bp->fw_rx_stats_ext_size = 0; 8021 bp->fw_tx_stats_ext_size = 0; 8022 } 8023 if (flags) 8024 goto qstats_done; 8025 8026 if (bp->fw_tx_stats_ext_size <= 8027 offsetof(struct tx_port_stats_ext, pfc_pri0_tx_duration_us) / 8) { 8028 mutex_unlock(&bp->hwrm_cmd_lock); 8029 bp->pri2cos_valid = 0; 8030 return rc; 8031 } 8032 8033 bnxt_hwrm_cmd_hdr_init(bp, &req2, HWRM_QUEUE_PRI2COS_QCFG, -1, -1); 8034 req2.flags = cpu_to_le32(QUEUE_PRI2COS_QCFG_REQ_FLAGS_IVLAN); 8035 8036 rc = _hwrm_send_message(bp, &req2, sizeof(req2), HWRM_CMD_TIMEOUT); 8037 if (!rc) { 8038 struct hwrm_queue_pri2cos_qcfg_output *resp2; 8039 u8 *pri2cos; 8040 int i, j; 8041 8042 resp2 = bp->hwrm_cmd_resp_addr; 8043 pri2cos = &resp2->pri0_cos_queue_id; 8044 for (i = 0; i < 8; i++) { 8045 u8 queue_id = pri2cos[i]; 8046 u8 queue_idx; 8047 8048 /* Per port queue IDs start from 0, 10, 20, etc */ 8049 queue_idx = queue_id % 10; 8050 if (queue_idx > BNXT_MAX_QUEUE) { 8051 bp->pri2cos_valid = false; 8052 goto qstats_done; 8053 } 8054 for (j = 0; j < bp->max_q; j++) { 8055 if (bp->q_ids[j] == queue_id) 8056 bp->pri2cos_idx[i] = queue_idx; 8057 } 8058 } 8059 bp->pri2cos_valid = 1; 8060 } 8061 qstats_done: 8062 mutex_unlock(&bp->hwrm_cmd_lock); 8063 return rc; 8064 } 8065 8066 static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp) 8067 { 8068 if (bp->vxlan_fw_dst_port_id != INVALID_HW_RING_ID) 8069 bnxt_hwrm_tunnel_dst_port_free( 8070 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN); 8071 if (bp->nge_fw_dst_port_id != INVALID_HW_RING_ID) 8072 bnxt_hwrm_tunnel_dst_port_free( 8073 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE); 8074 } 8075 8076 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa) 8077 { 8078 int rc, i; 8079 u32 tpa_flags = 0; 8080 8081 if (set_tpa) 8082 tpa_flags = bp->flags & BNXT_FLAG_TPA; 8083 else if (BNXT_NO_FW_ACCESS(bp)) 8084 return 0; 8085 for (i = 0; i < bp->nr_vnics; i++) { 8086 rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags); 8087 if (rc) { 8088 netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n", 8089 i, rc); 8090 return rc; 8091 } 8092 } 8093 return 0; 8094 } 8095 8096 static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp) 8097 { 8098 int i; 8099 8100 for (i = 0; i < bp->nr_vnics; i++) 8101 bnxt_hwrm_vnic_set_rss(bp, i, false); 8102 } 8103 8104 static void bnxt_clear_vnic(struct bnxt *bp) 8105 { 8106 if (!bp->vnic_info) 8107 return; 8108 8109 bnxt_hwrm_clear_vnic_filter(bp); 8110 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) { 8111 /* clear all RSS setting before free vnic ctx */ 8112 bnxt_hwrm_clear_vnic_rss(bp); 8113 bnxt_hwrm_vnic_ctx_free(bp); 8114 } 8115 /* before free the vnic, undo the vnic tpa settings */ 8116 if (bp->flags & BNXT_FLAG_TPA) 8117 bnxt_set_tpa(bp, false); 8118 bnxt_hwrm_vnic_free(bp); 8119 if (bp->flags & BNXT_FLAG_CHIP_P5) 8120 bnxt_hwrm_vnic_ctx_free(bp); 8121 } 8122 8123 static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path, 8124 bool irq_re_init) 8125 { 8126 bnxt_clear_vnic(bp); 8127 bnxt_hwrm_ring_free(bp, close_path); 8128 bnxt_hwrm_ring_grp_free(bp); 8129 if (irq_re_init) { 8130 bnxt_hwrm_stat_ctx_free(bp); 8131 bnxt_hwrm_free_tunnel_ports(bp); 8132 } 8133 } 8134 8135 static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode) 8136 { 8137 struct hwrm_func_cfg_input req = {0}; 8138 8139 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1); 8140 req.fid = cpu_to_le16(0xffff); 8141 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_EVB_MODE); 8142 if (br_mode == BRIDGE_MODE_VEB) 8143 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEB; 8144 else if (br_mode == BRIDGE_MODE_VEPA) 8145 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEPA; 8146 else 8147 return -EINVAL; 8148 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 8149 } 8150 8151 static int bnxt_hwrm_set_cache_line_size(struct bnxt *bp, int size) 8152 { 8153 struct hwrm_func_cfg_input req = {0}; 8154 8155 if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10803) 8156 return 0; 8157 8158 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1); 8159 req.fid = cpu_to_le16(0xffff); 8160 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_CACHE_LINESIZE); 8161 req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_64; 8162 if (size == 128) 8163 req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_128; 8164 8165 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 8166 } 8167 8168 static int __bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id) 8169 { 8170 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 8171 int rc; 8172 8173 if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) 8174 goto skip_rss_ctx; 8175 8176 /* allocate context for vnic */ 8177 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 0); 8178 if (rc) { 8179 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n", 8180 vnic_id, rc); 8181 goto vnic_setup_err; 8182 } 8183 bp->rsscos_nr_ctxs++; 8184 8185 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 8186 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 1); 8187 if (rc) { 8188 netdev_err(bp->dev, "hwrm vnic %d cos ctx alloc failure rc: %x\n", 8189 vnic_id, rc); 8190 goto vnic_setup_err; 8191 } 8192 bp->rsscos_nr_ctxs++; 8193 } 8194 8195 skip_rss_ctx: 8196 /* configure default vnic, ring grp */ 8197 rc = bnxt_hwrm_vnic_cfg(bp, vnic_id); 8198 if (rc) { 8199 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n", 8200 vnic_id, rc); 8201 goto vnic_setup_err; 8202 } 8203 8204 /* Enable RSS hashing on vnic */ 8205 rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true); 8206 if (rc) { 8207 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n", 8208 vnic_id, rc); 8209 goto vnic_setup_err; 8210 } 8211 8212 if (bp->flags & BNXT_FLAG_AGG_RINGS) { 8213 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id); 8214 if (rc) { 8215 netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n", 8216 vnic_id, rc); 8217 } 8218 } 8219 8220 vnic_setup_err: 8221 return rc; 8222 } 8223 8224 static int __bnxt_setup_vnic_p5(struct bnxt *bp, u16 vnic_id) 8225 { 8226 int rc, i, nr_ctxs; 8227 8228 nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings); 8229 for (i = 0; i < nr_ctxs; i++) { 8230 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, i); 8231 if (rc) { 8232 netdev_err(bp->dev, "hwrm vnic %d ctx %d alloc failure rc: %x\n", 8233 vnic_id, i, rc); 8234 break; 8235 } 8236 bp->rsscos_nr_ctxs++; 8237 } 8238 if (i < nr_ctxs) 8239 return -ENOMEM; 8240 8241 rc = bnxt_hwrm_vnic_set_rss_p5(bp, vnic_id, true); 8242 if (rc) { 8243 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %d\n", 8244 vnic_id, rc); 8245 return rc; 8246 } 8247 rc = bnxt_hwrm_vnic_cfg(bp, vnic_id); 8248 if (rc) { 8249 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n", 8250 vnic_id, rc); 8251 return rc; 8252 } 8253 if (bp->flags & BNXT_FLAG_AGG_RINGS) { 8254 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id); 8255 if (rc) { 8256 netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n", 8257 vnic_id, rc); 8258 } 8259 } 8260 return rc; 8261 } 8262 8263 static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id) 8264 { 8265 if (bp->flags & BNXT_FLAG_CHIP_P5) 8266 return __bnxt_setup_vnic_p5(bp, vnic_id); 8267 else 8268 return __bnxt_setup_vnic(bp, vnic_id); 8269 } 8270 8271 static int bnxt_alloc_rfs_vnics(struct bnxt *bp) 8272 { 8273 #ifdef CONFIG_RFS_ACCEL 8274 int i, rc = 0; 8275 8276 if (bp->flags & BNXT_FLAG_CHIP_P5) 8277 return 0; 8278 8279 for (i = 0; i < bp->rx_nr_rings; i++) { 8280 struct bnxt_vnic_info *vnic; 8281 u16 vnic_id = i + 1; 8282 u16 ring_id = i; 8283 8284 if (vnic_id >= bp->nr_vnics) 8285 break; 8286 8287 vnic = &bp->vnic_info[vnic_id]; 8288 vnic->flags |= BNXT_VNIC_RFS_FLAG; 8289 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP) 8290 vnic->flags |= BNXT_VNIC_RFS_NEW_RSS_FLAG; 8291 rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, ring_id, 1); 8292 if (rc) { 8293 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n", 8294 vnic_id, rc); 8295 break; 8296 } 8297 rc = bnxt_setup_vnic(bp, vnic_id); 8298 if (rc) 8299 break; 8300 } 8301 return rc; 8302 #else 8303 return 0; 8304 #endif 8305 } 8306 8307 /* Allow PF and VF with default VLAN to be in promiscuous mode */ 8308 static bool bnxt_promisc_ok(struct bnxt *bp) 8309 { 8310 #ifdef CONFIG_BNXT_SRIOV 8311 if (BNXT_VF(bp) && !bp->vf.vlan) 8312 return false; 8313 #endif 8314 return true; 8315 } 8316 8317 static int bnxt_setup_nitroa0_vnic(struct bnxt *bp) 8318 { 8319 unsigned int rc = 0; 8320 8321 rc = bnxt_hwrm_vnic_alloc(bp, 1, bp->rx_nr_rings - 1, 1); 8322 if (rc) { 8323 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n", 8324 rc); 8325 return rc; 8326 } 8327 8328 rc = bnxt_hwrm_vnic_cfg(bp, 1); 8329 if (rc) { 8330 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n", 8331 rc); 8332 return rc; 8333 } 8334 return rc; 8335 } 8336 8337 static int bnxt_cfg_rx_mode(struct bnxt *); 8338 static bool bnxt_mc_list_updated(struct bnxt *, u32 *); 8339 8340 static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init) 8341 { 8342 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 8343 int rc = 0; 8344 unsigned int rx_nr_rings = bp->rx_nr_rings; 8345 8346 if (irq_re_init) { 8347 rc = bnxt_hwrm_stat_ctx_alloc(bp); 8348 if (rc) { 8349 netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n", 8350 rc); 8351 goto err_out; 8352 } 8353 } 8354 8355 rc = bnxt_hwrm_ring_alloc(bp); 8356 if (rc) { 8357 netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc); 8358 goto err_out; 8359 } 8360 8361 rc = bnxt_hwrm_ring_grp_alloc(bp); 8362 if (rc) { 8363 netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc); 8364 goto err_out; 8365 } 8366 8367 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 8368 rx_nr_rings--; 8369 8370 /* default vnic 0 */ 8371 rc = bnxt_hwrm_vnic_alloc(bp, 0, 0, rx_nr_rings); 8372 if (rc) { 8373 netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc); 8374 goto err_out; 8375 } 8376 8377 rc = bnxt_setup_vnic(bp, 0); 8378 if (rc) 8379 goto err_out; 8380 8381 if (bp->flags & BNXT_FLAG_RFS) { 8382 rc = bnxt_alloc_rfs_vnics(bp); 8383 if (rc) 8384 goto err_out; 8385 } 8386 8387 if (bp->flags & BNXT_FLAG_TPA) { 8388 rc = bnxt_set_tpa(bp, true); 8389 if (rc) 8390 goto err_out; 8391 } 8392 8393 if (BNXT_VF(bp)) 8394 bnxt_update_vf_mac(bp); 8395 8396 /* Filter for default vnic 0 */ 8397 rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr); 8398 if (rc) { 8399 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc); 8400 goto err_out; 8401 } 8402 vnic->uc_filter_count = 1; 8403 8404 vnic->rx_mask = 0; 8405 if (bp->dev->flags & IFF_BROADCAST) 8406 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST; 8407 8408 if ((bp->dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp)) 8409 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 8410 8411 if (bp->dev->flags & IFF_ALLMULTI) { 8412 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 8413 vnic->mc_list_count = 0; 8414 } else { 8415 u32 mask = 0; 8416 8417 bnxt_mc_list_updated(bp, &mask); 8418 vnic->rx_mask |= mask; 8419 } 8420 8421 rc = bnxt_cfg_rx_mode(bp); 8422 if (rc) 8423 goto err_out; 8424 8425 rc = bnxt_hwrm_set_coal(bp); 8426 if (rc) 8427 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n", 8428 rc); 8429 8430 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 8431 rc = bnxt_setup_nitroa0_vnic(bp); 8432 if (rc) 8433 netdev_err(bp->dev, "Special vnic setup failure for NS2 A0 rc: %x\n", 8434 rc); 8435 } 8436 8437 if (BNXT_VF(bp)) { 8438 bnxt_hwrm_func_qcfg(bp); 8439 netdev_update_features(bp->dev); 8440 } 8441 8442 return 0; 8443 8444 err_out: 8445 bnxt_hwrm_resource_free(bp, 0, true); 8446 8447 return rc; 8448 } 8449 8450 static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init) 8451 { 8452 bnxt_hwrm_resource_free(bp, 1, irq_re_init); 8453 return 0; 8454 } 8455 8456 static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init) 8457 { 8458 bnxt_init_cp_rings(bp); 8459 bnxt_init_rx_rings(bp); 8460 bnxt_init_tx_rings(bp); 8461 bnxt_init_ring_grps(bp, irq_re_init); 8462 bnxt_init_vnics(bp); 8463 8464 return bnxt_init_chip(bp, irq_re_init); 8465 } 8466 8467 static int bnxt_set_real_num_queues(struct bnxt *bp) 8468 { 8469 int rc; 8470 struct net_device *dev = bp->dev; 8471 8472 rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings - 8473 bp->tx_nr_rings_xdp); 8474 if (rc) 8475 return rc; 8476 8477 rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings); 8478 if (rc) 8479 return rc; 8480 8481 #ifdef CONFIG_RFS_ACCEL 8482 if (bp->flags & BNXT_FLAG_RFS) 8483 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings); 8484 #endif 8485 8486 return rc; 8487 } 8488 8489 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max, 8490 bool shared) 8491 { 8492 int _rx = *rx, _tx = *tx; 8493 8494 if (shared) { 8495 *rx = min_t(int, _rx, max); 8496 *tx = min_t(int, _tx, max); 8497 } else { 8498 if (max < 2) 8499 return -ENOMEM; 8500 8501 while (_rx + _tx > max) { 8502 if (_rx > _tx && _rx > 1) 8503 _rx--; 8504 else if (_tx > 1) 8505 _tx--; 8506 } 8507 *rx = _rx; 8508 *tx = _tx; 8509 } 8510 return 0; 8511 } 8512 8513 static void bnxt_setup_msix(struct bnxt *bp) 8514 { 8515 const int len = sizeof(bp->irq_tbl[0].name); 8516 struct net_device *dev = bp->dev; 8517 int tcs, i; 8518 8519 tcs = netdev_get_num_tc(dev); 8520 if (tcs) { 8521 int i, off, count; 8522 8523 for (i = 0; i < tcs; i++) { 8524 count = bp->tx_nr_rings_per_tc; 8525 off = i * count; 8526 netdev_set_tc_queue(dev, i, count, off); 8527 } 8528 } 8529 8530 for (i = 0; i < bp->cp_nr_rings; i++) { 8531 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 8532 char *attr; 8533 8534 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 8535 attr = "TxRx"; 8536 else if (i < bp->rx_nr_rings) 8537 attr = "rx"; 8538 else 8539 attr = "tx"; 8540 8541 snprintf(bp->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name, 8542 attr, i); 8543 bp->irq_tbl[map_idx].handler = bnxt_msix; 8544 } 8545 } 8546 8547 static void bnxt_setup_inta(struct bnxt *bp) 8548 { 8549 const int len = sizeof(bp->irq_tbl[0].name); 8550 8551 if (netdev_get_num_tc(bp->dev)) 8552 netdev_reset_tc(bp->dev); 8553 8554 snprintf(bp->irq_tbl[0].name, len, "%s-%s-%d", bp->dev->name, "TxRx", 8555 0); 8556 bp->irq_tbl[0].handler = bnxt_inta; 8557 } 8558 8559 static int bnxt_init_int_mode(struct bnxt *bp); 8560 8561 static int bnxt_setup_int_mode(struct bnxt *bp) 8562 { 8563 int rc; 8564 8565 if (!bp->irq_tbl) { 8566 rc = bnxt_init_int_mode(bp); 8567 if (rc || !bp->irq_tbl) 8568 return rc ?: -ENODEV; 8569 } 8570 8571 if (bp->flags & BNXT_FLAG_USING_MSIX) 8572 bnxt_setup_msix(bp); 8573 else 8574 bnxt_setup_inta(bp); 8575 8576 rc = bnxt_set_real_num_queues(bp); 8577 return rc; 8578 } 8579 8580 #ifdef CONFIG_RFS_ACCEL 8581 static unsigned int bnxt_get_max_func_rss_ctxs(struct bnxt *bp) 8582 { 8583 return bp->hw_resc.max_rsscos_ctxs; 8584 } 8585 8586 static unsigned int bnxt_get_max_func_vnics(struct bnxt *bp) 8587 { 8588 return bp->hw_resc.max_vnics; 8589 } 8590 #endif 8591 8592 unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp) 8593 { 8594 return bp->hw_resc.max_stat_ctxs; 8595 } 8596 8597 unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp) 8598 { 8599 return bp->hw_resc.max_cp_rings; 8600 } 8601 8602 static unsigned int bnxt_get_max_func_cp_rings_for_en(struct bnxt *bp) 8603 { 8604 unsigned int cp = bp->hw_resc.max_cp_rings; 8605 8606 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 8607 cp -= bnxt_get_ulp_msix_num(bp); 8608 8609 return cp; 8610 } 8611 8612 static unsigned int bnxt_get_max_func_irqs(struct bnxt *bp) 8613 { 8614 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 8615 8616 if (bp->flags & BNXT_FLAG_CHIP_P5) 8617 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_nqs); 8618 8619 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_cp_rings); 8620 } 8621 8622 static void bnxt_set_max_func_irqs(struct bnxt *bp, unsigned int max_irqs) 8623 { 8624 bp->hw_resc.max_irqs = max_irqs; 8625 } 8626 8627 unsigned int bnxt_get_avail_cp_rings_for_en(struct bnxt *bp) 8628 { 8629 unsigned int cp; 8630 8631 cp = bnxt_get_max_func_cp_rings_for_en(bp); 8632 if (bp->flags & BNXT_FLAG_CHIP_P5) 8633 return cp - bp->rx_nr_rings - bp->tx_nr_rings; 8634 else 8635 return cp - bp->cp_nr_rings; 8636 } 8637 8638 unsigned int bnxt_get_avail_stat_ctxs_for_en(struct bnxt *bp) 8639 { 8640 return bnxt_get_max_func_stat_ctxs(bp) - bnxt_get_func_stat_ctxs(bp); 8641 } 8642 8643 int bnxt_get_avail_msix(struct bnxt *bp, int num) 8644 { 8645 int max_cp = bnxt_get_max_func_cp_rings(bp); 8646 int max_irq = bnxt_get_max_func_irqs(bp); 8647 int total_req = bp->cp_nr_rings + num; 8648 int max_idx, avail_msix; 8649 8650 max_idx = bp->total_irqs; 8651 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 8652 max_idx = min_t(int, bp->total_irqs, max_cp); 8653 avail_msix = max_idx - bp->cp_nr_rings; 8654 if (!BNXT_NEW_RM(bp) || avail_msix >= num) 8655 return avail_msix; 8656 8657 if (max_irq < total_req) { 8658 num = max_irq - bp->cp_nr_rings; 8659 if (num <= 0) 8660 return 0; 8661 } 8662 return num; 8663 } 8664 8665 static int bnxt_get_num_msix(struct bnxt *bp) 8666 { 8667 if (!BNXT_NEW_RM(bp)) 8668 return bnxt_get_max_func_irqs(bp); 8669 8670 return bnxt_nq_rings_in_use(bp); 8671 } 8672 8673 static int bnxt_init_msix(struct bnxt *bp) 8674 { 8675 int i, total_vecs, max, rc = 0, min = 1, ulp_msix; 8676 struct msix_entry *msix_ent; 8677 8678 total_vecs = bnxt_get_num_msix(bp); 8679 max = bnxt_get_max_func_irqs(bp); 8680 if (total_vecs > max) 8681 total_vecs = max; 8682 8683 if (!total_vecs) 8684 return 0; 8685 8686 msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL); 8687 if (!msix_ent) 8688 return -ENOMEM; 8689 8690 for (i = 0; i < total_vecs; i++) { 8691 msix_ent[i].entry = i; 8692 msix_ent[i].vector = 0; 8693 } 8694 8695 if (!(bp->flags & BNXT_FLAG_SHARED_RINGS)) 8696 min = 2; 8697 8698 total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs); 8699 ulp_msix = bnxt_get_ulp_msix_num(bp); 8700 if (total_vecs < 0 || total_vecs < ulp_msix) { 8701 rc = -ENODEV; 8702 goto msix_setup_exit; 8703 } 8704 8705 bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL); 8706 if (bp->irq_tbl) { 8707 for (i = 0; i < total_vecs; i++) 8708 bp->irq_tbl[i].vector = msix_ent[i].vector; 8709 8710 bp->total_irqs = total_vecs; 8711 /* Trim rings based upon num of vectors allocated */ 8712 rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings, 8713 total_vecs - ulp_msix, min == 1); 8714 if (rc) 8715 goto msix_setup_exit; 8716 8717 bp->cp_nr_rings = (min == 1) ? 8718 max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) : 8719 bp->tx_nr_rings + bp->rx_nr_rings; 8720 8721 } else { 8722 rc = -ENOMEM; 8723 goto msix_setup_exit; 8724 } 8725 bp->flags |= BNXT_FLAG_USING_MSIX; 8726 kfree(msix_ent); 8727 return 0; 8728 8729 msix_setup_exit: 8730 netdev_err(bp->dev, "bnxt_init_msix err: %x\n", rc); 8731 kfree(bp->irq_tbl); 8732 bp->irq_tbl = NULL; 8733 pci_disable_msix(bp->pdev); 8734 kfree(msix_ent); 8735 return rc; 8736 } 8737 8738 static int bnxt_init_inta(struct bnxt *bp) 8739 { 8740 bp->irq_tbl = kzalloc(sizeof(struct bnxt_irq), GFP_KERNEL); 8741 if (!bp->irq_tbl) 8742 return -ENOMEM; 8743 8744 bp->total_irqs = 1; 8745 bp->rx_nr_rings = 1; 8746 bp->tx_nr_rings = 1; 8747 bp->cp_nr_rings = 1; 8748 bp->flags |= BNXT_FLAG_SHARED_RINGS; 8749 bp->irq_tbl[0].vector = bp->pdev->irq; 8750 return 0; 8751 } 8752 8753 static int bnxt_init_int_mode(struct bnxt *bp) 8754 { 8755 int rc = -ENODEV; 8756 8757 if (bp->flags & BNXT_FLAG_MSIX_CAP) 8758 rc = bnxt_init_msix(bp); 8759 8760 if (!(bp->flags & BNXT_FLAG_USING_MSIX) && BNXT_PF(bp)) { 8761 /* fallback to INTA */ 8762 rc = bnxt_init_inta(bp); 8763 } 8764 return rc; 8765 } 8766 8767 static void bnxt_clear_int_mode(struct bnxt *bp) 8768 { 8769 if (bp->flags & BNXT_FLAG_USING_MSIX) 8770 pci_disable_msix(bp->pdev); 8771 8772 kfree(bp->irq_tbl); 8773 bp->irq_tbl = NULL; 8774 bp->flags &= ~BNXT_FLAG_USING_MSIX; 8775 } 8776 8777 int bnxt_reserve_rings(struct bnxt *bp, bool irq_re_init) 8778 { 8779 int tcs = netdev_get_num_tc(bp->dev); 8780 bool irq_cleared = false; 8781 int rc; 8782 8783 if (!bnxt_need_reserve_rings(bp)) 8784 return 0; 8785 8786 if (irq_re_init && BNXT_NEW_RM(bp) && 8787 bnxt_get_num_msix(bp) != bp->total_irqs) { 8788 bnxt_ulp_irq_stop(bp); 8789 bnxt_clear_int_mode(bp); 8790 irq_cleared = true; 8791 } 8792 rc = __bnxt_reserve_rings(bp); 8793 if (irq_cleared) { 8794 if (!rc) 8795 rc = bnxt_init_int_mode(bp); 8796 bnxt_ulp_irq_restart(bp, rc); 8797 } 8798 if (rc) { 8799 netdev_err(bp->dev, "ring reservation/IRQ init failure rc: %d\n", rc); 8800 return rc; 8801 } 8802 if (tcs && (bp->tx_nr_rings_per_tc * tcs != bp->tx_nr_rings)) { 8803 netdev_err(bp->dev, "tx ring reservation failure\n"); 8804 netdev_reset_tc(bp->dev); 8805 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 8806 return -ENOMEM; 8807 } 8808 return 0; 8809 } 8810 8811 static void bnxt_free_irq(struct bnxt *bp) 8812 { 8813 struct bnxt_irq *irq; 8814 int i; 8815 8816 #ifdef CONFIG_RFS_ACCEL 8817 free_irq_cpu_rmap(bp->dev->rx_cpu_rmap); 8818 bp->dev->rx_cpu_rmap = NULL; 8819 #endif 8820 if (!bp->irq_tbl || !bp->bnapi) 8821 return; 8822 8823 for (i = 0; i < bp->cp_nr_rings; i++) { 8824 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 8825 8826 irq = &bp->irq_tbl[map_idx]; 8827 if (irq->requested) { 8828 if (irq->have_cpumask) { 8829 irq_set_affinity_hint(irq->vector, NULL); 8830 free_cpumask_var(irq->cpu_mask); 8831 irq->have_cpumask = 0; 8832 } 8833 free_irq(irq->vector, bp->bnapi[i]); 8834 } 8835 8836 irq->requested = 0; 8837 } 8838 } 8839 8840 static int bnxt_request_irq(struct bnxt *bp) 8841 { 8842 int i, j, rc = 0; 8843 unsigned long flags = 0; 8844 #ifdef CONFIG_RFS_ACCEL 8845 struct cpu_rmap *rmap; 8846 #endif 8847 8848 rc = bnxt_setup_int_mode(bp); 8849 if (rc) { 8850 netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n", 8851 rc); 8852 return rc; 8853 } 8854 #ifdef CONFIG_RFS_ACCEL 8855 rmap = bp->dev->rx_cpu_rmap; 8856 #endif 8857 if (!(bp->flags & BNXT_FLAG_USING_MSIX)) 8858 flags = IRQF_SHARED; 8859 8860 for (i = 0, j = 0; i < bp->cp_nr_rings; i++) { 8861 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 8862 struct bnxt_irq *irq = &bp->irq_tbl[map_idx]; 8863 8864 #ifdef CONFIG_RFS_ACCEL 8865 if (rmap && bp->bnapi[i]->rx_ring) { 8866 rc = irq_cpu_rmap_add(rmap, irq->vector); 8867 if (rc) 8868 netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n", 8869 j); 8870 j++; 8871 } 8872 #endif 8873 rc = request_irq(irq->vector, irq->handler, flags, irq->name, 8874 bp->bnapi[i]); 8875 if (rc) 8876 break; 8877 8878 irq->requested = 1; 8879 8880 if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) { 8881 int numa_node = dev_to_node(&bp->pdev->dev); 8882 8883 irq->have_cpumask = 1; 8884 cpumask_set_cpu(cpumask_local_spread(i, numa_node), 8885 irq->cpu_mask); 8886 rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask); 8887 if (rc) { 8888 netdev_warn(bp->dev, 8889 "Set affinity failed, IRQ = %d\n", 8890 irq->vector); 8891 break; 8892 } 8893 } 8894 } 8895 return rc; 8896 } 8897 8898 static void bnxt_del_napi(struct bnxt *bp) 8899 { 8900 int i; 8901 8902 if (!bp->bnapi) 8903 return; 8904 8905 for (i = 0; i < bp->cp_nr_rings; i++) { 8906 struct bnxt_napi *bnapi = bp->bnapi[i]; 8907 8908 __netif_napi_del(&bnapi->napi); 8909 } 8910 /* We called __netif_napi_del(), we need 8911 * to respect an RCU grace period before freeing napi structures. 8912 */ 8913 synchronize_net(); 8914 } 8915 8916 static void bnxt_init_napi(struct bnxt *bp) 8917 { 8918 int i; 8919 unsigned int cp_nr_rings = bp->cp_nr_rings; 8920 struct bnxt_napi *bnapi; 8921 8922 if (bp->flags & BNXT_FLAG_USING_MSIX) { 8923 int (*poll_fn)(struct napi_struct *, int) = bnxt_poll; 8924 8925 if (bp->flags & BNXT_FLAG_CHIP_P5) 8926 poll_fn = bnxt_poll_p5; 8927 else if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 8928 cp_nr_rings--; 8929 for (i = 0; i < cp_nr_rings; i++) { 8930 bnapi = bp->bnapi[i]; 8931 netif_napi_add(bp->dev, &bnapi->napi, poll_fn, 64); 8932 } 8933 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 8934 bnapi = bp->bnapi[cp_nr_rings]; 8935 netif_napi_add(bp->dev, &bnapi->napi, 8936 bnxt_poll_nitroa0, 64); 8937 } 8938 } else { 8939 bnapi = bp->bnapi[0]; 8940 netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll, 64); 8941 } 8942 } 8943 8944 static void bnxt_disable_napi(struct bnxt *bp) 8945 { 8946 int i; 8947 8948 if (!bp->bnapi || 8949 test_and_set_bit(BNXT_STATE_NAPI_DISABLED, &bp->state)) 8950 return; 8951 8952 for (i = 0; i < bp->cp_nr_rings; i++) { 8953 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring; 8954 8955 if (bp->bnapi[i]->rx_ring) 8956 cancel_work_sync(&cpr->dim.work); 8957 8958 napi_disable(&bp->bnapi[i]->napi); 8959 } 8960 } 8961 8962 static void bnxt_enable_napi(struct bnxt *bp) 8963 { 8964 int i; 8965 8966 clear_bit(BNXT_STATE_NAPI_DISABLED, &bp->state); 8967 for (i = 0; i < bp->cp_nr_rings; i++) { 8968 struct bnxt_napi *bnapi = bp->bnapi[i]; 8969 struct bnxt_cp_ring_info *cpr; 8970 8971 cpr = &bnapi->cp_ring; 8972 if (bnapi->in_reset) 8973 cpr->sw_stats.rx.rx_resets++; 8974 bnapi->in_reset = false; 8975 8976 if (bnapi->rx_ring) { 8977 INIT_WORK(&cpr->dim.work, bnxt_dim_work); 8978 cpr->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; 8979 } 8980 napi_enable(&bnapi->napi); 8981 } 8982 } 8983 8984 void bnxt_tx_disable(struct bnxt *bp) 8985 { 8986 int i; 8987 struct bnxt_tx_ring_info *txr; 8988 8989 if (bp->tx_ring) { 8990 for (i = 0; i < bp->tx_nr_rings; i++) { 8991 txr = &bp->tx_ring[i]; 8992 txr->dev_state = BNXT_DEV_STATE_CLOSING; 8993 } 8994 } 8995 /* Drop carrier first to prevent TX timeout */ 8996 netif_carrier_off(bp->dev); 8997 /* Stop all TX queues */ 8998 netif_tx_disable(bp->dev); 8999 } 9000 9001 void bnxt_tx_enable(struct bnxt *bp) 9002 { 9003 int i; 9004 struct bnxt_tx_ring_info *txr; 9005 9006 for (i = 0; i < bp->tx_nr_rings; i++) { 9007 txr = &bp->tx_ring[i]; 9008 txr->dev_state = 0; 9009 } 9010 netif_tx_wake_all_queues(bp->dev); 9011 if (bp->link_info.link_up) 9012 netif_carrier_on(bp->dev); 9013 } 9014 9015 static char *bnxt_report_fec(struct bnxt_link_info *link_info) 9016 { 9017 u8 active_fec = link_info->active_fec_sig_mode & 9018 PORT_PHY_QCFG_RESP_ACTIVE_FEC_MASK; 9019 9020 switch (active_fec) { 9021 default: 9022 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_NONE_ACTIVE: 9023 return "None"; 9024 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE74_ACTIVE: 9025 return "Clause 74 BaseR"; 9026 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE91_ACTIVE: 9027 return "Clause 91 RS(528,514)"; 9028 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_1XN_ACTIVE: 9029 return "Clause 91 RS544_1XN"; 9030 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_IEEE_ACTIVE: 9031 return "Clause 91 RS(544,514)"; 9032 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_1XN_ACTIVE: 9033 return "Clause 91 RS272_1XN"; 9034 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_IEEE_ACTIVE: 9035 return "Clause 91 RS(272,257)"; 9036 } 9037 } 9038 9039 static void bnxt_report_link(struct bnxt *bp) 9040 { 9041 if (bp->link_info.link_up) { 9042 const char *duplex; 9043 const char *flow_ctrl; 9044 u32 speed; 9045 u16 fec; 9046 9047 netif_carrier_on(bp->dev); 9048 speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed); 9049 if (speed == SPEED_UNKNOWN) { 9050 netdev_info(bp->dev, "NIC Link is Up, speed unknown\n"); 9051 return; 9052 } 9053 if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL) 9054 duplex = "full"; 9055 else 9056 duplex = "half"; 9057 if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH) 9058 flow_ctrl = "ON - receive & transmit"; 9059 else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX) 9060 flow_ctrl = "ON - transmit"; 9061 else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX) 9062 flow_ctrl = "ON - receive"; 9063 else 9064 flow_ctrl = "none"; 9065 netdev_info(bp->dev, "NIC Link is Up, %u Mbps %s duplex, Flow control: %s\n", 9066 speed, duplex, flow_ctrl); 9067 if (bp->flags & BNXT_FLAG_EEE_CAP) 9068 netdev_info(bp->dev, "EEE is %s\n", 9069 bp->eee.eee_active ? "active" : 9070 "not active"); 9071 fec = bp->link_info.fec_cfg; 9072 if (!(fec & PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED)) 9073 netdev_info(bp->dev, "FEC autoneg %s encoding: %s\n", 9074 (fec & BNXT_FEC_AUTONEG) ? "on" : "off", 9075 bnxt_report_fec(&bp->link_info)); 9076 } else { 9077 netif_carrier_off(bp->dev); 9078 netdev_err(bp->dev, "NIC Link is Down\n"); 9079 } 9080 } 9081 9082 static bool bnxt_phy_qcaps_no_speed(struct hwrm_port_phy_qcaps_output *resp) 9083 { 9084 if (!resp->supported_speeds_auto_mode && 9085 !resp->supported_speeds_force_mode && 9086 !resp->supported_pam4_speeds_auto_mode && 9087 !resp->supported_pam4_speeds_force_mode) 9088 return true; 9089 return false; 9090 } 9091 9092 static int bnxt_hwrm_phy_qcaps(struct bnxt *bp) 9093 { 9094 int rc = 0; 9095 struct hwrm_port_phy_qcaps_input req = {0}; 9096 struct hwrm_port_phy_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 9097 struct bnxt_link_info *link_info = &bp->link_info; 9098 9099 bp->flags &= ~BNXT_FLAG_EEE_CAP; 9100 if (bp->test_info) 9101 bp->test_info->flags &= ~(BNXT_TEST_FL_EXT_LPBK | 9102 BNXT_TEST_FL_AN_PHY_LPBK); 9103 if (bp->hwrm_spec_code < 0x10201) 9104 return 0; 9105 9106 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCAPS, -1, -1); 9107 9108 mutex_lock(&bp->hwrm_cmd_lock); 9109 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 9110 if (rc) 9111 goto hwrm_phy_qcaps_exit; 9112 9113 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) { 9114 struct ethtool_eee *eee = &bp->eee; 9115 u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode); 9116 9117 bp->flags |= BNXT_FLAG_EEE_CAP; 9118 eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0); 9119 bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) & 9120 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK; 9121 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) & 9122 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK; 9123 } 9124 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EXTERNAL_LPBK_SUPPORTED) { 9125 if (bp->test_info) 9126 bp->test_info->flags |= BNXT_TEST_FL_EXT_LPBK; 9127 } 9128 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_AUTONEG_LPBK_SUPPORTED) { 9129 if (bp->test_info) 9130 bp->test_info->flags |= BNXT_TEST_FL_AN_PHY_LPBK; 9131 } 9132 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_SHARED_PHY_CFG_SUPPORTED) { 9133 if (BNXT_PF(bp)) 9134 bp->fw_cap |= BNXT_FW_CAP_SHARED_PORT_CFG; 9135 } 9136 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_CUMULATIVE_COUNTERS_ON_RESET) 9137 bp->fw_cap |= BNXT_FW_CAP_PORT_STATS_NO_RESET; 9138 9139 if (bp->hwrm_spec_code >= 0x10a01) { 9140 if (bnxt_phy_qcaps_no_speed(resp)) { 9141 link_info->phy_state = BNXT_PHY_STATE_DISABLED; 9142 netdev_warn(bp->dev, "Ethernet link disabled\n"); 9143 } else if (link_info->phy_state == BNXT_PHY_STATE_DISABLED) { 9144 link_info->phy_state = BNXT_PHY_STATE_ENABLED; 9145 netdev_info(bp->dev, "Ethernet link enabled\n"); 9146 /* Phy re-enabled, reprobe the speeds */ 9147 link_info->support_auto_speeds = 0; 9148 link_info->support_pam4_auto_speeds = 0; 9149 } 9150 } 9151 if (resp->supported_speeds_auto_mode) 9152 link_info->support_auto_speeds = 9153 le16_to_cpu(resp->supported_speeds_auto_mode); 9154 if (resp->supported_pam4_speeds_auto_mode) 9155 link_info->support_pam4_auto_speeds = 9156 le16_to_cpu(resp->supported_pam4_speeds_auto_mode); 9157 9158 bp->port_count = resp->port_cnt; 9159 9160 hwrm_phy_qcaps_exit: 9161 mutex_unlock(&bp->hwrm_cmd_lock); 9162 return rc; 9163 } 9164 9165 static bool bnxt_support_dropped(u16 advertising, u16 supported) 9166 { 9167 u16 diff = advertising ^ supported; 9168 9169 return ((supported | diff) != supported); 9170 } 9171 9172 int bnxt_update_link(struct bnxt *bp, bool chng_link_state) 9173 { 9174 int rc = 0; 9175 struct bnxt_link_info *link_info = &bp->link_info; 9176 struct hwrm_port_phy_qcfg_input req = {0}; 9177 struct hwrm_port_phy_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 9178 u8 link_up = link_info->link_up; 9179 bool support_changed = false; 9180 9181 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCFG, -1, -1); 9182 9183 mutex_lock(&bp->hwrm_cmd_lock); 9184 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 9185 if (rc) { 9186 mutex_unlock(&bp->hwrm_cmd_lock); 9187 return rc; 9188 } 9189 9190 memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp)); 9191 link_info->phy_link_status = resp->link; 9192 link_info->duplex = resp->duplex_cfg; 9193 if (bp->hwrm_spec_code >= 0x10800) 9194 link_info->duplex = resp->duplex_state; 9195 link_info->pause = resp->pause; 9196 link_info->auto_mode = resp->auto_mode; 9197 link_info->auto_pause_setting = resp->auto_pause; 9198 link_info->lp_pause = resp->link_partner_adv_pause; 9199 link_info->force_pause_setting = resp->force_pause; 9200 link_info->duplex_setting = resp->duplex_cfg; 9201 if (link_info->phy_link_status == BNXT_LINK_LINK) 9202 link_info->link_speed = le16_to_cpu(resp->link_speed); 9203 else 9204 link_info->link_speed = 0; 9205 link_info->force_link_speed = le16_to_cpu(resp->force_link_speed); 9206 link_info->force_pam4_link_speed = 9207 le16_to_cpu(resp->force_pam4_link_speed); 9208 link_info->support_speeds = le16_to_cpu(resp->support_speeds); 9209 link_info->support_pam4_speeds = le16_to_cpu(resp->support_pam4_speeds); 9210 link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask); 9211 link_info->auto_pam4_link_speeds = 9212 le16_to_cpu(resp->auto_pam4_link_speed_mask); 9213 link_info->lp_auto_link_speeds = 9214 le16_to_cpu(resp->link_partner_adv_speeds); 9215 link_info->lp_auto_pam4_link_speeds = 9216 resp->link_partner_pam4_adv_speeds; 9217 link_info->preemphasis = le32_to_cpu(resp->preemphasis); 9218 link_info->phy_ver[0] = resp->phy_maj; 9219 link_info->phy_ver[1] = resp->phy_min; 9220 link_info->phy_ver[2] = resp->phy_bld; 9221 link_info->media_type = resp->media_type; 9222 link_info->phy_type = resp->phy_type; 9223 link_info->transceiver = resp->xcvr_pkg_type; 9224 link_info->phy_addr = resp->eee_config_phy_addr & 9225 PORT_PHY_QCFG_RESP_PHY_ADDR_MASK; 9226 link_info->module_status = resp->module_status; 9227 9228 if (bp->flags & BNXT_FLAG_EEE_CAP) { 9229 struct ethtool_eee *eee = &bp->eee; 9230 u16 fw_speeds; 9231 9232 eee->eee_active = 0; 9233 if (resp->eee_config_phy_addr & 9234 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) { 9235 eee->eee_active = 1; 9236 fw_speeds = le16_to_cpu( 9237 resp->link_partner_adv_eee_link_speed_mask); 9238 eee->lp_advertised = 9239 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0); 9240 } 9241 9242 /* Pull initial EEE config */ 9243 if (!chng_link_state) { 9244 if (resp->eee_config_phy_addr & 9245 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED) 9246 eee->eee_enabled = 1; 9247 9248 fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask); 9249 eee->advertised = 9250 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0); 9251 9252 if (resp->eee_config_phy_addr & 9253 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) { 9254 __le32 tmr; 9255 9256 eee->tx_lpi_enabled = 1; 9257 tmr = resp->xcvr_identifier_type_tx_lpi_timer; 9258 eee->tx_lpi_timer = le32_to_cpu(tmr) & 9259 PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK; 9260 } 9261 } 9262 } 9263 9264 link_info->fec_cfg = PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED; 9265 if (bp->hwrm_spec_code >= 0x10504) { 9266 link_info->fec_cfg = le16_to_cpu(resp->fec_cfg); 9267 link_info->active_fec_sig_mode = resp->active_fec_signal_mode; 9268 } 9269 /* TODO: need to add more logic to report VF link */ 9270 if (chng_link_state) { 9271 if (link_info->phy_link_status == BNXT_LINK_LINK) 9272 link_info->link_up = 1; 9273 else 9274 link_info->link_up = 0; 9275 if (link_up != link_info->link_up) 9276 bnxt_report_link(bp); 9277 } else { 9278 /* alwasy link down if not require to update link state */ 9279 link_info->link_up = 0; 9280 } 9281 mutex_unlock(&bp->hwrm_cmd_lock); 9282 9283 if (!BNXT_PHY_CFG_ABLE(bp)) 9284 return 0; 9285 9286 /* Check if any advertised speeds are no longer supported. The caller 9287 * holds the link_lock mutex, so we can modify link_info settings. 9288 */ 9289 if (bnxt_support_dropped(link_info->advertising, 9290 link_info->support_auto_speeds)) { 9291 link_info->advertising = link_info->support_auto_speeds; 9292 support_changed = true; 9293 } 9294 if (bnxt_support_dropped(link_info->advertising_pam4, 9295 link_info->support_pam4_auto_speeds)) { 9296 link_info->advertising_pam4 = link_info->support_pam4_auto_speeds; 9297 support_changed = true; 9298 } 9299 if (support_changed && (link_info->autoneg & BNXT_AUTONEG_SPEED)) 9300 bnxt_hwrm_set_link_setting(bp, true, false); 9301 return 0; 9302 } 9303 9304 static void bnxt_get_port_module_status(struct bnxt *bp) 9305 { 9306 struct bnxt_link_info *link_info = &bp->link_info; 9307 struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp; 9308 u8 module_status; 9309 9310 if (bnxt_update_link(bp, true)) 9311 return; 9312 9313 module_status = link_info->module_status; 9314 switch (module_status) { 9315 case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX: 9316 case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN: 9317 case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG: 9318 netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n", 9319 bp->pf.port_id); 9320 if (bp->hwrm_spec_code >= 0x10201) { 9321 netdev_warn(bp->dev, "Module part number %s\n", 9322 resp->phy_vendor_partnumber); 9323 } 9324 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX) 9325 netdev_warn(bp->dev, "TX is disabled\n"); 9326 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN) 9327 netdev_warn(bp->dev, "SFP+ module is shutdown\n"); 9328 } 9329 } 9330 9331 static void 9332 bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req) 9333 { 9334 if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) { 9335 if (bp->hwrm_spec_code >= 0x10201) 9336 req->auto_pause = 9337 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE; 9338 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX) 9339 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX; 9340 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX) 9341 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX; 9342 req->enables |= 9343 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE); 9344 } else { 9345 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX) 9346 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX; 9347 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX) 9348 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX; 9349 req->enables |= 9350 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE); 9351 if (bp->hwrm_spec_code >= 0x10201) { 9352 req->auto_pause = req->force_pause; 9353 req->enables |= cpu_to_le32( 9354 PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE); 9355 } 9356 } 9357 } 9358 9359 static void bnxt_hwrm_set_link_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req) 9360 { 9361 if (bp->link_info.autoneg & BNXT_AUTONEG_SPEED) { 9362 req->auto_mode |= PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK; 9363 if (bp->link_info.advertising) { 9364 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK); 9365 req->auto_link_speed_mask = cpu_to_le16(bp->link_info.advertising); 9366 } 9367 if (bp->link_info.advertising_pam4) { 9368 req->enables |= 9369 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAM4_LINK_SPEED_MASK); 9370 req->auto_link_pam4_speed_mask = 9371 cpu_to_le16(bp->link_info.advertising_pam4); 9372 } 9373 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE); 9374 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG); 9375 } else { 9376 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE); 9377 if (bp->link_info.req_signal_mode == BNXT_SIG_MODE_PAM4) { 9378 req->force_pam4_link_speed = cpu_to_le16(bp->link_info.req_link_speed); 9379 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAM4_LINK_SPEED); 9380 } else { 9381 req->force_link_speed = cpu_to_le16(bp->link_info.req_link_speed); 9382 } 9383 } 9384 9385 /* tell chimp that the setting takes effect immediately */ 9386 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY); 9387 } 9388 9389 int bnxt_hwrm_set_pause(struct bnxt *bp) 9390 { 9391 struct hwrm_port_phy_cfg_input req = {0}; 9392 int rc; 9393 9394 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1); 9395 bnxt_hwrm_set_pause_common(bp, &req); 9396 9397 if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) || 9398 bp->link_info.force_link_chng) 9399 bnxt_hwrm_set_link_common(bp, &req); 9400 9401 mutex_lock(&bp->hwrm_cmd_lock); 9402 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 9403 if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) { 9404 /* since changing of pause setting doesn't trigger any link 9405 * change event, the driver needs to update the current pause 9406 * result upon successfully return of the phy_cfg command 9407 */ 9408 bp->link_info.pause = 9409 bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl; 9410 bp->link_info.auto_pause_setting = 0; 9411 if (!bp->link_info.force_link_chng) 9412 bnxt_report_link(bp); 9413 } 9414 bp->link_info.force_link_chng = false; 9415 mutex_unlock(&bp->hwrm_cmd_lock); 9416 return rc; 9417 } 9418 9419 static void bnxt_hwrm_set_eee(struct bnxt *bp, 9420 struct hwrm_port_phy_cfg_input *req) 9421 { 9422 struct ethtool_eee *eee = &bp->eee; 9423 9424 if (eee->eee_enabled) { 9425 u16 eee_speeds; 9426 u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE; 9427 9428 if (eee->tx_lpi_enabled) 9429 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE; 9430 else 9431 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE; 9432 9433 req->flags |= cpu_to_le32(flags); 9434 eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised); 9435 req->eee_link_speed_mask = cpu_to_le16(eee_speeds); 9436 req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer); 9437 } else { 9438 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE); 9439 } 9440 } 9441 9442 int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee) 9443 { 9444 struct hwrm_port_phy_cfg_input req = {0}; 9445 9446 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1); 9447 if (set_pause) 9448 bnxt_hwrm_set_pause_common(bp, &req); 9449 9450 bnxt_hwrm_set_link_common(bp, &req); 9451 9452 if (set_eee) 9453 bnxt_hwrm_set_eee(bp, &req); 9454 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 9455 } 9456 9457 static int bnxt_hwrm_shutdown_link(struct bnxt *bp) 9458 { 9459 struct hwrm_port_phy_cfg_input req = {0}; 9460 9461 if (!BNXT_SINGLE_PF(bp)) 9462 return 0; 9463 9464 if (pci_num_vf(bp->pdev)) 9465 return 0; 9466 9467 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1); 9468 req.flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN); 9469 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 9470 } 9471 9472 static int bnxt_fw_init_one(struct bnxt *bp); 9473 9474 static int bnxt_fw_reset_via_optee(struct bnxt *bp) 9475 { 9476 #ifdef CONFIG_TEE_BNXT_FW 9477 int rc = tee_bnxt_fw_load(); 9478 9479 if (rc) 9480 netdev_err(bp->dev, "Failed FW reset via OP-TEE, rc=%d\n", rc); 9481 9482 return rc; 9483 #else 9484 netdev_err(bp->dev, "OP-TEE not supported\n"); 9485 return -ENODEV; 9486 #endif 9487 } 9488 9489 static int bnxt_try_recover_fw(struct bnxt *bp) 9490 { 9491 if (bp->fw_health && bp->fw_health->status_reliable) { 9492 int retry = 0, rc; 9493 u32 sts; 9494 9495 mutex_lock(&bp->hwrm_cmd_lock); 9496 do { 9497 rc = __bnxt_hwrm_ver_get(bp, true); 9498 sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG); 9499 if (!sts || !BNXT_FW_IS_BOOTING(sts)) 9500 break; 9501 retry++; 9502 } while (rc == -EBUSY && retry < BNXT_FW_RETRY); 9503 mutex_unlock(&bp->hwrm_cmd_lock); 9504 9505 if (!BNXT_FW_IS_HEALTHY(sts)) { 9506 netdev_err(bp->dev, 9507 "Firmware not responding, status: 0x%x\n", 9508 sts); 9509 rc = -ENODEV; 9510 } 9511 if (sts & FW_STATUS_REG_CRASHED_NO_MASTER) { 9512 netdev_warn(bp->dev, "Firmware recover via OP-TEE requested\n"); 9513 return bnxt_fw_reset_via_optee(bp); 9514 } 9515 return rc; 9516 } 9517 9518 return -ENODEV; 9519 } 9520 9521 static int bnxt_hwrm_if_change(struct bnxt *bp, bool up) 9522 { 9523 struct hwrm_func_drv_if_change_output *resp = bp->hwrm_cmd_resp_addr; 9524 struct hwrm_func_drv_if_change_input req = {0}; 9525 bool fw_reset = !bp->irq_tbl; 9526 bool resc_reinit = false; 9527 int rc, retry = 0; 9528 u32 flags = 0; 9529 9530 if (!(bp->fw_cap & BNXT_FW_CAP_IF_CHANGE)) 9531 return 0; 9532 9533 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_IF_CHANGE, -1, -1); 9534 if (up) 9535 req.flags = cpu_to_le32(FUNC_DRV_IF_CHANGE_REQ_FLAGS_UP); 9536 mutex_lock(&bp->hwrm_cmd_lock); 9537 while (retry < BNXT_FW_IF_RETRY) { 9538 rc = _hwrm_send_message(bp, &req, sizeof(req), 9539 HWRM_CMD_TIMEOUT); 9540 if (rc != -EAGAIN) 9541 break; 9542 9543 msleep(50); 9544 retry++; 9545 } 9546 if (!rc) 9547 flags = le32_to_cpu(resp->flags); 9548 mutex_unlock(&bp->hwrm_cmd_lock); 9549 9550 if (rc == -EAGAIN) 9551 return rc; 9552 if (rc && up) { 9553 rc = bnxt_try_recover_fw(bp); 9554 fw_reset = true; 9555 } 9556 if (rc) 9557 return rc; 9558 9559 if (!up) 9560 return 0; 9561 9562 if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_RESC_CHANGE) 9563 resc_reinit = true; 9564 if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_HOT_FW_RESET_DONE) 9565 fw_reset = true; 9566 9567 if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state) && !fw_reset) { 9568 netdev_err(bp->dev, "RESET_DONE not set during FW reset.\n"); 9569 set_bit(BNXT_STATE_ABORT_ERR, &bp->state); 9570 return -ENODEV; 9571 } 9572 if (resc_reinit || fw_reset) { 9573 if (fw_reset) { 9574 if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 9575 bnxt_ulp_stop(bp); 9576 bnxt_free_ctx_mem(bp); 9577 kfree(bp->ctx); 9578 bp->ctx = NULL; 9579 bnxt_dcb_free(bp); 9580 rc = bnxt_fw_init_one(bp); 9581 if (rc) { 9582 set_bit(BNXT_STATE_ABORT_ERR, &bp->state); 9583 return rc; 9584 } 9585 bnxt_clear_int_mode(bp); 9586 rc = bnxt_init_int_mode(bp); 9587 if (rc) { 9588 netdev_err(bp->dev, "init int mode failed\n"); 9589 return rc; 9590 } 9591 set_bit(BNXT_STATE_FW_RESET_DET, &bp->state); 9592 } 9593 if (BNXT_NEW_RM(bp)) { 9594 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 9595 9596 rc = bnxt_hwrm_func_resc_qcaps(bp, true); 9597 hw_resc->resv_cp_rings = 0; 9598 hw_resc->resv_stat_ctxs = 0; 9599 hw_resc->resv_irqs = 0; 9600 hw_resc->resv_tx_rings = 0; 9601 hw_resc->resv_rx_rings = 0; 9602 hw_resc->resv_hw_ring_grps = 0; 9603 hw_resc->resv_vnics = 0; 9604 if (!fw_reset) { 9605 bp->tx_nr_rings = 0; 9606 bp->rx_nr_rings = 0; 9607 } 9608 } 9609 } 9610 return 0; 9611 } 9612 9613 static int bnxt_hwrm_port_led_qcaps(struct bnxt *bp) 9614 { 9615 struct hwrm_port_led_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 9616 struct hwrm_port_led_qcaps_input req = {0}; 9617 struct bnxt_pf_info *pf = &bp->pf; 9618 int rc; 9619 9620 bp->num_leds = 0; 9621 if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10601) 9622 return 0; 9623 9624 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_LED_QCAPS, -1, -1); 9625 req.port_id = cpu_to_le16(pf->port_id); 9626 mutex_lock(&bp->hwrm_cmd_lock); 9627 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 9628 if (rc) { 9629 mutex_unlock(&bp->hwrm_cmd_lock); 9630 return rc; 9631 } 9632 if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) { 9633 int i; 9634 9635 bp->num_leds = resp->num_leds; 9636 memcpy(bp->leds, &resp->led0_id, sizeof(bp->leds[0]) * 9637 bp->num_leds); 9638 for (i = 0; i < bp->num_leds; i++) { 9639 struct bnxt_led_info *led = &bp->leds[i]; 9640 __le16 caps = led->led_state_caps; 9641 9642 if (!led->led_group_id || 9643 !BNXT_LED_ALT_BLINK_CAP(caps)) { 9644 bp->num_leds = 0; 9645 break; 9646 } 9647 } 9648 } 9649 mutex_unlock(&bp->hwrm_cmd_lock); 9650 return 0; 9651 } 9652 9653 int bnxt_hwrm_alloc_wol_fltr(struct bnxt *bp) 9654 { 9655 struct hwrm_wol_filter_alloc_input req = {0}; 9656 struct hwrm_wol_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr; 9657 int rc; 9658 9659 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_ALLOC, -1, -1); 9660 req.port_id = cpu_to_le16(bp->pf.port_id); 9661 req.wol_type = WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT; 9662 req.enables = cpu_to_le32(WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS); 9663 memcpy(req.mac_address, bp->dev->dev_addr, ETH_ALEN); 9664 mutex_lock(&bp->hwrm_cmd_lock); 9665 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 9666 if (!rc) 9667 bp->wol_filter_id = resp->wol_filter_id; 9668 mutex_unlock(&bp->hwrm_cmd_lock); 9669 return rc; 9670 } 9671 9672 int bnxt_hwrm_free_wol_fltr(struct bnxt *bp) 9673 { 9674 struct hwrm_wol_filter_free_input req = {0}; 9675 9676 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_FREE, -1, -1); 9677 req.port_id = cpu_to_le16(bp->pf.port_id); 9678 req.enables = cpu_to_le32(WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID); 9679 req.wol_filter_id = bp->wol_filter_id; 9680 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 9681 } 9682 9683 static u16 bnxt_hwrm_get_wol_fltrs(struct bnxt *bp, u16 handle) 9684 { 9685 struct hwrm_wol_filter_qcfg_input req = {0}; 9686 struct hwrm_wol_filter_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 9687 u16 next_handle = 0; 9688 int rc; 9689 9690 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_QCFG, -1, -1); 9691 req.port_id = cpu_to_le16(bp->pf.port_id); 9692 req.handle = cpu_to_le16(handle); 9693 mutex_lock(&bp->hwrm_cmd_lock); 9694 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 9695 if (!rc) { 9696 next_handle = le16_to_cpu(resp->next_handle); 9697 if (next_handle != 0) { 9698 if (resp->wol_type == 9699 WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT) { 9700 bp->wol = 1; 9701 bp->wol_filter_id = resp->wol_filter_id; 9702 } 9703 } 9704 } 9705 mutex_unlock(&bp->hwrm_cmd_lock); 9706 return next_handle; 9707 } 9708 9709 static void bnxt_get_wol_settings(struct bnxt *bp) 9710 { 9711 u16 handle = 0; 9712 9713 bp->wol = 0; 9714 if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_WOL_CAP)) 9715 return; 9716 9717 do { 9718 handle = bnxt_hwrm_get_wol_fltrs(bp, handle); 9719 } while (handle && handle != 0xffff); 9720 } 9721 9722 #ifdef CONFIG_BNXT_HWMON 9723 static ssize_t bnxt_show_temp(struct device *dev, 9724 struct device_attribute *devattr, char *buf) 9725 { 9726 struct hwrm_temp_monitor_query_input req = {0}; 9727 struct hwrm_temp_monitor_query_output *resp; 9728 struct bnxt *bp = dev_get_drvdata(dev); 9729 u32 len = 0; 9730 int rc; 9731 9732 resp = bp->hwrm_cmd_resp_addr; 9733 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TEMP_MONITOR_QUERY, -1, -1); 9734 mutex_lock(&bp->hwrm_cmd_lock); 9735 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 9736 if (!rc) 9737 len = sprintf(buf, "%u\n", resp->temp * 1000); /* display millidegree */ 9738 mutex_unlock(&bp->hwrm_cmd_lock); 9739 return rc ?: len; 9740 } 9741 static SENSOR_DEVICE_ATTR(temp1_input, 0444, bnxt_show_temp, NULL, 0); 9742 9743 static struct attribute *bnxt_attrs[] = { 9744 &sensor_dev_attr_temp1_input.dev_attr.attr, 9745 NULL 9746 }; 9747 ATTRIBUTE_GROUPS(bnxt); 9748 9749 static void bnxt_hwmon_close(struct bnxt *bp) 9750 { 9751 if (bp->hwmon_dev) { 9752 hwmon_device_unregister(bp->hwmon_dev); 9753 bp->hwmon_dev = NULL; 9754 } 9755 } 9756 9757 static void bnxt_hwmon_open(struct bnxt *bp) 9758 { 9759 struct hwrm_temp_monitor_query_input req = {0}; 9760 struct pci_dev *pdev = bp->pdev; 9761 int rc; 9762 9763 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TEMP_MONITOR_QUERY, -1, -1); 9764 rc = hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 9765 if (rc == -EACCES || rc == -EOPNOTSUPP) { 9766 bnxt_hwmon_close(bp); 9767 return; 9768 } 9769 9770 if (bp->hwmon_dev) 9771 return; 9772 9773 bp->hwmon_dev = hwmon_device_register_with_groups(&pdev->dev, 9774 DRV_MODULE_NAME, bp, 9775 bnxt_groups); 9776 if (IS_ERR(bp->hwmon_dev)) { 9777 bp->hwmon_dev = NULL; 9778 dev_warn(&pdev->dev, "Cannot register hwmon device\n"); 9779 } 9780 } 9781 #else 9782 static void bnxt_hwmon_close(struct bnxt *bp) 9783 { 9784 } 9785 9786 static void bnxt_hwmon_open(struct bnxt *bp) 9787 { 9788 } 9789 #endif 9790 9791 static bool bnxt_eee_config_ok(struct bnxt *bp) 9792 { 9793 struct ethtool_eee *eee = &bp->eee; 9794 struct bnxt_link_info *link_info = &bp->link_info; 9795 9796 if (!(bp->flags & BNXT_FLAG_EEE_CAP)) 9797 return true; 9798 9799 if (eee->eee_enabled) { 9800 u32 advertising = 9801 _bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0); 9802 9803 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) { 9804 eee->eee_enabled = 0; 9805 return false; 9806 } 9807 if (eee->advertised & ~advertising) { 9808 eee->advertised = advertising & eee->supported; 9809 return false; 9810 } 9811 } 9812 return true; 9813 } 9814 9815 static int bnxt_update_phy_setting(struct bnxt *bp) 9816 { 9817 int rc; 9818 bool update_link = false; 9819 bool update_pause = false; 9820 bool update_eee = false; 9821 struct bnxt_link_info *link_info = &bp->link_info; 9822 9823 rc = bnxt_update_link(bp, true); 9824 if (rc) { 9825 netdev_err(bp->dev, "failed to update link (rc: %x)\n", 9826 rc); 9827 return rc; 9828 } 9829 if (!BNXT_SINGLE_PF(bp)) 9830 return 0; 9831 9832 if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) && 9833 (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) != 9834 link_info->req_flow_ctrl) 9835 update_pause = true; 9836 if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) && 9837 link_info->force_pause_setting != link_info->req_flow_ctrl) 9838 update_pause = true; 9839 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) { 9840 if (BNXT_AUTO_MODE(link_info->auto_mode)) 9841 update_link = true; 9842 if (link_info->req_signal_mode == BNXT_SIG_MODE_NRZ && 9843 link_info->req_link_speed != link_info->force_link_speed) 9844 update_link = true; 9845 else if (link_info->req_signal_mode == BNXT_SIG_MODE_PAM4 && 9846 link_info->req_link_speed != link_info->force_pam4_link_speed) 9847 update_link = true; 9848 if (link_info->req_duplex != link_info->duplex_setting) 9849 update_link = true; 9850 } else { 9851 if (link_info->auto_mode == BNXT_LINK_AUTO_NONE) 9852 update_link = true; 9853 if (link_info->advertising != link_info->auto_link_speeds || 9854 link_info->advertising_pam4 != link_info->auto_pam4_link_speeds) 9855 update_link = true; 9856 } 9857 9858 /* The last close may have shutdown the link, so need to call 9859 * PHY_CFG to bring it back up. 9860 */ 9861 if (!bp->link_info.link_up) 9862 update_link = true; 9863 9864 if (!bnxt_eee_config_ok(bp)) 9865 update_eee = true; 9866 9867 if (update_link) 9868 rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee); 9869 else if (update_pause) 9870 rc = bnxt_hwrm_set_pause(bp); 9871 if (rc) { 9872 netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n", 9873 rc); 9874 return rc; 9875 } 9876 9877 return rc; 9878 } 9879 9880 /* Common routine to pre-map certain register block to different GRC window. 9881 * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows 9882 * in PF and 3 windows in VF that can be customized to map in different 9883 * register blocks. 9884 */ 9885 static void bnxt_preset_reg_win(struct bnxt *bp) 9886 { 9887 if (BNXT_PF(bp)) { 9888 /* CAG registers map to GRC window #4 */ 9889 writel(BNXT_CAG_REG_BASE, 9890 bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12); 9891 } 9892 } 9893 9894 static int bnxt_init_dflt_ring_mode(struct bnxt *bp); 9895 9896 static int bnxt_reinit_after_abort(struct bnxt *bp) 9897 { 9898 int rc; 9899 9900 if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 9901 return -EBUSY; 9902 9903 if (bp->dev->reg_state == NETREG_UNREGISTERED) 9904 return -ENODEV; 9905 9906 rc = bnxt_fw_init_one(bp); 9907 if (!rc) { 9908 bnxt_clear_int_mode(bp); 9909 rc = bnxt_init_int_mode(bp); 9910 if (!rc) { 9911 clear_bit(BNXT_STATE_ABORT_ERR, &bp->state); 9912 set_bit(BNXT_STATE_FW_RESET_DET, &bp->state); 9913 } 9914 } 9915 return rc; 9916 } 9917 9918 static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) 9919 { 9920 int rc = 0; 9921 9922 bnxt_preset_reg_win(bp); 9923 netif_carrier_off(bp->dev); 9924 if (irq_re_init) { 9925 /* Reserve rings now if none were reserved at driver probe. */ 9926 rc = bnxt_init_dflt_ring_mode(bp); 9927 if (rc) { 9928 netdev_err(bp->dev, "Failed to reserve default rings at open\n"); 9929 return rc; 9930 } 9931 } 9932 rc = bnxt_reserve_rings(bp, irq_re_init); 9933 if (rc) 9934 return rc; 9935 if ((bp->flags & BNXT_FLAG_RFS) && 9936 !(bp->flags & BNXT_FLAG_USING_MSIX)) { 9937 /* disable RFS if falling back to INTA */ 9938 bp->dev->hw_features &= ~NETIF_F_NTUPLE; 9939 bp->flags &= ~BNXT_FLAG_RFS; 9940 } 9941 9942 rc = bnxt_alloc_mem(bp, irq_re_init); 9943 if (rc) { 9944 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc); 9945 goto open_err_free_mem; 9946 } 9947 9948 if (irq_re_init) { 9949 bnxt_init_napi(bp); 9950 rc = bnxt_request_irq(bp); 9951 if (rc) { 9952 netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc); 9953 goto open_err_irq; 9954 } 9955 } 9956 9957 rc = bnxt_init_nic(bp, irq_re_init); 9958 if (rc) { 9959 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc); 9960 goto open_err_irq; 9961 } 9962 9963 bnxt_enable_napi(bp); 9964 bnxt_debug_dev_init(bp); 9965 9966 if (link_re_init) { 9967 mutex_lock(&bp->link_lock); 9968 rc = bnxt_update_phy_setting(bp); 9969 mutex_unlock(&bp->link_lock); 9970 if (rc) { 9971 netdev_warn(bp->dev, "failed to update phy settings\n"); 9972 if (BNXT_SINGLE_PF(bp)) { 9973 bp->link_info.phy_retry = true; 9974 bp->link_info.phy_retry_expires = 9975 jiffies + 5 * HZ; 9976 } 9977 } 9978 } 9979 9980 if (irq_re_init) 9981 udp_tunnel_nic_reset_ntf(bp->dev); 9982 9983 set_bit(BNXT_STATE_OPEN, &bp->state); 9984 bnxt_enable_int(bp); 9985 /* Enable TX queues */ 9986 bnxt_tx_enable(bp); 9987 mod_timer(&bp->timer, jiffies + bp->current_interval); 9988 /* Poll link status and check for SFP+ module status */ 9989 bnxt_get_port_module_status(bp); 9990 9991 /* VF-reps may need to be re-opened after the PF is re-opened */ 9992 if (BNXT_PF(bp)) 9993 bnxt_vf_reps_open(bp); 9994 return 0; 9995 9996 open_err_irq: 9997 bnxt_del_napi(bp); 9998 9999 open_err_free_mem: 10000 bnxt_free_skbs(bp); 10001 bnxt_free_irq(bp); 10002 bnxt_free_mem(bp, true); 10003 return rc; 10004 } 10005 10006 /* rtnl_lock held */ 10007 int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) 10008 { 10009 int rc = 0; 10010 10011 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) 10012 rc = -EIO; 10013 if (!rc) 10014 rc = __bnxt_open_nic(bp, irq_re_init, link_re_init); 10015 if (rc) { 10016 netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc); 10017 dev_close(bp->dev); 10018 } 10019 return rc; 10020 } 10021 10022 /* rtnl_lock held, open the NIC half way by allocating all resources, but 10023 * NAPI, IRQ, and TX are not enabled. This is mainly used for offline 10024 * self tests. 10025 */ 10026 int bnxt_half_open_nic(struct bnxt *bp) 10027 { 10028 int rc = 0; 10029 10030 rc = bnxt_alloc_mem(bp, false); 10031 if (rc) { 10032 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc); 10033 goto half_open_err; 10034 } 10035 rc = bnxt_init_nic(bp, false); 10036 if (rc) { 10037 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc); 10038 goto half_open_err; 10039 } 10040 return 0; 10041 10042 half_open_err: 10043 bnxt_free_skbs(bp); 10044 bnxt_free_mem(bp, false); 10045 dev_close(bp->dev); 10046 return rc; 10047 } 10048 10049 /* rtnl_lock held, this call can only be made after a previous successful 10050 * call to bnxt_half_open_nic(). 10051 */ 10052 void bnxt_half_close_nic(struct bnxt *bp) 10053 { 10054 bnxt_hwrm_resource_free(bp, false, false); 10055 bnxt_free_skbs(bp); 10056 bnxt_free_mem(bp, false); 10057 } 10058 10059 static void bnxt_reenable_sriov(struct bnxt *bp) 10060 { 10061 if (BNXT_PF(bp)) { 10062 struct bnxt_pf_info *pf = &bp->pf; 10063 int n = pf->active_vfs; 10064 10065 if (n) 10066 bnxt_cfg_hw_sriov(bp, &n, true); 10067 } 10068 } 10069 10070 static int bnxt_open(struct net_device *dev) 10071 { 10072 struct bnxt *bp = netdev_priv(dev); 10073 int rc; 10074 10075 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) { 10076 rc = bnxt_reinit_after_abort(bp); 10077 if (rc) { 10078 if (rc == -EBUSY) 10079 netdev_err(bp->dev, "A previous firmware reset has not completed, aborting\n"); 10080 else 10081 netdev_err(bp->dev, "Failed to reinitialize after aborted firmware reset\n"); 10082 return -ENODEV; 10083 } 10084 } 10085 10086 rc = bnxt_hwrm_if_change(bp, true); 10087 if (rc) 10088 return rc; 10089 rc = __bnxt_open_nic(bp, true, true); 10090 if (rc) { 10091 bnxt_hwrm_if_change(bp, false); 10092 } else { 10093 if (test_and_clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state)) { 10094 if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) { 10095 bnxt_ulp_start(bp, 0); 10096 bnxt_reenable_sriov(bp); 10097 } 10098 } 10099 bnxt_hwmon_open(bp); 10100 } 10101 10102 return rc; 10103 } 10104 10105 static bool bnxt_drv_busy(struct bnxt *bp) 10106 { 10107 return (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state) || 10108 test_bit(BNXT_STATE_READ_STATS, &bp->state)); 10109 } 10110 10111 static void bnxt_get_ring_stats(struct bnxt *bp, 10112 struct rtnl_link_stats64 *stats); 10113 10114 static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init, 10115 bool link_re_init) 10116 { 10117 /* Close the VF-reps before closing PF */ 10118 if (BNXT_PF(bp)) 10119 bnxt_vf_reps_close(bp); 10120 10121 /* Change device state to avoid TX queue wake up's */ 10122 bnxt_tx_disable(bp); 10123 10124 clear_bit(BNXT_STATE_OPEN, &bp->state); 10125 smp_mb__after_atomic(); 10126 while (bnxt_drv_busy(bp)) 10127 msleep(20); 10128 10129 /* Flush rings and and disable interrupts */ 10130 bnxt_shutdown_nic(bp, irq_re_init); 10131 10132 /* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */ 10133 10134 bnxt_debug_dev_exit(bp); 10135 bnxt_disable_napi(bp); 10136 del_timer_sync(&bp->timer); 10137 bnxt_free_skbs(bp); 10138 10139 /* Save ring stats before shutdown */ 10140 if (bp->bnapi && irq_re_init) 10141 bnxt_get_ring_stats(bp, &bp->net_stats_prev); 10142 if (irq_re_init) { 10143 bnxt_free_irq(bp); 10144 bnxt_del_napi(bp); 10145 } 10146 bnxt_free_mem(bp, irq_re_init); 10147 } 10148 10149 int bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) 10150 { 10151 int rc = 0; 10152 10153 if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) { 10154 /* If we get here, it means firmware reset is in progress 10155 * while we are trying to close. We can safely proceed with 10156 * the close because we are holding rtnl_lock(). Some firmware 10157 * messages may fail as we proceed to close. We set the 10158 * ABORT_ERR flag here so that the FW reset thread will later 10159 * abort when it gets the rtnl_lock() and sees the flag. 10160 */ 10161 netdev_warn(bp->dev, "FW reset in progress during close, FW reset will be aborted\n"); 10162 set_bit(BNXT_STATE_ABORT_ERR, &bp->state); 10163 } 10164 10165 #ifdef CONFIG_BNXT_SRIOV 10166 if (bp->sriov_cfg) { 10167 rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait, 10168 !bp->sriov_cfg, 10169 BNXT_SRIOV_CFG_WAIT_TMO); 10170 if (rc) 10171 netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete!\n"); 10172 } 10173 #endif 10174 __bnxt_close_nic(bp, irq_re_init, link_re_init); 10175 return rc; 10176 } 10177 10178 static int bnxt_close(struct net_device *dev) 10179 { 10180 struct bnxt *bp = netdev_priv(dev); 10181 10182 bnxt_hwmon_close(bp); 10183 bnxt_close_nic(bp, true, true); 10184 bnxt_hwrm_shutdown_link(bp); 10185 bnxt_hwrm_if_change(bp, false); 10186 return 0; 10187 } 10188 10189 static int bnxt_hwrm_port_phy_read(struct bnxt *bp, u16 phy_addr, u16 reg, 10190 u16 *val) 10191 { 10192 struct hwrm_port_phy_mdio_read_output *resp = bp->hwrm_cmd_resp_addr; 10193 struct hwrm_port_phy_mdio_read_input req = {0}; 10194 int rc; 10195 10196 if (bp->hwrm_spec_code < 0x10a00) 10197 return -EOPNOTSUPP; 10198 10199 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_MDIO_READ, -1, -1); 10200 req.port_id = cpu_to_le16(bp->pf.port_id); 10201 req.phy_addr = phy_addr; 10202 req.reg_addr = cpu_to_le16(reg & 0x1f); 10203 if (mdio_phy_id_is_c45(phy_addr)) { 10204 req.cl45_mdio = 1; 10205 req.phy_addr = mdio_phy_id_prtad(phy_addr); 10206 req.dev_addr = mdio_phy_id_devad(phy_addr); 10207 req.reg_addr = cpu_to_le16(reg); 10208 } 10209 10210 mutex_lock(&bp->hwrm_cmd_lock); 10211 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 10212 if (!rc) 10213 *val = le16_to_cpu(resp->reg_data); 10214 mutex_unlock(&bp->hwrm_cmd_lock); 10215 return rc; 10216 } 10217 10218 static int bnxt_hwrm_port_phy_write(struct bnxt *bp, u16 phy_addr, u16 reg, 10219 u16 val) 10220 { 10221 struct hwrm_port_phy_mdio_write_input req = {0}; 10222 10223 if (bp->hwrm_spec_code < 0x10a00) 10224 return -EOPNOTSUPP; 10225 10226 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_MDIO_WRITE, -1, -1); 10227 req.port_id = cpu_to_le16(bp->pf.port_id); 10228 req.phy_addr = phy_addr; 10229 req.reg_addr = cpu_to_le16(reg & 0x1f); 10230 if (mdio_phy_id_is_c45(phy_addr)) { 10231 req.cl45_mdio = 1; 10232 req.phy_addr = mdio_phy_id_prtad(phy_addr); 10233 req.dev_addr = mdio_phy_id_devad(phy_addr); 10234 req.reg_addr = cpu_to_le16(reg); 10235 } 10236 req.reg_data = cpu_to_le16(val); 10237 10238 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 10239 } 10240 10241 /* rtnl_lock held */ 10242 static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) 10243 { 10244 struct mii_ioctl_data *mdio = if_mii(ifr); 10245 struct bnxt *bp = netdev_priv(dev); 10246 int rc; 10247 10248 switch (cmd) { 10249 case SIOCGMIIPHY: 10250 mdio->phy_id = bp->link_info.phy_addr; 10251 10252 fallthrough; 10253 case SIOCGMIIREG: { 10254 u16 mii_regval = 0; 10255 10256 if (!netif_running(dev)) 10257 return -EAGAIN; 10258 10259 rc = bnxt_hwrm_port_phy_read(bp, mdio->phy_id, mdio->reg_num, 10260 &mii_regval); 10261 mdio->val_out = mii_regval; 10262 return rc; 10263 } 10264 10265 case SIOCSMIIREG: 10266 if (!netif_running(dev)) 10267 return -EAGAIN; 10268 10269 return bnxt_hwrm_port_phy_write(bp, mdio->phy_id, mdio->reg_num, 10270 mdio->val_in); 10271 10272 default: 10273 /* do nothing */ 10274 break; 10275 } 10276 return -EOPNOTSUPP; 10277 } 10278 10279 static void bnxt_get_ring_stats(struct bnxt *bp, 10280 struct rtnl_link_stats64 *stats) 10281 { 10282 int i; 10283 10284 for (i = 0; i < bp->cp_nr_rings; i++) { 10285 struct bnxt_napi *bnapi = bp->bnapi[i]; 10286 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 10287 u64 *sw = cpr->stats.sw_stats; 10288 10289 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_ucast_pkts); 10290 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts); 10291 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_bcast_pkts); 10292 10293 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_ucast_pkts); 10294 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_mcast_pkts); 10295 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_bcast_pkts); 10296 10297 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_ucast_bytes); 10298 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_mcast_bytes); 10299 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_bcast_bytes); 10300 10301 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_ucast_bytes); 10302 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_mcast_bytes); 10303 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_bcast_bytes); 10304 10305 stats->rx_missed_errors += 10306 BNXT_GET_RING_STATS64(sw, rx_discard_pkts); 10307 10308 stats->multicast += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts); 10309 10310 stats->tx_dropped += BNXT_GET_RING_STATS64(sw, tx_error_pkts); 10311 } 10312 } 10313 10314 static void bnxt_add_prev_stats(struct bnxt *bp, 10315 struct rtnl_link_stats64 *stats) 10316 { 10317 struct rtnl_link_stats64 *prev_stats = &bp->net_stats_prev; 10318 10319 stats->rx_packets += prev_stats->rx_packets; 10320 stats->tx_packets += prev_stats->tx_packets; 10321 stats->rx_bytes += prev_stats->rx_bytes; 10322 stats->tx_bytes += prev_stats->tx_bytes; 10323 stats->rx_missed_errors += prev_stats->rx_missed_errors; 10324 stats->multicast += prev_stats->multicast; 10325 stats->tx_dropped += prev_stats->tx_dropped; 10326 } 10327 10328 static void 10329 bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) 10330 { 10331 struct bnxt *bp = netdev_priv(dev); 10332 10333 set_bit(BNXT_STATE_READ_STATS, &bp->state); 10334 /* Make sure bnxt_close_nic() sees that we are reading stats before 10335 * we check the BNXT_STATE_OPEN flag. 10336 */ 10337 smp_mb__after_atomic(); 10338 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 10339 clear_bit(BNXT_STATE_READ_STATS, &bp->state); 10340 *stats = bp->net_stats_prev; 10341 return; 10342 } 10343 10344 bnxt_get_ring_stats(bp, stats); 10345 bnxt_add_prev_stats(bp, stats); 10346 10347 if (bp->flags & BNXT_FLAG_PORT_STATS) { 10348 u64 *rx = bp->port_stats.sw_stats; 10349 u64 *tx = bp->port_stats.sw_stats + 10350 BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 10351 10352 stats->rx_crc_errors = 10353 BNXT_GET_RX_PORT_STATS64(rx, rx_fcs_err_frames); 10354 stats->rx_frame_errors = 10355 BNXT_GET_RX_PORT_STATS64(rx, rx_align_err_frames); 10356 stats->rx_length_errors = 10357 BNXT_GET_RX_PORT_STATS64(rx, rx_undrsz_frames) + 10358 BNXT_GET_RX_PORT_STATS64(rx, rx_ovrsz_frames) + 10359 BNXT_GET_RX_PORT_STATS64(rx, rx_runt_frames); 10360 stats->rx_errors = 10361 BNXT_GET_RX_PORT_STATS64(rx, rx_false_carrier_frames) + 10362 BNXT_GET_RX_PORT_STATS64(rx, rx_jbr_frames); 10363 stats->collisions = 10364 BNXT_GET_TX_PORT_STATS64(tx, tx_total_collisions); 10365 stats->tx_fifo_errors = 10366 BNXT_GET_TX_PORT_STATS64(tx, tx_fifo_underruns); 10367 stats->tx_errors = BNXT_GET_TX_PORT_STATS64(tx, tx_err); 10368 } 10369 clear_bit(BNXT_STATE_READ_STATS, &bp->state); 10370 } 10371 10372 static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask) 10373 { 10374 struct net_device *dev = bp->dev; 10375 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 10376 struct netdev_hw_addr *ha; 10377 u8 *haddr; 10378 int mc_count = 0; 10379 bool update = false; 10380 int off = 0; 10381 10382 netdev_for_each_mc_addr(ha, dev) { 10383 if (mc_count >= BNXT_MAX_MC_ADDRS) { 10384 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 10385 vnic->mc_list_count = 0; 10386 return false; 10387 } 10388 haddr = ha->addr; 10389 if (!ether_addr_equal(haddr, vnic->mc_list + off)) { 10390 memcpy(vnic->mc_list + off, haddr, ETH_ALEN); 10391 update = true; 10392 } 10393 off += ETH_ALEN; 10394 mc_count++; 10395 } 10396 if (mc_count) 10397 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST; 10398 10399 if (mc_count != vnic->mc_list_count) { 10400 vnic->mc_list_count = mc_count; 10401 update = true; 10402 } 10403 return update; 10404 } 10405 10406 static bool bnxt_uc_list_updated(struct bnxt *bp) 10407 { 10408 struct net_device *dev = bp->dev; 10409 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 10410 struct netdev_hw_addr *ha; 10411 int off = 0; 10412 10413 if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1)) 10414 return true; 10415 10416 netdev_for_each_uc_addr(ha, dev) { 10417 if (!ether_addr_equal(ha->addr, vnic->uc_list + off)) 10418 return true; 10419 10420 off += ETH_ALEN; 10421 } 10422 return false; 10423 } 10424 10425 static void bnxt_set_rx_mode(struct net_device *dev) 10426 { 10427 struct bnxt *bp = netdev_priv(dev); 10428 struct bnxt_vnic_info *vnic; 10429 bool mc_update = false; 10430 bool uc_update; 10431 u32 mask; 10432 10433 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) 10434 return; 10435 10436 vnic = &bp->vnic_info[0]; 10437 mask = vnic->rx_mask; 10438 mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS | 10439 CFA_L2_SET_RX_MASK_REQ_MASK_MCAST | 10440 CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST | 10441 CFA_L2_SET_RX_MASK_REQ_MASK_BCAST); 10442 10443 if ((dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp)) 10444 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 10445 10446 uc_update = bnxt_uc_list_updated(bp); 10447 10448 if (dev->flags & IFF_BROADCAST) 10449 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST; 10450 if (dev->flags & IFF_ALLMULTI) { 10451 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 10452 vnic->mc_list_count = 0; 10453 } else { 10454 mc_update = bnxt_mc_list_updated(bp, &mask); 10455 } 10456 10457 if (mask != vnic->rx_mask || uc_update || mc_update) { 10458 vnic->rx_mask = mask; 10459 10460 set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event); 10461 bnxt_queue_sp_work(bp); 10462 } 10463 } 10464 10465 static int bnxt_cfg_rx_mode(struct bnxt *bp) 10466 { 10467 struct net_device *dev = bp->dev; 10468 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 10469 struct netdev_hw_addr *ha; 10470 int i, off = 0, rc; 10471 bool uc_update; 10472 10473 netif_addr_lock_bh(dev); 10474 uc_update = bnxt_uc_list_updated(bp); 10475 netif_addr_unlock_bh(dev); 10476 10477 if (!uc_update) 10478 goto skip_uc; 10479 10480 mutex_lock(&bp->hwrm_cmd_lock); 10481 for (i = 1; i < vnic->uc_filter_count; i++) { 10482 struct hwrm_cfa_l2_filter_free_input req = {0}; 10483 10484 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_FREE, -1, 10485 -1); 10486 10487 req.l2_filter_id = vnic->fw_l2_filter_id[i]; 10488 10489 rc = _hwrm_send_message(bp, &req, sizeof(req), 10490 HWRM_CMD_TIMEOUT); 10491 } 10492 mutex_unlock(&bp->hwrm_cmd_lock); 10493 10494 vnic->uc_filter_count = 1; 10495 10496 netif_addr_lock_bh(dev); 10497 if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) { 10498 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 10499 } else { 10500 netdev_for_each_uc_addr(ha, dev) { 10501 memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN); 10502 off += ETH_ALEN; 10503 vnic->uc_filter_count++; 10504 } 10505 } 10506 netif_addr_unlock_bh(dev); 10507 10508 for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) { 10509 rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off); 10510 if (rc) { 10511 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", 10512 rc); 10513 vnic->uc_filter_count = i; 10514 return rc; 10515 } 10516 } 10517 10518 skip_uc: 10519 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0); 10520 if (rc && vnic->mc_list_count) { 10521 netdev_info(bp->dev, "Failed setting MC filters rc: %d, turning on ALL_MCAST mode\n", 10522 rc); 10523 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 10524 vnic->mc_list_count = 0; 10525 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0); 10526 } 10527 if (rc) 10528 netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %d\n", 10529 rc); 10530 10531 return rc; 10532 } 10533 10534 static bool bnxt_can_reserve_rings(struct bnxt *bp) 10535 { 10536 #ifdef CONFIG_BNXT_SRIOV 10537 if (BNXT_NEW_RM(bp) && BNXT_VF(bp)) { 10538 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 10539 10540 /* No minimum rings were provisioned by the PF. Don't 10541 * reserve rings by default when device is down. 10542 */ 10543 if (hw_resc->min_tx_rings || hw_resc->resv_tx_rings) 10544 return true; 10545 10546 if (!netif_running(bp->dev)) 10547 return false; 10548 } 10549 #endif 10550 return true; 10551 } 10552 10553 /* If the chip and firmware supports RFS */ 10554 static bool bnxt_rfs_supported(struct bnxt *bp) 10555 { 10556 if (bp->flags & BNXT_FLAG_CHIP_P5) { 10557 if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2) 10558 return true; 10559 return false; 10560 } 10561 if (BNXT_PF(bp) && !BNXT_CHIP_TYPE_NITRO_A0(bp)) 10562 return true; 10563 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP) 10564 return true; 10565 return false; 10566 } 10567 10568 /* If runtime conditions support RFS */ 10569 static bool bnxt_rfs_capable(struct bnxt *bp) 10570 { 10571 #ifdef CONFIG_RFS_ACCEL 10572 int vnics, max_vnics, max_rss_ctxs; 10573 10574 if (bp->flags & BNXT_FLAG_CHIP_P5) 10575 return bnxt_rfs_supported(bp); 10576 if (!(bp->flags & BNXT_FLAG_MSIX_CAP) || !bnxt_can_reserve_rings(bp)) 10577 return false; 10578 10579 vnics = 1 + bp->rx_nr_rings; 10580 max_vnics = bnxt_get_max_func_vnics(bp); 10581 max_rss_ctxs = bnxt_get_max_func_rss_ctxs(bp); 10582 10583 /* RSS contexts not a limiting factor */ 10584 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP) 10585 max_rss_ctxs = max_vnics; 10586 if (vnics > max_vnics || vnics > max_rss_ctxs) { 10587 if (bp->rx_nr_rings > 1) 10588 netdev_warn(bp->dev, 10589 "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n", 10590 min(max_rss_ctxs - 1, max_vnics - 1)); 10591 return false; 10592 } 10593 10594 if (!BNXT_NEW_RM(bp)) 10595 return true; 10596 10597 if (vnics == bp->hw_resc.resv_vnics) 10598 return true; 10599 10600 bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, vnics); 10601 if (vnics <= bp->hw_resc.resv_vnics) 10602 return true; 10603 10604 netdev_warn(bp->dev, "Unable to reserve resources to support NTUPLE filters.\n"); 10605 bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, 1); 10606 return false; 10607 #else 10608 return false; 10609 #endif 10610 } 10611 10612 static netdev_features_t bnxt_fix_features(struct net_device *dev, 10613 netdev_features_t features) 10614 { 10615 struct bnxt *bp = netdev_priv(dev); 10616 netdev_features_t vlan_features; 10617 10618 if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp)) 10619 features &= ~NETIF_F_NTUPLE; 10620 10621 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS) 10622 features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); 10623 10624 if (!(features & NETIF_F_GRO)) 10625 features &= ~NETIF_F_GRO_HW; 10626 10627 if (features & NETIF_F_GRO_HW) 10628 features &= ~NETIF_F_LRO; 10629 10630 /* Both CTAG and STAG VLAN accelaration on the RX side have to be 10631 * turned on or off together. 10632 */ 10633 vlan_features = features & BNXT_HW_FEATURE_VLAN_ALL_RX; 10634 if (vlan_features != BNXT_HW_FEATURE_VLAN_ALL_RX) { 10635 if (dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX) 10636 features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX; 10637 else if (vlan_features) 10638 features |= BNXT_HW_FEATURE_VLAN_ALL_RX; 10639 } 10640 #ifdef CONFIG_BNXT_SRIOV 10641 if (BNXT_VF(bp) && bp->vf.vlan) 10642 features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX; 10643 #endif 10644 return features; 10645 } 10646 10647 static int bnxt_set_features(struct net_device *dev, netdev_features_t features) 10648 { 10649 struct bnxt *bp = netdev_priv(dev); 10650 u32 flags = bp->flags; 10651 u32 changes; 10652 int rc = 0; 10653 bool re_init = false; 10654 bool update_tpa = false; 10655 10656 flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS; 10657 if (features & NETIF_F_GRO_HW) 10658 flags |= BNXT_FLAG_GRO; 10659 else if (features & NETIF_F_LRO) 10660 flags |= BNXT_FLAG_LRO; 10661 10662 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS) 10663 flags &= ~BNXT_FLAG_TPA; 10664 10665 if (features & BNXT_HW_FEATURE_VLAN_ALL_RX) 10666 flags |= BNXT_FLAG_STRIP_VLAN; 10667 10668 if (features & NETIF_F_NTUPLE) 10669 flags |= BNXT_FLAG_RFS; 10670 10671 changes = flags ^ bp->flags; 10672 if (changes & BNXT_FLAG_TPA) { 10673 update_tpa = true; 10674 if ((bp->flags & BNXT_FLAG_TPA) == 0 || 10675 (flags & BNXT_FLAG_TPA) == 0 || 10676 (bp->flags & BNXT_FLAG_CHIP_P5)) 10677 re_init = true; 10678 } 10679 10680 if (changes & ~BNXT_FLAG_TPA) 10681 re_init = true; 10682 10683 if (flags != bp->flags) { 10684 u32 old_flags = bp->flags; 10685 10686 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 10687 bp->flags = flags; 10688 if (update_tpa) 10689 bnxt_set_ring_params(bp); 10690 return rc; 10691 } 10692 10693 if (re_init) { 10694 bnxt_close_nic(bp, false, false); 10695 bp->flags = flags; 10696 if (update_tpa) 10697 bnxt_set_ring_params(bp); 10698 10699 return bnxt_open_nic(bp, false, false); 10700 } 10701 if (update_tpa) { 10702 bp->flags = flags; 10703 rc = bnxt_set_tpa(bp, 10704 (flags & BNXT_FLAG_TPA) ? 10705 true : false); 10706 if (rc) 10707 bp->flags = old_flags; 10708 } 10709 } 10710 return rc; 10711 } 10712 10713 int bnxt_dbg_hwrm_rd_reg(struct bnxt *bp, u32 reg_off, u16 num_words, 10714 u32 *reg_buf) 10715 { 10716 struct hwrm_dbg_read_direct_output *resp = bp->hwrm_cmd_resp_addr; 10717 struct hwrm_dbg_read_direct_input req = {0}; 10718 __le32 *dbg_reg_buf; 10719 dma_addr_t mapping; 10720 int rc, i; 10721 10722 dbg_reg_buf = dma_alloc_coherent(&bp->pdev->dev, num_words * 4, 10723 &mapping, GFP_KERNEL); 10724 if (!dbg_reg_buf) 10725 return -ENOMEM; 10726 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_DBG_READ_DIRECT, -1, -1); 10727 req.host_dest_addr = cpu_to_le64(mapping); 10728 req.read_addr = cpu_to_le32(reg_off + CHIMP_REG_VIEW_ADDR); 10729 req.read_len32 = cpu_to_le32(num_words); 10730 mutex_lock(&bp->hwrm_cmd_lock); 10731 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 10732 if (rc || resp->error_code) { 10733 rc = -EIO; 10734 goto dbg_rd_reg_exit; 10735 } 10736 for (i = 0; i < num_words; i++) 10737 reg_buf[i] = le32_to_cpu(dbg_reg_buf[i]); 10738 10739 dbg_rd_reg_exit: 10740 mutex_unlock(&bp->hwrm_cmd_lock); 10741 dma_free_coherent(&bp->pdev->dev, num_words * 4, dbg_reg_buf, mapping); 10742 return rc; 10743 } 10744 10745 static int bnxt_dbg_hwrm_ring_info_get(struct bnxt *bp, u8 ring_type, 10746 u32 ring_id, u32 *prod, u32 *cons) 10747 { 10748 struct hwrm_dbg_ring_info_get_output *resp = bp->hwrm_cmd_resp_addr; 10749 struct hwrm_dbg_ring_info_get_input req = {0}; 10750 int rc; 10751 10752 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_DBG_RING_INFO_GET, -1, -1); 10753 req.ring_type = ring_type; 10754 req.fw_ring_id = cpu_to_le32(ring_id); 10755 mutex_lock(&bp->hwrm_cmd_lock); 10756 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 10757 if (!rc) { 10758 *prod = le32_to_cpu(resp->producer_index); 10759 *cons = le32_to_cpu(resp->consumer_index); 10760 } 10761 mutex_unlock(&bp->hwrm_cmd_lock); 10762 return rc; 10763 } 10764 10765 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi) 10766 { 10767 struct bnxt_tx_ring_info *txr = bnapi->tx_ring; 10768 int i = bnapi->index; 10769 10770 if (!txr) 10771 return; 10772 10773 netdev_info(bnapi->bp->dev, "[%d]: tx{fw_ring: %d prod: %x cons: %x}\n", 10774 i, txr->tx_ring_struct.fw_ring_id, txr->tx_prod, 10775 txr->tx_cons); 10776 } 10777 10778 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi) 10779 { 10780 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 10781 int i = bnapi->index; 10782 10783 if (!rxr) 10784 return; 10785 10786 netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n", 10787 i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod, 10788 rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod, 10789 rxr->rx_sw_agg_prod); 10790 } 10791 10792 static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi) 10793 { 10794 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 10795 int i = bnapi->index; 10796 10797 netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n", 10798 i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons); 10799 } 10800 10801 static void bnxt_dbg_dump_states(struct bnxt *bp) 10802 { 10803 int i; 10804 struct bnxt_napi *bnapi; 10805 10806 for (i = 0; i < bp->cp_nr_rings; i++) { 10807 bnapi = bp->bnapi[i]; 10808 if (netif_msg_drv(bp)) { 10809 bnxt_dump_tx_sw_state(bnapi); 10810 bnxt_dump_rx_sw_state(bnapi); 10811 bnxt_dump_cp_sw_state(bnapi); 10812 } 10813 } 10814 } 10815 10816 static int bnxt_hwrm_rx_ring_reset(struct bnxt *bp, int ring_nr) 10817 { 10818 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr]; 10819 struct hwrm_ring_reset_input req = {0}; 10820 struct bnxt_napi *bnapi = rxr->bnapi; 10821 struct bnxt_cp_ring_info *cpr; 10822 u16 cp_ring_id; 10823 10824 cpr = &bnapi->cp_ring; 10825 cp_ring_id = cpr->cp_ring_struct.fw_ring_id; 10826 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_RESET, cp_ring_id, -1); 10827 req.ring_type = RING_RESET_REQ_RING_TYPE_RX_RING_GRP; 10828 req.ring_id = cpu_to_le16(bp->grp_info[bnapi->index].fw_grp_id); 10829 return hwrm_send_message_silent(bp, &req, sizeof(req), 10830 HWRM_CMD_TIMEOUT); 10831 } 10832 10833 static void bnxt_reset_task(struct bnxt *bp, bool silent) 10834 { 10835 if (!silent) 10836 bnxt_dbg_dump_states(bp); 10837 if (netif_running(bp->dev)) { 10838 int rc; 10839 10840 if (silent) { 10841 bnxt_close_nic(bp, false, false); 10842 bnxt_open_nic(bp, false, false); 10843 } else { 10844 bnxt_ulp_stop(bp); 10845 bnxt_close_nic(bp, true, false); 10846 rc = bnxt_open_nic(bp, true, false); 10847 bnxt_ulp_start(bp, rc); 10848 } 10849 } 10850 } 10851 10852 static void bnxt_tx_timeout(struct net_device *dev, unsigned int txqueue) 10853 { 10854 struct bnxt *bp = netdev_priv(dev); 10855 10856 netdev_err(bp->dev, "TX timeout detected, starting reset task!\n"); 10857 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event); 10858 bnxt_queue_sp_work(bp); 10859 } 10860 10861 static void bnxt_fw_health_check(struct bnxt *bp) 10862 { 10863 struct bnxt_fw_health *fw_health = bp->fw_health; 10864 u32 val; 10865 10866 if (!fw_health->enabled || test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 10867 return; 10868 10869 if (fw_health->tmr_counter) { 10870 fw_health->tmr_counter--; 10871 return; 10872 } 10873 10874 val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG); 10875 if (val == fw_health->last_fw_heartbeat) 10876 goto fw_reset; 10877 10878 fw_health->last_fw_heartbeat = val; 10879 10880 val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG); 10881 if (val != fw_health->last_fw_reset_cnt) 10882 goto fw_reset; 10883 10884 fw_health->tmr_counter = fw_health->tmr_multiplier; 10885 return; 10886 10887 fw_reset: 10888 set_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event); 10889 bnxt_queue_sp_work(bp); 10890 } 10891 10892 static void bnxt_timer(struct timer_list *t) 10893 { 10894 struct bnxt *bp = from_timer(bp, t, timer); 10895 struct net_device *dev = bp->dev; 10896 10897 if (!netif_running(dev) || !test_bit(BNXT_STATE_OPEN, &bp->state)) 10898 return; 10899 10900 if (atomic_read(&bp->intr_sem) != 0) 10901 goto bnxt_restart_timer; 10902 10903 if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) 10904 bnxt_fw_health_check(bp); 10905 10906 if (bp->link_info.link_up && bp->stats_coal_ticks) { 10907 set_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event); 10908 bnxt_queue_sp_work(bp); 10909 } 10910 10911 if (bnxt_tc_flower_enabled(bp)) { 10912 set_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event); 10913 bnxt_queue_sp_work(bp); 10914 } 10915 10916 #ifdef CONFIG_RFS_ACCEL 10917 if ((bp->flags & BNXT_FLAG_RFS) && bp->ntp_fltr_count) { 10918 set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event); 10919 bnxt_queue_sp_work(bp); 10920 } 10921 #endif /*CONFIG_RFS_ACCEL*/ 10922 10923 if (bp->link_info.phy_retry) { 10924 if (time_after(jiffies, bp->link_info.phy_retry_expires)) { 10925 bp->link_info.phy_retry = false; 10926 netdev_warn(bp->dev, "failed to update phy settings after maximum retries.\n"); 10927 } else { 10928 set_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event); 10929 bnxt_queue_sp_work(bp); 10930 } 10931 } 10932 10933 if ((bp->flags & BNXT_FLAG_CHIP_P5) && !bp->chip_rev && 10934 netif_carrier_ok(dev)) { 10935 set_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event); 10936 bnxt_queue_sp_work(bp); 10937 } 10938 bnxt_restart_timer: 10939 mod_timer(&bp->timer, jiffies + bp->current_interval); 10940 } 10941 10942 static void bnxt_rtnl_lock_sp(struct bnxt *bp) 10943 { 10944 /* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK 10945 * set. If the device is being closed, bnxt_close() may be holding 10946 * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear. So we 10947 * must clear BNXT_STATE_IN_SP_TASK before holding rtnl(). 10948 */ 10949 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 10950 rtnl_lock(); 10951 } 10952 10953 static void bnxt_rtnl_unlock_sp(struct bnxt *bp) 10954 { 10955 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 10956 rtnl_unlock(); 10957 } 10958 10959 /* Only called from bnxt_sp_task() */ 10960 static void bnxt_reset(struct bnxt *bp, bool silent) 10961 { 10962 bnxt_rtnl_lock_sp(bp); 10963 if (test_bit(BNXT_STATE_OPEN, &bp->state)) 10964 bnxt_reset_task(bp, silent); 10965 bnxt_rtnl_unlock_sp(bp); 10966 } 10967 10968 /* Only called from bnxt_sp_task() */ 10969 static void bnxt_rx_ring_reset(struct bnxt *bp) 10970 { 10971 int i; 10972 10973 bnxt_rtnl_lock_sp(bp); 10974 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 10975 bnxt_rtnl_unlock_sp(bp); 10976 return; 10977 } 10978 /* Disable and flush TPA before resetting the RX ring */ 10979 if (bp->flags & BNXT_FLAG_TPA) 10980 bnxt_set_tpa(bp, false); 10981 for (i = 0; i < bp->rx_nr_rings; i++) { 10982 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 10983 struct bnxt_cp_ring_info *cpr; 10984 int rc; 10985 10986 if (!rxr->bnapi->in_reset) 10987 continue; 10988 10989 rc = bnxt_hwrm_rx_ring_reset(bp, i); 10990 if (rc) { 10991 if (rc == -EINVAL || rc == -EOPNOTSUPP) 10992 netdev_info_once(bp->dev, "RX ring reset not supported by firmware, falling back to global reset\n"); 10993 else 10994 netdev_warn(bp->dev, "RX ring reset failed, rc = %d, falling back to global reset\n", 10995 rc); 10996 bnxt_reset_task(bp, true); 10997 break; 10998 } 10999 bnxt_free_one_rx_ring_skbs(bp, i); 11000 rxr->rx_prod = 0; 11001 rxr->rx_agg_prod = 0; 11002 rxr->rx_sw_agg_prod = 0; 11003 rxr->rx_next_cons = 0; 11004 rxr->bnapi->in_reset = false; 11005 bnxt_alloc_one_rx_ring(bp, i); 11006 cpr = &rxr->bnapi->cp_ring; 11007 cpr->sw_stats.rx.rx_resets++; 11008 if (bp->flags & BNXT_FLAG_AGG_RINGS) 11009 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); 11010 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 11011 } 11012 if (bp->flags & BNXT_FLAG_TPA) 11013 bnxt_set_tpa(bp, true); 11014 bnxt_rtnl_unlock_sp(bp); 11015 } 11016 11017 static void bnxt_fw_reset_close(struct bnxt *bp) 11018 { 11019 bnxt_ulp_stop(bp); 11020 /* When firmware is in fatal state, quiesce device and disable 11021 * bus master to prevent any potential bad DMAs before freeing 11022 * kernel memory. 11023 */ 11024 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) { 11025 u16 val = 0; 11026 11027 pci_read_config_word(bp->pdev, PCI_SUBSYSTEM_ID, &val); 11028 if (val == 0xffff) 11029 bp->fw_reset_min_dsecs = 0; 11030 bnxt_tx_disable(bp); 11031 bnxt_disable_napi(bp); 11032 bnxt_disable_int_sync(bp); 11033 bnxt_free_irq(bp); 11034 bnxt_clear_int_mode(bp); 11035 pci_disable_device(bp->pdev); 11036 } 11037 __bnxt_close_nic(bp, true, false); 11038 bnxt_clear_int_mode(bp); 11039 bnxt_hwrm_func_drv_unrgtr(bp); 11040 if (pci_is_enabled(bp->pdev)) 11041 pci_disable_device(bp->pdev); 11042 bnxt_free_ctx_mem(bp); 11043 kfree(bp->ctx); 11044 bp->ctx = NULL; 11045 } 11046 11047 static bool is_bnxt_fw_ok(struct bnxt *bp) 11048 { 11049 struct bnxt_fw_health *fw_health = bp->fw_health; 11050 bool no_heartbeat = false, has_reset = false; 11051 u32 val; 11052 11053 val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG); 11054 if (val == fw_health->last_fw_heartbeat) 11055 no_heartbeat = true; 11056 11057 val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG); 11058 if (val != fw_health->last_fw_reset_cnt) 11059 has_reset = true; 11060 11061 if (!no_heartbeat && has_reset) 11062 return true; 11063 11064 return false; 11065 } 11066 11067 /* rtnl_lock is acquired before calling this function */ 11068 static void bnxt_force_fw_reset(struct bnxt *bp) 11069 { 11070 struct bnxt_fw_health *fw_health = bp->fw_health; 11071 u32 wait_dsecs; 11072 11073 if (!test_bit(BNXT_STATE_OPEN, &bp->state) || 11074 test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 11075 return; 11076 11077 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 11078 bnxt_fw_reset_close(bp); 11079 wait_dsecs = fw_health->master_func_wait_dsecs; 11080 if (fw_health->master) { 11081 if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) 11082 wait_dsecs = 0; 11083 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW; 11084 } else { 11085 bp->fw_reset_timestamp = jiffies + wait_dsecs * HZ / 10; 11086 wait_dsecs = fw_health->normal_func_wait_dsecs; 11087 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 11088 } 11089 11090 bp->fw_reset_min_dsecs = fw_health->post_reset_wait_dsecs; 11091 bp->fw_reset_max_dsecs = fw_health->post_reset_max_wait_dsecs; 11092 bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10); 11093 } 11094 11095 void bnxt_fw_exception(struct bnxt *bp) 11096 { 11097 netdev_warn(bp->dev, "Detected firmware fatal condition, initiating reset\n"); 11098 set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state); 11099 bnxt_rtnl_lock_sp(bp); 11100 bnxt_force_fw_reset(bp); 11101 bnxt_rtnl_unlock_sp(bp); 11102 } 11103 11104 /* Returns the number of registered VFs, or 1 if VF configuration is pending, or 11105 * < 0 on error. 11106 */ 11107 static int bnxt_get_registered_vfs(struct bnxt *bp) 11108 { 11109 #ifdef CONFIG_BNXT_SRIOV 11110 int rc; 11111 11112 if (!BNXT_PF(bp)) 11113 return 0; 11114 11115 rc = bnxt_hwrm_func_qcfg(bp); 11116 if (rc) { 11117 netdev_err(bp->dev, "func_qcfg cmd failed, rc = %d\n", rc); 11118 return rc; 11119 } 11120 if (bp->pf.registered_vfs) 11121 return bp->pf.registered_vfs; 11122 if (bp->sriov_cfg) 11123 return 1; 11124 #endif 11125 return 0; 11126 } 11127 11128 void bnxt_fw_reset(struct bnxt *bp) 11129 { 11130 bnxt_rtnl_lock_sp(bp); 11131 if (test_bit(BNXT_STATE_OPEN, &bp->state) && 11132 !test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) { 11133 int n = 0, tmo; 11134 11135 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 11136 if (bp->pf.active_vfs && 11137 !test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) 11138 n = bnxt_get_registered_vfs(bp); 11139 if (n < 0) { 11140 netdev_err(bp->dev, "Firmware reset aborted, rc = %d\n", 11141 n); 11142 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 11143 dev_close(bp->dev); 11144 goto fw_reset_exit; 11145 } else if (n > 0) { 11146 u16 vf_tmo_dsecs = n * 10; 11147 11148 if (bp->fw_reset_max_dsecs < vf_tmo_dsecs) 11149 bp->fw_reset_max_dsecs = vf_tmo_dsecs; 11150 bp->fw_reset_state = 11151 BNXT_FW_RESET_STATE_POLL_VF; 11152 bnxt_queue_fw_reset_work(bp, HZ / 10); 11153 goto fw_reset_exit; 11154 } 11155 bnxt_fw_reset_close(bp); 11156 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) { 11157 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN; 11158 tmo = HZ / 10; 11159 } else { 11160 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 11161 tmo = bp->fw_reset_min_dsecs * HZ / 10; 11162 } 11163 bnxt_queue_fw_reset_work(bp, tmo); 11164 } 11165 fw_reset_exit: 11166 bnxt_rtnl_unlock_sp(bp); 11167 } 11168 11169 static void bnxt_chk_missed_irq(struct bnxt *bp) 11170 { 11171 int i; 11172 11173 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 11174 return; 11175 11176 for (i = 0; i < bp->cp_nr_rings; i++) { 11177 struct bnxt_napi *bnapi = bp->bnapi[i]; 11178 struct bnxt_cp_ring_info *cpr; 11179 u32 fw_ring_id; 11180 int j; 11181 11182 if (!bnapi) 11183 continue; 11184 11185 cpr = &bnapi->cp_ring; 11186 for (j = 0; j < 2; j++) { 11187 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j]; 11188 u32 val[2]; 11189 11190 if (!cpr2 || cpr2->has_more_work || 11191 !bnxt_has_work(bp, cpr2)) 11192 continue; 11193 11194 if (cpr2->cp_raw_cons != cpr2->last_cp_raw_cons) { 11195 cpr2->last_cp_raw_cons = cpr2->cp_raw_cons; 11196 continue; 11197 } 11198 fw_ring_id = cpr2->cp_ring_struct.fw_ring_id; 11199 bnxt_dbg_hwrm_ring_info_get(bp, 11200 DBG_RING_INFO_GET_REQ_RING_TYPE_L2_CMPL, 11201 fw_ring_id, &val[0], &val[1]); 11202 cpr->sw_stats.cmn.missed_irqs++; 11203 } 11204 } 11205 } 11206 11207 static void bnxt_cfg_ntp_filters(struct bnxt *); 11208 11209 static void bnxt_init_ethtool_link_settings(struct bnxt *bp) 11210 { 11211 struct bnxt_link_info *link_info = &bp->link_info; 11212 11213 if (BNXT_AUTO_MODE(link_info->auto_mode)) { 11214 link_info->autoneg = BNXT_AUTONEG_SPEED; 11215 if (bp->hwrm_spec_code >= 0x10201) { 11216 if (link_info->auto_pause_setting & 11217 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE) 11218 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL; 11219 } else { 11220 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL; 11221 } 11222 link_info->advertising = link_info->auto_link_speeds; 11223 link_info->advertising_pam4 = link_info->auto_pam4_link_speeds; 11224 } else { 11225 link_info->req_link_speed = link_info->force_link_speed; 11226 link_info->req_signal_mode = BNXT_SIG_MODE_NRZ; 11227 if (link_info->force_pam4_link_speed) { 11228 link_info->req_link_speed = 11229 link_info->force_pam4_link_speed; 11230 link_info->req_signal_mode = BNXT_SIG_MODE_PAM4; 11231 } 11232 link_info->req_duplex = link_info->duplex_setting; 11233 } 11234 if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) 11235 link_info->req_flow_ctrl = 11236 link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH; 11237 else 11238 link_info->req_flow_ctrl = link_info->force_pause_setting; 11239 } 11240 11241 static void bnxt_fw_echo_reply(struct bnxt *bp) 11242 { 11243 struct bnxt_fw_health *fw_health = bp->fw_health; 11244 struct hwrm_func_echo_response_input req = {0}; 11245 11246 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_ECHO_RESPONSE, -1, -1); 11247 req.event_data1 = cpu_to_le32(fw_health->echo_req_data1); 11248 req.event_data2 = cpu_to_le32(fw_health->echo_req_data2); 11249 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 11250 } 11251 11252 static void bnxt_sp_task(struct work_struct *work) 11253 { 11254 struct bnxt *bp = container_of(work, struct bnxt, sp_task); 11255 11256 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 11257 smp_mb__after_atomic(); 11258 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 11259 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 11260 return; 11261 } 11262 11263 if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event)) 11264 bnxt_cfg_rx_mode(bp); 11265 11266 if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event)) 11267 bnxt_cfg_ntp_filters(bp); 11268 if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event)) 11269 bnxt_hwrm_exec_fwd_req(bp); 11270 if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event)) { 11271 bnxt_hwrm_port_qstats(bp, 0); 11272 bnxt_hwrm_port_qstats_ext(bp, 0); 11273 bnxt_accumulate_all_stats(bp); 11274 } 11275 11276 if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) { 11277 int rc; 11278 11279 mutex_lock(&bp->link_lock); 11280 if (test_and_clear_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, 11281 &bp->sp_event)) 11282 bnxt_hwrm_phy_qcaps(bp); 11283 11284 rc = bnxt_update_link(bp, true); 11285 if (rc) 11286 netdev_err(bp->dev, "SP task can't update link (rc: %x)\n", 11287 rc); 11288 11289 if (test_and_clear_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT, 11290 &bp->sp_event)) 11291 bnxt_init_ethtool_link_settings(bp); 11292 mutex_unlock(&bp->link_lock); 11293 } 11294 if (test_and_clear_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event)) { 11295 int rc; 11296 11297 mutex_lock(&bp->link_lock); 11298 rc = bnxt_update_phy_setting(bp); 11299 mutex_unlock(&bp->link_lock); 11300 if (rc) { 11301 netdev_warn(bp->dev, "update phy settings retry failed\n"); 11302 } else { 11303 bp->link_info.phy_retry = false; 11304 netdev_info(bp->dev, "update phy settings retry succeeded\n"); 11305 } 11306 } 11307 if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event)) { 11308 mutex_lock(&bp->link_lock); 11309 bnxt_get_port_module_status(bp); 11310 mutex_unlock(&bp->link_lock); 11311 } 11312 11313 if (test_and_clear_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event)) 11314 bnxt_tc_flow_stats_work(bp); 11315 11316 if (test_and_clear_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event)) 11317 bnxt_chk_missed_irq(bp); 11318 11319 if (test_and_clear_bit(BNXT_FW_ECHO_REQUEST_SP_EVENT, &bp->sp_event)) 11320 bnxt_fw_echo_reply(bp); 11321 11322 /* These functions below will clear BNXT_STATE_IN_SP_TASK. They 11323 * must be the last functions to be called before exiting. 11324 */ 11325 if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event)) 11326 bnxt_reset(bp, false); 11327 11328 if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event)) 11329 bnxt_reset(bp, true); 11330 11331 if (test_and_clear_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event)) 11332 bnxt_rx_ring_reset(bp); 11333 11334 if (test_and_clear_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event)) 11335 bnxt_devlink_health_report(bp, BNXT_FW_RESET_NOTIFY_SP_EVENT); 11336 11337 if (test_and_clear_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event)) { 11338 if (!is_bnxt_fw_ok(bp)) 11339 bnxt_devlink_health_report(bp, 11340 BNXT_FW_EXCEPTION_SP_EVENT); 11341 } 11342 11343 smp_mb__before_atomic(); 11344 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 11345 } 11346 11347 /* Under rtnl_lock */ 11348 int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs, 11349 int tx_xdp) 11350 { 11351 int max_rx, max_tx, tx_sets = 1; 11352 int tx_rings_needed, stats; 11353 int rx_rings = rx; 11354 int cp, vnics, rc; 11355 11356 if (tcs) 11357 tx_sets = tcs; 11358 11359 rc = bnxt_get_max_rings(bp, &max_rx, &max_tx, sh); 11360 if (rc) 11361 return rc; 11362 11363 if (max_rx < rx) 11364 return -ENOMEM; 11365 11366 tx_rings_needed = tx * tx_sets + tx_xdp; 11367 if (max_tx < tx_rings_needed) 11368 return -ENOMEM; 11369 11370 vnics = 1; 11371 if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS) 11372 vnics += rx_rings; 11373 11374 if (bp->flags & BNXT_FLAG_AGG_RINGS) 11375 rx_rings <<= 1; 11376 cp = sh ? max_t(int, tx_rings_needed, rx) : tx_rings_needed + rx; 11377 stats = cp; 11378 if (BNXT_NEW_RM(bp)) { 11379 cp += bnxt_get_ulp_msix_num(bp); 11380 stats += bnxt_get_ulp_stat_ctxs(bp); 11381 } 11382 return bnxt_hwrm_check_rings(bp, tx_rings_needed, rx_rings, rx, cp, 11383 stats, vnics); 11384 } 11385 11386 static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev) 11387 { 11388 if (bp->bar2) { 11389 pci_iounmap(pdev, bp->bar2); 11390 bp->bar2 = NULL; 11391 } 11392 11393 if (bp->bar1) { 11394 pci_iounmap(pdev, bp->bar1); 11395 bp->bar1 = NULL; 11396 } 11397 11398 if (bp->bar0) { 11399 pci_iounmap(pdev, bp->bar0); 11400 bp->bar0 = NULL; 11401 } 11402 } 11403 11404 static void bnxt_cleanup_pci(struct bnxt *bp) 11405 { 11406 bnxt_unmap_bars(bp, bp->pdev); 11407 pci_release_regions(bp->pdev); 11408 if (pci_is_enabled(bp->pdev)) 11409 pci_disable_device(bp->pdev); 11410 } 11411 11412 static void bnxt_init_dflt_coal(struct bnxt *bp) 11413 { 11414 struct bnxt_coal *coal; 11415 11416 /* Tick values in micro seconds. 11417 * 1 coal_buf x bufs_per_record = 1 completion record. 11418 */ 11419 coal = &bp->rx_coal; 11420 coal->coal_ticks = 10; 11421 coal->coal_bufs = 30; 11422 coal->coal_ticks_irq = 1; 11423 coal->coal_bufs_irq = 2; 11424 coal->idle_thresh = 50; 11425 coal->bufs_per_record = 2; 11426 coal->budget = 64; /* NAPI budget */ 11427 11428 coal = &bp->tx_coal; 11429 coal->coal_ticks = 28; 11430 coal->coal_bufs = 30; 11431 coal->coal_ticks_irq = 2; 11432 coal->coal_bufs_irq = 2; 11433 coal->bufs_per_record = 1; 11434 11435 bp->stats_coal_ticks = BNXT_DEF_STATS_COAL_TICKS; 11436 } 11437 11438 static int bnxt_fw_init_one_p1(struct bnxt *bp) 11439 { 11440 int rc; 11441 11442 bp->fw_cap = 0; 11443 rc = bnxt_hwrm_ver_get(bp); 11444 bnxt_try_map_fw_health_reg(bp); 11445 if (rc) { 11446 rc = bnxt_try_recover_fw(bp); 11447 if (rc) 11448 return rc; 11449 rc = bnxt_hwrm_ver_get(bp); 11450 if (rc) 11451 return rc; 11452 } 11453 11454 if (bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL) { 11455 rc = bnxt_alloc_kong_hwrm_resources(bp); 11456 if (rc) 11457 bp->fw_cap &= ~BNXT_FW_CAP_KONG_MB_CHNL; 11458 } 11459 11460 if ((bp->fw_cap & BNXT_FW_CAP_SHORT_CMD) || 11461 bp->hwrm_max_ext_req_len > BNXT_HWRM_MAX_REQ_LEN) { 11462 rc = bnxt_alloc_hwrm_short_cmd_req(bp); 11463 if (rc) 11464 return rc; 11465 } 11466 bnxt_nvm_cfg_ver_get(bp); 11467 11468 rc = bnxt_hwrm_func_reset(bp); 11469 if (rc) 11470 return -ENODEV; 11471 11472 bnxt_hwrm_fw_set_time(bp); 11473 return 0; 11474 } 11475 11476 static int bnxt_fw_init_one_p2(struct bnxt *bp) 11477 { 11478 int rc; 11479 11480 /* Get the MAX capabilities for this function */ 11481 rc = bnxt_hwrm_func_qcaps(bp); 11482 if (rc) { 11483 netdev_err(bp->dev, "hwrm query capability failure rc: %x\n", 11484 rc); 11485 return -ENODEV; 11486 } 11487 11488 rc = bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(bp); 11489 if (rc) 11490 netdev_warn(bp->dev, "hwrm query adv flow mgnt failure rc: %d\n", 11491 rc); 11492 11493 if (bnxt_alloc_fw_health(bp)) { 11494 netdev_warn(bp->dev, "no memory for firmware error recovery\n"); 11495 } else { 11496 rc = bnxt_hwrm_error_recovery_qcfg(bp); 11497 if (rc) 11498 netdev_warn(bp->dev, "hwrm query error recovery failure rc: %d\n", 11499 rc); 11500 } 11501 11502 rc = bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false); 11503 if (rc) 11504 return -ENODEV; 11505 11506 bnxt_hwrm_func_qcfg(bp); 11507 bnxt_hwrm_vnic_qcaps(bp); 11508 bnxt_hwrm_port_led_qcaps(bp); 11509 bnxt_ethtool_init(bp); 11510 bnxt_dcb_init(bp); 11511 return 0; 11512 } 11513 11514 static void bnxt_set_dflt_rss_hash_type(struct bnxt *bp) 11515 { 11516 bp->flags &= ~BNXT_FLAG_UDP_RSS_CAP; 11517 bp->rss_hash_cfg = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 | 11518 VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 | 11519 VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 | 11520 VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6; 11521 if (BNXT_CHIP_P4_PLUS(bp) && bp->hwrm_spec_code >= 0x10501) { 11522 bp->flags |= BNXT_FLAG_UDP_RSS_CAP; 11523 bp->rss_hash_cfg |= VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 | 11524 VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6; 11525 } 11526 } 11527 11528 static void bnxt_set_dflt_rfs(struct bnxt *bp) 11529 { 11530 struct net_device *dev = bp->dev; 11531 11532 dev->hw_features &= ~NETIF_F_NTUPLE; 11533 dev->features &= ~NETIF_F_NTUPLE; 11534 bp->flags &= ~BNXT_FLAG_RFS; 11535 if (bnxt_rfs_supported(bp)) { 11536 dev->hw_features |= NETIF_F_NTUPLE; 11537 if (bnxt_rfs_capable(bp)) { 11538 bp->flags |= BNXT_FLAG_RFS; 11539 dev->features |= NETIF_F_NTUPLE; 11540 } 11541 } 11542 } 11543 11544 static void bnxt_fw_init_one_p3(struct bnxt *bp) 11545 { 11546 struct pci_dev *pdev = bp->pdev; 11547 11548 bnxt_set_dflt_rss_hash_type(bp); 11549 bnxt_set_dflt_rfs(bp); 11550 11551 bnxt_get_wol_settings(bp); 11552 if (bp->flags & BNXT_FLAG_WOL_CAP) 11553 device_set_wakeup_enable(&pdev->dev, bp->wol); 11554 else 11555 device_set_wakeup_capable(&pdev->dev, false); 11556 11557 bnxt_hwrm_set_cache_line_size(bp, cache_line_size()); 11558 bnxt_hwrm_coal_params_qcaps(bp); 11559 } 11560 11561 static int bnxt_fw_init_one(struct bnxt *bp) 11562 { 11563 int rc; 11564 11565 rc = bnxt_fw_init_one_p1(bp); 11566 if (rc) { 11567 netdev_err(bp->dev, "Firmware init phase 1 failed\n"); 11568 return rc; 11569 } 11570 rc = bnxt_fw_init_one_p2(bp); 11571 if (rc) { 11572 netdev_err(bp->dev, "Firmware init phase 2 failed\n"); 11573 return rc; 11574 } 11575 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, false); 11576 if (rc) 11577 return rc; 11578 11579 /* In case fw capabilities have changed, destroy the unneeded 11580 * reporters and create newly capable ones. 11581 */ 11582 bnxt_dl_fw_reporters_destroy(bp, false); 11583 bnxt_dl_fw_reporters_create(bp); 11584 bnxt_fw_init_one_p3(bp); 11585 return 0; 11586 } 11587 11588 static void bnxt_fw_reset_writel(struct bnxt *bp, int reg_idx) 11589 { 11590 struct bnxt_fw_health *fw_health = bp->fw_health; 11591 u32 reg = fw_health->fw_reset_seq_regs[reg_idx]; 11592 u32 val = fw_health->fw_reset_seq_vals[reg_idx]; 11593 u32 reg_type, reg_off, delay_msecs; 11594 11595 delay_msecs = fw_health->fw_reset_seq_delay_msec[reg_idx]; 11596 reg_type = BNXT_FW_HEALTH_REG_TYPE(reg); 11597 reg_off = BNXT_FW_HEALTH_REG_OFF(reg); 11598 switch (reg_type) { 11599 case BNXT_FW_HEALTH_REG_TYPE_CFG: 11600 pci_write_config_dword(bp->pdev, reg_off, val); 11601 break; 11602 case BNXT_FW_HEALTH_REG_TYPE_GRC: 11603 writel(reg_off & BNXT_GRC_BASE_MASK, 11604 bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4); 11605 reg_off = (reg_off & BNXT_GRC_OFFSET_MASK) + 0x2000; 11606 fallthrough; 11607 case BNXT_FW_HEALTH_REG_TYPE_BAR0: 11608 writel(val, bp->bar0 + reg_off); 11609 break; 11610 case BNXT_FW_HEALTH_REG_TYPE_BAR1: 11611 writel(val, bp->bar1 + reg_off); 11612 break; 11613 } 11614 if (delay_msecs) { 11615 pci_read_config_dword(bp->pdev, 0, &val); 11616 msleep(delay_msecs); 11617 } 11618 } 11619 11620 static void bnxt_reset_all(struct bnxt *bp) 11621 { 11622 struct bnxt_fw_health *fw_health = bp->fw_health; 11623 int i, rc; 11624 11625 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) { 11626 bnxt_fw_reset_via_optee(bp); 11627 bp->fw_reset_timestamp = jiffies; 11628 return; 11629 } 11630 11631 if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_HOST) { 11632 for (i = 0; i < fw_health->fw_reset_seq_cnt; i++) 11633 bnxt_fw_reset_writel(bp, i); 11634 } else if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) { 11635 struct hwrm_fw_reset_input req = {0}; 11636 11637 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FW_RESET, -1, -1); 11638 req.resp_addr = cpu_to_le64(bp->hwrm_cmd_kong_resp_dma_addr); 11639 req.embedded_proc_type = FW_RESET_REQ_EMBEDDED_PROC_TYPE_CHIP; 11640 req.selfrst_status = FW_RESET_REQ_SELFRST_STATUS_SELFRSTASAP; 11641 req.flags = FW_RESET_REQ_FLAGS_RESET_GRACEFUL; 11642 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 11643 if (rc) 11644 netdev_warn(bp->dev, "Unable to reset FW rc=%d\n", rc); 11645 } 11646 bp->fw_reset_timestamp = jiffies; 11647 } 11648 11649 static bool bnxt_fw_reset_timeout(struct bnxt *bp) 11650 { 11651 return time_after(jiffies, bp->fw_reset_timestamp + 11652 (bp->fw_reset_max_dsecs * HZ / 10)); 11653 } 11654 11655 static void bnxt_fw_reset_task(struct work_struct *work) 11656 { 11657 struct bnxt *bp = container_of(work, struct bnxt, fw_reset_task.work); 11658 int rc; 11659 11660 if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) { 11661 netdev_err(bp->dev, "bnxt_fw_reset_task() called when not in fw reset mode!\n"); 11662 return; 11663 } 11664 11665 switch (bp->fw_reset_state) { 11666 case BNXT_FW_RESET_STATE_POLL_VF: { 11667 int n = bnxt_get_registered_vfs(bp); 11668 int tmo; 11669 11670 if (n < 0) { 11671 netdev_err(bp->dev, "Firmware reset aborted, subsequent func_qcfg cmd failed, rc = %d, %d msecs since reset timestamp\n", 11672 n, jiffies_to_msecs(jiffies - 11673 bp->fw_reset_timestamp)); 11674 goto fw_reset_abort; 11675 } else if (n > 0) { 11676 if (bnxt_fw_reset_timeout(bp)) { 11677 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 11678 bp->fw_reset_state = 0; 11679 netdev_err(bp->dev, "Firmware reset aborted, bnxt_get_registered_vfs() returns %d\n", 11680 n); 11681 return; 11682 } 11683 bnxt_queue_fw_reset_work(bp, HZ / 10); 11684 return; 11685 } 11686 bp->fw_reset_timestamp = jiffies; 11687 rtnl_lock(); 11688 bnxt_fw_reset_close(bp); 11689 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) { 11690 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN; 11691 tmo = HZ / 10; 11692 } else { 11693 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 11694 tmo = bp->fw_reset_min_dsecs * HZ / 10; 11695 } 11696 rtnl_unlock(); 11697 bnxt_queue_fw_reset_work(bp, tmo); 11698 return; 11699 } 11700 case BNXT_FW_RESET_STATE_POLL_FW_DOWN: { 11701 u32 val; 11702 11703 val = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG); 11704 if (!(val & BNXT_FW_STATUS_SHUTDOWN) && 11705 !bnxt_fw_reset_timeout(bp)) { 11706 bnxt_queue_fw_reset_work(bp, HZ / 5); 11707 return; 11708 } 11709 11710 if (!bp->fw_health->master) { 11711 u32 wait_dsecs = bp->fw_health->normal_func_wait_dsecs; 11712 11713 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 11714 bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10); 11715 return; 11716 } 11717 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW; 11718 } 11719 fallthrough; 11720 case BNXT_FW_RESET_STATE_RESET_FW: 11721 bnxt_reset_all(bp); 11722 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 11723 bnxt_queue_fw_reset_work(bp, bp->fw_reset_min_dsecs * HZ / 10); 11724 return; 11725 case BNXT_FW_RESET_STATE_ENABLE_DEV: 11726 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) { 11727 u32 val; 11728 11729 if (!bp->fw_reset_min_dsecs) { 11730 u16 val; 11731 11732 pci_read_config_word(bp->pdev, PCI_SUBSYSTEM_ID, 11733 &val); 11734 if (val == 0xffff) { 11735 if (bnxt_fw_reset_timeout(bp)) { 11736 netdev_err(bp->dev, "Firmware reset aborted, PCI config space invalid\n"); 11737 goto fw_reset_abort; 11738 } 11739 bnxt_queue_fw_reset_work(bp, HZ / 1000); 11740 return; 11741 } 11742 } 11743 val = bnxt_fw_health_readl(bp, 11744 BNXT_FW_RESET_INPROG_REG); 11745 if (val) 11746 netdev_warn(bp->dev, "FW reset inprog %x after min wait time.\n", 11747 val); 11748 } 11749 clear_bit(BNXT_STATE_FW_FATAL_COND, &bp->state); 11750 if (pci_enable_device(bp->pdev)) { 11751 netdev_err(bp->dev, "Cannot re-enable PCI device\n"); 11752 goto fw_reset_abort; 11753 } 11754 pci_set_master(bp->pdev); 11755 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW; 11756 fallthrough; 11757 case BNXT_FW_RESET_STATE_POLL_FW: 11758 bp->hwrm_cmd_timeout = SHORT_HWRM_CMD_TIMEOUT; 11759 rc = __bnxt_hwrm_ver_get(bp, true); 11760 if (rc) { 11761 if (bnxt_fw_reset_timeout(bp)) { 11762 netdev_err(bp->dev, "Firmware reset aborted\n"); 11763 goto fw_reset_abort_status; 11764 } 11765 bnxt_queue_fw_reset_work(bp, HZ / 5); 11766 return; 11767 } 11768 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT; 11769 bp->fw_reset_state = BNXT_FW_RESET_STATE_OPENING; 11770 fallthrough; 11771 case BNXT_FW_RESET_STATE_OPENING: 11772 while (!rtnl_trylock()) { 11773 bnxt_queue_fw_reset_work(bp, HZ / 10); 11774 return; 11775 } 11776 rc = bnxt_open(bp->dev); 11777 if (rc) { 11778 netdev_err(bp->dev, "bnxt_open_nic() failed\n"); 11779 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 11780 dev_close(bp->dev); 11781 } 11782 11783 bp->fw_reset_state = 0; 11784 /* Make sure fw_reset_state is 0 before clearing the flag */ 11785 smp_mb__before_atomic(); 11786 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 11787 bnxt_ulp_start(bp, rc); 11788 if (!rc) 11789 bnxt_reenable_sriov(bp); 11790 bnxt_dl_health_recovery_done(bp); 11791 bnxt_dl_health_status_update(bp, true); 11792 rtnl_unlock(); 11793 break; 11794 } 11795 return; 11796 11797 fw_reset_abort_status: 11798 if (bp->fw_health->status_reliable || 11799 (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) { 11800 u32 sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG); 11801 11802 netdev_err(bp->dev, "fw_health_status 0x%x\n", sts); 11803 } 11804 fw_reset_abort: 11805 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 11806 if (bp->fw_reset_state != BNXT_FW_RESET_STATE_POLL_VF) 11807 bnxt_dl_health_status_update(bp, false); 11808 bp->fw_reset_state = 0; 11809 rtnl_lock(); 11810 dev_close(bp->dev); 11811 rtnl_unlock(); 11812 } 11813 11814 static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev) 11815 { 11816 int rc; 11817 struct bnxt *bp = netdev_priv(dev); 11818 11819 SET_NETDEV_DEV(dev, &pdev->dev); 11820 11821 /* enable device (incl. PCI PM wakeup), and bus-mastering */ 11822 rc = pci_enable_device(pdev); 11823 if (rc) { 11824 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n"); 11825 goto init_err; 11826 } 11827 11828 if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) { 11829 dev_err(&pdev->dev, 11830 "Cannot find PCI device base address, aborting\n"); 11831 rc = -ENODEV; 11832 goto init_err_disable; 11833 } 11834 11835 rc = pci_request_regions(pdev, DRV_MODULE_NAME); 11836 if (rc) { 11837 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n"); 11838 goto init_err_disable; 11839 } 11840 11841 if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 && 11842 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) { 11843 dev_err(&pdev->dev, "System does not support DMA, aborting\n"); 11844 rc = -EIO; 11845 goto init_err_release; 11846 } 11847 11848 pci_set_master(pdev); 11849 11850 bp->dev = dev; 11851 bp->pdev = pdev; 11852 11853 /* Doorbell BAR bp->bar1 is mapped after bnxt_fw_init_one_p2() 11854 * determines the BAR size. 11855 */ 11856 bp->bar0 = pci_ioremap_bar(pdev, 0); 11857 if (!bp->bar0) { 11858 dev_err(&pdev->dev, "Cannot map device registers, aborting\n"); 11859 rc = -ENOMEM; 11860 goto init_err_release; 11861 } 11862 11863 bp->bar2 = pci_ioremap_bar(pdev, 4); 11864 if (!bp->bar2) { 11865 dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n"); 11866 rc = -ENOMEM; 11867 goto init_err_release; 11868 } 11869 11870 pci_enable_pcie_error_reporting(pdev); 11871 11872 INIT_WORK(&bp->sp_task, bnxt_sp_task); 11873 INIT_DELAYED_WORK(&bp->fw_reset_task, bnxt_fw_reset_task); 11874 11875 spin_lock_init(&bp->ntp_fltr_lock); 11876 #if BITS_PER_LONG == 32 11877 spin_lock_init(&bp->db_lock); 11878 #endif 11879 11880 bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE; 11881 bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE; 11882 11883 bnxt_init_dflt_coal(bp); 11884 11885 timer_setup(&bp->timer, bnxt_timer, 0); 11886 bp->current_interval = BNXT_TIMER_INTERVAL; 11887 11888 bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID; 11889 bp->nge_fw_dst_port_id = INVALID_HW_RING_ID; 11890 11891 clear_bit(BNXT_STATE_OPEN, &bp->state); 11892 return 0; 11893 11894 init_err_release: 11895 bnxt_unmap_bars(bp, pdev); 11896 pci_release_regions(pdev); 11897 11898 init_err_disable: 11899 pci_disable_device(pdev); 11900 11901 init_err: 11902 return rc; 11903 } 11904 11905 /* rtnl_lock held */ 11906 static int bnxt_change_mac_addr(struct net_device *dev, void *p) 11907 { 11908 struct sockaddr *addr = p; 11909 struct bnxt *bp = netdev_priv(dev); 11910 int rc = 0; 11911 11912 if (!is_valid_ether_addr(addr->sa_data)) 11913 return -EADDRNOTAVAIL; 11914 11915 if (ether_addr_equal(addr->sa_data, dev->dev_addr)) 11916 return 0; 11917 11918 rc = bnxt_approve_mac(bp, addr->sa_data, true); 11919 if (rc) 11920 return rc; 11921 11922 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len); 11923 if (netif_running(dev)) { 11924 bnxt_close_nic(bp, false, false); 11925 rc = bnxt_open_nic(bp, false, false); 11926 } 11927 11928 return rc; 11929 } 11930 11931 /* rtnl_lock held */ 11932 static int bnxt_change_mtu(struct net_device *dev, int new_mtu) 11933 { 11934 struct bnxt *bp = netdev_priv(dev); 11935 11936 if (netif_running(dev)) 11937 bnxt_close_nic(bp, true, false); 11938 11939 dev->mtu = new_mtu; 11940 bnxt_set_ring_params(bp); 11941 11942 if (netif_running(dev)) 11943 return bnxt_open_nic(bp, true, false); 11944 11945 return 0; 11946 } 11947 11948 int bnxt_setup_mq_tc(struct net_device *dev, u8 tc) 11949 { 11950 struct bnxt *bp = netdev_priv(dev); 11951 bool sh = false; 11952 int rc; 11953 11954 if (tc > bp->max_tc) { 11955 netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n", 11956 tc, bp->max_tc); 11957 return -EINVAL; 11958 } 11959 11960 if (netdev_get_num_tc(dev) == tc) 11961 return 0; 11962 11963 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 11964 sh = true; 11965 11966 rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings, 11967 sh, tc, bp->tx_nr_rings_xdp); 11968 if (rc) 11969 return rc; 11970 11971 /* Needs to close the device and do hw resource re-allocations */ 11972 if (netif_running(bp->dev)) 11973 bnxt_close_nic(bp, true, false); 11974 11975 if (tc) { 11976 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc; 11977 netdev_set_num_tc(dev, tc); 11978 } else { 11979 bp->tx_nr_rings = bp->tx_nr_rings_per_tc; 11980 netdev_reset_tc(dev); 11981 } 11982 bp->tx_nr_rings += bp->tx_nr_rings_xdp; 11983 bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) : 11984 bp->tx_nr_rings + bp->rx_nr_rings; 11985 11986 if (netif_running(bp->dev)) 11987 return bnxt_open_nic(bp, true, false); 11988 11989 return 0; 11990 } 11991 11992 static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 11993 void *cb_priv) 11994 { 11995 struct bnxt *bp = cb_priv; 11996 11997 if (!bnxt_tc_flower_enabled(bp) || 11998 !tc_cls_can_offload_and_chain0(bp->dev, type_data)) 11999 return -EOPNOTSUPP; 12000 12001 switch (type) { 12002 case TC_SETUP_CLSFLOWER: 12003 return bnxt_tc_setup_flower(bp, bp->pf.fw_fid, type_data); 12004 default: 12005 return -EOPNOTSUPP; 12006 } 12007 } 12008 12009 LIST_HEAD(bnxt_block_cb_list); 12010 12011 static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type, 12012 void *type_data) 12013 { 12014 struct bnxt *bp = netdev_priv(dev); 12015 12016 switch (type) { 12017 case TC_SETUP_BLOCK: 12018 return flow_block_cb_setup_simple(type_data, 12019 &bnxt_block_cb_list, 12020 bnxt_setup_tc_block_cb, 12021 bp, bp, true); 12022 case TC_SETUP_QDISC_MQPRIO: { 12023 struct tc_mqprio_qopt *mqprio = type_data; 12024 12025 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 12026 12027 return bnxt_setup_mq_tc(dev, mqprio->num_tc); 12028 } 12029 default: 12030 return -EOPNOTSUPP; 12031 } 12032 } 12033 12034 #ifdef CONFIG_RFS_ACCEL 12035 static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1, 12036 struct bnxt_ntuple_filter *f2) 12037 { 12038 struct flow_keys *keys1 = &f1->fkeys; 12039 struct flow_keys *keys2 = &f2->fkeys; 12040 12041 if (keys1->basic.n_proto != keys2->basic.n_proto || 12042 keys1->basic.ip_proto != keys2->basic.ip_proto) 12043 return false; 12044 12045 if (keys1->basic.n_proto == htons(ETH_P_IP)) { 12046 if (keys1->addrs.v4addrs.src != keys2->addrs.v4addrs.src || 12047 keys1->addrs.v4addrs.dst != keys2->addrs.v4addrs.dst) 12048 return false; 12049 } else { 12050 if (memcmp(&keys1->addrs.v6addrs.src, &keys2->addrs.v6addrs.src, 12051 sizeof(keys1->addrs.v6addrs.src)) || 12052 memcmp(&keys1->addrs.v6addrs.dst, &keys2->addrs.v6addrs.dst, 12053 sizeof(keys1->addrs.v6addrs.dst))) 12054 return false; 12055 } 12056 12057 if (keys1->ports.ports == keys2->ports.ports && 12058 keys1->control.flags == keys2->control.flags && 12059 ether_addr_equal(f1->src_mac_addr, f2->src_mac_addr) && 12060 ether_addr_equal(f1->dst_mac_addr, f2->dst_mac_addr)) 12061 return true; 12062 12063 return false; 12064 } 12065 12066 static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb, 12067 u16 rxq_index, u32 flow_id) 12068 { 12069 struct bnxt *bp = netdev_priv(dev); 12070 struct bnxt_ntuple_filter *fltr, *new_fltr; 12071 struct flow_keys *fkeys; 12072 struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb); 12073 int rc = 0, idx, bit_id, l2_idx = 0; 12074 struct hlist_head *head; 12075 u32 flags; 12076 12077 if (!ether_addr_equal(dev->dev_addr, eth->h_dest)) { 12078 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 12079 int off = 0, j; 12080 12081 netif_addr_lock_bh(dev); 12082 for (j = 0; j < vnic->uc_filter_count; j++, off += ETH_ALEN) { 12083 if (ether_addr_equal(eth->h_dest, 12084 vnic->uc_list + off)) { 12085 l2_idx = j + 1; 12086 break; 12087 } 12088 } 12089 netif_addr_unlock_bh(dev); 12090 if (!l2_idx) 12091 return -EINVAL; 12092 } 12093 new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC); 12094 if (!new_fltr) 12095 return -ENOMEM; 12096 12097 fkeys = &new_fltr->fkeys; 12098 if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) { 12099 rc = -EPROTONOSUPPORT; 12100 goto err_free; 12101 } 12102 12103 if ((fkeys->basic.n_proto != htons(ETH_P_IP) && 12104 fkeys->basic.n_proto != htons(ETH_P_IPV6)) || 12105 ((fkeys->basic.ip_proto != IPPROTO_TCP) && 12106 (fkeys->basic.ip_proto != IPPROTO_UDP))) { 12107 rc = -EPROTONOSUPPORT; 12108 goto err_free; 12109 } 12110 if (fkeys->basic.n_proto == htons(ETH_P_IPV6) && 12111 bp->hwrm_spec_code < 0x10601) { 12112 rc = -EPROTONOSUPPORT; 12113 goto err_free; 12114 } 12115 flags = fkeys->control.flags; 12116 if (((flags & FLOW_DIS_ENCAPSULATION) && 12117 bp->hwrm_spec_code < 0x10601) || (flags & FLOW_DIS_IS_FRAGMENT)) { 12118 rc = -EPROTONOSUPPORT; 12119 goto err_free; 12120 } 12121 12122 memcpy(new_fltr->dst_mac_addr, eth->h_dest, ETH_ALEN); 12123 memcpy(new_fltr->src_mac_addr, eth->h_source, ETH_ALEN); 12124 12125 idx = skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK; 12126 head = &bp->ntp_fltr_hash_tbl[idx]; 12127 rcu_read_lock(); 12128 hlist_for_each_entry_rcu(fltr, head, hash) { 12129 if (bnxt_fltr_match(fltr, new_fltr)) { 12130 rcu_read_unlock(); 12131 rc = 0; 12132 goto err_free; 12133 } 12134 } 12135 rcu_read_unlock(); 12136 12137 spin_lock_bh(&bp->ntp_fltr_lock); 12138 bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap, 12139 BNXT_NTP_FLTR_MAX_FLTR, 0); 12140 if (bit_id < 0) { 12141 spin_unlock_bh(&bp->ntp_fltr_lock); 12142 rc = -ENOMEM; 12143 goto err_free; 12144 } 12145 12146 new_fltr->sw_id = (u16)bit_id; 12147 new_fltr->flow_id = flow_id; 12148 new_fltr->l2_fltr_idx = l2_idx; 12149 new_fltr->rxq = rxq_index; 12150 hlist_add_head_rcu(&new_fltr->hash, head); 12151 bp->ntp_fltr_count++; 12152 spin_unlock_bh(&bp->ntp_fltr_lock); 12153 12154 set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event); 12155 bnxt_queue_sp_work(bp); 12156 12157 return new_fltr->sw_id; 12158 12159 err_free: 12160 kfree(new_fltr); 12161 return rc; 12162 } 12163 12164 static void bnxt_cfg_ntp_filters(struct bnxt *bp) 12165 { 12166 int i; 12167 12168 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) { 12169 struct hlist_head *head; 12170 struct hlist_node *tmp; 12171 struct bnxt_ntuple_filter *fltr; 12172 int rc; 12173 12174 head = &bp->ntp_fltr_hash_tbl[i]; 12175 hlist_for_each_entry_safe(fltr, tmp, head, hash) { 12176 bool del = false; 12177 12178 if (test_bit(BNXT_FLTR_VALID, &fltr->state)) { 12179 if (rps_may_expire_flow(bp->dev, fltr->rxq, 12180 fltr->flow_id, 12181 fltr->sw_id)) { 12182 bnxt_hwrm_cfa_ntuple_filter_free(bp, 12183 fltr); 12184 del = true; 12185 } 12186 } else { 12187 rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp, 12188 fltr); 12189 if (rc) 12190 del = true; 12191 else 12192 set_bit(BNXT_FLTR_VALID, &fltr->state); 12193 } 12194 12195 if (del) { 12196 spin_lock_bh(&bp->ntp_fltr_lock); 12197 hlist_del_rcu(&fltr->hash); 12198 bp->ntp_fltr_count--; 12199 spin_unlock_bh(&bp->ntp_fltr_lock); 12200 synchronize_rcu(); 12201 clear_bit(fltr->sw_id, bp->ntp_fltr_bmap); 12202 kfree(fltr); 12203 } 12204 } 12205 } 12206 if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event)) 12207 netdev_info(bp->dev, "Receive PF driver unload event!\n"); 12208 } 12209 12210 #else 12211 12212 static void bnxt_cfg_ntp_filters(struct bnxt *bp) 12213 { 12214 } 12215 12216 #endif /* CONFIG_RFS_ACCEL */ 12217 12218 static int bnxt_udp_tunnel_sync(struct net_device *netdev, unsigned int table) 12219 { 12220 struct bnxt *bp = netdev_priv(netdev); 12221 struct udp_tunnel_info ti; 12222 unsigned int cmd; 12223 12224 udp_tunnel_nic_get_port(netdev, table, 0, &ti); 12225 if (ti.type == UDP_TUNNEL_TYPE_VXLAN) 12226 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN; 12227 else 12228 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE; 12229 12230 if (ti.port) 12231 return bnxt_hwrm_tunnel_dst_port_alloc(bp, ti.port, cmd); 12232 12233 return bnxt_hwrm_tunnel_dst_port_free(bp, cmd); 12234 } 12235 12236 static const struct udp_tunnel_nic_info bnxt_udp_tunnels = { 12237 .sync_table = bnxt_udp_tunnel_sync, 12238 .flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP | 12239 UDP_TUNNEL_NIC_INFO_OPEN_ONLY, 12240 .tables = { 12241 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN, }, 12242 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_GENEVE, }, 12243 }, 12244 }; 12245 12246 static int bnxt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 12247 struct net_device *dev, u32 filter_mask, 12248 int nlflags) 12249 { 12250 struct bnxt *bp = netdev_priv(dev); 12251 12252 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bp->br_mode, 0, 0, 12253 nlflags, filter_mask, NULL); 12254 } 12255 12256 static int bnxt_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh, 12257 u16 flags, struct netlink_ext_ack *extack) 12258 { 12259 struct bnxt *bp = netdev_priv(dev); 12260 struct nlattr *attr, *br_spec; 12261 int rem, rc = 0; 12262 12263 if (bp->hwrm_spec_code < 0x10708 || !BNXT_SINGLE_PF(bp)) 12264 return -EOPNOTSUPP; 12265 12266 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 12267 if (!br_spec) 12268 return -EINVAL; 12269 12270 nla_for_each_nested(attr, br_spec, rem) { 12271 u16 mode; 12272 12273 if (nla_type(attr) != IFLA_BRIDGE_MODE) 12274 continue; 12275 12276 if (nla_len(attr) < sizeof(mode)) 12277 return -EINVAL; 12278 12279 mode = nla_get_u16(attr); 12280 if (mode == bp->br_mode) 12281 break; 12282 12283 rc = bnxt_hwrm_set_br_mode(bp, mode); 12284 if (!rc) 12285 bp->br_mode = mode; 12286 break; 12287 } 12288 return rc; 12289 } 12290 12291 int bnxt_get_port_parent_id(struct net_device *dev, 12292 struct netdev_phys_item_id *ppid) 12293 { 12294 struct bnxt *bp = netdev_priv(dev); 12295 12296 if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV) 12297 return -EOPNOTSUPP; 12298 12299 /* The PF and it's VF-reps only support the switchdev framework */ 12300 if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_DSN_VALID)) 12301 return -EOPNOTSUPP; 12302 12303 ppid->id_len = sizeof(bp->dsn); 12304 memcpy(ppid->id, bp->dsn, ppid->id_len); 12305 12306 return 0; 12307 } 12308 12309 static struct devlink_port *bnxt_get_devlink_port(struct net_device *dev) 12310 { 12311 struct bnxt *bp = netdev_priv(dev); 12312 12313 return &bp->dl_port; 12314 } 12315 12316 static const struct net_device_ops bnxt_netdev_ops = { 12317 .ndo_open = bnxt_open, 12318 .ndo_start_xmit = bnxt_start_xmit, 12319 .ndo_stop = bnxt_close, 12320 .ndo_get_stats64 = bnxt_get_stats64, 12321 .ndo_set_rx_mode = bnxt_set_rx_mode, 12322 .ndo_do_ioctl = bnxt_ioctl, 12323 .ndo_validate_addr = eth_validate_addr, 12324 .ndo_set_mac_address = bnxt_change_mac_addr, 12325 .ndo_change_mtu = bnxt_change_mtu, 12326 .ndo_fix_features = bnxt_fix_features, 12327 .ndo_set_features = bnxt_set_features, 12328 .ndo_tx_timeout = bnxt_tx_timeout, 12329 #ifdef CONFIG_BNXT_SRIOV 12330 .ndo_get_vf_config = bnxt_get_vf_config, 12331 .ndo_set_vf_mac = bnxt_set_vf_mac, 12332 .ndo_set_vf_vlan = bnxt_set_vf_vlan, 12333 .ndo_set_vf_rate = bnxt_set_vf_bw, 12334 .ndo_set_vf_link_state = bnxt_set_vf_link_state, 12335 .ndo_set_vf_spoofchk = bnxt_set_vf_spoofchk, 12336 .ndo_set_vf_trust = bnxt_set_vf_trust, 12337 #endif 12338 .ndo_setup_tc = bnxt_setup_tc, 12339 #ifdef CONFIG_RFS_ACCEL 12340 .ndo_rx_flow_steer = bnxt_rx_flow_steer, 12341 #endif 12342 .ndo_bpf = bnxt_xdp, 12343 .ndo_xdp_xmit = bnxt_xdp_xmit, 12344 .ndo_bridge_getlink = bnxt_bridge_getlink, 12345 .ndo_bridge_setlink = bnxt_bridge_setlink, 12346 .ndo_get_devlink_port = bnxt_get_devlink_port, 12347 }; 12348 12349 static void bnxt_remove_one(struct pci_dev *pdev) 12350 { 12351 struct net_device *dev = pci_get_drvdata(pdev); 12352 struct bnxt *bp = netdev_priv(dev); 12353 12354 if (BNXT_PF(bp)) 12355 bnxt_sriov_disable(bp); 12356 12357 if (BNXT_PF(bp)) 12358 devlink_port_type_clear(&bp->dl_port); 12359 pci_disable_pcie_error_reporting(pdev); 12360 unregister_netdev(dev); 12361 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 12362 /* Flush any pending tasks */ 12363 cancel_work_sync(&bp->sp_task); 12364 cancel_delayed_work_sync(&bp->fw_reset_task); 12365 bp->sp_event = 0; 12366 12367 bnxt_dl_fw_reporters_destroy(bp, true); 12368 bnxt_dl_unregister(bp); 12369 bnxt_shutdown_tc(bp); 12370 12371 bnxt_clear_int_mode(bp); 12372 bnxt_hwrm_func_drv_unrgtr(bp); 12373 bnxt_free_hwrm_resources(bp); 12374 bnxt_free_hwrm_short_cmd_req(bp); 12375 bnxt_ethtool_free(bp); 12376 bnxt_dcb_free(bp); 12377 kfree(bp->edev); 12378 bp->edev = NULL; 12379 kfree(bp->fw_health); 12380 bp->fw_health = NULL; 12381 bnxt_cleanup_pci(bp); 12382 bnxt_free_ctx_mem(bp); 12383 kfree(bp->ctx); 12384 bp->ctx = NULL; 12385 kfree(bp->rss_indir_tbl); 12386 bp->rss_indir_tbl = NULL; 12387 bnxt_free_port_stats(bp); 12388 free_netdev(dev); 12389 } 12390 12391 static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt) 12392 { 12393 int rc = 0; 12394 struct bnxt_link_info *link_info = &bp->link_info; 12395 12396 rc = bnxt_hwrm_phy_qcaps(bp); 12397 if (rc) { 12398 netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n", 12399 rc); 12400 return rc; 12401 } 12402 if (!fw_dflt) 12403 return 0; 12404 12405 rc = bnxt_update_link(bp, false); 12406 if (rc) { 12407 netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n", 12408 rc); 12409 return rc; 12410 } 12411 12412 /* Older firmware does not have supported_auto_speeds, so assume 12413 * that all supported speeds can be autonegotiated. 12414 */ 12415 if (link_info->auto_link_speeds && !link_info->support_auto_speeds) 12416 link_info->support_auto_speeds = link_info->support_speeds; 12417 12418 bnxt_init_ethtool_link_settings(bp); 12419 return 0; 12420 } 12421 12422 static int bnxt_get_max_irq(struct pci_dev *pdev) 12423 { 12424 u16 ctrl; 12425 12426 if (!pdev->msix_cap) 12427 return 1; 12428 12429 pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl); 12430 return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1; 12431 } 12432 12433 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, 12434 int *max_cp) 12435 { 12436 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 12437 int max_ring_grps = 0, max_irq; 12438 12439 *max_tx = hw_resc->max_tx_rings; 12440 *max_rx = hw_resc->max_rx_rings; 12441 *max_cp = bnxt_get_max_func_cp_rings_for_en(bp); 12442 max_irq = min_t(int, bnxt_get_max_func_irqs(bp) - 12443 bnxt_get_ulp_msix_num(bp), 12444 hw_resc->max_stat_ctxs - bnxt_get_ulp_stat_ctxs(bp)); 12445 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 12446 *max_cp = min_t(int, *max_cp, max_irq); 12447 max_ring_grps = hw_resc->max_hw_ring_grps; 12448 if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) { 12449 *max_cp -= 1; 12450 *max_rx -= 2; 12451 } 12452 if (bp->flags & BNXT_FLAG_AGG_RINGS) 12453 *max_rx >>= 1; 12454 if (bp->flags & BNXT_FLAG_CHIP_P5) { 12455 bnxt_trim_rings(bp, max_rx, max_tx, *max_cp, false); 12456 /* On P5 chips, max_cp output param should be available NQs */ 12457 *max_cp = max_irq; 12458 } 12459 *max_rx = min_t(int, *max_rx, max_ring_grps); 12460 } 12461 12462 int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared) 12463 { 12464 int rx, tx, cp; 12465 12466 _bnxt_get_max_rings(bp, &rx, &tx, &cp); 12467 *max_rx = rx; 12468 *max_tx = tx; 12469 if (!rx || !tx || !cp) 12470 return -ENOMEM; 12471 12472 return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared); 12473 } 12474 12475 static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx, 12476 bool shared) 12477 { 12478 int rc; 12479 12480 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared); 12481 if (rc && (bp->flags & BNXT_FLAG_AGG_RINGS)) { 12482 /* Not enough rings, try disabling agg rings. */ 12483 bp->flags &= ~BNXT_FLAG_AGG_RINGS; 12484 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared); 12485 if (rc) { 12486 /* set BNXT_FLAG_AGG_RINGS back for consistency */ 12487 bp->flags |= BNXT_FLAG_AGG_RINGS; 12488 return rc; 12489 } 12490 bp->flags |= BNXT_FLAG_NO_AGG_RINGS; 12491 bp->dev->hw_features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); 12492 bp->dev->features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); 12493 bnxt_set_ring_params(bp); 12494 } 12495 12496 if (bp->flags & BNXT_FLAG_ROCE_CAP) { 12497 int max_cp, max_stat, max_irq; 12498 12499 /* Reserve minimum resources for RoCE */ 12500 max_cp = bnxt_get_max_func_cp_rings(bp); 12501 max_stat = bnxt_get_max_func_stat_ctxs(bp); 12502 max_irq = bnxt_get_max_func_irqs(bp); 12503 if (max_cp <= BNXT_MIN_ROCE_CP_RINGS || 12504 max_irq <= BNXT_MIN_ROCE_CP_RINGS || 12505 max_stat <= BNXT_MIN_ROCE_STAT_CTXS) 12506 return 0; 12507 12508 max_cp -= BNXT_MIN_ROCE_CP_RINGS; 12509 max_irq -= BNXT_MIN_ROCE_CP_RINGS; 12510 max_stat -= BNXT_MIN_ROCE_STAT_CTXS; 12511 max_cp = min_t(int, max_cp, max_irq); 12512 max_cp = min_t(int, max_cp, max_stat); 12513 rc = bnxt_trim_rings(bp, max_rx, max_tx, max_cp, shared); 12514 if (rc) 12515 rc = 0; 12516 } 12517 return rc; 12518 } 12519 12520 /* In initial default shared ring setting, each shared ring must have a 12521 * RX/TX ring pair. 12522 */ 12523 static void bnxt_trim_dflt_sh_rings(struct bnxt *bp) 12524 { 12525 bp->cp_nr_rings = min_t(int, bp->tx_nr_rings_per_tc, bp->rx_nr_rings); 12526 bp->rx_nr_rings = bp->cp_nr_rings; 12527 bp->tx_nr_rings_per_tc = bp->cp_nr_rings; 12528 bp->tx_nr_rings = bp->tx_nr_rings_per_tc; 12529 } 12530 12531 static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh) 12532 { 12533 int dflt_rings, max_rx_rings, max_tx_rings, rc; 12534 12535 if (!bnxt_can_reserve_rings(bp)) 12536 return 0; 12537 12538 if (sh) 12539 bp->flags |= BNXT_FLAG_SHARED_RINGS; 12540 dflt_rings = is_kdump_kernel() ? 1 : netif_get_num_default_rss_queues(); 12541 /* Reduce default rings on multi-port cards so that total default 12542 * rings do not exceed CPU count. 12543 */ 12544 if (bp->port_count > 1) { 12545 int max_rings = 12546 max_t(int, num_online_cpus() / bp->port_count, 1); 12547 12548 dflt_rings = min_t(int, dflt_rings, max_rings); 12549 } 12550 rc = bnxt_get_dflt_rings(bp, &max_rx_rings, &max_tx_rings, sh); 12551 if (rc) 12552 return rc; 12553 bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings); 12554 bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings); 12555 if (sh) 12556 bnxt_trim_dflt_sh_rings(bp); 12557 else 12558 bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings; 12559 bp->tx_nr_rings = bp->tx_nr_rings_per_tc; 12560 12561 rc = __bnxt_reserve_rings(bp); 12562 if (rc) 12563 netdev_warn(bp->dev, "Unable to reserve tx rings\n"); 12564 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 12565 if (sh) 12566 bnxt_trim_dflt_sh_rings(bp); 12567 12568 /* Rings may have been trimmed, re-reserve the trimmed rings. */ 12569 if (bnxt_need_reserve_rings(bp)) { 12570 rc = __bnxt_reserve_rings(bp); 12571 if (rc) 12572 netdev_warn(bp->dev, "2nd rings reservation failed.\n"); 12573 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 12574 } 12575 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 12576 bp->rx_nr_rings++; 12577 bp->cp_nr_rings++; 12578 } 12579 if (rc) { 12580 bp->tx_nr_rings = 0; 12581 bp->rx_nr_rings = 0; 12582 } 12583 return rc; 12584 } 12585 12586 static int bnxt_init_dflt_ring_mode(struct bnxt *bp) 12587 { 12588 int rc; 12589 12590 if (bp->tx_nr_rings) 12591 return 0; 12592 12593 bnxt_ulp_irq_stop(bp); 12594 bnxt_clear_int_mode(bp); 12595 rc = bnxt_set_dflt_rings(bp, true); 12596 if (rc) { 12597 netdev_err(bp->dev, "Not enough rings available.\n"); 12598 goto init_dflt_ring_err; 12599 } 12600 rc = bnxt_init_int_mode(bp); 12601 if (rc) 12602 goto init_dflt_ring_err; 12603 12604 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 12605 if (bnxt_rfs_supported(bp) && bnxt_rfs_capable(bp)) { 12606 bp->flags |= BNXT_FLAG_RFS; 12607 bp->dev->features |= NETIF_F_NTUPLE; 12608 } 12609 init_dflt_ring_err: 12610 bnxt_ulp_irq_restart(bp, rc); 12611 return rc; 12612 } 12613 12614 int bnxt_restore_pf_fw_resources(struct bnxt *bp) 12615 { 12616 int rc; 12617 12618 ASSERT_RTNL(); 12619 bnxt_hwrm_func_qcaps(bp); 12620 12621 if (netif_running(bp->dev)) 12622 __bnxt_close_nic(bp, true, false); 12623 12624 bnxt_ulp_irq_stop(bp); 12625 bnxt_clear_int_mode(bp); 12626 rc = bnxt_init_int_mode(bp); 12627 bnxt_ulp_irq_restart(bp, rc); 12628 12629 if (netif_running(bp->dev)) { 12630 if (rc) 12631 dev_close(bp->dev); 12632 else 12633 rc = bnxt_open_nic(bp, true, false); 12634 } 12635 12636 return rc; 12637 } 12638 12639 static int bnxt_init_mac_addr(struct bnxt *bp) 12640 { 12641 int rc = 0; 12642 12643 if (BNXT_PF(bp)) { 12644 memcpy(bp->dev->dev_addr, bp->pf.mac_addr, ETH_ALEN); 12645 } else { 12646 #ifdef CONFIG_BNXT_SRIOV 12647 struct bnxt_vf_info *vf = &bp->vf; 12648 bool strict_approval = true; 12649 12650 if (is_valid_ether_addr(vf->mac_addr)) { 12651 /* overwrite netdev dev_addr with admin VF MAC */ 12652 memcpy(bp->dev->dev_addr, vf->mac_addr, ETH_ALEN); 12653 /* Older PF driver or firmware may not approve this 12654 * correctly. 12655 */ 12656 strict_approval = false; 12657 } else { 12658 eth_hw_addr_random(bp->dev); 12659 } 12660 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, strict_approval); 12661 #endif 12662 } 12663 return rc; 12664 } 12665 12666 #define BNXT_VPD_LEN 512 12667 static void bnxt_vpd_read_info(struct bnxt *bp) 12668 { 12669 struct pci_dev *pdev = bp->pdev; 12670 int i, len, pos, ro_size, size; 12671 ssize_t vpd_size; 12672 u8 *vpd_data; 12673 12674 vpd_data = kmalloc(BNXT_VPD_LEN, GFP_KERNEL); 12675 if (!vpd_data) 12676 return; 12677 12678 vpd_size = pci_read_vpd(pdev, 0, BNXT_VPD_LEN, vpd_data); 12679 if (vpd_size <= 0) { 12680 netdev_err(bp->dev, "Unable to read VPD\n"); 12681 goto exit; 12682 } 12683 12684 i = pci_vpd_find_tag(vpd_data, 0, vpd_size, PCI_VPD_LRDT_RO_DATA); 12685 if (i < 0) { 12686 netdev_err(bp->dev, "VPD READ-Only not found\n"); 12687 goto exit; 12688 } 12689 12690 ro_size = pci_vpd_lrdt_size(&vpd_data[i]); 12691 i += PCI_VPD_LRDT_TAG_SIZE; 12692 if (i + ro_size > vpd_size) 12693 goto exit; 12694 12695 pos = pci_vpd_find_info_keyword(vpd_data, i, ro_size, 12696 PCI_VPD_RO_KEYWORD_PARTNO); 12697 if (pos < 0) 12698 goto read_sn; 12699 12700 len = pci_vpd_info_field_size(&vpd_data[pos]); 12701 pos += PCI_VPD_INFO_FLD_HDR_SIZE; 12702 if (len + pos > vpd_size) 12703 goto read_sn; 12704 12705 size = min(len, BNXT_VPD_FLD_LEN - 1); 12706 memcpy(bp->board_partno, &vpd_data[pos], size); 12707 12708 read_sn: 12709 pos = pci_vpd_find_info_keyword(vpd_data, i, ro_size, 12710 PCI_VPD_RO_KEYWORD_SERIALNO); 12711 if (pos < 0) 12712 goto exit; 12713 12714 len = pci_vpd_info_field_size(&vpd_data[pos]); 12715 pos += PCI_VPD_INFO_FLD_HDR_SIZE; 12716 if (len + pos > vpd_size) 12717 goto exit; 12718 12719 size = min(len, BNXT_VPD_FLD_LEN - 1); 12720 memcpy(bp->board_serialno, &vpd_data[pos], size); 12721 exit: 12722 kfree(vpd_data); 12723 } 12724 12725 static int bnxt_pcie_dsn_get(struct bnxt *bp, u8 dsn[]) 12726 { 12727 struct pci_dev *pdev = bp->pdev; 12728 u64 qword; 12729 12730 qword = pci_get_dsn(pdev); 12731 if (!qword) { 12732 netdev_info(bp->dev, "Unable to read adapter's DSN\n"); 12733 return -EOPNOTSUPP; 12734 } 12735 12736 put_unaligned_le64(qword, dsn); 12737 12738 bp->flags |= BNXT_FLAG_DSN_VALID; 12739 return 0; 12740 } 12741 12742 static int bnxt_map_db_bar(struct bnxt *bp) 12743 { 12744 if (!bp->db_size) 12745 return -ENODEV; 12746 bp->bar1 = pci_iomap(bp->pdev, 2, bp->db_size); 12747 if (!bp->bar1) 12748 return -ENOMEM; 12749 return 0; 12750 } 12751 12752 static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) 12753 { 12754 struct net_device *dev; 12755 struct bnxt *bp; 12756 int rc, max_irqs; 12757 12758 if (pci_is_bridge(pdev)) 12759 return -ENODEV; 12760 12761 /* Clear any pending DMA transactions from crash kernel 12762 * while loading driver in capture kernel. 12763 */ 12764 if (is_kdump_kernel()) { 12765 pci_clear_master(pdev); 12766 pcie_flr(pdev); 12767 } 12768 12769 max_irqs = bnxt_get_max_irq(pdev); 12770 dev = alloc_etherdev_mq(sizeof(*bp), max_irqs); 12771 if (!dev) 12772 return -ENOMEM; 12773 12774 bp = netdev_priv(dev); 12775 bp->msg_enable = BNXT_DEF_MSG_ENABLE; 12776 bnxt_set_max_func_irqs(bp, max_irqs); 12777 12778 if (bnxt_vf_pciid(ent->driver_data)) 12779 bp->flags |= BNXT_FLAG_VF; 12780 12781 if (pdev->msix_cap) 12782 bp->flags |= BNXT_FLAG_MSIX_CAP; 12783 12784 rc = bnxt_init_board(pdev, dev); 12785 if (rc < 0) 12786 goto init_err_free; 12787 12788 dev->netdev_ops = &bnxt_netdev_ops; 12789 dev->watchdog_timeo = BNXT_TX_TIMEOUT; 12790 dev->ethtool_ops = &bnxt_ethtool_ops; 12791 pci_set_drvdata(pdev, dev); 12792 12793 rc = bnxt_alloc_hwrm_resources(bp); 12794 if (rc) 12795 goto init_err_pci_clean; 12796 12797 mutex_init(&bp->hwrm_cmd_lock); 12798 mutex_init(&bp->link_lock); 12799 12800 rc = bnxt_fw_init_one_p1(bp); 12801 if (rc) 12802 goto init_err_pci_clean; 12803 12804 if (BNXT_PF(bp)) 12805 bnxt_vpd_read_info(bp); 12806 12807 if (BNXT_CHIP_P5(bp)) { 12808 bp->flags |= BNXT_FLAG_CHIP_P5; 12809 if (BNXT_CHIP_SR2(bp)) 12810 bp->flags |= BNXT_FLAG_CHIP_SR2; 12811 } 12812 12813 rc = bnxt_alloc_rss_indir_tbl(bp); 12814 if (rc) 12815 goto init_err_pci_clean; 12816 12817 rc = bnxt_fw_init_one_p2(bp); 12818 if (rc) 12819 goto init_err_pci_clean; 12820 12821 rc = bnxt_map_db_bar(bp); 12822 if (rc) { 12823 dev_err(&pdev->dev, "Cannot map doorbell BAR rc = %d, aborting\n", 12824 rc); 12825 goto init_err_pci_clean; 12826 } 12827 12828 dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG | 12829 NETIF_F_TSO | NETIF_F_TSO6 | 12830 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE | 12831 NETIF_F_GSO_IPXIP4 | 12832 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM | 12833 NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH | 12834 NETIF_F_RXCSUM | NETIF_F_GRO; 12835 12836 if (BNXT_SUPPORTS_TPA(bp)) 12837 dev->hw_features |= NETIF_F_LRO; 12838 12839 dev->hw_enc_features = 12840 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG | 12841 NETIF_F_TSO | NETIF_F_TSO6 | 12842 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE | 12843 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM | 12844 NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL; 12845 dev->udp_tunnel_nic_info = &bnxt_udp_tunnels; 12846 12847 dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM | 12848 NETIF_F_GSO_GRE_CSUM; 12849 dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA; 12850 if (bp->fw_cap & BNXT_FW_CAP_VLAN_RX_STRIP) 12851 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_RX; 12852 if (bp->fw_cap & BNXT_FW_CAP_VLAN_TX_INSERT) 12853 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_TX; 12854 if (BNXT_SUPPORTS_TPA(bp)) 12855 dev->hw_features |= NETIF_F_GRO_HW; 12856 dev->features |= dev->hw_features | NETIF_F_HIGHDMA; 12857 if (dev->features & NETIF_F_GRO_HW) 12858 dev->features &= ~NETIF_F_LRO; 12859 dev->priv_flags |= IFF_UNICAST_FLT; 12860 12861 #ifdef CONFIG_BNXT_SRIOV 12862 init_waitqueue_head(&bp->sriov_cfg_wait); 12863 mutex_init(&bp->sriov_lock); 12864 #endif 12865 if (BNXT_SUPPORTS_TPA(bp)) { 12866 bp->gro_func = bnxt_gro_func_5730x; 12867 if (BNXT_CHIP_P4(bp)) 12868 bp->gro_func = bnxt_gro_func_5731x; 12869 else if (BNXT_CHIP_P5(bp)) 12870 bp->gro_func = bnxt_gro_func_5750x; 12871 } 12872 if (!BNXT_CHIP_P4_PLUS(bp)) 12873 bp->flags |= BNXT_FLAG_DOUBLE_DB; 12874 12875 bp->ulp_probe = bnxt_ulp_probe; 12876 12877 rc = bnxt_init_mac_addr(bp); 12878 if (rc) { 12879 dev_err(&pdev->dev, "Unable to initialize mac address.\n"); 12880 rc = -EADDRNOTAVAIL; 12881 goto init_err_pci_clean; 12882 } 12883 12884 if (BNXT_PF(bp)) { 12885 /* Read the adapter's DSN to use as the eswitch switch_id */ 12886 rc = bnxt_pcie_dsn_get(bp, bp->dsn); 12887 } 12888 12889 /* MTU range: 60 - FW defined max */ 12890 dev->min_mtu = ETH_ZLEN; 12891 dev->max_mtu = bp->max_mtu; 12892 12893 rc = bnxt_probe_phy(bp, true); 12894 if (rc) 12895 goto init_err_pci_clean; 12896 12897 bnxt_set_rx_skb_mode(bp, false); 12898 bnxt_set_tpa_flags(bp); 12899 bnxt_set_ring_params(bp); 12900 rc = bnxt_set_dflt_rings(bp, true); 12901 if (rc) { 12902 netdev_err(bp->dev, "Not enough rings available.\n"); 12903 rc = -ENOMEM; 12904 goto init_err_pci_clean; 12905 } 12906 12907 bnxt_fw_init_one_p3(bp); 12908 12909 if (dev->hw_features & BNXT_HW_FEATURE_VLAN_ALL_RX) 12910 bp->flags |= BNXT_FLAG_STRIP_VLAN; 12911 12912 rc = bnxt_init_int_mode(bp); 12913 if (rc) 12914 goto init_err_pci_clean; 12915 12916 /* No TC has been set yet and rings may have been trimmed due to 12917 * limited MSIX, so we re-initialize the TX rings per TC. 12918 */ 12919 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 12920 12921 if (BNXT_PF(bp)) { 12922 if (!bnxt_pf_wq) { 12923 bnxt_pf_wq = 12924 create_singlethread_workqueue("bnxt_pf_wq"); 12925 if (!bnxt_pf_wq) { 12926 dev_err(&pdev->dev, "Unable to create workqueue.\n"); 12927 rc = -ENOMEM; 12928 goto init_err_pci_clean; 12929 } 12930 } 12931 rc = bnxt_init_tc(bp); 12932 if (rc) 12933 netdev_err(dev, "Failed to initialize TC flower offload, err = %d.\n", 12934 rc); 12935 } 12936 12937 bnxt_dl_register(bp); 12938 12939 rc = register_netdev(dev); 12940 if (rc) 12941 goto init_err_cleanup; 12942 12943 if (BNXT_PF(bp)) 12944 devlink_port_type_eth_set(&bp->dl_port, bp->dev); 12945 bnxt_dl_fw_reporters_create(bp); 12946 12947 netdev_info(dev, "%s found at mem %lx, node addr %pM\n", 12948 board_info[ent->driver_data].name, 12949 (long)pci_resource_start(pdev, 0), dev->dev_addr); 12950 pcie_print_link_status(pdev); 12951 12952 pci_save_state(pdev); 12953 return 0; 12954 12955 init_err_cleanup: 12956 bnxt_dl_unregister(bp); 12957 bnxt_shutdown_tc(bp); 12958 bnxt_clear_int_mode(bp); 12959 12960 init_err_pci_clean: 12961 bnxt_hwrm_func_drv_unrgtr(bp); 12962 bnxt_free_hwrm_short_cmd_req(bp); 12963 bnxt_free_hwrm_resources(bp); 12964 kfree(bp->fw_health); 12965 bp->fw_health = NULL; 12966 bnxt_cleanup_pci(bp); 12967 bnxt_free_ctx_mem(bp); 12968 kfree(bp->ctx); 12969 bp->ctx = NULL; 12970 kfree(bp->rss_indir_tbl); 12971 bp->rss_indir_tbl = NULL; 12972 12973 init_err_free: 12974 free_netdev(dev); 12975 return rc; 12976 } 12977 12978 static void bnxt_shutdown(struct pci_dev *pdev) 12979 { 12980 struct net_device *dev = pci_get_drvdata(pdev); 12981 struct bnxt *bp; 12982 12983 if (!dev) 12984 return; 12985 12986 rtnl_lock(); 12987 bp = netdev_priv(dev); 12988 if (!bp) 12989 goto shutdown_exit; 12990 12991 if (netif_running(dev)) 12992 dev_close(dev); 12993 12994 bnxt_ulp_shutdown(bp); 12995 bnxt_clear_int_mode(bp); 12996 pci_disable_device(pdev); 12997 12998 if (system_state == SYSTEM_POWER_OFF) { 12999 pci_wake_from_d3(pdev, bp->wol); 13000 pci_set_power_state(pdev, PCI_D3hot); 13001 } 13002 13003 shutdown_exit: 13004 rtnl_unlock(); 13005 } 13006 13007 #ifdef CONFIG_PM_SLEEP 13008 static int bnxt_suspend(struct device *device) 13009 { 13010 struct net_device *dev = dev_get_drvdata(device); 13011 struct bnxt *bp = netdev_priv(dev); 13012 int rc = 0; 13013 13014 rtnl_lock(); 13015 bnxt_ulp_stop(bp); 13016 if (netif_running(dev)) { 13017 netif_device_detach(dev); 13018 rc = bnxt_close(dev); 13019 } 13020 bnxt_hwrm_func_drv_unrgtr(bp); 13021 pci_disable_device(bp->pdev); 13022 bnxt_free_ctx_mem(bp); 13023 kfree(bp->ctx); 13024 bp->ctx = NULL; 13025 rtnl_unlock(); 13026 return rc; 13027 } 13028 13029 static int bnxt_resume(struct device *device) 13030 { 13031 struct net_device *dev = dev_get_drvdata(device); 13032 struct bnxt *bp = netdev_priv(dev); 13033 int rc = 0; 13034 13035 rtnl_lock(); 13036 rc = pci_enable_device(bp->pdev); 13037 if (rc) { 13038 netdev_err(dev, "Cannot re-enable PCI device during resume, err = %d\n", 13039 rc); 13040 goto resume_exit; 13041 } 13042 pci_set_master(bp->pdev); 13043 if (bnxt_hwrm_ver_get(bp)) { 13044 rc = -ENODEV; 13045 goto resume_exit; 13046 } 13047 rc = bnxt_hwrm_func_reset(bp); 13048 if (rc) { 13049 rc = -EBUSY; 13050 goto resume_exit; 13051 } 13052 13053 rc = bnxt_hwrm_func_qcaps(bp); 13054 if (rc) 13055 goto resume_exit; 13056 13057 if (bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false)) { 13058 rc = -ENODEV; 13059 goto resume_exit; 13060 } 13061 13062 bnxt_get_wol_settings(bp); 13063 if (netif_running(dev)) { 13064 rc = bnxt_open(dev); 13065 if (!rc) 13066 netif_device_attach(dev); 13067 } 13068 13069 resume_exit: 13070 bnxt_ulp_start(bp, rc); 13071 if (!rc) 13072 bnxt_reenable_sriov(bp); 13073 rtnl_unlock(); 13074 return rc; 13075 } 13076 13077 static SIMPLE_DEV_PM_OPS(bnxt_pm_ops, bnxt_suspend, bnxt_resume); 13078 #define BNXT_PM_OPS (&bnxt_pm_ops) 13079 13080 #else 13081 13082 #define BNXT_PM_OPS NULL 13083 13084 #endif /* CONFIG_PM_SLEEP */ 13085 13086 /** 13087 * bnxt_io_error_detected - called when PCI error is detected 13088 * @pdev: Pointer to PCI device 13089 * @state: The current pci connection state 13090 * 13091 * This function is called after a PCI bus error affecting 13092 * this device has been detected. 13093 */ 13094 static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev, 13095 pci_channel_state_t state) 13096 { 13097 struct net_device *netdev = pci_get_drvdata(pdev); 13098 struct bnxt *bp = netdev_priv(netdev); 13099 13100 netdev_info(netdev, "PCI I/O error detected\n"); 13101 13102 rtnl_lock(); 13103 netif_device_detach(netdev); 13104 13105 bnxt_ulp_stop(bp); 13106 13107 if (state == pci_channel_io_perm_failure) { 13108 rtnl_unlock(); 13109 return PCI_ERS_RESULT_DISCONNECT; 13110 } 13111 13112 if (state == pci_channel_io_frozen) 13113 set_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, &bp->state); 13114 13115 if (netif_running(netdev)) 13116 bnxt_close(netdev); 13117 13118 pci_disable_device(pdev); 13119 bnxt_free_ctx_mem(bp); 13120 kfree(bp->ctx); 13121 bp->ctx = NULL; 13122 rtnl_unlock(); 13123 13124 /* Request a slot slot reset. */ 13125 return PCI_ERS_RESULT_NEED_RESET; 13126 } 13127 13128 /** 13129 * bnxt_io_slot_reset - called after the pci bus has been reset. 13130 * @pdev: Pointer to PCI device 13131 * 13132 * Restart the card from scratch, as if from a cold-boot. 13133 * At this point, the card has exprienced a hard reset, 13134 * followed by fixups by BIOS, and has its config space 13135 * set up identically to what it was at cold boot. 13136 */ 13137 static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev) 13138 { 13139 pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT; 13140 struct net_device *netdev = pci_get_drvdata(pdev); 13141 struct bnxt *bp = netdev_priv(netdev); 13142 int err = 0, off; 13143 13144 netdev_info(bp->dev, "PCI Slot Reset\n"); 13145 13146 rtnl_lock(); 13147 13148 if (pci_enable_device(pdev)) { 13149 dev_err(&pdev->dev, 13150 "Cannot re-enable PCI device after reset.\n"); 13151 } else { 13152 pci_set_master(pdev); 13153 /* Upon fatal error, our device internal logic that latches to 13154 * BAR value is getting reset and will restore only upon 13155 * rewritting the BARs. 13156 * 13157 * As pci_restore_state() does not re-write the BARs if the 13158 * value is same as saved value earlier, driver needs to 13159 * write the BARs to 0 to force restore, in case of fatal error. 13160 */ 13161 if (test_and_clear_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, 13162 &bp->state)) { 13163 for (off = PCI_BASE_ADDRESS_0; 13164 off <= PCI_BASE_ADDRESS_5; off += 4) 13165 pci_write_config_dword(bp->pdev, off, 0); 13166 } 13167 pci_restore_state(pdev); 13168 pci_save_state(pdev); 13169 13170 err = bnxt_hwrm_func_reset(bp); 13171 if (!err) 13172 result = PCI_ERS_RESULT_RECOVERED; 13173 } 13174 13175 rtnl_unlock(); 13176 13177 return result; 13178 } 13179 13180 /** 13181 * bnxt_io_resume - called when traffic can start flowing again. 13182 * @pdev: Pointer to PCI device 13183 * 13184 * This callback is called when the error recovery driver tells 13185 * us that its OK to resume normal operation. 13186 */ 13187 static void bnxt_io_resume(struct pci_dev *pdev) 13188 { 13189 struct net_device *netdev = pci_get_drvdata(pdev); 13190 struct bnxt *bp = netdev_priv(netdev); 13191 int err; 13192 13193 netdev_info(bp->dev, "PCI Slot Resume\n"); 13194 rtnl_lock(); 13195 13196 err = bnxt_hwrm_func_qcaps(bp); 13197 if (!err && netif_running(netdev)) 13198 err = bnxt_open(netdev); 13199 13200 bnxt_ulp_start(bp, err); 13201 if (!err) { 13202 bnxt_reenable_sriov(bp); 13203 netif_device_attach(netdev); 13204 } 13205 13206 rtnl_unlock(); 13207 } 13208 13209 static const struct pci_error_handlers bnxt_err_handler = { 13210 .error_detected = bnxt_io_error_detected, 13211 .slot_reset = bnxt_io_slot_reset, 13212 .resume = bnxt_io_resume 13213 }; 13214 13215 static struct pci_driver bnxt_pci_driver = { 13216 .name = DRV_MODULE_NAME, 13217 .id_table = bnxt_pci_tbl, 13218 .probe = bnxt_init_one, 13219 .remove = bnxt_remove_one, 13220 .shutdown = bnxt_shutdown, 13221 .driver.pm = BNXT_PM_OPS, 13222 .err_handler = &bnxt_err_handler, 13223 #if defined(CONFIG_BNXT_SRIOV) 13224 .sriov_configure = bnxt_sriov_configure, 13225 #endif 13226 }; 13227 13228 static int __init bnxt_init(void) 13229 { 13230 bnxt_debug_init(); 13231 return pci_register_driver(&bnxt_pci_driver); 13232 } 13233 13234 static void __exit bnxt_exit(void) 13235 { 13236 pci_unregister_driver(&bnxt_pci_driver); 13237 if (bnxt_pf_wq) 13238 destroy_workqueue(bnxt_pf_wq); 13239 bnxt_debug_exit(); 13240 } 13241 13242 module_init(bnxt_init); 13243 module_exit(bnxt_exit); 13244