1 /* Broadcom NetXtreme-C/E network driver. 2 * 3 * Copyright (c) 2014-2016 Broadcom Corporation 4 * Copyright (c) 2016-2019 Broadcom Limited 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License as published by 8 * the Free Software Foundation. 9 */ 10 11 #include <linux/module.h> 12 13 #include <linux/stringify.h> 14 #include <linux/kernel.h> 15 #include <linux/timer.h> 16 #include <linux/errno.h> 17 #include <linux/ioport.h> 18 #include <linux/slab.h> 19 #include <linux/vmalloc.h> 20 #include <linux/interrupt.h> 21 #include <linux/pci.h> 22 #include <linux/netdevice.h> 23 #include <linux/etherdevice.h> 24 #include <linux/skbuff.h> 25 #include <linux/dma-mapping.h> 26 #include <linux/bitops.h> 27 #include <linux/io.h> 28 #include <linux/irq.h> 29 #include <linux/delay.h> 30 #include <asm/byteorder.h> 31 #include <asm/page.h> 32 #include <linux/time.h> 33 #include <linux/mii.h> 34 #include <linux/mdio.h> 35 #include <linux/if.h> 36 #include <linux/if_vlan.h> 37 #include <linux/if_bridge.h> 38 #include <linux/rtc.h> 39 #include <linux/bpf.h> 40 #include <net/ip.h> 41 #include <net/tcp.h> 42 #include <net/udp.h> 43 #include <net/checksum.h> 44 #include <net/ip6_checksum.h> 45 #include <net/udp_tunnel.h> 46 #include <linux/workqueue.h> 47 #include <linux/prefetch.h> 48 #include <linux/cache.h> 49 #include <linux/log2.h> 50 #include <linux/aer.h> 51 #include <linux/bitmap.h> 52 #include <linux/cpu_rmap.h> 53 #include <linux/cpumask.h> 54 #include <net/pkt_cls.h> 55 #include <linux/hwmon.h> 56 #include <linux/hwmon-sysfs.h> 57 #include <net/page_pool.h> 58 59 #include "bnxt_hsi.h" 60 #include "bnxt.h" 61 #include "bnxt_ulp.h" 62 #include "bnxt_sriov.h" 63 #include "bnxt_ethtool.h" 64 #include "bnxt_dcb.h" 65 #include "bnxt_xdp.h" 66 #include "bnxt_vfr.h" 67 #include "bnxt_tc.h" 68 #include "bnxt_devlink.h" 69 #include "bnxt_debugfs.h" 70 71 #define BNXT_TX_TIMEOUT (5 * HZ) 72 #define BNXT_DEF_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_HW) 73 74 MODULE_LICENSE("GPL"); 75 MODULE_DESCRIPTION("Broadcom BCM573xx network driver"); 76 77 #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN) 78 #define BNXT_RX_DMA_OFFSET NET_SKB_PAD 79 #define BNXT_RX_COPY_THRESH 256 80 81 #define BNXT_TX_PUSH_THRESH 164 82 83 enum board_idx { 84 BCM57301, 85 BCM57302, 86 BCM57304, 87 BCM57417_NPAR, 88 BCM58700, 89 BCM57311, 90 BCM57312, 91 BCM57402, 92 BCM57404, 93 BCM57406, 94 BCM57402_NPAR, 95 BCM57407, 96 BCM57412, 97 BCM57414, 98 BCM57416, 99 BCM57417, 100 BCM57412_NPAR, 101 BCM57314, 102 BCM57417_SFP, 103 BCM57416_SFP, 104 BCM57404_NPAR, 105 BCM57406_NPAR, 106 BCM57407_SFP, 107 BCM57407_NPAR, 108 BCM57414_NPAR, 109 BCM57416_NPAR, 110 BCM57452, 111 BCM57454, 112 BCM5745x_NPAR, 113 BCM57508, 114 BCM57504, 115 BCM57502, 116 BCM57508_NPAR, 117 BCM57504_NPAR, 118 BCM57502_NPAR, 119 BCM58802, 120 BCM58804, 121 BCM58808, 122 NETXTREME_E_VF, 123 NETXTREME_C_VF, 124 NETXTREME_S_VF, 125 NETXTREME_E_P5_VF, 126 }; 127 128 /* indexed by enum above */ 129 static const struct { 130 char *name; 131 } board_info[] = { 132 [BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" }, 133 [BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" }, 134 [BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" }, 135 [BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" }, 136 [BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" }, 137 [BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" }, 138 [BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" }, 139 [BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" }, 140 [BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" }, 141 [BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" }, 142 [BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" }, 143 [BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" }, 144 [BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" }, 145 [BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" }, 146 [BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" }, 147 [BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" }, 148 [BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" }, 149 [BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" }, 150 [BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" }, 151 [BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" }, 152 [BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" }, 153 [BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" }, 154 [BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" }, 155 [BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" }, 156 [BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" }, 157 [BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" }, 158 [BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" }, 159 [BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" }, 160 [BCM5745x_NPAR] = { "Broadcom BCM5745x NetXtreme-E Ethernet Partition" }, 161 [BCM57508] = { "Broadcom BCM57508 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" }, 162 [BCM57504] = { "Broadcom BCM57504 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" }, 163 [BCM57502] = { "Broadcom BCM57502 NetXtreme-E 10Gb/25Gb/50Gb Ethernet" }, 164 [BCM57508_NPAR] = { "Broadcom BCM57508 NetXtreme-E Ethernet Partition" }, 165 [BCM57504_NPAR] = { "Broadcom BCM57504 NetXtreme-E Ethernet Partition" }, 166 [BCM57502_NPAR] = { "Broadcom BCM57502 NetXtreme-E Ethernet Partition" }, 167 [BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" }, 168 [BCM58804] = { "Broadcom BCM58804 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" }, 169 [BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" }, 170 [NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" }, 171 [NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" }, 172 [NETXTREME_S_VF] = { "Broadcom NetXtreme-S Ethernet Virtual Function" }, 173 [NETXTREME_E_P5_VF] = { "Broadcom BCM5750X NetXtreme-E Ethernet Virtual Function" }, 174 }; 175 176 static const struct pci_device_id bnxt_pci_tbl[] = { 177 { PCI_VDEVICE(BROADCOM, 0x1604), .driver_data = BCM5745x_NPAR }, 178 { PCI_VDEVICE(BROADCOM, 0x1605), .driver_data = BCM5745x_NPAR }, 179 { PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 }, 180 { PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR }, 181 { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 }, 182 { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 }, 183 { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 }, 184 { PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR }, 185 { PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 }, 186 { PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 }, 187 { PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 }, 188 { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 }, 189 { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 }, 190 { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 }, 191 { PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR }, 192 { PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 }, 193 { PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 }, 194 { PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 }, 195 { PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 }, 196 { PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 }, 197 { PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR }, 198 { PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 }, 199 { PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP }, 200 { PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP }, 201 { PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR }, 202 { PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR }, 203 { PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP }, 204 { PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR }, 205 { PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR }, 206 { PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR }, 207 { PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR }, 208 { PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR }, 209 { PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR }, 210 { PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 }, 211 { PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 }, 212 { PCI_VDEVICE(BROADCOM, 0x1750), .driver_data = BCM57508 }, 213 { PCI_VDEVICE(BROADCOM, 0x1751), .driver_data = BCM57504 }, 214 { PCI_VDEVICE(BROADCOM, 0x1752), .driver_data = BCM57502 }, 215 { PCI_VDEVICE(BROADCOM, 0x1800), .driver_data = BCM57508_NPAR }, 216 { PCI_VDEVICE(BROADCOM, 0x1801), .driver_data = BCM57504_NPAR }, 217 { PCI_VDEVICE(BROADCOM, 0x1802), .driver_data = BCM57502_NPAR }, 218 { PCI_VDEVICE(BROADCOM, 0x1803), .driver_data = BCM57508_NPAR }, 219 { PCI_VDEVICE(BROADCOM, 0x1804), .driver_data = BCM57504_NPAR }, 220 { PCI_VDEVICE(BROADCOM, 0x1805), .driver_data = BCM57502_NPAR }, 221 { PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 }, 222 { PCI_VDEVICE(BROADCOM, 0xd804), .driver_data = BCM58804 }, 223 #ifdef CONFIG_BNXT_SRIOV 224 { PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF }, 225 { PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF }, 226 { PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF }, 227 { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF }, 228 { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF }, 229 { PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF }, 230 { PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF }, 231 { PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF }, 232 { PCI_VDEVICE(BROADCOM, 0x1806), .driver_data = NETXTREME_E_P5_VF }, 233 { PCI_VDEVICE(BROADCOM, 0x1807), .driver_data = NETXTREME_E_P5_VF }, 234 { PCI_VDEVICE(BROADCOM, 0xd800), .driver_data = NETXTREME_S_VF }, 235 #endif 236 { 0 } 237 }; 238 239 MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl); 240 241 static const u16 bnxt_vf_req_snif[] = { 242 HWRM_FUNC_CFG, 243 HWRM_FUNC_VF_CFG, 244 HWRM_PORT_PHY_QCFG, 245 HWRM_CFA_L2_FILTER_ALLOC, 246 }; 247 248 static const u16 bnxt_async_events_arr[] = { 249 ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE, 250 ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE, 251 ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD, 252 ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED, 253 ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE, 254 ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE, 255 ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE, 256 ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY, 257 ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY, 258 ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION, 259 ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG, 260 ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST, 261 }; 262 263 static struct workqueue_struct *bnxt_pf_wq; 264 265 static bool bnxt_vf_pciid(enum board_idx idx) 266 { 267 return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF || 268 idx == NETXTREME_S_VF || idx == NETXTREME_E_P5_VF); 269 } 270 271 #define DB_CP_REARM_FLAGS (DB_KEY_CP | DB_IDX_VALID) 272 #define DB_CP_FLAGS (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS) 273 #define DB_CP_IRQ_DIS_FLAGS (DB_KEY_CP | DB_IRQ_DIS) 274 275 #define BNXT_CP_DB_IRQ_DIS(db) \ 276 writel(DB_CP_IRQ_DIS_FLAGS, db) 277 278 #define BNXT_DB_CQ(db, idx) \ 279 writel(DB_CP_FLAGS | RING_CMP(idx), (db)->doorbell) 280 281 #define BNXT_DB_NQ_P5(db, idx) \ 282 writeq((db)->db_key64 | DBR_TYPE_NQ | RING_CMP(idx), (db)->doorbell) 283 284 #define BNXT_DB_CQ_ARM(db, idx) \ 285 writel(DB_CP_REARM_FLAGS | RING_CMP(idx), (db)->doorbell) 286 287 #define BNXT_DB_NQ_ARM_P5(db, idx) \ 288 writeq((db)->db_key64 | DBR_TYPE_NQ_ARM | RING_CMP(idx), (db)->doorbell) 289 290 static void bnxt_db_nq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx) 291 { 292 if (bp->flags & BNXT_FLAG_CHIP_P5) 293 BNXT_DB_NQ_P5(db, idx); 294 else 295 BNXT_DB_CQ(db, idx); 296 } 297 298 static void bnxt_db_nq_arm(struct bnxt *bp, struct bnxt_db_info *db, u32 idx) 299 { 300 if (bp->flags & BNXT_FLAG_CHIP_P5) 301 BNXT_DB_NQ_ARM_P5(db, idx); 302 else 303 BNXT_DB_CQ_ARM(db, idx); 304 } 305 306 static void bnxt_db_cq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx) 307 { 308 if (bp->flags & BNXT_FLAG_CHIP_P5) 309 writeq(db->db_key64 | DBR_TYPE_CQ_ARMALL | RING_CMP(idx), 310 db->doorbell); 311 else 312 BNXT_DB_CQ(db, idx); 313 } 314 315 const u16 bnxt_lhint_arr[] = { 316 TX_BD_FLAGS_LHINT_512_AND_SMALLER, 317 TX_BD_FLAGS_LHINT_512_TO_1023, 318 TX_BD_FLAGS_LHINT_1024_TO_2047, 319 TX_BD_FLAGS_LHINT_1024_TO_2047, 320 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 321 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 322 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 323 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 324 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 325 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 326 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 327 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 328 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 329 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 330 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 331 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 332 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 333 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 334 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 335 }; 336 337 static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb) 338 { 339 struct metadata_dst *md_dst = skb_metadata_dst(skb); 340 341 if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX) 342 return 0; 343 344 return md_dst->u.port_info.port_id; 345 } 346 347 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev) 348 { 349 struct bnxt *bp = netdev_priv(dev); 350 struct tx_bd *txbd; 351 struct tx_bd_ext *txbd1; 352 struct netdev_queue *txq; 353 int i; 354 dma_addr_t mapping; 355 unsigned int length, pad = 0; 356 u32 len, free_size, vlan_tag_flags, cfa_action, flags; 357 u16 prod, last_frag; 358 struct pci_dev *pdev = bp->pdev; 359 struct bnxt_tx_ring_info *txr; 360 struct bnxt_sw_tx_bd *tx_buf; 361 362 i = skb_get_queue_mapping(skb); 363 if (unlikely(i >= bp->tx_nr_rings)) { 364 dev_kfree_skb_any(skb); 365 return NETDEV_TX_OK; 366 } 367 368 txq = netdev_get_tx_queue(dev, i); 369 txr = &bp->tx_ring[bp->tx_ring_map[i]]; 370 prod = txr->tx_prod; 371 372 free_size = bnxt_tx_avail(bp, txr); 373 if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) { 374 netif_tx_stop_queue(txq); 375 return NETDEV_TX_BUSY; 376 } 377 378 length = skb->len; 379 len = skb_headlen(skb); 380 last_frag = skb_shinfo(skb)->nr_frags; 381 382 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 383 384 txbd->tx_bd_opaque = prod; 385 386 tx_buf = &txr->tx_buf_ring[prod]; 387 tx_buf->skb = skb; 388 tx_buf->nr_frags = last_frag; 389 390 vlan_tag_flags = 0; 391 cfa_action = bnxt_xmit_get_cfa_action(skb); 392 if (skb_vlan_tag_present(skb)) { 393 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN | 394 skb_vlan_tag_get(skb); 395 /* Currently supports 8021Q, 8021AD vlan offloads 396 * QINQ1, QINQ2, QINQ3 vlan headers are deprecated 397 */ 398 if (skb->vlan_proto == htons(ETH_P_8021Q)) 399 vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT; 400 } 401 402 if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh) { 403 struct tx_push_buffer *tx_push_buf = txr->tx_push; 404 struct tx_push_bd *tx_push = &tx_push_buf->push_bd; 405 struct tx_bd_ext *tx_push1 = &tx_push->txbd2; 406 void __iomem *db = txr->tx_db.doorbell; 407 void *pdata = tx_push_buf->data; 408 u64 *end; 409 int j, push_len; 410 411 /* Set COAL_NOW to be ready quickly for the next push */ 412 tx_push->tx_bd_len_flags_type = 413 cpu_to_le32((length << TX_BD_LEN_SHIFT) | 414 TX_BD_TYPE_LONG_TX_BD | 415 TX_BD_FLAGS_LHINT_512_AND_SMALLER | 416 TX_BD_FLAGS_COAL_NOW | 417 TX_BD_FLAGS_PACKET_END | 418 (2 << TX_BD_FLAGS_BD_CNT_SHIFT)); 419 420 if (skb->ip_summed == CHECKSUM_PARTIAL) 421 tx_push1->tx_bd_hsize_lflags = 422 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM); 423 else 424 tx_push1->tx_bd_hsize_lflags = 0; 425 426 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags); 427 tx_push1->tx_bd_cfa_action = 428 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT); 429 430 end = pdata + length; 431 end = PTR_ALIGN(end, 8) - 1; 432 *end = 0; 433 434 skb_copy_from_linear_data(skb, pdata, len); 435 pdata += len; 436 for (j = 0; j < last_frag; j++) { 437 skb_frag_t *frag = &skb_shinfo(skb)->frags[j]; 438 void *fptr; 439 440 fptr = skb_frag_address_safe(frag); 441 if (!fptr) 442 goto normal_tx; 443 444 memcpy(pdata, fptr, skb_frag_size(frag)); 445 pdata += skb_frag_size(frag); 446 } 447 448 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type; 449 txbd->tx_bd_haddr = txr->data_mapping; 450 prod = NEXT_TX(prod); 451 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 452 memcpy(txbd, tx_push1, sizeof(*txbd)); 453 prod = NEXT_TX(prod); 454 tx_push->doorbell = 455 cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | prod); 456 txr->tx_prod = prod; 457 458 tx_buf->is_push = 1; 459 netdev_tx_sent_queue(txq, skb->len); 460 wmb(); /* Sync is_push and byte queue before pushing data */ 461 462 push_len = (length + sizeof(*tx_push) + 7) / 8; 463 if (push_len > 16) { 464 __iowrite64_copy(db, tx_push_buf, 16); 465 __iowrite32_copy(db + 4, tx_push_buf + 1, 466 (push_len - 16) << 1); 467 } else { 468 __iowrite64_copy(db, tx_push_buf, push_len); 469 } 470 471 goto tx_done; 472 } 473 474 normal_tx: 475 if (length < BNXT_MIN_PKT_SIZE) { 476 pad = BNXT_MIN_PKT_SIZE - length; 477 if (skb_pad(skb, pad)) { 478 /* SKB already freed. */ 479 tx_buf->skb = NULL; 480 return NETDEV_TX_OK; 481 } 482 length = BNXT_MIN_PKT_SIZE; 483 } 484 485 mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE); 486 487 if (unlikely(dma_mapping_error(&pdev->dev, mapping))) { 488 dev_kfree_skb_any(skb); 489 tx_buf->skb = NULL; 490 return NETDEV_TX_OK; 491 } 492 493 dma_unmap_addr_set(tx_buf, mapping, mapping); 494 flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD | 495 ((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT); 496 497 txbd->tx_bd_haddr = cpu_to_le64(mapping); 498 499 prod = NEXT_TX(prod); 500 txbd1 = (struct tx_bd_ext *) 501 &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 502 503 txbd1->tx_bd_hsize_lflags = 0; 504 if (skb_is_gso(skb)) { 505 u32 hdr_len; 506 507 if (skb->encapsulation) 508 hdr_len = skb_inner_network_offset(skb) + 509 skb_inner_network_header_len(skb) + 510 inner_tcp_hdrlen(skb); 511 else 512 hdr_len = skb_transport_offset(skb) + 513 tcp_hdrlen(skb); 514 515 txbd1->tx_bd_hsize_lflags = cpu_to_le32(TX_BD_FLAGS_LSO | 516 TX_BD_FLAGS_T_IPID | 517 (hdr_len << (TX_BD_HSIZE_SHIFT - 1))); 518 length = skb_shinfo(skb)->gso_size; 519 txbd1->tx_bd_mss = cpu_to_le32(length); 520 length += hdr_len; 521 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { 522 txbd1->tx_bd_hsize_lflags = 523 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM); 524 txbd1->tx_bd_mss = 0; 525 } 526 527 length >>= 9; 528 if (unlikely(length >= ARRAY_SIZE(bnxt_lhint_arr))) { 529 dev_warn_ratelimited(&pdev->dev, "Dropped oversize %d bytes TX packet.\n", 530 skb->len); 531 i = 0; 532 goto tx_dma_error; 533 } 534 flags |= bnxt_lhint_arr[length]; 535 txbd->tx_bd_len_flags_type = cpu_to_le32(flags); 536 537 txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags); 538 txbd1->tx_bd_cfa_action = 539 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT); 540 for (i = 0; i < last_frag; i++) { 541 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 542 543 prod = NEXT_TX(prod); 544 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 545 546 len = skb_frag_size(frag); 547 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len, 548 DMA_TO_DEVICE); 549 550 if (unlikely(dma_mapping_error(&pdev->dev, mapping))) 551 goto tx_dma_error; 552 553 tx_buf = &txr->tx_buf_ring[prod]; 554 dma_unmap_addr_set(tx_buf, mapping, mapping); 555 556 txbd->tx_bd_haddr = cpu_to_le64(mapping); 557 558 flags = len << TX_BD_LEN_SHIFT; 559 txbd->tx_bd_len_flags_type = cpu_to_le32(flags); 560 } 561 562 flags &= ~TX_BD_LEN; 563 txbd->tx_bd_len_flags_type = 564 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags | 565 TX_BD_FLAGS_PACKET_END); 566 567 netdev_tx_sent_queue(txq, skb->len); 568 569 /* Sync BD data before updating doorbell */ 570 wmb(); 571 572 prod = NEXT_TX(prod); 573 txr->tx_prod = prod; 574 575 if (!netdev_xmit_more() || netif_xmit_stopped(txq)) 576 bnxt_db_write(bp, &txr->tx_db, prod); 577 578 tx_done: 579 580 if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) { 581 if (netdev_xmit_more() && !tx_buf->is_push) 582 bnxt_db_write(bp, &txr->tx_db, prod); 583 584 netif_tx_stop_queue(txq); 585 586 /* netif_tx_stop_queue() must be done before checking 587 * tx index in bnxt_tx_avail() below, because in 588 * bnxt_tx_int(), we update tx index before checking for 589 * netif_tx_queue_stopped(). 590 */ 591 smp_mb(); 592 if (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh) 593 netif_tx_wake_queue(txq); 594 } 595 return NETDEV_TX_OK; 596 597 tx_dma_error: 598 last_frag = i; 599 600 /* start back at beginning and unmap skb */ 601 prod = txr->tx_prod; 602 tx_buf = &txr->tx_buf_ring[prod]; 603 tx_buf->skb = NULL; 604 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping), 605 skb_headlen(skb), PCI_DMA_TODEVICE); 606 prod = NEXT_TX(prod); 607 608 /* unmap remaining mapped pages */ 609 for (i = 0; i < last_frag; i++) { 610 prod = NEXT_TX(prod); 611 tx_buf = &txr->tx_buf_ring[prod]; 612 dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping), 613 skb_frag_size(&skb_shinfo(skb)->frags[i]), 614 PCI_DMA_TODEVICE); 615 } 616 617 dev_kfree_skb_any(skb); 618 return NETDEV_TX_OK; 619 } 620 621 static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts) 622 { 623 struct bnxt_tx_ring_info *txr = bnapi->tx_ring; 624 struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index); 625 u16 cons = txr->tx_cons; 626 struct pci_dev *pdev = bp->pdev; 627 int i; 628 unsigned int tx_bytes = 0; 629 630 for (i = 0; i < nr_pkts; i++) { 631 struct bnxt_sw_tx_bd *tx_buf; 632 struct sk_buff *skb; 633 int j, last; 634 635 tx_buf = &txr->tx_buf_ring[cons]; 636 cons = NEXT_TX(cons); 637 skb = tx_buf->skb; 638 tx_buf->skb = NULL; 639 640 if (tx_buf->is_push) { 641 tx_buf->is_push = 0; 642 goto next_tx_int; 643 } 644 645 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping), 646 skb_headlen(skb), PCI_DMA_TODEVICE); 647 last = tx_buf->nr_frags; 648 649 for (j = 0; j < last; j++) { 650 cons = NEXT_TX(cons); 651 tx_buf = &txr->tx_buf_ring[cons]; 652 dma_unmap_page( 653 &pdev->dev, 654 dma_unmap_addr(tx_buf, mapping), 655 skb_frag_size(&skb_shinfo(skb)->frags[j]), 656 PCI_DMA_TODEVICE); 657 } 658 659 next_tx_int: 660 cons = NEXT_TX(cons); 661 662 tx_bytes += skb->len; 663 dev_kfree_skb_any(skb); 664 } 665 666 netdev_tx_completed_queue(txq, nr_pkts, tx_bytes); 667 txr->tx_cons = cons; 668 669 /* Need to make the tx_cons update visible to bnxt_start_xmit() 670 * before checking for netif_tx_queue_stopped(). Without the 671 * memory barrier, there is a small possibility that bnxt_start_xmit() 672 * will miss it and cause the queue to be stopped forever. 673 */ 674 smp_mb(); 675 676 if (unlikely(netif_tx_queue_stopped(txq)) && 677 (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)) { 678 __netif_tx_lock(txq, smp_processor_id()); 679 if (netif_tx_queue_stopped(txq) && 680 bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh && 681 txr->dev_state != BNXT_DEV_STATE_CLOSING) 682 netif_tx_wake_queue(txq); 683 __netif_tx_unlock(txq); 684 } 685 } 686 687 static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping, 688 struct bnxt_rx_ring_info *rxr, 689 gfp_t gfp) 690 { 691 struct device *dev = &bp->pdev->dev; 692 struct page *page; 693 694 page = page_pool_dev_alloc_pages(rxr->page_pool); 695 if (!page) 696 return NULL; 697 698 *mapping = dma_map_page_attrs(dev, page, 0, PAGE_SIZE, bp->rx_dir, 699 DMA_ATTR_WEAK_ORDERING); 700 if (dma_mapping_error(dev, *mapping)) { 701 page_pool_recycle_direct(rxr->page_pool, page); 702 return NULL; 703 } 704 *mapping += bp->rx_dma_offset; 705 return page; 706 } 707 708 static inline u8 *__bnxt_alloc_rx_data(struct bnxt *bp, dma_addr_t *mapping, 709 gfp_t gfp) 710 { 711 u8 *data; 712 struct pci_dev *pdev = bp->pdev; 713 714 data = kmalloc(bp->rx_buf_size, gfp); 715 if (!data) 716 return NULL; 717 718 *mapping = dma_map_single_attrs(&pdev->dev, data + bp->rx_dma_offset, 719 bp->rx_buf_use_size, bp->rx_dir, 720 DMA_ATTR_WEAK_ORDERING); 721 722 if (dma_mapping_error(&pdev->dev, *mapping)) { 723 kfree(data); 724 data = NULL; 725 } 726 return data; 727 } 728 729 int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, 730 u16 prod, gfp_t gfp) 731 { 732 struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 733 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod]; 734 dma_addr_t mapping; 735 736 if (BNXT_RX_PAGE_MODE(bp)) { 737 struct page *page = 738 __bnxt_alloc_rx_page(bp, &mapping, rxr, gfp); 739 740 if (!page) 741 return -ENOMEM; 742 743 rx_buf->data = page; 744 rx_buf->data_ptr = page_address(page) + bp->rx_offset; 745 } else { 746 u8 *data = __bnxt_alloc_rx_data(bp, &mapping, gfp); 747 748 if (!data) 749 return -ENOMEM; 750 751 rx_buf->data = data; 752 rx_buf->data_ptr = data + bp->rx_offset; 753 } 754 rx_buf->mapping = mapping; 755 756 rxbd->rx_bd_haddr = cpu_to_le64(mapping); 757 return 0; 758 } 759 760 void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data) 761 { 762 u16 prod = rxr->rx_prod; 763 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf; 764 struct rx_bd *cons_bd, *prod_bd; 765 766 prod_rx_buf = &rxr->rx_buf_ring[prod]; 767 cons_rx_buf = &rxr->rx_buf_ring[cons]; 768 769 prod_rx_buf->data = data; 770 prod_rx_buf->data_ptr = cons_rx_buf->data_ptr; 771 772 prod_rx_buf->mapping = cons_rx_buf->mapping; 773 774 prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 775 cons_bd = &rxr->rx_desc_ring[RX_RING(cons)][RX_IDX(cons)]; 776 777 prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr; 778 } 779 780 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx) 781 { 782 u16 next, max = rxr->rx_agg_bmap_size; 783 784 next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx); 785 if (next >= max) 786 next = find_first_zero_bit(rxr->rx_agg_bmap, max); 787 return next; 788 } 789 790 static inline int bnxt_alloc_rx_page(struct bnxt *bp, 791 struct bnxt_rx_ring_info *rxr, 792 u16 prod, gfp_t gfp) 793 { 794 struct rx_bd *rxbd = 795 &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 796 struct bnxt_sw_rx_agg_bd *rx_agg_buf; 797 struct pci_dev *pdev = bp->pdev; 798 struct page *page; 799 dma_addr_t mapping; 800 u16 sw_prod = rxr->rx_sw_agg_prod; 801 unsigned int offset = 0; 802 803 if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) { 804 page = rxr->rx_page; 805 if (!page) { 806 page = alloc_page(gfp); 807 if (!page) 808 return -ENOMEM; 809 rxr->rx_page = page; 810 rxr->rx_page_offset = 0; 811 } 812 offset = rxr->rx_page_offset; 813 rxr->rx_page_offset += BNXT_RX_PAGE_SIZE; 814 if (rxr->rx_page_offset == PAGE_SIZE) 815 rxr->rx_page = NULL; 816 else 817 get_page(page); 818 } else { 819 page = alloc_page(gfp); 820 if (!page) 821 return -ENOMEM; 822 } 823 824 mapping = dma_map_page_attrs(&pdev->dev, page, offset, 825 BNXT_RX_PAGE_SIZE, PCI_DMA_FROMDEVICE, 826 DMA_ATTR_WEAK_ORDERING); 827 if (dma_mapping_error(&pdev->dev, mapping)) { 828 __free_page(page); 829 return -EIO; 830 } 831 832 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap))) 833 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod); 834 835 __set_bit(sw_prod, rxr->rx_agg_bmap); 836 rx_agg_buf = &rxr->rx_agg_ring[sw_prod]; 837 rxr->rx_sw_agg_prod = NEXT_RX_AGG(sw_prod); 838 839 rx_agg_buf->page = page; 840 rx_agg_buf->offset = offset; 841 rx_agg_buf->mapping = mapping; 842 rxbd->rx_bd_haddr = cpu_to_le64(mapping); 843 rxbd->rx_bd_opaque = sw_prod; 844 return 0; 845 } 846 847 static struct rx_agg_cmp *bnxt_get_agg(struct bnxt *bp, 848 struct bnxt_cp_ring_info *cpr, 849 u16 cp_cons, u16 curr) 850 { 851 struct rx_agg_cmp *agg; 852 853 cp_cons = RING_CMP(ADV_RAW_CMP(cp_cons, curr)); 854 agg = (struct rx_agg_cmp *) 855 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 856 return agg; 857 } 858 859 static struct rx_agg_cmp *bnxt_get_tpa_agg_p5(struct bnxt *bp, 860 struct bnxt_rx_ring_info *rxr, 861 u16 agg_id, u16 curr) 862 { 863 struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[agg_id]; 864 865 return &tpa_info->agg_arr[curr]; 866 } 867 868 static void bnxt_reuse_rx_agg_bufs(struct bnxt_cp_ring_info *cpr, u16 idx, 869 u16 start, u32 agg_bufs, bool tpa) 870 { 871 struct bnxt_napi *bnapi = cpr->bnapi; 872 struct bnxt *bp = bnapi->bp; 873 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 874 u16 prod = rxr->rx_agg_prod; 875 u16 sw_prod = rxr->rx_sw_agg_prod; 876 bool p5_tpa = false; 877 u32 i; 878 879 if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa) 880 p5_tpa = true; 881 882 for (i = 0; i < agg_bufs; i++) { 883 u16 cons; 884 struct rx_agg_cmp *agg; 885 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf; 886 struct rx_bd *prod_bd; 887 struct page *page; 888 889 if (p5_tpa) 890 agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, start + i); 891 else 892 agg = bnxt_get_agg(bp, cpr, idx, start + i); 893 cons = agg->rx_agg_cmp_opaque; 894 __clear_bit(cons, rxr->rx_agg_bmap); 895 896 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap))) 897 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod); 898 899 __set_bit(sw_prod, rxr->rx_agg_bmap); 900 prod_rx_buf = &rxr->rx_agg_ring[sw_prod]; 901 cons_rx_buf = &rxr->rx_agg_ring[cons]; 902 903 /* It is possible for sw_prod to be equal to cons, so 904 * set cons_rx_buf->page to NULL first. 905 */ 906 page = cons_rx_buf->page; 907 cons_rx_buf->page = NULL; 908 prod_rx_buf->page = page; 909 prod_rx_buf->offset = cons_rx_buf->offset; 910 911 prod_rx_buf->mapping = cons_rx_buf->mapping; 912 913 prod_bd = &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 914 915 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping); 916 prod_bd->rx_bd_opaque = sw_prod; 917 918 prod = NEXT_RX_AGG(prod); 919 sw_prod = NEXT_RX_AGG(sw_prod); 920 } 921 rxr->rx_agg_prod = prod; 922 rxr->rx_sw_agg_prod = sw_prod; 923 } 924 925 static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp, 926 struct bnxt_rx_ring_info *rxr, 927 u16 cons, void *data, u8 *data_ptr, 928 dma_addr_t dma_addr, 929 unsigned int offset_and_len) 930 { 931 unsigned int payload = offset_and_len >> 16; 932 unsigned int len = offset_and_len & 0xffff; 933 skb_frag_t *frag; 934 struct page *page = data; 935 u16 prod = rxr->rx_prod; 936 struct sk_buff *skb; 937 int off, err; 938 939 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC); 940 if (unlikely(err)) { 941 bnxt_reuse_rx_data(rxr, cons, data); 942 return NULL; 943 } 944 dma_addr -= bp->rx_dma_offset; 945 dma_unmap_page_attrs(&bp->pdev->dev, dma_addr, PAGE_SIZE, bp->rx_dir, 946 DMA_ATTR_WEAK_ORDERING); 947 page_pool_release_page(rxr->page_pool, page); 948 949 if (unlikely(!payload)) 950 payload = eth_get_headlen(bp->dev, data_ptr, len); 951 952 skb = napi_alloc_skb(&rxr->bnapi->napi, payload); 953 if (!skb) { 954 __free_page(page); 955 return NULL; 956 } 957 958 off = (void *)data_ptr - page_address(page); 959 skb_add_rx_frag(skb, 0, page, off, len, PAGE_SIZE); 960 memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN, 961 payload + NET_IP_ALIGN); 962 963 frag = &skb_shinfo(skb)->frags[0]; 964 skb_frag_size_sub(frag, payload); 965 skb_frag_off_add(frag, payload); 966 skb->data_len -= payload; 967 skb->tail += payload; 968 969 return skb; 970 } 971 972 static struct sk_buff *bnxt_rx_skb(struct bnxt *bp, 973 struct bnxt_rx_ring_info *rxr, u16 cons, 974 void *data, u8 *data_ptr, 975 dma_addr_t dma_addr, 976 unsigned int offset_and_len) 977 { 978 u16 prod = rxr->rx_prod; 979 struct sk_buff *skb; 980 int err; 981 982 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC); 983 if (unlikely(err)) { 984 bnxt_reuse_rx_data(rxr, cons, data); 985 return NULL; 986 } 987 988 skb = build_skb(data, 0); 989 dma_unmap_single_attrs(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size, 990 bp->rx_dir, DMA_ATTR_WEAK_ORDERING); 991 if (!skb) { 992 kfree(data); 993 return NULL; 994 } 995 996 skb_reserve(skb, bp->rx_offset); 997 skb_put(skb, offset_and_len & 0xffff); 998 return skb; 999 } 1000 1001 static struct sk_buff *bnxt_rx_pages(struct bnxt *bp, 1002 struct bnxt_cp_ring_info *cpr, 1003 struct sk_buff *skb, u16 idx, 1004 u32 agg_bufs, bool tpa) 1005 { 1006 struct bnxt_napi *bnapi = cpr->bnapi; 1007 struct pci_dev *pdev = bp->pdev; 1008 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1009 u16 prod = rxr->rx_agg_prod; 1010 bool p5_tpa = false; 1011 u32 i; 1012 1013 if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa) 1014 p5_tpa = true; 1015 1016 for (i = 0; i < agg_bufs; i++) { 1017 u16 cons, frag_len; 1018 struct rx_agg_cmp *agg; 1019 struct bnxt_sw_rx_agg_bd *cons_rx_buf; 1020 struct page *page; 1021 dma_addr_t mapping; 1022 1023 if (p5_tpa) 1024 agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, i); 1025 else 1026 agg = bnxt_get_agg(bp, cpr, idx, i); 1027 cons = agg->rx_agg_cmp_opaque; 1028 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) & 1029 RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT; 1030 1031 cons_rx_buf = &rxr->rx_agg_ring[cons]; 1032 skb_fill_page_desc(skb, i, cons_rx_buf->page, 1033 cons_rx_buf->offset, frag_len); 1034 __clear_bit(cons, rxr->rx_agg_bmap); 1035 1036 /* It is possible for bnxt_alloc_rx_page() to allocate 1037 * a sw_prod index that equals the cons index, so we 1038 * need to clear the cons entry now. 1039 */ 1040 mapping = cons_rx_buf->mapping; 1041 page = cons_rx_buf->page; 1042 cons_rx_buf->page = NULL; 1043 1044 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) { 1045 struct skb_shared_info *shinfo; 1046 unsigned int nr_frags; 1047 1048 shinfo = skb_shinfo(skb); 1049 nr_frags = --shinfo->nr_frags; 1050 __skb_frag_set_page(&shinfo->frags[nr_frags], NULL); 1051 1052 dev_kfree_skb(skb); 1053 1054 cons_rx_buf->page = page; 1055 1056 /* Update prod since possibly some pages have been 1057 * allocated already. 1058 */ 1059 rxr->rx_agg_prod = prod; 1060 bnxt_reuse_rx_agg_bufs(cpr, idx, i, agg_bufs - i, tpa); 1061 return NULL; 1062 } 1063 1064 dma_unmap_page_attrs(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE, 1065 PCI_DMA_FROMDEVICE, 1066 DMA_ATTR_WEAK_ORDERING); 1067 1068 skb->data_len += frag_len; 1069 skb->len += frag_len; 1070 skb->truesize += PAGE_SIZE; 1071 1072 prod = NEXT_RX_AGG(prod); 1073 } 1074 rxr->rx_agg_prod = prod; 1075 return skb; 1076 } 1077 1078 static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 1079 u8 agg_bufs, u32 *raw_cons) 1080 { 1081 u16 last; 1082 struct rx_agg_cmp *agg; 1083 1084 *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs); 1085 last = RING_CMP(*raw_cons); 1086 agg = (struct rx_agg_cmp *) 1087 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)]; 1088 return RX_AGG_CMP_VALID(agg, *raw_cons); 1089 } 1090 1091 static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data, 1092 unsigned int len, 1093 dma_addr_t mapping) 1094 { 1095 struct bnxt *bp = bnapi->bp; 1096 struct pci_dev *pdev = bp->pdev; 1097 struct sk_buff *skb; 1098 1099 skb = napi_alloc_skb(&bnapi->napi, len); 1100 if (!skb) 1101 return NULL; 1102 1103 dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copy_thresh, 1104 bp->rx_dir); 1105 1106 memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN, 1107 len + NET_IP_ALIGN); 1108 1109 dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copy_thresh, 1110 bp->rx_dir); 1111 1112 skb_put(skb, len); 1113 return skb; 1114 } 1115 1116 static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 1117 u32 *raw_cons, void *cmp) 1118 { 1119 struct rx_cmp *rxcmp = cmp; 1120 u32 tmp_raw_cons = *raw_cons; 1121 u8 cmp_type, agg_bufs = 0; 1122 1123 cmp_type = RX_CMP_TYPE(rxcmp); 1124 1125 if (cmp_type == CMP_TYPE_RX_L2_CMP) { 1126 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) & 1127 RX_CMP_AGG_BUFS) >> 1128 RX_CMP_AGG_BUFS_SHIFT; 1129 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) { 1130 struct rx_tpa_end_cmp *tpa_end = cmp; 1131 1132 if (bp->flags & BNXT_FLAG_CHIP_P5) 1133 return 0; 1134 1135 agg_bufs = TPA_END_AGG_BUFS(tpa_end); 1136 } 1137 1138 if (agg_bufs) { 1139 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons)) 1140 return -EBUSY; 1141 } 1142 *raw_cons = tmp_raw_cons; 1143 return 0; 1144 } 1145 1146 static void bnxt_queue_fw_reset_work(struct bnxt *bp, unsigned long delay) 1147 { 1148 if (!(test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))) 1149 return; 1150 1151 if (BNXT_PF(bp)) 1152 queue_delayed_work(bnxt_pf_wq, &bp->fw_reset_task, delay); 1153 else 1154 schedule_delayed_work(&bp->fw_reset_task, delay); 1155 } 1156 1157 static void bnxt_queue_sp_work(struct bnxt *bp) 1158 { 1159 if (BNXT_PF(bp)) 1160 queue_work(bnxt_pf_wq, &bp->sp_task); 1161 else 1162 schedule_work(&bp->sp_task); 1163 } 1164 1165 static void bnxt_sched_reset(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) 1166 { 1167 if (!rxr->bnapi->in_reset) { 1168 rxr->bnapi->in_reset = true; 1169 if (bp->flags & BNXT_FLAG_CHIP_P5) 1170 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event); 1171 else 1172 set_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event); 1173 bnxt_queue_sp_work(bp); 1174 } 1175 rxr->rx_next_cons = 0xffff; 1176 } 1177 1178 static u16 bnxt_alloc_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id) 1179 { 1180 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map; 1181 u16 idx = agg_id & MAX_TPA_P5_MASK; 1182 1183 if (test_bit(idx, map->agg_idx_bmap)) 1184 idx = find_first_zero_bit(map->agg_idx_bmap, 1185 BNXT_AGG_IDX_BMAP_SIZE); 1186 __set_bit(idx, map->agg_idx_bmap); 1187 map->agg_id_tbl[agg_id] = idx; 1188 return idx; 1189 } 1190 1191 static void bnxt_free_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx) 1192 { 1193 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map; 1194 1195 __clear_bit(idx, map->agg_idx_bmap); 1196 } 1197 1198 static u16 bnxt_lookup_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id) 1199 { 1200 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map; 1201 1202 return map->agg_id_tbl[agg_id]; 1203 } 1204 1205 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, 1206 struct rx_tpa_start_cmp *tpa_start, 1207 struct rx_tpa_start_cmp_ext *tpa_start1) 1208 { 1209 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf; 1210 struct bnxt_tpa_info *tpa_info; 1211 u16 cons, prod, agg_id; 1212 struct rx_bd *prod_bd; 1213 dma_addr_t mapping; 1214 1215 if (bp->flags & BNXT_FLAG_CHIP_P5) { 1216 agg_id = TPA_START_AGG_ID_P5(tpa_start); 1217 agg_id = bnxt_alloc_agg_idx(rxr, agg_id); 1218 } else { 1219 agg_id = TPA_START_AGG_ID(tpa_start); 1220 } 1221 cons = tpa_start->rx_tpa_start_cmp_opaque; 1222 prod = rxr->rx_prod; 1223 cons_rx_buf = &rxr->rx_buf_ring[cons]; 1224 prod_rx_buf = &rxr->rx_buf_ring[prod]; 1225 tpa_info = &rxr->rx_tpa[agg_id]; 1226 1227 if (unlikely(cons != rxr->rx_next_cons || 1228 TPA_START_ERROR(tpa_start))) { 1229 netdev_warn(bp->dev, "TPA cons %x, expected cons %x, error code %x\n", 1230 cons, rxr->rx_next_cons, 1231 TPA_START_ERROR_CODE(tpa_start1)); 1232 bnxt_sched_reset(bp, rxr); 1233 return; 1234 } 1235 /* Store cfa_code in tpa_info to use in tpa_end 1236 * completion processing. 1237 */ 1238 tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1); 1239 prod_rx_buf->data = tpa_info->data; 1240 prod_rx_buf->data_ptr = tpa_info->data_ptr; 1241 1242 mapping = tpa_info->mapping; 1243 prod_rx_buf->mapping = mapping; 1244 1245 prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 1246 1247 prod_bd->rx_bd_haddr = cpu_to_le64(mapping); 1248 1249 tpa_info->data = cons_rx_buf->data; 1250 tpa_info->data_ptr = cons_rx_buf->data_ptr; 1251 cons_rx_buf->data = NULL; 1252 tpa_info->mapping = cons_rx_buf->mapping; 1253 1254 tpa_info->len = 1255 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >> 1256 RX_TPA_START_CMP_LEN_SHIFT; 1257 if (likely(TPA_START_HASH_VALID(tpa_start))) { 1258 u32 hash_type = TPA_START_HASH_TYPE(tpa_start); 1259 1260 tpa_info->hash_type = PKT_HASH_TYPE_L4; 1261 tpa_info->gso_type = SKB_GSO_TCPV4; 1262 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */ 1263 if (hash_type == 3 || TPA_START_IS_IPV6(tpa_start1)) 1264 tpa_info->gso_type = SKB_GSO_TCPV6; 1265 tpa_info->rss_hash = 1266 le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash); 1267 } else { 1268 tpa_info->hash_type = PKT_HASH_TYPE_NONE; 1269 tpa_info->gso_type = 0; 1270 netif_warn(bp, rx_err, bp->dev, "TPA packet without valid hash\n"); 1271 } 1272 tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2); 1273 tpa_info->metadata = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata); 1274 tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info); 1275 tpa_info->agg_count = 0; 1276 1277 rxr->rx_prod = NEXT_RX(prod); 1278 cons = NEXT_RX(cons); 1279 rxr->rx_next_cons = NEXT_RX(cons); 1280 cons_rx_buf = &rxr->rx_buf_ring[cons]; 1281 1282 bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data); 1283 rxr->rx_prod = NEXT_RX(rxr->rx_prod); 1284 cons_rx_buf->data = NULL; 1285 } 1286 1287 static void bnxt_abort_tpa(struct bnxt_cp_ring_info *cpr, u16 idx, u32 agg_bufs) 1288 { 1289 if (agg_bufs) 1290 bnxt_reuse_rx_agg_bufs(cpr, idx, 0, agg_bufs, true); 1291 } 1292 1293 #ifdef CONFIG_INET 1294 static void bnxt_gro_tunnel(struct sk_buff *skb, __be16 ip_proto) 1295 { 1296 struct udphdr *uh = NULL; 1297 1298 if (ip_proto == htons(ETH_P_IP)) { 1299 struct iphdr *iph = (struct iphdr *)skb->data; 1300 1301 if (iph->protocol == IPPROTO_UDP) 1302 uh = (struct udphdr *)(iph + 1); 1303 } else { 1304 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data; 1305 1306 if (iph->nexthdr == IPPROTO_UDP) 1307 uh = (struct udphdr *)(iph + 1); 1308 } 1309 if (uh) { 1310 if (uh->check) 1311 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM; 1312 else 1313 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL; 1314 } 1315 } 1316 #endif 1317 1318 static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info, 1319 int payload_off, int tcp_ts, 1320 struct sk_buff *skb) 1321 { 1322 #ifdef CONFIG_INET 1323 struct tcphdr *th; 1324 int len, nw_off; 1325 u16 outer_ip_off, inner_ip_off, inner_mac_off; 1326 u32 hdr_info = tpa_info->hdr_info; 1327 bool loopback = false; 1328 1329 inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info); 1330 inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info); 1331 outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info); 1332 1333 /* If the packet is an internal loopback packet, the offsets will 1334 * have an extra 4 bytes. 1335 */ 1336 if (inner_mac_off == 4) { 1337 loopback = true; 1338 } else if (inner_mac_off > 4) { 1339 __be16 proto = *((__be16 *)(skb->data + inner_ip_off - 1340 ETH_HLEN - 2)); 1341 1342 /* We only support inner iPv4/ipv6. If we don't see the 1343 * correct protocol ID, it must be a loopback packet where 1344 * the offsets are off by 4. 1345 */ 1346 if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6)) 1347 loopback = true; 1348 } 1349 if (loopback) { 1350 /* internal loopback packet, subtract all offsets by 4 */ 1351 inner_ip_off -= 4; 1352 inner_mac_off -= 4; 1353 outer_ip_off -= 4; 1354 } 1355 1356 nw_off = inner_ip_off - ETH_HLEN; 1357 skb_set_network_header(skb, nw_off); 1358 if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) { 1359 struct ipv6hdr *iph = ipv6_hdr(skb); 1360 1361 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr)); 1362 len = skb->len - skb_transport_offset(skb); 1363 th = tcp_hdr(skb); 1364 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0); 1365 } else { 1366 struct iphdr *iph = ip_hdr(skb); 1367 1368 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr)); 1369 len = skb->len - skb_transport_offset(skb); 1370 th = tcp_hdr(skb); 1371 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0); 1372 } 1373 1374 if (inner_mac_off) { /* tunnel */ 1375 __be16 proto = *((__be16 *)(skb->data + outer_ip_off - 1376 ETH_HLEN - 2)); 1377 1378 bnxt_gro_tunnel(skb, proto); 1379 } 1380 #endif 1381 return skb; 1382 } 1383 1384 static struct sk_buff *bnxt_gro_func_5750x(struct bnxt_tpa_info *tpa_info, 1385 int payload_off, int tcp_ts, 1386 struct sk_buff *skb) 1387 { 1388 #ifdef CONFIG_INET 1389 u16 outer_ip_off, inner_ip_off, inner_mac_off; 1390 u32 hdr_info = tpa_info->hdr_info; 1391 int iphdr_len, nw_off; 1392 1393 inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info); 1394 inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info); 1395 outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info); 1396 1397 nw_off = inner_ip_off - ETH_HLEN; 1398 skb_set_network_header(skb, nw_off); 1399 iphdr_len = (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) ? 1400 sizeof(struct ipv6hdr) : sizeof(struct iphdr); 1401 skb_set_transport_header(skb, nw_off + iphdr_len); 1402 1403 if (inner_mac_off) { /* tunnel */ 1404 __be16 proto = *((__be16 *)(skb->data + outer_ip_off - 1405 ETH_HLEN - 2)); 1406 1407 bnxt_gro_tunnel(skb, proto); 1408 } 1409 #endif 1410 return skb; 1411 } 1412 1413 #define BNXT_IPV4_HDR_SIZE (sizeof(struct iphdr) + sizeof(struct tcphdr)) 1414 #define BNXT_IPV6_HDR_SIZE (sizeof(struct ipv6hdr) + sizeof(struct tcphdr)) 1415 1416 static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info, 1417 int payload_off, int tcp_ts, 1418 struct sk_buff *skb) 1419 { 1420 #ifdef CONFIG_INET 1421 struct tcphdr *th; 1422 int len, nw_off, tcp_opt_len = 0; 1423 1424 if (tcp_ts) 1425 tcp_opt_len = 12; 1426 1427 if (tpa_info->gso_type == SKB_GSO_TCPV4) { 1428 struct iphdr *iph; 1429 1430 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len - 1431 ETH_HLEN; 1432 skb_set_network_header(skb, nw_off); 1433 iph = ip_hdr(skb); 1434 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr)); 1435 len = skb->len - skb_transport_offset(skb); 1436 th = tcp_hdr(skb); 1437 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0); 1438 } else if (tpa_info->gso_type == SKB_GSO_TCPV6) { 1439 struct ipv6hdr *iph; 1440 1441 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len - 1442 ETH_HLEN; 1443 skb_set_network_header(skb, nw_off); 1444 iph = ipv6_hdr(skb); 1445 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr)); 1446 len = skb->len - skb_transport_offset(skb); 1447 th = tcp_hdr(skb); 1448 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0); 1449 } else { 1450 dev_kfree_skb_any(skb); 1451 return NULL; 1452 } 1453 1454 if (nw_off) /* tunnel */ 1455 bnxt_gro_tunnel(skb, skb->protocol); 1456 #endif 1457 return skb; 1458 } 1459 1460 static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp, 1461 struct bnxt_tpa_info *tpa_info, 1462 struct rx_tpa_end_cmp *tpa_end, 1463 struct rx_tpa_end_cmp_ext *tpa_end1, 1464 struct sk_buff *skb) 1465 { 1466 #ifdef CONFIG_INET 1467 int payload_off; 1468 u16 segs; 1469 1470 segs = TPA_END_TPA_SEGS(tpa_end); 1471 if (segs == 1) 1472 return skb; 1473 1474 NAPI_GRO_CB(skb)->count = segs; 1475 skb_shinfo(skb)->gso_size = 1476 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len); 1477 skb_shinfo(skb)->gso_type = tpa_info->gso_type; 1478 if (bp->flags & BNXT_FLAG_CHIP_P5) 1479 payload_off = TPA_END_PAYLOAD_OFF_P5(tpa_end1); 1480 else 1481 payload_off = TPA_END_PAYLOAD_OFF(tpa_end); 1482 skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb); 1483 if (likely(skb)) 1484 tcp_gro_complete(skb); 1485 #endif 1486 return skb; 1487 } 1488 1489 /* Given the cfa_code of a received packet determine which 1490 * netdev (vf-rep or PF) the packet is destined to. 1491 */ 1492 static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code) 1493 { 1494 struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code); 1495 1496 /* if vf-rep dev is NULL, the must belongs to the PF */ 1497 return dev ? dev : bp->dev; 1498 } 1499 1500 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp, 1501 struct bnxt_cp_ring_info *cpr, 1502 u32 *raw_cons, 1503 struct rx_tpa_end_cmp *tpa_end, 1504 struct rx_tpa_end_cmp_ext *tpa_end1, 1505 u8 *event) 1506 { 1507 struct bnxt_napi *bnapi = cpr->bnapi; 1508 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1509 u8 *data_ptr, agg_bufs; 1510 unsigned int len; 1511 struct bnxt_tpa_info *tpa_info; 1512 dma_addr_t mapping; 1513 struct sk_buff *skb; 1514 u16 idx = 0, agg_id; 1515 void *data; 1516 bool gro; 1517 1518 if (unlikely(bnapi->in_reset)) { 1519 int rc = bnxt_discard_rx(bp, cpr, raw_cons, tpa_end); 1520 1521 if (rc < 0) 1522 return ERR_PTR(-EBUSY); 1523 return NULL; 1524 } 1525 1526 if (bp->flags & BNXT_FLAG_CHIP_P5) { 1527 agg_id = TPA_END_AGG_ID_P5(tpa_end); 1528 agg_id = bnxt_lookup_agg_idx(rxr, agg_id); 1529 agg_bufs = TPA_END_AGG_BUFS_P5(tpa_end1); 1530 tpa_info = &rxr->rx_tpa[agg_id]; 1531 if (unlikely(agg_bufs != tpa_info->agg_count)) { 1532 netdev_warn(bp->dev, "TPA end agg_buf %d != expected agg_bufs %d\n", 1533 agg_bufs, tpa_info->agg_count); 1534 agg_bufs = tpa_info->agg_count; 1535 } 1536 tpa_info->agg_count = 0; 1537 *event |= BNXT_AGG_EVENT; 1538 bnxt_free_agg_idx(rxr, agg_id); 1539 idx = agg_id; 1540 gro = !!(bp->flags & BNXT_FLAG_GRO); 1541 } else { 1542 agg_id = TPA_END_AGG_ID(tpa_end); 1543 agg_bufs = TPA_END_AGG_BUFS(tpa_end); 1544 tpa_info = &rxr->rx_tpa[agg_id]; 1545 idx = RING_CMP(*raw_cons); 1546 if (agg_bufs) { 1547 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons)) 1548 return ERR_PTR(-EBUSY); 1549 1550 *event |= BNXT_AGG_EVENT; 1551 idx = NEXT_CMP(idx); 1552 } 1553 gro = !!TPA_END_GRO(tpa_end); 1554 } 1555 data = tpa_info->data; 1556 data_ptr = tpa_info->data_ptr; 1557 prefetch(data_ptr); 1558 len = tpa_info->len; 1559 mapping = tpa_info->mapping; 1560 1561 if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) { 1562 bnxt_abort_tpa(cpr, idx, agg_bufs); 1563 if (agg_bufs > MAX_SKB_FRAGS) 1564 netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n", 1565 agg_bufs, (int)MAX_SKB_FRAGS); 1566 return NULL; 1567 } 1568 1569 if (len <= bp->rx_copy_thresh) { 1570 skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping); 1571 if (!skb) { 1572 bnxt_abort_tpa(cpr, idx, agg_bufs); 1573 return NULL; 1574 } 1575 } else { 1576 u8 *new_data; 1577 dma_addr_t new_mapping; 1578 1579 new_data = __bnxt_alloc_rx_data(bp, &new_mapping, GFP_ATOMIC); 1580 if (!new_data) { 1581 bnxt_abort_tpa(cpr, idx, agg_bufs); 1582 return NULL; 1583 } 1584 1585 tpa_info->data = new_data; 1586 tpa_info->data_ptr = new_data + bp->rx_offset; 1587 tpa_info->mapping = new_mapping; 1588 1589 skb = build_skb(data, 0); 1590 dma_unmap_single_attrs(&bp->pdev->dev, mapping, 1591 bp->rx_buf_use_size, bp->rx_dir, 1592 DMA_ATTR_WEAK_ORDERING); 1593 1594 if (!skb) { 1595 kfree(data); 1596 bnxt_abort_tpa(cpr, idx, agg_bufs); 1597 return NULL; 1598 } 1599 skb_reserve(skb, bp->rx_offset); 1600 skb_put(skb, len); 1601 } 1602 1603 if (agg_bufs) { 1604 skb = bnxt_rx_pages(bp, cpr, skb, idx, agg_bufs, true); 1605 if (!skb) { 1606 /* Page reuse already handled by bnxt_rx_pages(). */ 1607 return NULL; 1608 } 1609 } 1610 1611 skb->protocol = 1612 eth_type_trans(skb, bnxt_get_pkt_dev(bp, tpa_info->cfa_code)); 1613 1614 if (tpa_info->hash_type != PKT_HASH_TYPE_NONE) 1615 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type); 1616 1617 if ((tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) && 1618 (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) { 1619 u16 vlan_proto = tpa_info->metadata >> 1620 RX_CMP_FLAGS2_METADATA_TPID_SFT; 1621 u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_TCI_MASK; 1622 1623 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag); 1624 } 1625 1626 skb_checksum_none_assert(skb); 1627 if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) { 1628 skb->ip_summed = CHECKSUM_UNNECESSARY; 1629 skb->csum_level = 1630 (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3; 1631 } 1632 1633 if (gro) 1634 skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb); 1635 1636 return skb; 1637 } 1638 1639 static void bnxt_tpa_agg(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, 1640 struct rx_agg_cmp *rx_agg) 1641 { 1642 u16 agg_id = TPA_AGG_AGG_ID(rx_agg); 1643 struct bnxt_tpa_info *tpa_info; 1644 1645 agg_id = bnxt_lookup_agg_idx(rxr, agg_id); 1646 tpa_info = &rxr->rx_tpa[agg_id]; 1647 BUG_ON(tpa_info->agg_count >= MAX_SKB_FRAGS); 1648 tpa_info->agg_arr[tpa_info->agg_count++] = *rx_agg; 1649 } 1650 1651 static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi, 1652 struct sk_buff *skb) 1653 { 1654 if (skb->dev != bp->dev) { 1655 /* this packet belongs to a vf-rep */ 1656 bnxt_vf_rep_rx(bp, skb); 1657 return; 1658 } 1659 skb_record_rx_queue(skb, bnapi->index); 1660 napi_gro_receive(&bnapi->napi, skb); 1661 } 1662 1663 /* returns the following: 1664 * 1 - 1 packet successfully received 1665 * 0 - successful TPA_START, packet not completed yet 1666 * -EBUSY - completion ring does not have all the agg buffers yet 1667 * -ENOMEM - packet aborted due to out of memory 1668 * -EIO - packet aborted due to hw error indicated in BD 1669 */ 1670 static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 1671 u32 *raw_cons, u8 *event) 1672 { 1673 struct bnxt_napi *bnapi = cpr->bnapi; 1674 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1675 struct net_device *dev = bp->dev; 1676 struct rx_cmp *rxcmp; 1677 struct rx_cmp_ext *rxcmp1; 1678 u32 tmp_raw_cons = *raw_cons; 1679 u16 cfa_code, cons, prod, cp_cons = RING_CMP(tmp_raw_cons); 1680 struct bnxt_sw_rx_bd *rx_buf; 1681 unsigned int len; 1682 u8 *data_ptr, agg_bufs, cmp_type; 1683 dma_addr_t dma_addr; 1684 struct sk_buff *skb; 1685 void *data; 1686 int rc = 0; 1687 u32 misc; 1688 1689 rxcmp = (struct rx_cmp *) 1690 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1691 1692 cmp_type = RX_CMP_TYPE(rxcmp); 1693 1694 if (cmp_type == CMP_TYPE_RX_TPA_AGG_CMP) { 1695 bnxt_tpa_agg(bp, rxr, (struct rx_agg_cmp *)rxcmp); 1696 goto next_rx_no_prod_no_len; 1697 } 1698 1699 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons); 1700 cp_cons = RING_CMP(tmp_raw_cons); 1701 rxcmp1 = (struct rx_cmp_ext *) 1702 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1703 1704 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons)) 1705 return -EBUSY; 1706 1707 prod = rxr->rx_prod; 1708 1709 if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) { 1710 bnxt_tpa_start(bp, rxr, (struct rx_tpa_start_cmp *)rxcmp, 1711 (struct rx_tpa_start_cmp_ext *)rxcmp1); 1712 1713 *event |= BNXT_RX_EVENT; 1714 goto next_rx_no_prod_no_len; 1715 1716 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) { 1717 skb = bnxt_tpa_end(bp, cpr, &tmp_raw_cons, 1718 (struct rx_tpa_end_cmp *)rxcmp, 1719 (struct rx_tpa_end_cmp_ext *)rxcmp1, event); 1720 1721 if (IS_ERR(skb)) 1722 return -EBUSY; 1723 1724 rc = -ENOMEM; 1725 if (likely(skb)) { 1726 bnxt_deliver_skb(bp, bnapi, skb); 1727 rc = 1; 1728 } 1729 *event |= BNXT_RX_EVENT; 1730 goto next_rx_no_prod_no_len; 1731 } 1732 1733 cons = rxcmp->rx_cmp_opaque; 1734 if (unlikely(cons != rxr->rx_next_cons)) { 1735 int rc1 = bnxt_discard_rx(bp, cpr, raw_cons, rxcmp); 1736 1737 /* 0xffff is forced error, don't print it */ 1738 if (rxr->rx_next_cons != 0xffff) 1739 netdev_warn(bp->dev, "RX cons %x != expected cons %x\n", 1740 cons, rxr->rx_next_cons); 1741 bnxt_sched_reset(bp, rxr); 1742 return rc1; 1743 } 1744 rx_buf = &rxr->rx_buf_ring[cons]; 1745 data = rx_buf->data; 1746 data_ptr = rx_buf->data_ptr; 1747 prefetch(data_ptr); 1748 1749 misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1); 1750 agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT; 1751 1752 if (agg_bufs) { 1753 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons)) 1754 return -EBUSY; 1755 1756 cp_cons = NEXT_CMP(cp_cons); 1757 *event |= BNXT_AGG_EVENT; 1758 } 1759 *event |= BNXT_RX_EVENT; 1760 1761 rx_buf->data = NULL; 1762 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) { 1763 u32 rx_err = le32_to_cpu(rxcmp1->rx_cmp_cfa_code_errors_v2); 1764 1765 bnxt_reuse_rx_data(rxr, cons, data); 1766 if (agg_bufs) 1767 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, agg_bufs, 1768 false); 1769 1770 rc = -EIO; 1771 if (rx_err & RX_CMPL_ERRORS_BUFFER_ERROR_MASK) { 1772 bnapi->cp_ring.sw_stats.rx.rx_buf_errors++; 1773 if (!(bp->flags & BNXT_FLAG_CHIP_P5) && 1774 !(bp->fw_cap & BNXT_FW_CAP_RING_MONITOR)) { 1775 netdev_warn_once(bp->dev, "RX buffer error %x\n", 1776 rx_err); 1777 bnxt_sched_reset(bp, rxr); 1778 } 1779 } 1780 goto next_rx_no_len; 1781 } 1782 1783 len = le32_to_cpu(rxcmp->rx_cmp_len_flags_type) >> RX_CMP_LEN_SHIFT; 1784 dma_addr = rx_buf->mapping; 1785 1786 if (bnxt_rx_xdp(bp, rxr, cons, data, &data_ptr, &len, event)) { 1787 rc = 1; 1788 goto next_rx; 1789 } 1790 1791 if (len <= bp->rx_copy_thresh) { 1792 skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr); 1793 bnxt_reuse_rx_data(rxr, cons, data); 1794 if (!skb) { 1795 if (agg_bufs) 1796 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, 1797 agg_bufs, false); 1798 rc = -ENOMEM; 1799 goto next_rx; 1800 } 1801 } else { 1802 u32 payload; 1803 1804 if (rx_buf->data_ptr == data_ptr) 1805 payload = misc & RX_CMP_PAYLOAD_OFFSET; 1806 else 1807 payload = 0; 1808 skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr, 1809 payload | len); 1810 if (!skb) { 1811 rc = -ENOMEM; 1812 goto next_rx; 1813 } 1814 } 1815 1816 if (agg_bufs) { 1817 skb = bnxt_rx_pages(bp, cpr, skb, cp_cons, agg_bufs, false); 1818 if (!skb) { 1819 rc = -ENOMEM; 1820 goto next_rx; 1821 } 1822 } 1823 1824 if (RX_CMP_HASH_VALID(rxcmp)) { 1825 u32 hash_type = RX_CMP_HASH_TYPE(rxcmp); 1826 enum pkt_hash_types type = PKT_HASH_TYPE_L4; 1827 1828 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */ 1829 if (hash_type != 1 && hash_type != 3) 1830 type = PKT_HASH_TYPE_L3; 1831 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type); 1832 } 1833 1834 cfa_code = RX_CMP_CFA_CODE(rxcmp1); 1835 skb->protocol = eth_type_trans(skb, bnxt_get_pkt_dev(bp, cfa_code)); 1836 1837 if ((rxcmp1->rx_cmp_flags2 & 1838 cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) && 1839 (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) { 1840 u32 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data); 1841 u16 vtag = meta_data & RX_CMP_FLAGS2_METADATA_TCI_MASK; 1842 u16 vlan_proto = meta_data >> RX_CMP_FLAGS2_METADATA_TPID_SFT; 1843 1844 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag); 1845 } 1846 1847 skb_checksum_none_assert(skb); 1848 if (RX_CMP_L4_CS_OK(rxcmp1)) { 1849 if (dev->features & NETIF_F_RXCSUM) { 1850 skb->ip_summed = CHECKSUM_UNNECESSARY; 1851 skb->csum_level = RX_CMP_ENCAP(rxcmp1); 1852 } 1853 } else { 1854 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) { 1855 if (dev->features & NETIF_F_RXCSUM) 1856 bnapi->cp_ring.sw_stats.rx.rx_l4_csum_errors++; 1857 } 1858 } 1859 1860 bnxt_deliver_skb(bp, bnapi, skb); 1861 rc = 1; 1862 1863 next_rx: 1864 cpr->rx_packets += 1; 1865 cpr->rx_bytes += len; 1866 1867 next_rx_no_len: 1868 rxr->rx_prod = NEXT_RX(prod); 1869 rxr->rx_next_cons = NEXT_RX(cons); 1870 1871 next_rx_no_prod_no_len: 1872 *raw_cons = tmp_raw_cons; 1873 1874 return rc; 1875 } 1876 1877 /* In netpoll mode, if we are using a combined completion ring, we need to 1878 * discard the rx packets and recycle the buffers. 1879 */ 1880 static int bnxt_force_rx_discard(struct bnxt *bp, 1881 struct bnxt_cp_ring_info *cpr, 1882 u32 *raw_cons, u8 *event) 1883 { 1884 u32 tmp_raw_cons = *raw_cons; 1885 struct rx_cmp_ext *rxcmp1; 1886 struct rx_cmp *rxcmp; 1887 u16 cp_cons; 1888 u8 cmp_type; 1889 1890 cp_cons = RING_CMP(tmp_raw_cons); 1891 rxcmp = (struct rx_cmp *) 1892 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1893 1894 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons); 1895 cp_cons = RING_CMP(tmp_raw_cons); 1896 rxcmp1 = (struct rx_cmp_ext *) 1897 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1898 1899 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons)) 1900 return -EBUSY; 1901 1902 cmp_type = RX_CMP_TYPE(rxcmp); 1903 if (cmp_type == CMP_TYPE_RX_L2_CMP) { 1904 rxcmp1->rx_cmp_cfa_code_errors_v2 |= 1905 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR); 1906 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) { 1907 struct rx_tpa_end_cmp_ext *tpa_end1; 1908 1909 tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1; 1910 tpa_end1->rx_tpa_end_cmp_errors_v2 |= 1911 cpu_to_le32(RX_TPA_END_CMP_ERRORS); 1912 } 1913 return bnxt_rx_pkt(bp, cpr, raw_cons, event); 1914 } 1915 1916 u32 bnxt_fw_health_readl(struct bnxt *bp, int reg_idx) 1917 { 1918 struct bnxt_fw_health *fw_health = bp->fw_health; 1919 u32 reg = fw_health->regs[reg_idx]; 1920 u32 reg_type, reg_off, val = 0; 1921 1922 reg_type = BNXT_FW_HEALTH_REG_TYPE(reg); 1923 reg_off = BNXT_FW_HEALTH_REG_OFF(reg); 1924 switch (reg_type) { 1925 case BNXT_FW_HEALTH_REG_TYPE_CFG: 1926 pci_read_config_dword(bp->pdev, reg_off, &val); 1927 break; 1928 case BNXT_FW_HEALTH_REG_TYPE_GRC: 1929 reg_off = fw_health->mapped_regs[reg_idx]; 1930 fallthrough; 1931 case BNXT_FW_HEALTH_REG_TYPE_BAR0: 1932 val = readl(bp->bar0 + reg_off); 1933 break; 1934 case BNXT_FW_HEALTH_REG_TYPE_BAR1: 1935 val = readl(bp->bar1 + reg_off); 1936 break; 1937 } 1938 if (reg_idx == BNXT_FW_RESET_INPROG_REG) 1939 val &= fw_health->fw_reset_inprog_reg_mask; 1940 return val; 1941 } 1942 1943 static u16 bnxt_agg_ring_id_to_grp_idx(struct bnxt *bp, u16 ring_id) 1944 { 1945 int i; 1946 1947 for (i = 0; i < bp->rx_nr_rings; i++) { 1948 u16 grp_idx = bp->rx_ring[i].bnapi->index; 1949 struct bnxt_ring_grp_info *grp_info; 1950 1951 grp_info = &bp->grp_info[grp_idx]; 1952 if (grp_info->agg_fw_ring_id == ring_id) 1953 return grp_idx; 1954 } 1955 return INVALID_HW_RING_ID; 1956 } 1957 1958 #define BNXT_GET_EVENT_PORT(data) \ 1959 ((data) & \ 1960 ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK) 1961 1962 #define BNXT_EVENT_RING_TYPE(data2) \ 1963 ((data2) & \ 1964 ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_MASK) 1965 1966 #define BNXT_EVENT_RING_TYPE_RX(data2) \ 1967 (BNXT_EVENT_RING_TYPE(data2) == \ 1968 ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_RX) 1969 1970 static int bnxt_async_event_process(struct bnxt *bp, 1971 struct hwrm_async_event_cmpl *cmpl) 1972 { 1973 u16 event_id = le16_to_cpu(cmpl->event_id); 1974 u32 data1 = le32_to_cpu(cmpl->event_data1); 1975 u32 data2 = le32_to_cpu(cmpl->event_data2); 1976 1977 /* TODO CHIMP_FW: Define event id's for link change, error etc */ 1978 switch (event_id) { 1979 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: { 1980 struct bnxt_link_info *link_info = &bp->link_info; 1981 1982 if (BNXT_VF(bp)) 1983 goto async_event_process_exit; 1984 1985 /* print unsupported speed warning in forced speed mode only */ 1986 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED) && 1987 (data1 & 0x20000)) { 1988 u16 fw_speed = link_info->force_link_speed; 1989 u32 speed = bnxt_fw_to_ethtool_speed(fw_speed); 1990 1991 if (speed != SPEED_UNKNOWN) 1992 netdev_warn(bp->dev, "Link speed %d no longer supported\n", 1993 speed); 1994 } 1995 set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event); 1996 } 1997 fallthrough; 1998 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE: 1999 case ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE: 2000 set_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT, &bp->sp_event); 2001 fallthrough; 2002 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE: 2003 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event); 2004 break; 2005 case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD: 2006 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event); 2007 break; 2008 case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: { 2009 u16 port_id = BNXT_GET_EVENT_PORT(data1); 2010 2011 if (BNXT_VF(bp)) 2012 break; 2013 2014 if (bp->pf.port_id != port_id) 2015 break; 2016 2017 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event); 2018 break; 2019 } 2020 case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE: 2021 if (BNXT_PF(bp)) 2022 goto async_event_process_exit; 2023 set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event); 2024 break; 2025 case ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY: { 2026 char *fatal_str = "non-fatal"; 2027 2028 if (!bp->fw_health) 2029 goto async_event_process_exit; 2030 2031 bp->fw_reset_timestamp = jiffies; 2032 bp->fw_reset_min_dsecs = cmpl->timestamp_lo; 2033 if (!bp->fw_reset_min_dsecs) 2034 bp->fw_reset_min_dsecs = BNXT_DFLT_FW_RST_MIN_DSECS; 2035 bp->fw_reset_max_dsecs = le16_to_cpu(cmpl->timestamp_hi); 2036 if (!bp->fw_reset_max_dsecs) 2037 bp->fw_reset_max_dsecs = BNXT_DFLT_FW_RST_MAX_DSECS; 2038 if (EVENT_DATA1_RESET_NOTIFY_FATAL(data1)) { 2039 fatal_str = "fatal"; 2040 set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state); 2041 } 2042 netif_warn(bp, hw, bp->dev, 2043 "Firmware %s reset event, data1: 0x%x, data2: 0x%x, min wait %u ms, max wait %u ms\n", 2044 fatal_str, data1, data2, 2045 bp->fw_reset_min_dsecs * 100, 2046 bp->fw_reset_max_dsecs * 100); 2047 set_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event); 2048 break; 2049 } 2050 case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY: { 2051 struct bnxt_fw_health *fw_health = bp->fw_health; 2052 2053 if (!fw_health) 2054 goto async_event_process_exit; 2055 2056 fw_health->enabled = EVENT_DATA1_RECOVERY_ENABLED(data1); 2057 fw_health->master = EVENT_DATA1_RECOVERY_MASTER_FUNC(data1); 2058 if (!fw_health->enabled) { 2059 netif_info(bp, drv, bp->dev, 2060 "Error recovery info: error recovery[0]\n"); 2061 break; 2062 } 2063 fw_health->tmr_multiplier = 2064 DIV_ROUND_UP(fw_health->polling_dsecs * HZ, 2065 bp->current_interval * 10); 2066 fw_health->tmr_counter = fw_health->tmr_multiplier; 2067 fw_health->last_fw_heartbeat = 2068 bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG); 2069 fw_health->last_fw_reset_cnt = 2070 bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG); 2071 netif_info(bp, drv, bp->dev, 2072 "Error recovery info: error recovery[1], master[%d], reset count[%u], health status: 0x%x\n", 2073 fw_health->master, fw_health->last_fw_reset_cnt, 2074 bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG)); 2075 goto async_event_process_exit; 2076 } 2077 case ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION: 2078 netif_notice(bp, hw, bp->dev, 2079 "Received firmware debug notification, data1: 0x%x, data2: 0x%x\n", 2080 data1, data2); 2081 goto async_event_process_exit; 2082 case ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG: { 2083 struct bnxt_rx_ring_info *rxr; 2084 u16 grp_idx; 2085 2086 if (bp->flags & BNXT_FLAG_CHIP_P5) 2087 goto async_event_process_exit; 2088 2089 netdev_warn(bp->dev, "Ring monitor event, ring type %lu id 0x%x\n", 2090 BNXT_EVENT_RING_TYPE(data2), data1); 2091 if (!BNXT_EVENT_RING_TYPE_RX(data2)) 2092 goto async_event_process_exit; 2093 2094 grp_idx = bnxt_agg_ring_id_to_grp_idx(bp, data1); 2095 if (grp_idx == INVALID_HW_RING_ID) { 2096 netdev_warn(bp->dev, "Unknown RX agg ring id 0x%x\n", 2097 data1); 2098 goto async_event_process_exit; 2099 } 2100 rxr = bp->bnapi[grp_idx]->rx_ring; 2101 bnxt_sched_reset(bp, rxr); 2102 goto async_event_process_exit; 2103 } 2104 case ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST: { 2105 struct bnxt_fw_health *fw_health = bp->fw_health; 2106 2107 netif_notice(bp, hw, bp->dev, 2108 "Received firmware echo request, data1: 0x%x, data2: 0x%x\n", 2109 data1, data2); 2110 if (fw_health) { 2111 fw_health->echo_req_data1 = data1; 2112 fw_health->echo_req_data2 = data2; 2113 set_bit(BNXT_FW_ECHO_REQUEST_SP_EVENT, &bp->sp_event); 2114 break; 2115 } 2116 goto async_event_process_exit; 2117 } 2118 default: 2119 goto async_event_process_exit; 2120 } 2121 bnxt_queue_sp_work(bp); 2122 async_event_process_exit: 2123 bnxt_ulp_async_events(bp, cmpl); 2124 return 0; 2125 } 2126 2127 static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp) 2128 { 2129 u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id; 2130 struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp; 2131 struct hwrm_fwd_req_cmpl *fwd_req_cmpl = 2132 (struct hwrm_fwd_req_cmpl *)txcmp; 2133 2134 switch (cmpl_type) { 2135 case CMPL_BASE_TYPE_HWRM_DONE: 2136 seq_id = le16_to_cpu(h_cmpl->sequence_id); 2137 if (seq_id == bp->hwrm_intr_seq_id) 2138 bp->hwrm_intr_seq_id = (u16)~bp->hwrm_intr_seq_id; 2139 else 2140 netdev_err(bp->dev, "Invalid hwrm seq id %d\n", seq_id); 2141 break; 2142 2143 case CMPL_BASE_TYPE_HWRM_FWD_REQ: 2144 vf_id = le16_to_cpu(fwd_req_cmpl->source_id); 2145 2146 if ((vf_id < bp->pf.first_vf_id) || 2147 (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) { 2148 netdev_err(bp->dev, "Msg contains invalid VF id %x\n", 2149 vf_id); 2150 return -EINVAL; 2151 } 2152 2153 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap); 2154 set_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event); 2155 bnxt_queue_sp_work(bp); 2156 break; 2157 2158 case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT: 2159 bnxt_async_event_process(bp, 2160 (struct hwrm_async_event_cmpl *)txcmp); 2161 2162 default: 2163 break; 2164 } 2165 2166 return 0; 2167 } 2168 2169 static irqreturn_t bnxt_msix(int irq, void *dev_instance) 2170 { 2171 struct bnxt_napi *bnapi = dev_instance; 2172 struct bnxt *bp = bnapi->bp; 2173 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2174 u32 cons = RING_CMP(cpr->cp_raw_cons); 2175 2176 cpr->event_ctr++; 2177 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]); 2178 napi_schedule(&bnapi->napi); 2179 return IRQ_HANDLED; 2180 } 2181 2182 static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr) 2183 { 2184 u32 raw_cons = cpr->cp_raw_cons; 2185 u16 cons = RING_CMP(raw_cons); 2186 struct tx_cmp *txcmp; 2187 2188 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]; 2189 2190 return TX_CMP_VALID(txcmp, raw_cons); 2191 } 2192 2193 static irqreturn_t bnxt_inta(int irq, void *dev_instance) 2194 { 2195 struct bnxt_napi *bnapi = dev_instance; 2196 struct bnxt *bp = bnapi->bp; 2197 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2198 u32 cons = RING_CMP(cpr->cp_raw_cons); 2199 u32 int_status; 2200 2201 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]); 2202 2203 if (!bnxt_has_work(bp, cpr)) { 2204 int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS); 2205 /* return if erroneous interrupt */ 2206 if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id))) 2207 return IRQ_NONE; 2208 } 2209 2210 /* disable ring IRQ */ 2211 BNXT_CP_DB_IRQ_DIS(cpr->cp_db.doorbell); 2212 2213 /* Return here if interrupt is shared and is disabled. */ 2214 if (unlikely(atomic_read(&bp->intr_sem) != 0)) 2215 return IRQ_HANDLED; 2216 2217 napi_schedule(&bnapi->napi); 2218 return IRQ_HANDLED; 2219 } 2220 2221 static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 2222 int budget) 2223 { 2224 struct bnxt_napi *bnapi = cpr->bnapi; 2225 u32 raw_cons = cpr->cp_raw_cons; 2226 u32 cons; 2227 int tx_pkts = 0; 2228 int rx_pkts = 0; 2229 u8 event = 0; 2230 struct tx_cmp *txcmp; 2231 2232 cpr->has_more_work = 0; 2233 cpr->had_work_done = 1; 2234 while (1) { 2235 int rc; 2236 2237 cons = RING_CMP(raw_cons); 2238 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]; 2239 2240 if (!TX_CMP_VALID(txcmp, raw_cons)) 2241 break; 2242 2243 /* The valid test of the entry must be done first before 2244 * reading any further. 2245 */ 2246 dma_rmb(); 2247 if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) { 2248 tx_pkts++; 2249 /* return full budget so NAPI will complete. */ 2250 if (unlikely(tx_pkts > bp->tx_wake_thresh)) { 2251 rx_pkts = budget; 2252 raw_cons = NEXT_RAW_CMP(raw_cons); 2253 if (budget) 2254 cpr->has_more_work = 1; 2255 break; 2256 } 2257 } else if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) { 2258 if (likely(budget)) 2259 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event); 2260 else 2261 rc = bnxt_force_rx_discard(bp, cpr, &raw_cons, 2262 &event); 2263 if (likely(rc >= 0)) 2264 rx_pkts += rc; 2265 /* Increment rx_pkts when rc is -ENOMEM to count towards 2266 * the NAPI budget. Otherwise, we may potentially loop 2267 * here forever if we consistently cannot allocate 2268 * buffers. 2269 */ 2270 else if (rc == -ENOMEM && budget) 2271 rx_pkts++; 2272 else if (rc == -EBUSY) /* partial completion */ 2273 break; 2274 } else if (unlikely((TX_CMP_TYPE(txcmp) == 2275 CMPL_BASE_TYPE_HWRM_DONE) || 2276 (TX_CMP_TYPE(txcmp) == 2277 CMPL_BASE_TYPE_HWRM_FWD_REQ) || 2278 (TX_CMP_TYPE(txcmp) == 2279 CMPL_BASE_TYPE_HWRM_ASYNC_EVENT))) { 2280 bnxt_hwrm_handler(bp, txcmp); 2281 } 2282 raw_cons = NEXT_RAW_CMP(raw_cons); 2283 2284 if (rx_pkts && rx_pkts == budget) { 2285 cpr->has_more_work = 1; 2286 break; 2287 } 2288 } 2289 2290 if (event & BNXT_REDIRECT_EVENT) 2291 xdp_do_flush_map(); 2292 2293 if (event & BNXT_TX_EVENT) { 2294 struct bnxt_tx_ring_info *txr = bnapi->tx_ring; 2295 u16 prod = txr->tx_prod; 2296 2297 /* Sync BD data before updating doorbell */ 2298 wmb(); 2299 2300 bnxt_db_write_relaxed(bp, &txr->tx_db, prod); 2301 } 2302 2303 cpr->cp_raw_cons = raw_cons; 2304 bnapi->tx_pkts += tx_pkts; 2305 bnapi->events |= event; 2306 return rx_pkts; 2307 } 2308 2309 static void __bnxt_poll_work_done(struct bnxt *bp, struct bnxt_napi *bnapi) 2310 { 2311 if (bnapi->tx_pkts) { 2312 bnapi->tx_int(bp, bnapi, bnapi->tx_pkts); 2313 bnapi->tx_pkts = 0; 2314 } 2315 2316 if ((bnapi->events & BNXT_RX_EVENT) && !(bnapi->in_reset)) { 2317 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 2318 2319 if (bnapi->events & BNXT_AGG_EVENT) 2320 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); 2321 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 2322 } 2323 bnapi->events = 0; 2324 } 2325 2326 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 2327 int budget) 2328 { 2329 struct bnxt_napi *bnapi = cpr->bnapi; 2330 int rx_pkts; 2331 2332 rx_pkts = __bnxt_poll_work(bp, cpr, budget); 2333 2334 /* ACK completion ring before freeing tx ring and producing new 2335 * buffers in rx/agg rings to prevent overflowing the completion 2336 * ring. 2337 */ 2338 bnxt_db_cq(bp, &cpr->cp_db, cpr->cp_raw_cons); 2339 2340 __bnxt_poll_work_done(bp, bnapi); 2341 return rx_pkts; 2342 } 2343 2344 static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget) 2345 { 2346 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi); 2347 struct bnxt *bp = bnapi->bp; 2348 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2349 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 2350 struct tx_cmp *txcmp; 2351 struct rx_cmp_ext *rxcmp1; 2352 u32 cp_cons, tmp_raw_cons; 2353 u32 raw_cons = cpr->cp_raw_cons; 2354 u32 rx_pkts = 0; 2355 u8 event = 0; 2356 2357 while (1) { 2358 int rc; 2359 2360 cp_cons = RING_CMP(raw_cons); 2361 txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 2362 2363 if (!TX_CMP_VALID(txcmp, raw_cons)) 2364 break; 2365 2366 if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) { 2367 tmp_raw_cons = NEXT_RAW_CMP(raw_cons); 2368 cp_cons = RING_CMP(tmp_raw_cons); 2369 rxcmp1 = (struct rx_cmp_ext *) 2370 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 2371 2372 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons)) 2373 break; 2374 2375 /* force an error to recycle the buffer */ 2376 rxcmp1->rx_cmp_cfa_code_errors_v2 |= 2377 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR); 2378 2379 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event); 2380 if (likely(rc == -EIO) && budget) 2381 rx_pkts++; 2382 else if (rc == -EBUSY) /* partial completion */ 2383 break; 2384 } else if (unlikely(TX_CMP_TYPE(txcmp) == 2385 CMPL_BASE_TYPE_HWRM_DONE)) { 2386 bnxt_hwrm_handler(bp, txcmp); 2387 } else { 2388 netdev_err(bp->dev, 2389 "Invalid completion received on special ring\n"); 2390 } 2391 raw_cons = NEXT_RAW_CMP(raw_cons); 2392 2393 if (rx_pkts == budget) 2394 break; 2395 } 2396 2397 cpr->cp_raw_cons = raw_cons; 2398 BNXT_DB_CQ(&cpr->cp_db, cpr->cp_raw_cons); 2399 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 2400 2401 if (event & BNXT_AGG_EVENT) 2402 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); 2403 2404 if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) { 2405 napi_complete_done(napi, rx_pkts); 2406 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons); 2407 } 2408 return rx_pkts; 2409 } 2410 2411 static int bnxt_poll(struct napi_struct *napi, int budget) 2412 { 2413 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi); 2414 struct bnxt *bp = bnapi->bp; 2415 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2416 int work_done = 0; 2417 2418 if (unlikely(test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))) { 2419 napi_complete(napi); 2420 return 0; 2421 } 2422 while (1) { 2423 work_done += bnxt_poll_work(bp, cpr, budget - work_done); 2424 2425 if (work_done >= budget) { 2426 if (!budget) 2427 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons); 2428 break; 2429 } 2430 2431 if (!bnxt_has_work(bp, cpr)) { 2432 if (napi_complete_done(napi, work_done)) 2433 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons); 2434 break; 2435 } 2436 } 2437 if (bp->flags & BNXT_FLAG_DIM) { 2438 struct dim_sample dim_sample = {}; 2439 2440 dim_update_sample(cpr->event_ctr, 2441 cpr->rx_packets, 2442 cpr->rx_bytes, 2443 &dim_sample); 2444 net_dim(&cpr->dim, dim_sample); 2445 } 2446 return work_done; 2447 } 2448 2449 static int __bnxt_poll_cqs(struct bnxt *bp, struct bnxt_napi *bnapi, int budget) 2450 { 2451 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2452 int i, work_done = 0; 2453 2454 for (i = 0; i < 2; i++) { 2455 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i]; 2456 2457 if (cpr2) { 2458 work_done += __bnxt_poll_work(bp, cpr2, 2459 budget - work_done); 2460 cpr->has_more_work |= cpr2->has_more_work; 2461 } 2462 } 2463 return work_done; 2464 } 2465 2466 static void __bnxt_poll_cqs_done(struct bnxt *bp, struct bnxt_napi *bnapi, 2467 u64 dbr_type) 2468 { 2469 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2470 int i; 2471 2472 for (i = 0; i < 2; i++) { 2473 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i]; 2474 struct bnxt_db_info *db; 2475 2476 if (cpr2 && cpr2->had_work_done) { 2477 db = &cpr2->cp_db; 2478 writeq(db->db_key64 | dbr_type | 2479 RING_CMP(cpr2->cp_raw_cons), db->doorbell); 2480 cpr2->had_work_done = 0; 2481 } 2482 } 2483 __bnxt_poll_work_done(bp, bnapi); 2484 } 2485 2486 static int bnxt_poll_p5(struct napi_struct *napi, int budget) 2487 { 2488 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi); 2489 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2490 u32 raw_cons = cpr->cp_raw_cons; 2491 struct bnxt *bp = bnapi->bp; 2492 struct nqe_cn *nqcmp; 2493 int work_done = 0; 2494 u32 cons; 2495 2496 if (unlikely(test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))) { 2497 napi_complete(napi); 2498 return 0; 2499 } 2500 if (cpr->has_more_work) { 2501 cpr->has_more_work = 0; 2502 work_done = __bnxt_poll_cqs(bp, bnapi, budget); 2503 } 2504 while (1) { 2505 cons = RING_CMP(raw_cons); 2506 nqcmp = &cpr->nq_desc_ring[CP_RING(cons)][CP_IDX(cons)]; 2507 2508 if (!NQ_CMP_VALID(nqcmp, raw_cons)) { 2509 if (cpr->has_more_work) 2510 break; 2511 2512 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ_ARMALL); 2513 cpr->cp_raw_cons = raw_cons; 2514 if (napi_complete_done(napi, work_done)) 2515 BNXT_DB_NQ_ARM_P5(&cpr->cp_db, 2516 cpr->cp_raw_cons); 2517 return work_done; 2518 } 2519 2520 /* The valid test of the entry must be done first before 2521 * reading any further. 2522 */ 2523 dma_rmb(); 2524 2525 if (nqcmp->type == cpu_to_le16(NQ_CN_TYPE_CQ_NOTIFICATION)) { 2526 u32 idx = le32_to_cpu(nqcmp->cq_handle_low); 2527 struct bnxt_cp_ring_info *cpr2; 2528 2529 cpr2 = cpr->cp_ring_arr[idx]; 2530 work_done += __bnxt_poll_work(bp, cpr2, 2531 budget - work_done); 2532 cpr->has_more_work |= cpr2->has_more_work; 2533 } else { 2534 bnxt_hwrm_handler(bp, (struct tx_cmp *)nqcmp); 2535 } 2536 raw_cons = NEXT_RAW_CMP(raw_cons); 2537 } 2538 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ); 2539 if (raw_cons != cpr->cp_raw_cons) { 2540 cpr->cp_raw_cons = raw_cons; 2541 BNXT_DB_NQ_P5(&cpr->cp_db, raw_cons); 2542 } 2543 return work_done; 2544 } 2545 2546 static void bnxt_free_tx_skbs(struct bnxt *bp) 2547 { 2548 int i, max_idx; 2549 struct pci_dev *pdev = bp->pdev; 2550 2551 if (!bp->tx_ring) 2552 return; 2553 2554 max_idx = bp->tx_nr_pages * TX_DESC_CNT; 2555 for (i = 0; i < bp->tx_nr_rings; i++) { 2556 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 2557 int j; 2558 2559 for (j = 0; j < max_idx;) { 2560 struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j]; 2561 struct sk_buff *skb; 2562 int k, last; 2563 2564 if (i < bp->tx_nr_rings_xdp && 2565 tx_buf->action == XDP_REDIRECT) { 2566 dma_unmap_single(&pdev->dev, 2567 dma_unmap_addr(tx_buf, mapping), 2568 dma_unmap_len(tx_buf, len), 2569 PCI_DMA_TODEVICE); 2570 xdp_return_frame(tx_buf->xdpf); 2571 tx_buf->action = 0; 2572 tx_buf->xdpf = NULL; 2573 j++; 2574 continue; 2575 } 2576 2577 skb = tx_buf->skb; 2578 if (!skb) { 2579 j++; 2580 continue; 2581 } 2582 2583 tx_buf->skb = NULL; 2584 2585 if (tx_buf->is_push) { 2586 dev_kfree_skb(skb); 2587 j += 2; 2588 continue; 2589 } 2590 2591 dma_unmap_single(&pdev->dev, 2592 dma_unmap_addr(tx_buf, mapping), 2593 skb_headlen(skb), 2594 PCI_DMA_TODEVICE); 2595 2596 last = tx_buf->nr_frags; 2597 j += 2; 2598 for (k = 0; k < last; k++, j++) { 2599 int ring_idx = j & bp->tx_ring_mask; 2600 skb_frag_t *frag = &skb_shinfo(skb)->frags[k]; 2601 2602 tx_buf = &txr->tx_buf_ring[ring_idx]; 2603 dma_unmap_page( 2604 &pdev->dev, 2605 dma_unmap_addr(tx_buf, mapping), 2606 skb_frag_size(frag), PCI_DMA_TODEVICE); 2607 } 2608 dev_kfree_skb(skb); 2609 } 2610 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i)); 2611 } 2612 } 2613 2614 static void bnxt_free_one_rx_ring_skbs(struct bnxt *bp, int ring_nr) 2615 { 2616 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr]; 2617 struct pci_dev *pdev = bp->pdev; 2618 struct bnxt_tpa_idx_map *map; 2619 int i, max_idx, max_agg_idx; 2620 2621 max_idx = bp->rx_nr_pages * RX_DESC_CNT; 2622 max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT; 2623 if (!rxr->rx_tpa) 2624 goto skip_rx_tpa_free; 2625 2626 for (i = 0; i < bp->max_tpa; i++) { 2627 struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[i]; 2628 u8 *data = tpa_info->data; 2629 2630 if (!data) 2631 continue; 2632 2633 dma_unmap_single_attrs(&pdev->dev, tpa_info->mapping, 2634 bp->rx_buf_use_size, bp->rx_dir, 2635 DMA_ATTR_WEAK_ORDERING); 2636 2637 tpa_info->data = NULL; 2638 2639 kfree(data); 2640 } 2641 2642 skip_rx_tpa_free: 2643 for (i = 0; i < max_idx; i++) { 2644 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[i]; 2645 dma_addr_t mapping = rx_buf->mapping; 2646 void *data = rx_buf->data; 2647 2648 if (!data) 2649 continue; 2650 2651 rx_buf->data = NULL; 2652 if (BNXT_RX_PAGE_MODE(bp)) { 2653 mapping -= bp->rx_dma_offset; 2654 dma_unmap_page_attrs(&pdev->dev, mapping, PAGE_SIZE, 2655 bp->rx_dir, 2656 DMA_ATTR_WEAK_ORDERING); 2657 page_pool_recycle_direct(rxr->page_pool, data); 2658 } else { 2659 dma_unmap_single_attrs(&pdev->dev, mapping, 2660 bp->rx_buf_use_size, bp->rx_dir, 2661 DMA_ATTR_WEAK_ORDERING); 2662 kfree(data); 2663 } 2664 } 2665 for (i = 0; i < max_agg_idx; i++) { 2666 struct bnxt_sw_rx_agg_bd *rx_agg_buf = &rxr->rx_agg_ring[i]; 2667 struct page *page = rx_agg_buf->page; 2668 2669 if (!page) 2670 continue; 2671 2672 dma_unmap_page_attrs(&pdev->dev, rx_agg_buf->mapping, 2673 BNXT_RX_PAGE_SIZE, PCI_DMA_FROMDEVICE, 2674 DMA_ATTR_WEAK_ORDERING); 2675 2676 rx_agg_buf->page = NULL; 2677 __clear_bit(i, rxr->rx_agg_bmap); 2678 2679 __free_page(page); 2680 } 2681 if (rxr->rx_page) { 2682 __free_page(rxr->rx_page); 2683 rxr->rx_page = NULL; 2684 } 2685 map = rxr->rx_tpa_idx_map; 2686 if (map) 2687 memset(map->agg_idx_bmap, 0, sizeof(map->agg_idx_bmap)); 2688 } 2689 2690 static void bnxt_free_rx_skbs(struct bnxt *bp) 2691 { 2692 int i; 2693 2694 if (!bp->rx_ring) 2695 return; 2696 2697 for (i = 0; i < bp->rx_nr_rings; i++) 2698 bnxt_free_one_rx_ring_skbs(bp, i); 2699 } 2700 2701 static void bnxt_free_skbs(struct bnxt *bp) 2702 { 2703 bnxt_free_tx_skbs(bp); 2704 bnxt_free_rx_skbs(bp); 2705 } 2706 2707 static void bnxt_init_ctx_mem(struct bnxt_mem_init *mem_init, void *p, int len) 2708 { 2709 u8 init_val = mem_init->init_val; 2710 u16 offset = mem_init->offset; 2711 u8 *p2 = p; 2712 int i; 2713 2714 if (!init_val) 2715 return; 2716 if (offset == BNXT_MEM_INVALID_OFFSET) { 2717 memset(p, init_val, len); 2718 return; 2719 } 2720 for (i = 0; i < len; i += mem_init->size) 2721 *(p2 + i + offset) = init_val; 2722 } 2723 2724 static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem) 2725 { 2726 struct pci_dev *pdev = bp->pdev; 2727 int i; 2728 2729 for (i = 0; i < rmem->nr_pages; i++) { 2730 if (!rmem->pg_arr[i]) 2731 continue; 2732 2733 dma_free_coherent(&pdev->dev, rmem->page_size, 2734 rmem->pg_arr[i], rmem->dma_arr[i]); 2735 2736 rmem->pg_arr[i] = NULL; 2737 } 2738 if (rmem->pg_tbl) { 2739 size_t pg_tbl_size = rmem->nr_pages * 8; 2740 2741 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG) 2742 pg_tbl_size = rmem->page_size; 2743 dma_free_coherent(&pdev->dev, pg_tbl_size, 2744 rmem->pg_tbl, rmem->pg_tbl_map); 2745 rmem->pg_tbl = NULL; 2746 } 2747 if (rmem->vmem_size && *rmem->vmem) { 2748 vfree(*rmem->vmem); 2749 *rmem->vmem = NULL; 2750 } 2751 } 2752 2753 static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem) 2754 { 2755 struct pci_dev *pdev = bp->pdev; 2756 u64 valid_bit = 0; 2757 int i; 2758 2759 if (rmem->flags & (BNXT_RMEM_VALID_PTE_FLAG | BNXT_RMEM_RING_PTE_FLAG)) 2760 valid_bit = PTU_PTE_VALID; 2761 if ((rmem->nr_pages > 1 || rmem->depth > 0) && !rmem->pg_tbl) { 2762 size_t pg_tbl_size = rmem->nr_pages * 8; 2763 2764 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG) 2765 pg_tbl_size = rmem->page_size; 2766 rmem->pg_tbl = dma_alloc_coherent(&pdev->dev, pg_tbl_size, 2767 &rmem->pg_tbl_map, 2768 GFP_KERNEL); 2769 if (!rmem->pg_tbl) 2770 return -ENOMEM; 2771 } 2772 2773 for (i = 0; i < rmem->nr_pages; i++) { 2774 u64 extra_bits = valid_bit; 2775 2776 rmem->pg_arr[i] = dma_alloc_coherent(&pdev->dev, 2777 rmem->page_size, 2778 &rmem->dma_arr[i], 2779 GFP_KERNEL); 2780 if (!rmem->pg_arr[i]) 2781 return -ENOMEM; 2782 2783 if (rmem->mem_init) 2784 bnxt_init_ctx_mem(rmem->mem_init, rmem->pg_arr[i], 2785 rmem->page_size); 2786 if (rmem->nr_pages > 1 || rmem->depth > 0) { 2787 if (i == rmem->nr_pages - 2 && 2788 (rmem->flags & BNXT_RMEM_RING_PTE_FLAG)) 2789 extra_bits |= PTU_PTE_NEXT_TO_LAST; 2790 else if (i == rmem->nr_pages - 1 && 2791 (rmem->flags & BNXT_RMEM_RING_PTE_FLAG)) 2792 extra_bits |= PTU_PTE_LAST; 2793 rmem->pg_tbl[i] = 2794 cpu_to_le64(rmem->dma_arr[i] | extra_bits); 2795 } 2796 } 2797 2798 if (rmem->vmem_size) { 2799 *rmem->vmem = vzalloc(rmem->vmem_size); 2800 if (!(*rmem->vmem)) 2801 return -ENOMEM; 2802 } 2803 return 0; 2804 } 2805 2806 static void bnxt_free_tpa_info(struct bnxt *bp) 2807 { 2808 int i; 2809 2810 for (i = 0; i < bp->rx_nr_rings; i++) { 2811 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 2812 2813 kfree(rxr->rx_tpa_idx_map); 2814 rxr->rx_tpa_idx_map = NULL; 2815 if (rxr->rx_tpa) { 2816 kfree(rxr->rx_tpa[0].agg_arr); 2817 rxr->rx_tpa[0].agg_arr = NULL; 2818 } 2819 kfree(rxr->rx_tpa); 2820 rxr->rx_tpa = NULL; 2821 } 2822 } 2823 2824 static int bnxt_alloc_tpa_info(struct bnxt *bp) 2825 { 2826 int i, j, total_aggs = 0; 2827 2828 bp->max_tpa = MAX_TPA; 2829 if (bp->flags & BNXT_FLAG_CHIP_P5) { 2830 if (!bp->max_tpa_v2) 2831 return 0; 2832 bp->max_tpa = max_t(u16, bp->max_tpa_v2, MAX_TPA_P5); 2833 total_aggs = bp->max_tpa * MAX_SKB_FRAGS; 2834 } 2835 2836 for (i = 0; i < bp->rx_nr_rings; i++) { 2837 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 2838 struct rx_agg_cmp *agg; 2839 2840 rxr->rx_tpa = kcalloc(bp->max_tpa, sizeof(struct bnxt_tpa_info), 2841 GFP_KERNEL); 2842 if (!rxr->rx_tpa) 2843 return -ENOMEM; 2844 2845 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 2846 continue; 2847 agg = kcalloc(total_aggs, sizeof(*agg), GFP_KERNEL); 2848 rxr->rx_tpa[0].agg_arr = agg; 2849 if (!agg) 2850 return -ENOMEM; 2851 for (j = 1; j < bp->max_tpa; j++) 2852 rxr->rx_tpa[j].agg_arr = agg + j * MAX_SKB_FRAGS; 2853 rxr->rx_tpa_idx_map = kzalloc(sizeof(*rxr->rx_tpa_idx_map), 2854 GFP_KERNEL); 2855 if (!rxr->rx_tpa_idx_map) 2856 return -ENOMEM; 2857 } 2858 return 0; 2859 } 2860 2861 static void bnxt_free_rx_rings(struct bnxt *bp) 2862 { 2863 int i; 2864 2865 if (!bp->rx_ring) 2866 return; 2867 2868 bnxt_free_tpa_info(bp); 2869 for (i = 0; i < bp->rx_nr_rings; i++) { 2870 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 2871 struct bnxt_ring_struct *ring; 2872 2873 if (rxr->xdp_prog) 2874 bpf_prog_put(rxr->xdp_prog); 2875 2876 if (xdp_rxq_info_is_reg(&rxr->xdp_rxq)) 2877 xdp_rxq_info_unreg(&rxr->xdp_rxq); 2878 2879 page_pool_destroy(rxr->page_pool); 2880 rxr->page_pool = NULL; 2881 2882 kfree(rxr->rx_agg_bmap); 2883 rxr->rx_agg_bmap = NULL; 2884 2885 ring = &rxr->rx_ring_struct; 2886 bnxt_free_ring(bp, &ring->ring_mem); 2887 2888 ring = &rxr->rx_agg_ring_struct; 2889 bnxt_free_ring(bp, &ring->ring_mem); 2890 } 2891 } 2892 2893 static int bnxt_alloc_rx_page_pool(struct bnxt *bp, 2894 struct bnxt_rx_ring_info *rxr) 2895 { 2896 struct page_pool_params pp = { 0 }; 2897 2898 pp.pool_size = bp->rx_ring_size; 2899 pp.nid = dev_to_node(&bp->pdev->dev); 2900 pp.dev = &bp->pdev->dev; 2901 pp.dma_dir = DMA_BIDIRECTIONAL; 2902 2903 rxr->page_pool = page_pool_create(&pp); 2904 if (IS_ERR(rxr->page_pool)) { 2905 int err = PTR_ERR(rxr->page_pool); 2906 2907 rxr->page_pool = NULL; 2908 return err; 2909 } 2910 return 0; 2911 } 2912 2913 static int bnxt_alloc_rx_rings(struct bnxt *bp) 2914 { 2915 int i, rc = 0, agg_rings = 0; 2916 2917 if (!bp->rx_ring) 2918 return -ENOMEM; 2919 2920 if (bp->flags & BNXT_FLAG_AGG_RINGS) 2921 agg_rings = 1; 2922 2923 for (i = 0; i < bp->rx_nr_rings; i++) { 2924 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 2925 struct bnxt_ring_struct *ring; 2926 2927 ring = &rxr->rx_ring_struct; 2928 2929 rc = bnxt_alloc_rx_page_pool(bp, rxr); 2930 if (rc) 2931 return rc; 2932 2933 rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i, 0); 2934 if (rc < 0) 2935 return rc; 2936 2937 rc = xdp_rxq_info_reg_mem_model(&rxr->xdp_rxq, 2938 MEM_TYPE_PAGE_POOL, 2939 rxr->page_pool); 2940 if (rc) { 2941 xdp_rxq_info_unreg(&rxr->xdp_rxq); 2942 return rc; 2943 } 2944 2945 rc = bnxt_alloc_ring(bp, &ring->ring_mem); 2946 if (rc) 2947 return rc; 2948 2949 ring->grp_idx = i; 2950 if (agg_rings) { 2951 u16 mem_size; 2952 2953 ring = &rxr->rx_agg_ring_struct; 2954 rc = bnxt_alloc_ring(bp, &ring->ring_mem); 2955 if (rc) 2956 return rc; 2957 2958 ring->grp_idx = i; 2959 rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1; 2960 mem_size = rxr->rx_agg_bmap_size / 8; 2961 rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL); 2962 if (!rxr->rx_agg_bmap) 2963 return -ENOMEM; 2964 } 2965 } 2966 if (bp->flags & BNXT_FLAG_TPA) 2967 rc = bnxt_alloc_tpa_info(bp); 2968 return rc; 2969 } 2970 2971 static void bnxt_free_tx_rings(struct bnxt *bp) 2972 { 2973 int i; 2974 struct pci_dev *pdev = bp->pdev; 2975 2976 if (!bp->tx_ring) 2977 return; 2978 2979 for (i = 0; i < bp->tx_nr_rings; i++) { 2980 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 2981 struct bnxt_ring_struct *ring; 2982 2983 if (txr->tx_push) { 2984 dma_free_coherent(&pdev->dev, bp->tx_push_size, 2985 txr->tx_push, txr->tx_push_mapping); 2986 txr->tx_push = NULL; 2987 } 2988 2989 ring = &txr->tx_ring_struct; 2990 2991 bnxt_free_ring(bp, &ring->ring_mem); 2992 } 2993 } 2994 2995 static int bnxt_alloc_tx_rings(struct bnxt *bp) 2996 { 2997 int i, j, rc; 2998 struct pci_dev *pdev = bp->pdev; 2999 3000 bp->tx_push_size = 0; 3001 if (bp->tx_push_thresh) { 3002 int push_size; 3003 3004 push_size = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) + 3005 bp->tx_push_thresh); 3006 3007 if (push_size > 256) { 3008 push_size = 0; 3009 bp->tx_push_thresh = 0; 3010 } 3011 3012 bp->tx_push_size = push_size; 3013 } 3014 3015 for (i = 0, j = 0; i < bp->tx_nr_rings; i++) { 3016 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 3017 struct bnxt_ring_struct *ring; 3018 u8 qidx; 3019 3020 ring = &txr->tx_ring_struct; 3021 3022 rc = bnxt_alloc_ring(bp, &ring->ring_mem); 3023 if (rc) 3024 return rc; 3025 3026 ring->grp_idx = txr->bnapi->index; 3027 if (bp->tx_push_size) { 3028 dma_addr_t mapping; 3029 3030 /* One pre-allocated DMA buffer to backup 3031 * TX push operation 3032 */ 3033 txr->tx_push = dma_alloc_coherent(&pdev->dev, 3034 bp->tx_push_size, 3035 &txr->tx_push_mapping, 3036 GFP_KERNEL); 3037 3038 if (!txr->tx_push) 3039 return -ENOMEM; 3040 3041 mapping = txr->tx_push_mapping + 3042 sizeof(struct tx_push_bd); 3043 txr->data_mapping = cpu_to_le64(mapping); 3044 } 3045 qidx = bp->tc_to_qidx[j]; 3046 ring->queue_id = bp->q_info[qidx].queue_id; 3047 if (i < bp->tx_nr_rings_xdp) 3048 continue; 3049 if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1)) 3050 j++; 3051 } 3052 return 0; 3053 } 3054 3055 static void bnxt_free_cp_rings(struct bnxt *bp) 3056 { 3057 int i; 3058 3059 if (!bp->bnapi) 3060 return; 3061 3062 for (i = 0; i < bp->cp_nr_rings; i++) { 3063 struct bnxt_napi *bnapi = bp->bnapi[i]; 3064 struct bnxt_cp_ring_info *cpr; 3065 struct bnxt_ring_struct *ring; 3066 int j; 3067 3068 if (!bnapi) 3069 continue; 3070 3071 cpr = &bnapi->cp_ring; 3072 ring = &cpr->cp_ring_struct; 3073 3074 bnxt_free_ring(bp, &ring->ring_mem); 3075 3076 for (j = 0; j < 2; j++) { 3077 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j]; 3078 3079 if (cpr2) { 3080 ring = &cpr2->cp_ring_struct; 3081 bnxt_free_ring(bp, &ring->ring_mem); 3082 kfree(cpr2); 3083 cpr->cp_ring_arr[j] = NULL; 3084 } 3085 } 3086 } 3087 } 3088 3089 static struct bnxt_cp_ring_info *bnxt_alloc_cp_sub_ring(struct bnxt *bp) 3090 { 3091 struct bnxt_ring_mem_info *rmem; 3092 struct bnxt_ring_struct *ring; 3093 struct bnxt_cp_ring_info *cpr; 3094 int rc; 3095 3096 cpr = kzalloc(sizeof(*cpr), GFP_KERNEL); 3097 if (!cpr) 3098 return NULL; 3099 3100 ring = &cpr->cp_ring_struct; 3101 rmem = &ring->ring_mem; 3102 rmem->nr_pages = bp->cp_nr_pages; 3103 rmem->page_size = HW_CMPD_RING_SIZE; 3104 rmem->pg_arr = (void **)cpr->cp_desc_ring; 3105 rmem->dma_arr = cpr->cp_desc_mapping; 3106 rmem->flags = BNXT_RMEM_RING_PTE_FLAG; 3107 rc = bnxt_alloc_ring(bp, rmem); 3108 if (rc) { 3109 bnxt_free_ring(bp, rmem); 3110 kfree(cpr); 3111 cpr = NULL; 3112 } 3113 return cpr; 3114 } 3115 3116 static int bnxt_alloc_cp_rings(struct bnxt *bp) 3117 { 3118 bool sh = !!(bp->flags & BNXT_FLAG_SHARED_RINGS); 3119 int i, rc, ulp_base_vec, ulp_msix; 3120 3121 ulp_msix = bnxt_get_ulp_msix_num(bp); 3122 ulp_base_vec = bnxt_get_ulp_msix_base(bp); 3123 for (i = 0; i < bp->cp_nr_rings; i++) { 3124 struct bnxt_napi *bnapi = bp->bnapi[i]; 3125 struct bnxt_cp_ring_info *cpr; 3126 struct bnxt_ring_struct *ring; 3127 3128 if (!bnapi) 3129 continue; 3130 3131 cpr = &bnapi->cp_ring; 3132 cpr->bnapi = bnapi; 3133 ring = &cpr->cp_ring_struct; 3134 3135 rc = bnxt_alloc_ring(bp, &ring->ring_mem); 3136 if (rc) 3137 return rc; 3138 3139 if (ulp_msix && i >= ulp_base_vec) 3140 ring->map_idx = i + ulp_msix; 3141 else 3142 ring->map_idx = i; 3143 3144 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 3145 continue; 3146 3147 if (i < bp->rx_nr_rings) { 3148 struct bnxt_cp_ring_info *cpr2 = 3149 bnxt_alloc_cp_sub_ring(bp); 3150 3151 cpr->cp_ring_arr[BNXT_RX_HDL] = cpr2; 3152 if (!cpr2) 3153 return -ENOMEM; 3154 cpr2->bnapi = bnapi; 3155 } 3156 if ((sh && i < bp->tx_nr_rings) || 3157 (!sh && i >= bp->rx_nr_rings)) { 3158 struct bnxt_cp_ring_info *cpr2 = 3159 bnxt_alloc_cp_sub_ring(bp); 3160 3161 cpr->cp_ring_arr[BNXT_TX_HDL] = cpr2; 3162 if (!cpr2) 3163 return -ENOMEM; 3164 cpr2->bnapi = bnapi; 3165 } 3166 } 3167 return 0; 3168 } 3169 3170 static void bnxt_init_ring_struct(struct bnxt *bp) 3171 { 3172 int i; 3173 3174 for (i = 0; i < bp->cp_nr_rings; i++) { 3175 struct bnxt_napi *bnapi = bp->bnapi[i]; 3176 struct bnxt_ring_mem_info *rmem; 3177 struct bnxt_cp_ring_info *cpr; 3178 struct bnxt_rx_ring_info *rxr; 3179 struct bnxt_tx_ring_info *txr; 3180 struct bnxt_ring_struct *ring; 3181 3182 if (!bnapi) 3183 continue; 3184 3185 cpr = &bnapi->cp_ring; 3186 ring = &cpr->cp_ring_struct; 3187 rmem = &ring->ring_mem; 3188 rmem->nr_pages = bp->cp_nr_pages; 3189 rmem->page_size = HW_CMPD_RING_SIZE; 3190 rmem->pg_arr = (void **)cpr->cp_desc_ring; 3191 rmem->dma_arr = cpr->cp_desc_mapping; 3192 rmem->vmem_size = 0; 3193 3194 rxr = bnapi->rx_ring; 3195 if (!rxr) 3196 goto skip_rx; 3197 3198 ring = &rxr->rx_ring_struct; 3199 rmem = &ring->ring_mem; 3200 rmem->nr_pages = bp->rx_nr_pages; 3201 rmem->page_size = HW_RXBD_RING_SIZE; 3202 rmem->pg_arr = (void **)rxr->rx_desc_ring; 3203 rmem->dma_arr = rxr->rx_desc_mapping; 3204 rmem->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages; 3205 rmem->vmem = (void **)&rxr->rx_buf_ring; 3206 3207 ring = &rxr->rx_agg_ring_struct; 3208 rmem = &ring->ring_mem; 3209 rmem->nr_pages = bp->rx_agg_nr_pages; 3210 rmem->page_size = HW_RXBD_RING_SIZE; 3211 rmem->pg_arr = (void **)rxr->rx_agg_desc_ring; 3212 rmem->dma_arr = rxr->rx_agg_desc_mapping; 3213 rmem->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages; 3214 rmem->vmem = (void **)&rxr->rx_agg_ring; 3215 3216 skip_rx: 3217 txr = bnapi->tx_ring; 3218 if (!txr) 3219 continue; 3220 3221 ring = &txr->tx_ring_struct; 3222 rmem = &ring->ring_mem; 3223 rmem->nr_pages = bp->tx_nr_pages; 3224 rmem->page_size = HW_RXBD_RING_SIZE; 3225 rmem->pg_arr = (void **)txr->tx_desc_ring; 3226 rmem->dma_arr = txr->tx_desc_mapping; 3227 rmem->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages; 3228 rmem->vmem = (void **)&txr->tx_buf_ring; 3229 } 3230 } 3231 3232 static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type) 3233 { 3234 int i; 3235 u32 prod; 3236 struct rx_bd **rx_buf_ring; 3237 3238 rx_buf_ring = (struct rx_bd **)ring->ring_mem.pg_arr; 3239 for (i = 0, prod = 0; i < ring->ring_mem.nr_pages; i++) { 3240 int j; 3241 struct rx_bd *rxbd; 3242 3243 rxbd = rx_buf_ring[i]; 3244 if (!rxbd) 3245 continue; 3246 3247 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) { 3248 rxbd->rx_bd_len_flags_type = cpu_to_le32(type); 3249 rxbd->rx_bd_opaque = prod; 3250 } 3251 } 3252 } 3253 3254 static int bnxt_alloc_one_rx_ring(struct bnxt *bp, int ring_nr) 3255 { 3256 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr]; 3257 struct net_device *dev = bp->dev; 3258 u32 prod; 3259 int i; 3260 3261 prod = rxr->rx_prod; 3262 for (i = 0; i < bp->rx_ring_size; i++) { 3263 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL)) { 3264 netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n", 3265 ring_nr, i, bp->rx_ring_size); 3266 break; 3267 } 3268 prod = NEXT_RX(prod); 3269 } 3270 rxr->rx_prod = prod; 3271 3272 if (!(bp->flags & BNXT_FLAG_AGG_RINGS)) 3273 return 0; 3274 3275 prod = rxr->rx_agg_prod; 3276 for (i = 0; i < bp->rx_agg_ring_size; i++) { 3277 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL)) { 3278 netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n", 3279 ring_nr, i, bp->rx_ring_size); 3280 break; 3281 } 3282 prod = NEXT_RX_AGG(prod); 3283 } 3284 rxr->rx_agg_prod = prod; 3285 3286 if (rxr->rx_tpa) { 3287 dma_addr_t mapping; 3288 u8 *data; 3289 3290 for (i = 0; i < bp->max_tpa; i++) { 3291 data = __bnxt_alloc_rx_data(bp, &mapping, GFP_KERNEL); 3292 if (!data) 3293 return -ENOMEM; 3294 3295 rxr->rx_tpa[i].data = data; 3296 rxr->rx_tpa[i].data_ptr = data + bp->rx_offset; 3297 rxr->rx_tpa[i].mapping = mapping; 3298 } 3299 } 3300 return 0; 3301 } 3302 3303 static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr) 3304 { 3305 struct bnxt_rx_ring_info *rxr; 3306 struct bnxt_ring_struct *ring; 3307 u32 type; 3308 3309 type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) | 3310 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP; 3311 3312 if (NET_IP_ALIGN == 2) 3313 type |= RX_BD_FLAGS_SOP; 3314 3315 rxr = &bp->rx_ring[ring_nr]; 3316 ring = &rxr->rx_ring_struct; 3317 bnxt_init_rxbd_pages(ring, type); 3318 3319 if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) { 3320 bpf_prog_add(bp->xdp_prog, 1); 3321 rxr->xdp_prog = bp->xdp_prog; 3322 } 3323 ring->fw_ring_id = INVALID_HW_RING_ID; 3324 3325 ring = &rxr->rx_agg_ring_struct; 3326 ring->fw_ring_id = INVALID_HW_RING_ID; 3327 3328 if ((bp->flags & BNXT_FLAG_AGG_RINGS)) { 3329 type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) | 3330 RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP; 3331 3332 bnxt_init_rxbd_pages(ring, type); 3333 } 3334 3335 return bnxt_alloc_one_rx_ring(bp, ring_nr); 3336 } 3337 3338 static void bnxt_init_cp_rings(struct bnxt *bp) 3339 { 3340 int i, j; 3341 3342 for (i = 0; i < bp->cp_nr_rings; i++) { 3343 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring; 3344 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 3345 3346 ring->fw_ring_id = INVALID_HW_RING_ID; 3347 cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks; 3348 cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs; 3349 for (j = 0; j < 2; j++) { 3350 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j]; 3351 3352 if (!cpr2) 3353 continue; 3354 3355 ring = &cpr2->cp_ring_struct; 3356 ring->fw_ring_id = INVALID_HW_RING_ID; 3357 cpr2->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks; 3358 cpr2->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs; 3359 } 3360 } 3361 } 3362 3363 static int bnxt_init_rx_rings(struct bnxt *bp) 3364 { 3365 int i, rc = 0; 3366 3367 if (BNXT_RX_PAGE_MODE(bp)) { 3368 bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM; 3369 bp->rx_dma_offset = XDP_PACKET_HEADROOM; 3370 } else { 3371 bp->rx_offset = BNXT_RX_OFFSET; 3372 bp->rx_dma_offset = BNXT_RX_DMA_OFFSET; 3373 } 3374 3375 for (i = 0; i < bp->rx_nr_rings; i++) { 3376 rc = bnxt_init_one_rx_ring(bp, i); 3377 if (rc) 3378 break; 3379 } 3380 3381 return rc; 3382 } 3383 3384 static int bnxt_init_tx_rings(struct bnxt *bp) 3385 { 3386 u16 i; 3387 3388 bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2, 3389 MAX_SKB_FRAGS + 1); 3390 3391 for (i = 0; i < bp->tx_nr_rings; i++) { 3392 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 3393 struct bnxt_ring_struct *ring = &txr->tx_ring_struct; 3394 3395 ring->fw_ring_id = INVALID_HW_RING_ID; 3396 } 3397 3398 return 0; 3399 } 3400 3401 static void bnxt_free_ring_grps(struct bnxt *bp) 3402 { 3403 kfree(bp->grp_info); 3404 bp->grp_info = NULL; 3405 } 3406 3407 static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init) 3408 { 3409 int i; 3410 3411 if (irq_re_init) { 3412 bp->grp_info = kcalloc(bp->cp_nr_rings, 3413 sizeof(struct bnxt_ring_grp_info), 3414 GFP_KERNEL); 3415 if (!bp->grp_info) 3416 return -ENOMEM; 3417 } 3418 for (i = 0; i < bp->cp_nr_rings; i++) { 3419 if (irq_re_init) 3420 bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID; 3421 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID; 3422 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID; 3423 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID; 3424 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID; 3425 } 3426 return 0; 3427 } 3428 3429 static void bnxt_free_vnics(struct bnxt *bp) 3430 { 3431 kfree(bp->vnic_info); 3432 bp->vnic_info = NULL; 3433 bp->nr_vnics = 0; 3434 } 3435 3436 static int bnxt_alloc_vnics(struct bnxt *bp) 3437 { 3438 int num_vnics = 1; 3439 3440 #ifdef CONFIG_RFS_ACCEL 3441 if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS) 3442 num_vnics += bp->rx_nr_rings; 3443 #endif 3444 3445 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 3446 num_vnics++; 3447 3448 bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info), 3449 GFP_KERNEL); 3450 if (!bp->vnic_info) 3451 return -ENOMEM; 3452 3453 bp->nr_vnics = num_vnics; 3454 return 0; 3455 } 3456 3457 static void bnxt_init_vnics(struct bnxt *bp) 3458 { 3459 int i; 3460 3461 for (i = 0; i < bp->nr_vnics; i++) { 3462 struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; 3463 int j; 3464 3465 vnic->fw_vnic_id = INVALID_HW_RING_ID; 3466 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) 3467 vnic->fw_rss_cos_lb_ctx[j] = INVALID_HW_RING_ID; 3468 3469 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID; 3470 3471 if (bp->vnic_info[i].rss_hash_key) { 3472 if (i == 0) 3473 prandom_bytes(vnic->rss_hash_key, 3474 HW_HASH_KEY_SIZE); 3475 else 3476 memcpy(vnic->rss_hash_key, 3477 bp->vnic_info[0].rss_hash_key, 3478 HW_HASH_KEY_SIZE); 3479 } 3480 } 3481 } 3482 3483 static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg) 3484 { 3485 int pages; 3486 3487 pages = ring_size / desc_per_pg; 3488 3489 if (!pages) 3490 return 1; 3491 3492 pages++; 3493 3494 while (pages & (pages - 1)) 3495 pages++; 3496 3497 return pages; 3498 } 3499 3500 void bnxt_set_tpa_flags(struct bnxt *bp) 3501 { 3502 bp->flags &= ~BNXT_FLAG_TPA; 3503 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS) 3504 return; 3505 if (bp->dev->features & NETIF_F_LRO) 3506 bp->flags |= BNXT_FLAG_LRO; 3507 else if (bp->dev->features & NETIF_F_GRO_HW) 3508 bp->flags |= BNXT_FLAG_GRO; 3509 } 3510 3511 /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must 3512 * be set on entry. 3513 */ 3514 void bnxt_set_ring_params(struct bnxt *bp) 3515 { 3516 u32 ring_size, rx_size, rx_space, max_rx_cmpl; 3517 u32 agg_factor = 0, agg_ring_size = 0; 3518 3519 /* 8 for CRC and VLAN */ 3520 rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8); 3521 3522 rx_space = rx_size + NET_SKB_PAD + 3523 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 3524 3525 bp->rx_copy_thresh = BNXT_RX_COPY_THRESH; 3526 ring_size = bp->rx_ring_size; 3527 bp->rx_agg_ring_size = 0; 3528 bp->rx_agg_nr_pages = 0; 3529 3530 if (bp->flags & BNXT_FLAG_TPA) 3531 agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE); 3532 3533 bp->flags &= ~BNXT_FLAG_JUMBO; 3534 if (rx_space > PAGE_SIZE && !(bp->flags & BNXT_FLAG_NO_AGG_RINGS)) { 3535 u32 jumbo_factor; 3536 3537 bp->flags |= BNXT_FLAG_JUMBO; 3538 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT; 3539 if (jumbo_factor > agg_factor) 3540 agg_factor = jumbo_factor; 3541 } 3542 agg_ring_size = ring_size * agg_factor; 3543 3544 if (agg_ring_size) { 3545 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size, 3546 RX_DESC_CNT); 3547 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) { 3548 u32 tmp = agg_ring_size; 3549 3550 bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES; 3551 agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1; 3552 netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n", 3553 tmp, agg_ring_size); 3554 } 3555 bp->rx_agg_ring_size = agg_ring_size; 3556 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1; 3557 rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN); 3558 rx_space = rx_size + NET_SKB_PAD + 3559 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 3560 } 3561 3562 bp->rx_buf_use_size = rx_size; 3563 bp->rx_buf_size = rx_space; 3564 3565 bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT); 3566 bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1; 3567 3568 ring_size = bp->tx_ring_size; 3569 bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT); 3570 bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1; 3571 3572 max_rx_cmpl = bp->rx_ring_size; 3573 /* MAX TPA needs to be added because TPA_START completions are 3574 * immediately recycled, so the TPA completions are not bound by 3575 * the RX ring size. 3576 */ 3577 if (bp->flags & BNXT_FLAG_TPA) 3578 max_rx_cmpl += bp->max_tpa; 3579 /* RX and TPA completions are 32-byte, all others are 16-byte */ 3580 ring_size = max_rx_cmpl * 2 + agg_ring_size + bp->tx_ring_size; 3581 bp->cp_ring_size = ring_size; 3582 3583 bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT); 3584 if (bp->cp_nr_pages > MAX_CP_PAGES) { 3585 bp->cp_nr_pages = MAX_CP_PAGES; 3586 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1; 3587 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n", 3588 ring_size, bp->cp_ring_size); 3589 } 3590 bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT; 3591 bp->cp_ring_mask = bp->cp_bit - 1; 3592 } 3593 3594 /* Changing allocation mode of RX rings. 3595 * TODO: Update when extending xdp_rxq_info to support allocation modes. 3596 */ 3597 int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode) 3598 { 3599 if (page_mode) { 3600 if (bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU) 3601 return -EOPNOTSUPP; 3602 bp->dev->max_mtu = 3603 min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU); 3604 bp->flags &= ~BNXT_FLAG_AGG_RINGS; 3605 bp->flags |= BNXT_FLAG_NO_AGG_RINGS | BNXT_FLAG_RX_PAGE_MODE; 3606 bp->rx_dir = DMA_BIDIRECTIONAL; 3607 bp->rx_skb_func = bnxt_rx_page_skb; 3608 /* Disable LRO or GRO_HW */ 3609 netdev_update_features(bp->dev); 3610 } else { 3611 bp->dev->max_mtu = bp->max_mtu; 3612 bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE; 3613 bp->rx_dir = DMA_FROM_DEVICE; 3614 bp->rx_skb_func = bnxt_rx_skb; 3615 } 3616 return 0; 3617 } 3618 3619 static void bnxt_free_vnic_attributes(struct bnxt *bp) 3620 { 3621 int i; 3622 struct bnxt_vnic_info *vnic; 3623 struct pci_dev *pdev = bp->pdev; 3624 3625 if (!bp->vnic_info) 3626 return; 3627 3628 for (i = 0; i < bp->nr_vnics; i++) { 3629 vnic = &bp->vnic_info[i]; 3630 3631 kfree(vnic->fw_grp_ids); 3632 vnic->fw_grp_ids = NULL; 3633 3634 kfree(vnic->uc_list); 3635 vnic->uc_list = NULL; 3636 3637 if (vnic->mc_list) { 3638 dma_free_coherent(&pdev->dev, vnic->mc_list_size, 3639 vnic->mc_list, vnic->mc_list_mapping); 3640 vnic->mc_list = NULL; 3641 } 3642 3643 if (vnic->rss_table) { 3644 dma_free_coherent(&pdev->dev, vnic->rss_table_size, 3645 vnic->rss_table, 3646 vnic->rss_table_dma_addr); 3647 vnic->rss_table = NULL; 3648 } 3649 3650 vnic->rss_hash_key = NULL; 3651 vnic->flags = 0; 3652 } 3653 } 3654 3655 static int bnxt_alloc_vnic_attributes(struct bnxt *bp) 3656 { 3657 int i, rc = 0, size; 3658 struct bnxt_vnic_info *vnic; 3659 struct pci_dev *pdev = bp->pdev; 3660 int max_rings; 3661 3662 for (i = 0; i < bp->nr_vnics; i++) { 3663 vnic = &bp->vnic_info[i]; 3664 3665 if (vnic->flags & BNXT_VNIC_UCAST_FLAG) { 3666 int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN; 3667 3668 if (mem_size > 0) { 3669 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL); 3670 if (!vnic->uc_list) { 3671 rc = -ENOMEM; 3672 goto out; 3673 } 3674 } 3675 } 3676 3677 if (vnic->flags & BNXT_VNIC_MCAST_FLAG) { 3678 vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN; 3679 vnic->mc_list = 3680 dma_alloc_coherent(&pdev->dev, 3681 vnic->mc_list_size, 3682 &vnic->mc_list_mapping, 3683 GFP_KERNEL); 3684 if (!vnic->mc_list) { 3685 rc = -ENOMEM; 3686 goto out; 3687 } 3688 } 3689 3690 if (bp->flags & BNXT_FLAG_CHIP_P5) 3691 goto vnic_skip_grps; 3692 3693 if (vnic->flags & BNXT_VNIC_RSS_FLAG) 3694 max_rings = bp->rx_nr_rings; 3695 else 3696 max_rings = 1; 3697 3698 vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL); 3699 if (!vnic->fw_grp_ids) { 3700 rc = -ENOMEM; 3701 goto out; 3702 } 3703 vnic_skip_grps: 3704 if ((bp->flags & BNXT_FLAG_NEW_RSS_CAP) && 3705 !(vnic->flags & BNXT_VNIC_RSS_FLAG)) 3706 continue; 3707 3708 /* Allocate rss table and hash key */ 3709 size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16)); 3710 if (bp->flags & BNXT_FLAG_CHIP_P5) 3711 size = L1_CACHE_ALIGN(BNXT_MAX_RSS_TABLE_SIZE_P5); 3712 3713 vnic->rss_table_size = size + HW_HASH_KEY_SIZE; 3714 vnic->rss_table = dma_alloc_coherent(&pdev->dev, 3715 vnic->rss_table_size, 3716 &vnic->rss_table_dma_addr, 3717 GFP_KERNEL); 3718 if (!vnic->rss_table) { 3719 rc = -ENOMEM; 3720 goto out; 3721 } 3722 3723 vnic->rss_hash_key = ((void *)vnic->rss_table) + size; 3724 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size; 3725 } 3726 return 0; 3727 3728 out: 3729 return rc; 3730 } 3731 3732 static void bnxt_free_hwrm_resources(struct bnxt *bp) 3733 { 3734 struct pci_dev *pdev = bp->pdev; 3735 3736 if (bp->hwrm_cmd_resp_addr) { 3737 dma_free_coherent(&pdev->dev, PAGE_SIZE, bp->hwrm_cmd_resp_addr, 3738 bp->hwrm_cmd_resp_dma_addr); 3739 bp->hwrm_cmd_resp_addr = NULL; 3740 } 3741 3742 if (bp->hwrm_cmd_kong_resp_addr) { 3743 dma_free_coherent(&pdev->dev, PAGE_SIZE, 3744 bp->hwrm_cmd_kong_resp_addr, 3745 bp->hwrm_cmd_kong_resp_dma_addr); 3746 bp->hwrm_cmd_kong_resp_addr = NULL; 3747 } 3748 } 3749 3750 static int bnxt_alloc_kong_hwrm_resources(struct bnxt *bp) 3751 { 3752 struct pci_dev *pdev = bp->pdev; 3753 3754 if (bp->hwrm_cmd_kong_resp_addr) 3755 return 0; 3756 3757 bp->hwrm_cmd_kong_resp_addr = 3758 dma_alloc_coherent(&pdev->dev, PAGE_SIZE, 3759 &bp->hwrm_cmd_kong_resp_dma_addr, 3760 GFP_KERNEL); 3761 if (!bp->hwrm_cmd_kong_resp_addr) 3762 return -ENOMEM; 3763 3764 return 0; 3765 } 3766 3767 static int bnxt_alloc_hwrm_resources(struct bnxt *bp) 3768 { 3769 struct pci_dev *pdev = bp->pdev; 3770 3771 bp->hwrm_cmd_resp_addr = dma_alloc_coherent(&pdev->dev, PAGE_SIZE, 3772 &bp->hwrm_cmd_resp_dma_addr, 3773 GFP_KERNEL); 3774 if (!bp->hwrm_cmd_resp_addr) 3775 return -ENOMEM; 3776 3777 return 0; 3778 } 3779 3780 static void bnxt_free_hwrm_short_cmd_req(struct bnxt *bp) 3781 { 3782 if (bp->hwrm_short_cmd_req_addr) { 3783 struct pci_dev *pdev = bp->pdev; 3784 3785 dma_free_coherent(&pdev->dev, bp->hwrm_max_ext_req_len, 3786 bp->hwrm_short_cmd_req_addr, 3787 bp->hwrm_short_cmd_req_dma_addr); 3788 bp->hwrm_short_cmd_req_addr = NULL; 3789 } 3790 } 3791 3792 static int bnxt_alloc_hwrm_short_cmd_req(struct bnxt *bp) 3793 { 3794 struct pci_dev *pdev = bp->pdev; 3795 3796 if (bp->hwrm_short_cmd_req_addr) 3797 return 0; 3798 3799 bp->hwrm_short_cmd_req_addr = 3800 dma_alloc_coherent(&pdev->dev, bp->hwrm_max_ext_req_len, 3801 &bp->hwrm_short_cmd_req_dma_addr, 3802 GFP_KERNEL); 3803 if (!bp->hwrm_short_cmd_req_addr) 3804 return -ENOMEM; 3805 3806 return 0; 3807 } 3808 3809 static void bnxt_free_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats) 3810 { 3811 kfree(stats->hw_masks); 3812 stats->hw_masks = NULL; 3813 kfree(stats->sw_stats); 3814 stats->sw_stats = NULL; 3815 if (stats->hw_stats) { 3816 dma_free_coherent(&bp->pdev->dev, stats->len, stats->hw_stats, 3817 stats->hw_stats_map); 3818 stats->hw_stats = NULL; 3819 } 3820 } 3821 3822 static int bnxt_alloc_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats, 3823 bool alloc_masks) 3824 { 3825 stats->hw_stats = dma_alloc_coherent(&bp->pdev->dev, stats->len, 3826 &stats->hw_stats_map, GFP_KERNEL); 3827 if (!stats->hw_stats) 3828 return -ENOMEM; 3829 3830 stats->sw_stats = kzalloc(stats->len, GFP_KERNEL); 3831 if (!stats->sw_stats) 3832 goto stats_mem_err; 3833 3834 if (alloc_masks) { 3835 stats->hw_masks = kzalloc(stats->len, GFP_KERNEL); 3836 if (!stats->hw_masks) 3837 goto stats_mem_err; 3838 } 3839 return 0; 3840 3841 stats_mem_err: 3842 bnxt_free_stats_mem(bp, stats); 3843 return -ENOMEM; 3844 } 3845 3846 static void bnxt_fill_masks(u64 *mask_arr, u64 mask, int count) 3847 { 3848 int i; 3849 3850 for (i = 0; i < count; i++) 3851 mask_arr[i] = mask; 3852 } 3853 3854 static void bnxt_copy_hw_masks(u64 *mask_arr, __le64 *hw_mask_arr, int count) 3855 { 3856 int i; 3857 3858 for (i = 0; i < count; i++) 3859 mask_arr[i] = le64_to_cpu(hw_mask_arr[i]); 3860 } 3861 3862 static int bnxt_hwrm_func_qstat_ext(struct bnxt *bp, 3863 struct bnxt_stats_mem *stats) 3864 { 3865 struct hwrm_func_qstats_ext_output *resp = bp->hwrm_cmd_resp_addr; 3866 struct hwrm_func_qstats_ext_input req = {0}; 3867 __le64 *hw_masks; 3868 int rc; 3869 3870 if (!(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED) || 3871 !(bp->flags & BNXT_FLAG_CHIP_P5)) 3872 return -EOPNOTSUPP; 3873 3874 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QSTATS_EXT, -1, -1); 3875 req.fid = cpu_to_le16(0xffff); 3876 req.flags = FUNC_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK; 3877 mutex_lock(&bp->hwrm_cmd_lock); 3878 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3879 if (rc) 3880 goto qstat_exit; 3881 3882 hw_masks = &resp->rx_ucast_pkts; 3883 bnxt_copy_hw_masks(stats->hw_masks, hw_masks, stats->len / 8); 3884 3885 qstat_exit: 3886 mutex_unlock(&bp->hwrm_cmd_lock); 3887 return rc; 3888 } 3889 3890 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags); 3891 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags); 3892 3893 static void bnxt_init_stats(struct bnxt *bp) 3894 { 3895 struct bnxt_napi *bnapi = bp->bnapi[0]; 3896 struct bnxt_cp_ring_info *cpr; 3897 struct bnxt_stats_mem *stats; 3898 __le64 *rx_stats, *tx_stats; 3899 int rc, rx_count, tx_count; 3900 u64 *rx_masks, *tx_masks; 3901 u64 mask; 3902 u8 flags; 3903 3904 cpr = &bnapi->cp_ring; 3905 stats = &cpr->stats; 3906 rc = bnxt_hwrm_func_qstat_ext(bp, stats); 3907 if (rc) { 3908 if (bp->flags & BNXT_FLAG_CHIP_P5) 3909 mask = (1ULL << 48) - 1; 3910 else 3911 mask = -1ULL; 3912 bnxt_fill_masks(stats->hw_masks, mask, stats->len / 8); 3913 } 3914 if (bp->flags & BNXT_FLAG_PORT_STATS) { 3915 stats = &bp->port_stats; 3916 rx_stats = stats->hw_stats; 3917 rx_masks = stats->hw_masks; 3918 rx_count = sizeof(struct rx_port_stats) / 8; 3919 tx_stats = rx_stats + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 3920 tx_masks = rx_masks + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 3921 tx_count = sizeof(struct tx_port_stats) / 8; 3922 3923 flags = PORT_QSTATS_REQ_FLAGS_COUNTER_MASK; 3924 rc = bnxt_hwrm_port_qstats(bp, flags); 3925 if (rc) { 3926 mask = (1ULL << 40) - 1; 3927 3928 bnxt_fill_masks(rx_masks, mask, rx_count); 3929 bnxt_fill_masks(tx_masks, mask, tx_count); 3930 } else { 3931 bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count); 3932 bnxt_copy_hw_masks(tx_masks, tx_stats, tx_count); 3933 bnxt_hwrm_port_qstats(bp, 0); 3934 } 3935 } 3936 if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) { 3937 stats = &bp->rx_port_stats_ext; 3938 rx_stats = stats->hw_stats; 3939 rx_masks = stats->hw_masks; 3940 rx_count = sizeof(struct rx_port_stats_ext) / 8; 3941 stats = &bp->tx_port_stats_ext; 3942 tx_stats = stats->hw_stats; 3943 tx_masks = stats->hw_masks; 3944 tx_count = sizeof(struct tx_port_stats_ext) / 8; 3945 3946 flags = PORT_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK; 3947 rc = bnxt_hwrm_port_qstats_ext(bp, flags); 3948 if (rc) { 3949 mask = (1ULL << 40) - 1; 3950 3951 bnxt_fill_masks(rx_masks, mask, rx_count); 3952 if (tx_stats) 3953 bnxt_fill_masks(tx_masks, mask, tx_count); 3954 } else { 3955 bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count); 3956 if (tx_stats) 3957 bnxt_copy_hw_masks(tx_masks, tx_stats, 3958 tx_count); 3959 bnxt_hwrm_port_qstats_ext(bp, 0); 3960 } 3961 } 3962 } 3963 3964 static void bnxt_free_port_stats(struct bnxt *bp) 3965 { 3966 bp->flags &= ~BNXT_FLAG_PORT_STATS; 3967 bp->flags &= ~BNXT_FLAG_PORT_STATS_EXT; 3968 3969 bnxt_free_stats_mem(bp, &bp->port_stats); 3970 bnxt_free_stats_mem(bp, &bp->rx_port_stats_ext); 3971 bnxt_free_stats_mem(bp, &bp->tx_port_stats_ext); 3972 } 3973 3974 static void bnxt_free_ring_stats(struct bnxt *bp) 3975 { 3976 int i; 3977 3978 if (!bp->bnapi) 3979 return; 3980 3981 for (i = 0; i < bp->cp_nr_rings; i++) { 3982 struct bnxt_napi *bnapi = bp->bnapi[i]; 3983 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 3984 3985 bnxt_free_stats_mem(bp, &cpr->stats); 3986 } 3987 } 3988 3989 static int bnxt_alloc_stats(struct bnxt *bp) 3990 { 3991 u32 size, i; 3992 int rc; 3993 3994 size = bp->hw_ring_stats_size; 3995 3996 for (i = 0; i < bp->cp_nr_rings; i++) { 3997 struct bnxt_napi *bnapi = bp->bnapi[i]; 3998 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 3999 4000 cpr->stats.len = size; 4001 rc = bnxt_alloc_stats_mem(bp, &cpr->stats, !i); 4002 if (rc) 4003 return rc; 4004 4005 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID; 4006 } 4007 4008 if (BNXT_VF(bp) || bp->chip_num == CHIP_NUM_58700) 4009 return 0; 4010 4011 if (bp->port_stats.hw_stats) 4012 goto alloc_ext_stats; 4013 4014 bp->port_stats.len = BNXT_PORT_STATS_SIZE; 4015 rc = bnxt_alloc_stats_mem(bp, &bp->port_stats, true); 4016 if (rc) 4017 return rc; 4018 4019 bp->flags |= BNXT_FLAG_PORT_STATS; 4020 4021 alloc_ext_stats: 4022 /* Display extended statistics only if FW supports it */ 4023 if (bp->hwrm_spec_code < 0x10804 || bp->hwrm_spec_code == 0x10900) 4024 if (!(bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED)) 4025 return 0; 4026 4027 if (bp->rx_port_stats_ext.hw_stats) 4028 goto alloc_tx_ext_stats; 4029 4030 bp->rx_port_stats_ext.len = sizeof(struct rx_port_stats_ext); 4031 rc = bnxt_alloc_stats_mem(bp, &bp->rx_port_stats_ext, true); 4032 /* Extended stats are optional */ 4033 if (rc) 4034 return 0; 4035 4036 alloc_tx_ext_stats: 4037 if (bp->tx_port_stats_ext.hw_stats) 4038 return 0; 4039 4040 if (bp->hwrm_spec_code >= 0x10902 || 4041 (bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED)) { 4042 bp->tx_port_stats_ext.len = sizeof(struct tx_port_stats_ext); 4043 rc = bnxt_alloc_stats_mem(bp, &bp->tx_port_stats_ext, true); 4044 /* Extended stats are optional */ 4045 if (rc) 4046 return 0; 4047 } 4048 bp->flags |= BNXT_FLAG_PORT_STATS_EXT; 4049 return 0; 4050 } 4051 4052 static void bnxt_clear_ring_indices(struct bnxt *bp) 4053 { 4054 int i; 4055 4056 if (!bp->bnapi) 4057 return; 4058 4059 for (i = 0; i < bp->cp_nr_rings; i++) { 4060 struct bnxt_napi *bnapi = bp->bnapi[i]; 4061 struct bnxt_cp_ring_info *cpr; 4062 struct bnxt_rx_ring_info *rxr; 4063 struct bnxt_tx_ring_info *txr; 4064 4065 if (!bnapi) 4066 continue; 4067 4068 cpr = &bnapi->cp_ring; 4069 cpr->cp_raw_cons = 0; 4070 4071 txr = bnapi->tx_ring; 4072 if (txr) { 4073 txr->tx_prod = 0; 4074 txr->tx_cons = 0; 4075 } 4076 4077 rxr = bnapi->rx_ring; 4078 if (rxr) { 4079 rxr->rx_prod = 0; 4080 rxr->rx_agg_prod = 0; 4081 rxr->rx_sw_agg_prod = 0; 4082 rxr->rx_next_cons = 0; 4083 } 4084 } 4085 } 4086 4087 static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool irq_reinit) 4088 { 4089 #ifdef CONFIG_RFS_ACCEL 4090 int i; 4091 4092 /* Under rtnl_lock and all our NAPIs have been disabled. It's 4093 * safe to delete the hash table. 4094 */ 4095 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) { 4096 struct hlist_head *head; 4097 struct hlist_node *tmp; 4098 struct bnxt_ntuple_filter *fltr; 4099 4100 head = &bp->ntp_fltr_hash_tbl[i]; 4101 hlist_for_each_entry_safe(fltr, tmp, head, hash) { 4102 hlist_del(&fltr->hash); 4103 kfree(fltr); 4104 } 4105 } 4106 if (irq_reinit) { 4107 kfree(bp->ntp_fltr_bmap); 4108 bp->ntp_fltr_bmap = NULL; 4109 } 4110 bp->ntp_fltr_count = 0; 4111 #endif 4112 } 4113 4114 static int bnxt_alloc_ntp_fltrs(struct bnxt *bp) 4115 { 4116 #ifdef CONFIG_RFS_ACCEL 4117 int i, rc = 0; 4118 4119 if (!(bp->flags & BNXT_FLAG_RFS)) 4120 return 0; 4121 4122 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) 4123 INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]); 4124 4125 bp->ntp_fltr_count = 0; 4126 bp->ntp_fltr_bmap = kcalloc(BITS_TO_LONGS(BNXT_NTP_FLTR_MAX_FLTR), 4127 sizeof(long), 4128 GFP_KERNEL); 4129 4130 if (!bp->ntp_fltr_bmap) 4131 rc = -ENOMEM; 4132 4133 return rc; 4134 #else 4135 return 0; 4136 #endif 4137 } 4138 4139 static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init) 4140 { 4141 bnxt_free_vnic_attributes(bp); 4142 bnxt_free_tx_rings(bp); 4143 bnxt_free_rx_rings(bp); 4144 bnxt_free_cp_rings(bp); 4145 bnxt_free_ntp_fltrs(bp, irq_re_init); 4146 if (irq_re_init) { 4147 bnxt_free_ring_stats(bp); 4148 if (!(bp->fw_cap & BNXT_FW_CAP_PORT_STATS_NO_RESET) || 4149 test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 4150 bnxt_free_port_stats(bp); 4151 bnxt_free_ring_grps(bp); 4152 bnxt_free_vnics(bp); 4153 kfree(bp->tx_ring_map); 4154 bp->tx_ring_map = NULL; 4155 kfree(bp->tx_ring); 4156 bp->tx_ring = NULL; 4157 kfree(bp->rx_ring); 4158 bp->rx_ring = NULL; 4159 kfree(bp->bnapi); 4160 bp->bnapi = NULL; 4161 } else { 4162 bnxt_clear_ring_indices(bp); 4163 } 4164 } 4165 4166 static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init) 4167 { 4168 int i, j, rc, size, arr_size; 4169 void *bnapi; 4170 4171 if (irq_re_init) { 4172 /* Allocate bnapi mem pointer array and mem block for 4173 * all queues 4174 */ 4175 arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) * 4176 bp->cp_nr_rings); 4177 size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi)); 4178 bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL); 4179 if (!bnapi) 4180 return -ENOMEM; 4181 4182 bp->bnapi = bnapi; 4183 bnapi += arr_size; 4184 for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) { 4185 bp->bnapi[i] = bnapi; 4186 bp->bnapi[i]->index = i; 4187 bp->bnapi[i]->bp = bp; 4188 if (bp->flags & BNXT_FLAG_CHIP_P5) { 4189 struct bnxt_cp_ring_info *cpr = 4190 &bp->bnapi[i]->cp_ring; 4191 4192 cpr->cp_ring_struct.ring_mem.flags = 4193 BNXT_RMEM_RING_PTE_FLAG; 4194 } 4195 } 4196 4197 bp->rx_ring = kcalloc(bp->rx_nr_rings, 4198 sizeof(struct bnxt_rx_ring_info), 4199 GFP_KERNEL); 4200 if (!bp->rx_ring) 4201 return -ENOMEM; 4202 4203 for (i = 0; i < bp->rx_nr_rings; i++) { 4204 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 4205 4206 if (bp->flags & BNXT_FLAG_CHIP_P5) { 4207 rxr->rx_ring_struct.ring_mem.flags = 4208 BNXT_RMEM_RING_PTE_FLAG; 4209 rxr->rx_agg_ring_struct.ring_mem.flags = 4210 BNXT_RMEM_RING_PTE_FLAG; 4211 } 4212 rxr->bnapi = bp->bnapi[i]; 4213 bp->bnapi[i]->rx_ring = &bp->rx_ring[i]; 4214 } 4215 4216 bp->tx_ring = kcalloc(bp->tx_nr_rings, 4217 sizeof(struct bnxt_tx_ring_info), 4218 GFP_KERNEL); 4219 if (!bp->tx_ring) 4220 return -ENOMEM; 4221 4222 bp->tx_ring_map = kcalloc(bp->tx_nr_rings, sizeof(u16), 4223 GFP_KERNEL); 4224 4225 if (!bp->tx_ring_map) 4226 return -ENOMEM; 4227 4228 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 4229 j = 0; 4230 else 4231 j = bp->rx_nr_rings; 4232 4233 for (i = 0; i < bp->tx_nr_rings; i++, j++) { 4234 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 4235 4236 if (bp->flags & BNXT_FLAG_CHIP_P5) 4237 txr->tx_ring_struct.ring_mem.flags = 4238 BNXT_RMEM_RING_PTE_FLAG; 4239 txr->bnapi = bp->bnapi[j]; 4240 bp->bnapi[j]->tx_ring = txr; 4241 bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i; 4242 if (i >= bp->tx_nr_rings_xdp) { 4243 txr->txq_index = i - bp->tx_nr_rings_xdp; 4244 bp->bnapi[j]->tx_int = bnxt_tx_int; 4245 } else { 4246 bp->bnapi[j]->flags |= BNXT_NAPI_FLAG_XDP; 4247 bp->bnapi[j]->tx_int = bnxt_tx_int_xdp; 4248 } 4249 } 4250 4251 rc = bnxt_alloc_stats(bp); 4252 if (rc) 4253 goto alloc_mem_err; 4254 bnxt_init_stats(bp); 4255 4256 rc = bnxt_alloc_ntp_fltrs(bp); 4257 if (rc) 4258 goto alloc_mem_err; 4259 4260 rc = bnxt_alloc_vnics(bp); 4261 if (rc) 4262 goto alloc_mem_err; 4263 } 4264 4265 bnxt_init_ring_struct(bp); 4266 4267 rc = bnxt_alloc_rx_rings(bp); 4268 if (rc) 4269 goto alloc_mem_err; 4270 4271 rc = bnxt_alloc_tx_rings(bp); 4272 if (rc) 4273 goto alloc_mem_err; 4274 4275 rc = bnxt_alloc_cp_rings(bp); 4276 if (rc) 4277 goto alloc_mem_err; 4278 4279 bp->vnic_info[0].flags |= BNXT_VNIC_RSS_FLAG | BNXT_VNIC_MCAST_FLAG | 4280 BNXT_VNIC_UCAST_FLAG; 4281 rc = bnxt_alloc_vnic_attributes(bp); 4282 if (rc) 4283 goto alloc_mem_err; 4284 return 0; 4285 4286 alloc_mem_err: 4287 bnxt_free_mem(bp, true); 4288 return rc; 4289 } 4290 4291 static void bnxt_disable_int(struct bnxt *bp) 4292 { 4293 int i; 4294 4295 if (!bp->bnapi) 4296 return; 4297 4298 for (i = 0; i < bp->cp_nr_rings; i++) { 4299 struct bnxt_napi *bnapi = bp->bnapi[i]; 4300 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 4301 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 4302 4303 if (ring->fw_ring_id != INVALID_HW_RING_ID) 4304 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons); 4305 } 4306 } 4307 4308 static int bnxt_cp_num_to_irq_num(struct bnxt *bp, int n) 4309 { 4310 struct bnxt_napi *bnapi = bp->bnapi[n]; 4311 struct bnxt_cp_ring_info *cpr; 4312 4313 cpr = &bnapi->cp_ring; 4314 return cpr->cp_ring_struct.map_idx; 4315 } 4316 4317 static void bnxt_disable_int_sync(struct bnxt *bp) 4318 { 4319 int i; 4320 4321 if (!bp->irq_tbl) 4322 return; 4323 4324 atomic_inc(&bp->intr_sem); 4325 4326 bnxt_disable_int(bp); 4327 for (i = 0; i < bp->cp_nr_rings; i++) { 4328 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 4329 4330 synchronize_irq(bp->irq_tbl[map_idx].vector); 4331 } 4332 } 4333 4334 static void bnxt_enable_int(struct bnxt *bp) 4335 { 4336 int i; 4337 4338 atomic_set(&bp->intr_sem, 0); 4339 for (i = 0; i < bp->cp_nr_rings; i++) { 4340 struct bnxt_napi *bnapi = bp->bnapi[i]; 4341 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 4342 4343 bnxt_db_nq_arm(bp, &cpr->cp_db, cpr->cp_raw_cons); 4344 } 4345 } 4346 4347 void bnxt_hwrm_cmd_hdr_init(struct bnxt *bp, void *request, u16 req_type, 4348 u16 cmpl_ring, u16 target_id) 4349 { 4350 struct input *req = request; 4351 4352 req->req_type = cpu_to_le16(req_type); 4353 req->cmpl_ring = cpu_to_le16(cmpl_ring); 4354 req->target_id = cpu_to_le16(target_id); 4355 if (bnxt_kong_hwrm_message(bp, req)) 4356 req->resp_addr = cpu_to_le64(bp->hwrm_cmd_kong_resp_dma_addr); 4357 else 4358 req->resp_addr = cpu_to_le64(bp->hwrm_cmd_resp_dma_addr); 4359 } 4360 4361 static int bnxt_hwrm_to_stderr(u32 hwrm_err) 4362 { 4363 switch (hwrm_err) { 4364 case HWRM_ERR_CODE_SUCCESS: 4365 return 0; 4366 case HWRM_ERR_CODE_RESOURCE_LOCKED: 4367 return -EROFS; 4368 case HWRM_ERR_CODE_RESOURCE_ACCESS_DENIED: 4369 return -EACCES; 4370 case HWRM_ERR_CODE_RESOURCE_ALLOC_ERROR: 4371 return -ENOSPC; 4372 case HWRM_ERR_CODE_INVALID_PARAMS: 4373 case HWRM_ERR_CODE_INVALID_FLAGS: 4374 case HWRM_ERR_CODE_INVALID_ENABLES: 4375 case HWRM_ERR_CODE_UNSUPPORTED_TLV: 4376 case HWRM_ERR_CODE_UNSUPPORTED_OPTION_ERR: 4377 return -EINVAL; 4378 case HWRM_ERR_CODE_NO_BUFFER: 4379 return -ENOMEM; 4380 case HWRM_ERR_CODE_HOT_RESET_PROGRESS: 4381 case HWRM_ERR_CODE_BUSY: 4382 return -EAGAIN; 4383 case HWRM_ERR_CODE_CMD_NOT_SUPPORTED: 4384 return -EOPNOTSUPP; 4385 default: 4386 return -EIO; 4387 } 4388 } 4389 4390 static int bnxt_hwrm_do_send_msg(struct bnxt *bp, void *msg, u32 msg_len, 4391 int timeout, bool silent) 4392 { 4393 int i, intr_process, rc, tmo_count; 4394 struct input *req = msg; 4395 u32 *data = msg; 4396 u8 *valid; 4397 u16 cp_ring_id, len = 0; 4398 struct hwrm_err_output *resp = bp->hwrm_cmd_resp_addr; 4399 u16 max_req_len = BNXT_HWRM_MAX_REQ_LEN; 4400 struct hwrm_short_input short_input = {0}; 4401 u32 doorbell_offset = BNXT_GRCPF_REG_CHIMP_COMM_TRIGGER; 4402 u32 bar_offset = BNXT_GRCPF_REG_CHIMP_COMM; 4403 u16 dst = BNXT_HWRM_CHNL_CHIMP; 4404 4405 if (BNXT_NO_FW_ACCESS(bp) && 4406 le16_to_cpu(req->req_type) != HWRM_FUNC_RESET) 4407 return -EBUSY; 4408 4409 if (msg_len > BNXT_HWRM_MAX_REQ_LEN) { 4410 if (msg_len > bp->hwrm_max_ext_req_len || 4411 !bp->hwrm_short_cmd_req_addr) 4412 return -EINVAL; 4413 } 4414 4415 if (bnxt_hwrm_kong_chnl(bp, req)) { 4416 dst = BNXT_HWRM_CHNL_KONG; 4417 bar_offset = BNXT_GRCPF_REG_KONG_COMM; 4418 doorbell_offset = BNXT_GRCPF_REG_KONG_COMM_TRIGGER; 4419 resp = bp->hwrm_cmd_kong_resp_addr; 4420 } 4421 4422 memset(resp, 0, PAGE_SIZE); 4423 cp_ring_id = le16_to_cpu(req->cmpl_ring); 4424 intr_process = (cp_ring_id == INVALID_HW_RING_ID) ? 0 : 1; 4425 4426 req->seq_id = cpu_to_le16(bnxt_get_hwrm_seq_id(bp, dst)); 4427 /* currently supports only one outstanding message */ 4428 if (intr_process) 4429 bp->hwrm_intr_seq_id = le16_to_cpu(req->seq_id); 4430 4431 if ((bp->fw_cap & BNXT_FW_CAP_SHORT_CMD) || 4432 msg_len > BNXT_HWRM_MAX_REQ_LEN) { 4433 void *short_cmd_req = bp->hwrm_short_cmd_req_addr; 4434 u16 max_msg_len; 4435 4436 /* Set boundary for maximum extended request length for short 4437 * cmd format. If passed up from device use the max supported 4438 * internal req length. 4439 */ 4440 max_msg_len = bp->hwrm_max_ext_req_len; 4441 4442 memcpy(short_cmd_req, req, msg_len); 4443 if (msg_len < max_msg_len) 4444 memset(short_cmd_req + msg_len, 0, 4445 max_msg_len - msg_len); 4446 4447 short_input.req_type = req->req_type; 4448 short_input.signature = 4449 cpu_to_le16(SHORT_REQ_SIGNATURE_SHORT_CMD); 4450 short_input.size = cpu_to_le16(msg_len); 4451 short_input.req_addr = 4452 cpu_to_le64(bp->hwrm_short_cmd_req_dma_addr); 4453 4454 data = (u32 *)&short_input; 4455 msg_len = sizeof(short_input); 4456 4457 /* Sync memory write before updating doorbell */ 4458 wmb(); 4459 4460 max_req_len = BNXT_HWRM_SHORT_REQ_LEN; 4461 } 4462 4463 /* Write request msg to hwrm channel */ 4464 __iowrite32_copy(bp->bar0 + bar_offset, data, msg_len / 4); 4465 4466 for (i = msg_len; i < max_req_len; i += 4) 4467 writel(0, bp->bar0 + bar_offset + i); 4468 4469 /* Ring channel doorbell */ 4470 writel(1, bp->bar0 + doorbell_offset); 4471 4472 if (!pci_is_enabled(bp->pdev)) 4473 return 0; 4474 4475 if (!timeout) 4476 timeout = DFLT_HWRM_CMD_TIMEOUT; 4477 /* Limit timeout to an upper limit */ 4478 timeout = min(timeout, HWRM_CMD_MAX_TIMEOUT); 4479 /* convert timeout to usec */ 4480 timeout *= 1000; 4481 4482 i = 0; 4483 /* Short timeout for the first few iterations: 4484 * number of loops = number of loops for short timeout + 4485 * number of loops for standard timeout. 4486 */ 4487 tmo_count = HWRM_SHORT_TIMEOUT_COUNTER; 4488 timeout = timeout - HWRM_SHORT_MIN_TIMEOUT * HWRM_SHORT_TIMEOUT_COUNTER; 4489 tmo_count += DIV_ROUND_UP(timeout, HWRM_MIN_TIMEOUT); 4490 4491 if (intr_process) { 4492 u16 seq_id = bp->hwrm_intr_seq_id; 4493 4494 /* Wait until hwrm response cmpl interrupt is processed */ 4495 while (bp->hwrm_intr_seq_id != (u16)~seq_id && 4496 i++ < tmo_count) { 4497 /* Abort the wait for completion if the FW health 4498 * check has failed. 4499 */ 4500 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) 4501 return -EBUSY; 4502 /* on first few passes, just barely sleep */ 4503 if (i < HWRM_SHORT_TIMEOUT_COUNTER) 4504 usleep_range(HWRM_SHORT_MIN_TIMEOUT, 4505 HWRM_SHORT_MAX_TIMEOUT); 4506 else 4507 usleep_range(HWRM_MIN_TIMEOUT, 4508 HWRM_MAX_TIMEOUT); 4509 } 4510 4511 if (bp->hwrm_intr_seq_id != (u16)~seq_id) { 4512 if (!silent) 4513 netdev_err(bp->dev, "Resp cmpl intr err msg: 0x%x\n", 4514 le16_to_cpu(req->req_type)); 4515 return -EBUSY; 4516 } 4517 len = le16_to_cpu(resp->resp_len); 4518 valid = ((u8 *)resp) + len - 1; 4519 } else { 4520 int j; 4521 4522 /* Check if response len is updated */ 4523 for (i = 0; i < tmo_count; i++) { 4524 /* Abort the wait for completion if the FW health 4525 * check has failed. 4526 */ 4527 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) 4528 return -EBUSY; 4529 len = le16_to_cpu(resp->resp_len); 4530 if (len) 4531 break; 4532 /* on first few passes, just barely sleep */ 4533 if (i < HWRM_SHORT_TIMEOUT_COUNTER) 4534 usleep_range(HWRM_SHORT_MIN_TIMEOUT, 4535 HWRM_SHORT_MAX_TIMEOUT); 4536 else 4537 usleep_range(HWRM_MIN_TIMEOUT, 4538 HWRM_MAX_TIMEOUT); 4539 } 4540 4541 if (i >= tmo_count) { 4542 if (!silent) 4543 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d\n", 4544 HWRM_TOTAL_TIMEOUT(i), 4545 le16_to_cpu(req->req_type), 4546 le16_to_cpu(req->seq_id), len); 4547 return -EBUSY; 4548 } 4549 4550 /* Last byte of resp contains valid bit */ 4551 valid = ((u8 *)resp) + len - 1; 4552 for (j = 0; j < HWRM_VALID_BIT_DELAY_USEC; j++) { 4553 /* make sure we read from updated DMA memory */ 4554 dma_rmb(); 4555 if (*valid) 4556 break; 4557 usleep_range(1, 5); 4558 } 4559 4560 if (j >= HWRM_VALID_BIT_DELAY_USEC) { 4561 if (!silent) 4562 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d v:%d\n", 4563 HWRM_TOTAL_TIMEOUT(i), 4564 le16_to_cpu(req->req_type), 4565 le16_to_cpu(req->seq_id), len, 4566 *valid); 4567 return -EBUSY; 4568 } 4569 } 4570 4571 /* Zero valid bit for compatibility. Valid bit in an older spec 4572 * may become a new field in a newer spec. We must make sure that 4573 * a new field not implemented by old spec will read zero. 4574 */ 4575 *valid = 0; 4576 rc = le16_to_cpu(resp->error_code); 4577 if (rc && !silent) 4578 netdev_err(bp->dev, "hwrm req_type 0x%x seq id 0x%x error 0x%x\n", 4579 le16_to_cpu(resp->req_type), 4580 le16_to_cpu(resp->seq_id), rc); 4581 return bnxt_hwrm_to_stderr(rc); 4582 } 4583 4584 int _hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout) 4585 { 4586 return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, false); 4587 } 4588 4589 int _hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len, 4590 int timeout) 4591 { 4592 return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true); 4593 } 4594 4595 int hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout) 4596 { 4597 int rc; 4598 4599 mutex_lock(&bp->hwrm_cmd_lock); 4600 rc = _hwrm_send_message(bp, msg, msg_len, timeout); 4601 mutex_unlock(&bp->hwrm_cmd_lock); 4602 return rc; 4603 } 4604 4605 int hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len, 4606 int timeout) 4607 { 4608 int rc; 4609 4610 mutex_lock(&bp->hwrm_cmd_lock); 4611 rc = bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true); 4612 mutex_unlock(&bp->hwrm_cmd_lock); 4613 return rc; 4614 } 4615 4616 int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp, unsigned long *bmap, int bmap_size, 4617 bool async_only) 4618 { 4619 struct hwrm_func_drv_rgtr_output *resp = bp->hwrm_cmd_resp_addr; 4620 struct hwrm_func_drv_rgtr_input req = {0}; 4621 DECLARE_BITMAP(async_events_bmap, 256); 4622 u32 *events = (u32 *)async_events_bmap; 4623 u32 flags; 4624 int rc, i; 4625 4626 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1); 4627 4628 req.enables = 4629 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE | 4630 FUNC_DRV_RGTR_REQ_ENABLES_VER | 4631 FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD); 4632 4633 req.os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX); 4634 flags = FUNC_DRV_RGTR_REQ_FLAGS_16BIT_VER_MODE; 4635 if (bp->fw_cap & BNXT_FW_CAP_HOT_RESET) 4636 flags |= FUNC_DRV_RGTR_REQ_FLAGS_HOT_RESET_SUPPORT; 4637 if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) 4638 flags |= FUNC_DRV_RGTR_REQ_FLAGS_ERROR_RECOVERY_SUPPORT | 4639 FUNC_DRV_RGTR_REQ_FLAGS_MASTER_SUPPORT; 4640 req.flags = cpu_to_le32(flags); 4641 req.ver_maj_8b = DRV_VER_MAJ; 4642 req.ver_min_8b = DRV_VER_MIN; 4643 req.ver_upd_8b = DRV_VER_UPD; 4644 req.ver_maj = cpu_to_le16(DRV_VER_MAJ); 4645 req.ver_min = cpu_to_le16(DRV_VER_MIN); 4646 req.ver_upd = cpu_to_le16(DRV_VER_UPD); 4647 4648 if (BNXT_PF(bp)) { 4649 u32 data[8]; 4650 int i; 4651 4652 memset(data, 0, sizeof(data)); 4653 for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) { 4654 u16 cmd = bnxt_vf_req_snif[i]; 4655 unsigned int bit, idx; 4656 4657 idx = cmd / 32; 4658 bit = cmd % 32; 4659 data[idx] |= 1 << bit; 4660 } 4661 4662 for (i = 0; i < 8; i++) 4663 req.vf_req_fwd[i] = cpu_to_le32(data[i]); 4664 4665 req.enables |= 4666 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD); 4667 } 4668 4669 if (bp->fw_cap & BNXT_FW_CAP_OVS_64BIT_HANDLE) 4670 req.flags |= cpu_to_le32( 4671 FUNC_DRV_RGTR_REQ_FLAGS_FLOW_HANDLE_64BIT_MODE); 4672 4673 memset(async_events_bmap, 0, sizeof(async_events_bmap)); 4674 for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++) { 4675 u16 event_id = bnxt_async_events_arr[i]; 4676 4677 if (event_id == ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY && 4678 !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) 4679 continue; 4680 __set_bit(bnxt_async_events_arr[i], async_events_bmap); 4681 } 4682 if (bmap && bmap_size) { 4683 for (i = 0; i < bmap_size; i++) { 4684 if (test_bit(i, bmap)) 4685 __set_bit(i, async_events_bmap); 4686 } 4687 } 4688 for (i = 0; i < 8; i++) 4689 req.async_event_fwd[i] |= cpu_to_le32(events[i]); 4690 4691 if (async_only) 4692 req.enables = 4693 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD); 4694 4695 mutex_lock(&bp->hwrm_cmd_lock); 4696 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4697 if (!rc) { 4698 set_bit(BNXT_STATE_DRV_REGISTERED, &bp->state); 4699 if (resp->flags & 4700 cpu_to_le32(FUNC_DRV_RGTR_RESP_FLAGS_IF_CHANGE_SUPPORTED)) 4701 bp->fw_cap |= BNXT_FW_CAP_IF_CHANGE; 4702 } 4703 mutex_unlock(&bp->hwrm_cmd_lock); 4704 return rc; 4705 } 4706 4707 static int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp) 4708 { 4709 struct hwrm_func_drv_unrgtr_input req = {0}; 4710 4711 if (!test_and_clear_bit(BNXT_STATE_DRV_REGISTERED, &bp->state)) 4712 return 0; 4713 4714 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_UNRGTR, -1, -1); 4715 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4716 } 4717 4718 static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type) 4719 { 4720 u32 rc = 0; 4721 struct hwrm_tunnel_dst_port_free_input req = {0}; 4722 4723 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_FREE, -1, -1); 4724 req.tunnel_type = tunnel_type; 4725 4726 switch (tunnel_type) { 4727 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN: 4728 req.tunnel_dst_port_id = cpu_to_le16(bp->vxlan_fw_dst_port_id); 4729 bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID; 4730 break; 4731 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE: 4732 req.tunnel_dst_port_id = cpu_to_le16(bp->nge_fw_dst_port_id); 4733 bp->nge_fw_dst_port_id = INVALID_HW_RING_ID; 4734 break; 4735 default: 4736 break; 4737 } 4738 4739 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4740 if (rc) 4741 netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n", 4742 rc); 4743 return rc; 4744 } 4745 4746 static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port, 4747 u8 tunnel_type) 4748 { 4749 u32 rc = 0; 4750 struct hwrm_tunnel_dst_port_alloc_input req = {0}; 4751 struct hwrm_tunnel_dst_port_alloc_output *resp = bp->hwrm_cmd_resp_addr; 4752 4753 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_ALLOC, -1, -1); 4754 4755 req.tunnel_type = tunnel_type; 4756 req.tunnel_dst_port_val = port; 4757 4758 mutex_lock(&bp->hwrm_cmd_lock); 4759 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4760 if (rc) { 4761 netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n", 4762 rc); 4763 goto err_out; 4764 } 4765 4766 switch (tunnel_type) { 4767 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN: 4768 bp->vxlan_fw_dst_port_id = 4769 le16_to_cpu(resp->tunnel_dst_port_id); 4770 break; 4771 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE: 4772 bp->nge_fw_dst_port_id = le16_to_cpu(resp->tunnel_dst_port_id); 4773 break; 4774 default: 4775 break; 4776 } 4777 4778 err_out: 4779 mutex_unlock(&bp->hwrm_cmd_lock); 4780 return rc; 4781 } 4782 4783 static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id) 4784 { 4785 struct hwrm_cfa_l2_set_rx_mask_input req = {0}; 4786 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 4787 4788 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_SET_RX_MASK, -1, -1); 4789 req.vnic_id = cpu_to_le32(vnic->fw_vnic_id); 4790 4791 req.num_mc_entries = cpu_to_le32(vnic->mc_list_count); 4792 req.mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping); 4793 req.mask = cpu_to_le32(vnic->rx_mask); 4794 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4795 } 4796 4797 #ifdef CONFIG_RFS_ACCEL 4798 static int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp, 4799 struct bnxt_ntuple_filter *fltr) 4800 { 4801 struct hwrm_cfa_ntuple_filter_free_input req = {0}; 4802 4803 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_FREE, -1, -1); 4804 req.ntuple_filter_id = fltr->filter_id; 4805 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4806 } 4807 4808 #define BNXT_NTP_FLTR_FLAGS \ 4809 (CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID | \ 4810 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE | \ 4811 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR | \ 4812 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE | \ 4813 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR | \ 4814 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK | \ 4815 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR | \ 4816 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK | \ 4817 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL | \ 4818 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT | \ 4819 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK | \ 4820 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT | \ 4821 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK | \ 4822 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID) 4823 4824 #define BNXT_NTP_TUNNEL_FLTR_FLAG \ 4825 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE 4826 4827 static int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp, 4828 struct bnxt_ntuple_filter *fltr) 4829 { 4830 struct hwrm_cfa_ntuple_filter_alloc_input req = {0}; 4831 struct hwrm_cfa_ntuple_filter_alloc_output *resp; 4832 struct flow_keys *keys = &fltr->fkeys; 4833 struct bnxt_vnic_info *vnic; 4834 u32 flags = 0; 4835 int rc = 0; 4836 4837 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_ALLOC, -1, -1); 4838 req.l2_filter_id = bp->vnic_info[0].fw_l2_filter_id[fltr->l2_fltr_idx]; 4839 4840 if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2) { 4841 flags = CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DEST_RFS_RING_IDX; 4842 req.dst_id = cpu_to_le16(fltr->rxq); 4843 } else { 4844 vnic = &bp->vnic_info[fltr->rxq + 1]; 4845 req.dst_id = cpu_to_le16(vnic->fw_vnic_id); 4846 } 4847 req.flags = cpu_to_le32(flags); 4848 req.enables = cpu_to_le32(BNXT_NTP_FLTR_FLAGS); 4849 4850 req.ethertype = htons(ETH_P_IP); 4851 memcpy(req.src_macaddr, fltr->src_mac_addr, ETH_ALEN); 4852 req.ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4; 4853 req.ip_protocol = keys->basic.ip_proto; 4854 4855 if (keys->basic.n_proto == htons(ETH_P_IPV6)) { 4856 int i; 4857 4858 req.ethertype = htons(ETH_P_IPV6); 4859 req.ip_addr_type = 4860 CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6; 4861 *(struct in6_addr *)&req.src_ipaddr[0] = 4862 keys->addrs.v6addrs.src; 4863 *(struct in6_addr *)&req.dst_ipaddr[0] = 4864 keys->addrs.v6addrs.dst; 4865 for (i = 0; i < 4; i++) { 4866 req.src_ipaddr_mask[i] = cpu_to_be32(0xffffffff); 4867 req.dst_ipaddr_mask[i] = cpu_to_be32(0xffffffff); 4868 } 4869 } else { 4870 req.src_ipaddr[0] = keys->addrs.v4addrs.src; 4871 req.src_ipaddr_mask[0] = cpu_to_be32(0xffffffff); 4872 req.dst_ipaddr[0] = keys->addrs.v4addrs.dst; 4873 req.dst_ipaddr_mask[0] = cpu_to_be32(0xffffffff); 4874 } 4875 if (keys->control.flags & FLOW_DIS_ENCAPSULATION) { 4876 req.enables |= cpu_to_le32(BNXT_NTP_TUNNEL_FLTR_FLAG); 4877 req.tunnel_type = 4878 CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL; 4879 } 4880 4881 req.src_port = keys->ports.src; 4882 req.src_port_mask = cpu_to_be16(0xffff); 4883 req.dst_port = keys->ports.dst; 4884 req.dst_port_mask = cpu_to_be16(0xffff); 4885 4886 mutex_lock(&bp->hwrm_cmd_lock); 4887 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4888 if (!rc) { 4889 resp = bnxt_get_hwrm_resp_addr(bp, &req); 4890 fltr->filter_id = resp->ntuple_filter_id; 4891 } 4892 mutex_unlock(&bp->hwrm_cmd_lock); 4893 return rc; 4894 } 4895 #endif 4896 4897 static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx, 4898 u8 *mac_addr) 4899 { 4900 u32 rc = 0; 4901 struct hwrm_cfa_l2_filter_alloc_input req = {0}; 4902 struct hwrm_cfa_l2_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr; 4903 4904 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_ALLOC, -1, -1); 4905 req.flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX); 4906 if (!BNXT_CHIP_TYPE_NITRO_A0(bp)) 4907 req.flags |= 4908 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST); 4909 req.dst_id = cpu_to_le16(bp->vnic_info[vnic_id].fw_vnic_id); 4910 req.enables = 4911 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR | 4912 CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID | 4913 CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK); 4914 memcpy(req.l2_addr, mac_addr, ETH_ALEN); 4915 req.l2_addr_mask[0] = 0xff; 4916 req.l2_addr_mask[1] = 0xff; 4917 req.l2_addr_mask[2] = 0xff; 4918 req.l2_addr_mask[3] = 0xff; 4919 req.l2_addr_mask[4] = 0xff; 4920 req.l2_addr_mask[5] = 0xff; 4921 4922 mutex_lock(&bp->hwrm_cmd_lock); 4923 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4924 if (!rc) 4925 bp->vnic_info[vnic_id].fw_l2_filter_id[idx] = 4926 resp->l2_filter_id; 4927 mutex_unlock(&bp->hwrm_cmd_lock); 4928 return rc; 4929 } 4930 4931 static int bnxt_hwrm_clear_vnic_filter(struct bnxt *bp) 4932 { 4933 u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */ 4934 int rc = 0; 4935 4936 /* Any associated ntuple filters will also be cleared by firmware. */ 4937 mutex_lock(&bp->hwrm_cmd_lock); 4938 for (i = 0; i < num_of_vnics; i++) { 4939 struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; 4940 4941 for (j = 0; j < vnic->uc_filter_count; j++) { 4942 struct hwrm_cfa_l2_filter_free_input req = {0}; 4943 4944 bnxt_hwrm_cmd_hdr_init(bp, &req, 4945 HWRM_CFA_L2_FILTER_FREE, -1, -1); 4946 4947 req.l2_filter_id = vnic->fw_l2_filter_id[j]; 4948 4949 rc = _hwrm_send_message(bp, &req, sizeof(req), 4950 HWRM_CMD_TIMEOUT); 4951 } 4952 vnic->uc_filter_count = 0; 4953 } 4954 mutex_unlock(&bp->hwrm_cmd_lock); 4955 4956 return rc; 4957 } 4958 4959 static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags) 4960 { 4961 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 4962 u16 max_aggs = VNIC_TPA_CFG_REQ_MAX_AGGS_MAX; 4963 struct hwrm_vnic_tpa_cfg_input req = {0}; 4964 4965 if (vnic->fw_vnic_id == INVALID_HW_RING_ID) 4966 return 0; 4967 4968 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_TPA_CFG, -1, -1); 4969 4970 if (tpa_flags) { 4971 u16 mss = bp->dev->mtu - 40; 4972 u32 nsegs, n, segs = 0, flags; 4973 4974 flags = VNIC_TPA_CFG_REQ_FLAGS_TPA | 4975 VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA | 4976 VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE | 4977 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN | 4978 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ; 4979 if (tpa_flags & BNXT_FLAG_GRO) 4980 flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO; 4981 4982 req.flags = cpu_to_le32(flags); 4983 4984 req.enables = 4985 cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS | 4986 VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS | 4987 VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN); 4988 4989 /* Number of segs are log2 units, and first packet is not 4990 * included as part of this units. 4991 */ 4992 if (mss <= BNXT_RX_PAGE_SIZE) { 4993 n = BNXT_RX_PAGE_SIZE / mss; 4994 nsegs = (MAX_SKB_FRAGS - 1) * n; 4995 } else { 4996 n = mss / BNXT_RX_PAGE_SIZE; 4997 if (mss & (BNXT_RX_PAGE_SIZE - 1)) 4998 n++; 4999 nsegs = (MAX_SKB_FRAGS - n) / n; 5000 } 5001 5002 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5003 segs = MAX_TPA_SEGS_P5; 5004 max_aggs = bp->max_tpa; 5005 } else { 5006 segs = ilog2(nsegs); 5007 } 5008 req.max_agg_segs = cpu_to_le16(segs); 5009 req.max_aggs = cpu_to_le16(max_aggs); 5010 5011 req.min_agg_len = cpu_to_le32(512); 5012 } 5013 req.vnic_id = cpu_to_le16(vnic->fw_vnic_id); 5014 5015 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5016 } 5017 5018 static u16 bnxt_cp_ring_from_grp(struct bnxt *bp, struct bnxt_ring_struct *ring) 5019 { 5020 struct bnxt_ring_grp_info *grp_info; 5021 5022 grp_info = &bp->grp_info[ring->grp_idx]; 5023 return grp_info->cp_fw_ring_id; 5024 } 5025 5026 static u16 bnxt_cp_ring_for_rx(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) 5027 { 5028 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5029 struct bnxt_napi *bnapi = rxr->bnapi; 5030 struct bnxt_cp_ring_info *cpr; 5031 5032 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_RX_HDL]; 5033 return cpr->cp_ring_struct.fw_ring_id; 5034 } else { 5035 return bnxt_cp_ring_from_grp(bp, &rxr->rx_ring_struct); 5036 } 5037 } 5038 5039 static u16 bnxt_cp_ring_for_tx(struct bnxt *bp, struct bnxt_tx_ring_info *txr) 5040 { 5041 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5042 struct bnxt_napi *bnapi = txr->bnapi; 5043 struct bnxt_cp_ring_info *cpr; 5044 5045 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_TX_HDL]; 5046 return cpr->cp_ring_struct.fw_ring_id; 5047 } else { 5048 return bnxt_cp_ring_from_grp(bp, &txr->tx_ring_struct); 5049 } 5050 } 5051 5052 static int bnxt_alloc_rss_indir_tbl(struct bnxt *bp) 5053 { 5054 int entries; 5055 5056 if (bp->flags & BNXT_FLAG_CHIP_P5) 5057 entries = BNXT_MAX_RSS_TABLE_ENTRIES_P5; 5058 else 5059 entries = HW_HASH_INDEX_SIZE; 5060 5061 bp->rss_indir_tbl_entries = entries; 5062 bp->rss_indir_tbl = kmalloc_array(entries, sizeof(*bp->rss_indir_tbl), 5063 GFP_KERNEL); 5064 if (!bp->rss_indir_tbl) 5065 return -ENOMEM; 5066 return 0; 5067 } 5068 5069 static void bnxt_set_dflt_rss_indir_tbl(struct bnxt *bp) 5070 { 5071 u16 max_rings, max_entries, pad, i; 5072 5073 if (!bp->rx_nr_rings) 5074 return; 5075 5076 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 5077 max_rings = bp->rx_nr_rings - 1; 5078 else 5079 max_rings = bp->rx_nr_rings; 5080 5081 max_entries = bnxt_get_rxfh_indir_size(bp->dev); 5082 5083 for (i = 0; i < max_entries; i++) 5084 bp->rss_indir_tbl[i] = ethtool_rxfh_indir_default(i, max_rings); 5085 5086 pad = bp->rss_indir_tbl_entries - max_entries; 5087 if (pad) 5088 memset(&bp->rss_indir_tbl[i], 0, pad * sizeof(u16)); 5089 } 5090 5091 static u16 bnxt_get_max_rss_ring(struct bnxt *bp) 5092 { 5093 u16 i, tbl_size, max_ring = 0; 5094 5095 if (!bp->rss_indir_tbl) 5096 return 0; 5097 5098 tbl_size = bnxt_get_rxfh_indir_size(bp->dev); 5099 for (i = 0; i < tbl_size; i++) 5100 max_ring = max(max_ring, bp->rss_indir_tbl[i]); 5101 return max_ring; 5102 } 5103 5104 int bnxt_get_nr_rss_ctxs(struct bnxt *bp, int rx_rings) 5105 { 5106 if (bp->flags & BNXT_FLAG_CHIP_P5) 5107 return DIV_ROUND_UP(rx_rings, BNXT_RSS_TABLE_ENTRIES_P5); 5108 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 5109 return 2; 5110 return 1; 5111 } 5112 5113 static void __bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic) 5114 { 5115 bool no_rss = !(vnic->flags & BNXT_VNIC_RSS_FLAG); 5116 u16 i, j; 5117 5118 /* Fill the RSS indirection table with ring group ids */ 5119 for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++) { 5120 if (!no_rss) 5121 j = bp->rss_indir_tbl[i]; 5122 vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]); 5123 } 5124 } 5125 5126 static void __bnxt_fill_hw_rss_tbl_p5(struct bnxt *bp, 5127 struct bnxt_vnic_info *vnic) 5128 { 5129 __le16 *ring_tbl = vnic->rss_table; 5130 struct bnxt_rx_ring_info *rxr; 5131 u16 tbl_size, i; 5132 5133 tbl_size = bnxt_get_rxfh_indir_size(bp->dev); 5134 5135 for (i = 0; i < tbl_size; i++) { 5136 u16 ring_id, j; 5137 5138 j = bp->rss_indir_tbl[i]; 5139 rxr = &bp->rx_ring[j]; 5140 5141 ring_id = rxr->rx_ring_struct.fw_ring_id; 5142 *ring_tbl++ = cpu_to_le16(ring_id); 5143 ring_id = bnxt_cp_ring_for_rx(bp, rxr); 5144 *ring_tbl++ = cpu_to_le16(ring_id); 5145 } 5146 } 5147 5148 static void bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic) 5149 { 5150 if (bp->flags & BNXT_FLAG_CHIP_P5) 5151 __bnxt_fill_hw_rss_tbl_p5(bp, vnic); 5152 else 5153 __bnxt_fill_hw_rss_tbl(bp, vnic); 5154 } 5155 5156 static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss) 5157 { 5158 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5159 struct hwrm_vnic_rss_cfg_input req = {0}; 5160 5161 if ((bp->flags & BNXT_FLAG_CHIP_P5) || 5162 vnic->fw_rss_cos_lb_ctx[0] == INVALID_HW_RING_ID) 5163 return 0; 5164 5165 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1); 5166 if (set_rss) { 5167 bnxt_fill_hw_rss_tbl(bp, vnic); 5168 req.hash_type = cpu_to_le32(bp->rss_hash_cfg); 5169 req.hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT; 5170 req.ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr); 5171 req.hash_key_tbl_addr = 5172 cpu_to_le64(vnic->rss_hash_key_dma_addr); 5173 } 5174 req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]); 5175 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5176 } 5177 5178 static int bnxt_hwrm_vnic_set_rss_p5(struct bnxt *bp, u16 vnic_id, bool set_rss) 5179 { 5180 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5181 struct hwrm_vnic_rss_cfg_input req = {0}; 5182 dma_addr_t ring_tbl_map; 5183 u32 i, nr_ctxs; 5184 5185 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1); 5186 req.vnic_id = cpu_to_le16(vnic->fw_vnic_id); 5187 if (!set_rss) { 5188 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5189 return 0; 5190 } 5191 bnxt_fill_hw_rss_tbl(bp, vnic); 5192 req.hash_type = cpu_to_le32(bp->rss_hash_cfg); 5193 req.hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT; 5194 req.hash_key_tbl_addr = cpu_to_le64(vnic->rss_hash_key_dma_addr); 5195 ring_tbl_map = vnic->rss_table_dma_addr; 5196 nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings); 5197 for (i = 0; i < nr_ctxs; ring_tbl_map += BNXT_RSS_TABLE_SIZE_P5, i++) { 5198 int rc; 5199 5200 req.ring_grp_tbl_addr = cpu_to_le64(ring_tbl_map); 5201 req.ring_table_pair_index = i; 5202 req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[i]); 5203 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5204 if (rc) 5205 return rc; 5206 } 5207 return 0; 5208 } 5209 5210 static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id) 5211 { 5212 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5213 struct hwrm_vnic_plcmodes_cfg_input req = {0}; 5214 5215 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_PLCMODES_CFG, -1, -1); 5216 req.flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT | 5217 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 | 5218 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6); 5219 req.enables = 5220 cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID | 5221 VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID); 5222 /* thresholds not implemented in firmware yet */ 5223 req.jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh); 5224 req.hds_threshold = cpu_to_le16(bp->rx_copy_thresh); 5225 req.vnic_id = cpu_to_le32(vnic->fw_vnic_id); 5226 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5227 } 5228 5229 static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id, 5230 u16 ctx_idx) 5231 { 5232 struct hwrm_vnic_rss_cos_lb_ctx_free_input req = {0}; 5233 5234 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_FREE, -1, -1); 5235 req.rss_cos_lb_ctx_id = 5236 cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx]); 5237 5238 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5239 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID; 5240 } 5241 5242 static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp) 5243 { 5244 int i, j; 5245 5246 for (i = 0; i < bp->nr_vnics; i++) { 5247 struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; 5248 5249 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) { 5250 if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID) 5251 bnxt_hwrm_vnic_ctx_free_one(bp, i, j); 5252 } 5253 } 5254 bp->rsscos_nr_ctxs = 0; 5255 } 5256 5257 static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id, u16 ctx_idx) 5258 { 5259 int rc; 5260 struct hwrm_vnic_rss_cos_lb_ctx_alloc_input req = {0}; 5261 struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp = 5262 bp->hwrm_cmd_resp_addr; 5263 5264 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC, -1, 5265 -1); 5266 5267 mutex_lock(&bp->hwrm_cmd_lock); 5268 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5269 if (!rc) 5270 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = 5271 le16_to_cpu(resp->rss_cos_lb_ctx_id); 5272 mutex_unlock(&bp->hwrm_cmd_lock); 5273 5274 return rc; 5275 } 5276 5277 static u32 bnxt_get_roce_vnic_mode(struct bnxt *bp) 5278 { 5279 if (bp->flags & BNXT_FLAG_ROCE_MIRROR_CAP) 5280 return VNIC_CFG_REQ_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE; 5281 return VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE; 5282 } 5283 5284 int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id) 5285 { 5286 unsigned int ring = 0, grp_idx; 5287 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5288 struct hwrm_vnic_cfg_input req = {0}; 5289 u16 def_vlan = 0; 5290 5291 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_CFG, -1, -1); 5292 5293 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5294 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0]; 5295 5296 req.default_rx_ring_id = 5297 cpu_to_le16(rxr->rx_ring_struct.fw_ring_id); 5298 req.default_cmpl_ring_id = 5299 cpu_to_le16(bnxt_cp_ring_for_rx(bp, rxr)); 5300 req.enables = 5301 cpu_to_le32(VNIC_CFG_REQ_ENABLES_DEFAULT_RX_RING_ID | 5302 VNIC_CFG_REQ_ENABLES_DEFAULT_CMPL_RING_ID); 5303 goto vnic_mru; 5304 } 5305 req.enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP); 5306 /* Only RSS support for now TBD: COS & LB */ 5307 if (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID) { 5308 req.rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]); 5309 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE | 5310 VNIC_CFG_REQ_ENABLES_MRU); 5311 } else if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) { 5312 req.rss_rule = 5313 cpu_to_le16(bp->vnic_info[0].fw_rss_cos_lb_ctx[0]); 5314 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE | 5315 VNIC_CFG_REQ_ENABLES_MRU); 5316 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE); 5317 } else { 5318 req.rss_rule = cpu_to_le16(0xffff); 5319 } 5320 5321 if (BNXT_CHIP_TYPE_NITRO_A0(bp) && 5322 (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID)) { 5323 req.cos_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[1]); 5324 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_COS_RULE); 5325 } else { 5326 req.cos_rule = cpu_to_le16(0xffff); 5327 } 5328 5329 if (vnic->flags & BNXT_VNIC_RSS_FLAG) 5330 ring = 0; 5331 else if (vnic->flags & BNXT_VNIC_RFS_FLAG) 5332 ring = vnic_id - 1; 5333 else if ((vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp)) 5334 ring = bp->rx_nr_rings - 1; 5335 5336 grp_idx = bp->rx_ring[ring].bnapi->index; 5337 req.dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id); 5338 req.lb_rule = cpu_to_le16(0xffff); 5339 vnic_mru: 5340 req.mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + VLAN_HLEN); 5341 5342 req.vnic_id = cpu_to_le16(vnic->fw_vnic_id); 5343 #ifdef CONFIG_BNXT_SRIOV 5344 if (BNXT_VF(bp)) 5345 def_vlan = bp->vf.vlan; 5346 #endif 5347 if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan) 5348 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE); 5349 if (!vnic_id && bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP)) 5350 req.flags |= cpu_to_le32(bnxt_get_roce_vnic_mode(bp)); 5351 5352 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5353 } 5354 5355 static void bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id) 5356 { 5357 if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) { 5358 struct hwrm_vnic_free_input req = {0}; 5359 5360 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_FREE, -1, -1); 5361 req.vnic_id = 5362 cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id); 5363 5364 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5365 bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID; 5366 } 5367 } 5368 5369 static void bnxt_hwrm_vnic_free(struct bnxt *bp) 5370 { 5371 u16 i; 5372 5373 for (i = 0; i < bp->nr_vnics; i++) 5374 bnxt_hwrm_vnic_free_one(bp, i); 5375 } 5376 5377 static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id, 5378 unsigned int start_rx_ring_idx, 5379 unsigned int nr_rings) 5380 { 5381 int rc = 0; 5382 unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings; 5383 struct hwrm_vnic_alloc_input req = {0}; 5384 struct hwrm_vnic_alloc_output *resp = bp->hwrm_cmd_resp_addr; 5385 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5386 5387 if (bp->flags & BNXT_FLAG_CHIP_P5) 5388 goto vnic_no_ring_grps; 5389 5390 /* map ring groups to this vnic */ 5391 for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) { 5392 grp_idx = bp->rx_ring[i].bnapi->index; 5393 if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) { 5394 netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n", 5395 j, nr_rings); 5396 break; 5397 } 5398 vnic->fw_grp_ids[j] = bp->grp_info[grp_idx].fw_grp_id; 5399 } 5400 5401 vnic_no_ring_grps: 5402 for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++) 5403 vnic->fw_rss_cos_lb_ctx[i] = INVALID_HW_RING_ID; 5404 if (vnic_id == 0) 5405 req.flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT); 5406 5407 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_ALLOC, -1, -1); 5408 5409 mutex_lock(&bp->hwrm_cmd_lock); 5410 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5411 if (!rc) 5412 vnic->fw_vnic_id = le32_to_cpu(resp->vnic_id); 5413 mutex_unlock(&bp->hwrm_cmd_lock); 5414 return rc; 5415 } 5416 5417 static int bnxt_hwrm_vnic_qcaps(struct bnxt *bp) 5418 { 5419 struct hwrm_vnic_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 5420 struct hwrm_vnic_qcaps_input req = {0}; 5421 int rc; 5422 5423 bp->hw_ring_stats_size = sizeof(struct ctx_hw_stats); 5424 bp->flags &= ~(BNXT_FLAG_NEW_RSS_CAP | BNXT_FLAG_ROCE_MIRROR_CAP); 5425 if (bp->hwrm_spec_code < 0x10600) 5426 return 0; 5427 5428 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_QCAPS, -1, -1); 5429 mutex_lock(&bp->hwrm_cmd_lock); 5430 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5431 if (!rc) { 5432 u32 flags = le32_to_cpu(resp->flags); 5433 5434 if (!(bp->flags & BNXT_FLAG_CHIP_P5) && 5435 (flags & VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP)) 5436 bp->flags |= BNXT_FLAG_NEW_RSS_CAP; 5437 if (flags & 5438 VNIC_QCAPS_RESP_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_CAP) 5439 bp->flags |= BNXT_FLAG_ROCE_MIRROR_CAP; 5440 5441 /* Older P5 fw before EXT_HW_STATS support did not set 5442 * VLAN_STRIP_CAP properly. 5443 */ 5444 if ((flags & VNIC_QCAPS_RESP_FLAGS_VLAN_STRIP_CAP) || 5445 (BNXT_CHIP_P5_THOR(bp) && 5446 !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))) 5447 bp->fw_cap |= BNXT_FW_CAP_VLAN_RX_STRIP; 5448 bp->max_tpa_v2 = le16_to_cpu(resp->max_aggs_supported); 5449 if (bp->max_tpa_v2) { 5450 if (BNXT_CHIP_P5_THOR(bp)) 5451 bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P5; 5452 else 5453 bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P5_SR2; 5454 } 5455 } 5456 mutex_unlock(&bp->hwrm_cmd_lock); 5457 return rc; 5458 } 5459 5460 static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp) 5461 { 5462 u16 i; 5463 u32 rc = 0; 5464 5465 if (bp->flags & BNXT_FLAG_CHIP_P5) 5466 return 0; 5467 5468 mutex_lock(&bp->hwrm_cmd_lock); 5469 for (i = 0; i < bp->rx_nr_rings; i++) { 5470 struct hwrm_ring_grp_alloc_input req = {0}; 5471 struct hwrm_ring_grp_alloc_output *resp = 5472 bp->hwrm_cmd_resp_addr; 5473 unsigned int grp_idx = bp->rx_ring[i].bnapi->index; 5474 5475 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_ALLOC, -1, -1); 5476 5477 req.cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id); 5478 req.rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id); 5479 req.ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id); 5480 req.sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx); 5481 5482 rc = _hwrm_send_message(bp, &req, sizeof(req), 5483 HWRM_CMD_TIMEOUT); 5484 if (rc) 5485 break; 5486 5487 bp->grp_info[grp_idx].fw_grp_id = 5488 le32_to_cpu(resp->ring_group_id); 5489 } 5490 mutex_unlock(&bp->hwrm_cmd_lock); 5491 return rc; 5492 } 5493 5494 static void bnxt_hwrm_ring_grp_free(struct bnxt *bp) 5495 { 5496 u16 i; 5497 struct hwrm_ring_grp_free_input req = {0}; 5498 5499 if (!bp->grp_info || (bp->flags & BNXT_FLAG_CHIP_P5)) 5500 return; 5501 5502 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_FREE, -1, -1); 5503 5504 mutex_lock(&bp->hwrm_cmd_lock); 5505 for (i = 0; i < bp->cp_nr_rings; i++) { 5506 if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID) 5507 continue; 5508 req.ring_group_id = 5509 cpu_to_le32(bp->grp_info[i].fw_grp_id); 5510 5511 _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5512 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID; 5513 } 5514 mutex_unlock(&bp->hwrm_cmd_lock); 5515 } 5516 5517 static int hwrm_ring_alloc_send_msg(struct bnxt *bp, 5518 struct bnxt_ring_struct *ring, 5519 u32 ring_type, u32 map_index) 5520 { 5521 int rc = 0, err = 0; 5522 struct hwrm_ring_alloc_input req = {0}; 5523 struct hwrm_ring_alloc_output *resp = bp->hwrm_cmd_resp_addr; 5524 struct bnxt_ring_mem_info *rmem = &ring->ring_mem; 5525 struct bnxt_ring_grp_info *grp_info; 5526 u16 ring_id; 5527 5528 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_ALLOC, -1, -1); 5529 5530 req.enables = 0; 5531 if (rmem->nr_pages > 1) { 5532 req.page_tbl_addr = cpu_to_le64(rmem->pg_tbl_map); 5533 /* Page size is in log2 units */ 5534 req.page_size = BNXT_PAGE_SHIFT; 5535 req.page_tbl_depth = 1; 5536 } else { 5537 req.page_tbl_addr = cpu_to_le64(rmem->dma_arr[0]); 5538 } 5539 req.fbo = 0; 5540 /* Association of ring index with doorbell index and MSIX number */ 5541 req.logical_id = cpu_to_le16(map_index); 5542 5543 switch (ring_type) { 5544 case HWRM_RING_ALLOC_TX: { 5545 struct bnxt_tx_ring_info *txr; 5546 5547 txr = container_of(ring, struct bnxt_tx_ring_info, 5548 tx_ring_struct); 5549 req.ring_type = RING_ALLOC_REQ_RING_TYPE_TX; 5550 /* Association of transmit ring with completion ring */ 5551 grp_info = &bp->grp_info[ring->grp_idx]; 5552 req.cmpl_ring_id = cpu_to_le16(bnxt_cp_ring_for_tx(bp, txr)); 5553 req.length = cpu_to_le32(bp->tx_ring_mask + 1); 5554 req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx); 5555 req.queue_id = cpu_to_le16(ring->queue_id); 5556 break; 5557 } 5558 case HWRM_RING_ALLOC_RX: 5559 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX; 5560 req.length = cpu_to_le32(bp->rx_ring_mask + 1); 5561 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5562 u16 flags = 0; 5563 5564 /* Association of rx ring with stats context */ 5565 grp_info = &bp->grp_info[ring->grp_idx]; 5566 req.rx_buf_size = cpu_to_le16(bp->rx_buf_use_size); 5567 req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx); 5568 req.enables |= cpu_to_le32( 5569 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID); 5570 if (NET_IP_ALIGN == 2) 5571 flags = RING_ALLOC_REQ_FLAGS_RX_SOP_PAD; 5572 req.flags = cpu_to_le16(flags); 5573 } 5574 break; 5575 case HWRM_RING_ALLOC_AGG: 5576 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5577 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX_AGG; 5578 /* Association of agg ring with rx ring */ 5579 grp_info = &bp->grp_info[ring->grp_idx]; 5580 req.rx_ring_id = cpu_to_le16(grp_info->rx_fw_ring_id); 5581 req.rx_buf_size = cpu_to_le16(BNXT_RX_PAGE_SIZE); 5582 req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx); 5583 req.enables |= cpu_to_le32( 5584 RING_ALLOC_REQ_ENABLES_RX_RING_ID_VALID | 5585 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID); 5586 } else { 5587 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX; 5588 } 5589 req.length = cpu_to_le32(bp->rx_agg_ring_mask + 1); 5590 break; 5591 case HWRM_RING_ALLOC_CMPL: 5592 req.ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL; 5593 req.length = cpu_to_le32(bp->cp_ring_mask + 1); 5594 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5595 /* Association of cp ring with nq */ 5596 grp_info = &bp->grp_info[map_index]; 5597 req.nq_ring_id = cpu_to_le16(grp_info->cp_fw_ring_id); 5598 req.cq_handle = cpu_to_le64(ring->handle); 5599 req.enables |= cpu_to_le32( 5600 RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID); 5601 } else if (bp->flags & BNXT_FLAG_USING_MSIX) { 5602 req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX; 5603 } 5604 break; 5605 case HWRM_RING_ALLOC_NQ: 5606 req.ring_type = RING_ALLOC_REQ_RING_TYPE_NQ; 5607 req.length = cpu_to_le32(bp->cp_ring_mask + 1); 5608 if (bp->flags & BNXT_FLAG_USING_MSIX) 5609 req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX; 5610 break; 5611 default: 5612 netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n", 5613 ring_type); 5614 return -1; 5615 } 5616 5617 mutex_lock(&bp->hwrm_cmd_lock); 5618 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5619 err = le16_to_cpu(resp->error_code); 5620 ring_id = le16_to_cpu(resp->ring_id); 5621 mutex_unlock(&bp->hwrm_cmd_lock); 5622 5623 if (rc || err) { 5624 netdev_err(bp->dev, "hwrm_ring_alloc type %d failed. rc:%x err:%x\n", 5625 ring_type, rc, err); 5626 return -EIO; 5627 } 5628 ring->fw_ring_id = ring_id; 5629 return rc; 5630 } 5631 5632 static int bnxt_hwrm_set_async_event_cr(struct bnxt *bp, int idx) 5633 { 5634 int rc; 5635 5636 if (BNXT_PF(bp)) { 5637 struct hwrm_func_cfg_input req = {0}; 5638 5639 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1); 5640 req.fid = cpu_to_le16(0xffff); 5641 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR); 5642 req.async_event_cr = cpu_to_le16(idx); 5643 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5644 } else { 5645 struct hwrm_func_vf_cfg_input req = {0}; 5646 5647 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_VF_CFG, -1, -1); 5648 req.enables = 5649 cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR); 5650 req.async_event_cr = cpu_to_le16(idx); 5651 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5652 } 5653 return rc; 5654 } 5655 5656 static void bnxt_set_db(struct bnxt *bp, struct bnxt_db_info *db, u32 ring_type, 5657 u32 map_idx, u32 xid) 5658 { 5659 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5660 if (BNXT_PF(bp)) 5661 db->doorbell = bp->bar1 + DB_PF_OFFSET_P5; 5662 else 5663 db->doorbell = bp->bar1 + DB_VF_OFFSET_P5; 5664 switch (ring_type) { 5665 case HWRM_RING_ALLOC_TX: 5666 db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SQ; 5667 break; 5668 case HWRM_RING_ALLOC_RX: 5669 case HWRM_RING_ALLOC_AGG: 5670 db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SRQ; 5671 break; 5672 case HWRM_RING_ALLOC_CMPL: 5673 db->db_key64 = DBR_PATH_L2; 5674 break; 5675 case HWRM_RING_ALLOC_NQ: 5676 db->db_key64 = DBR_PATH_L2; 5677 break; 5678 } 5679 db->db_key64 |= (u64)xid << DBR_XID_SFT; 5680 } else { 5681 db->doorbell = bp->bar1 + map_idx * 0x80; 5682 switch (ring_type) { 5683 case HWRM_RING_ALLOC_TX: 5684 db->db_key32 = DB_KEY_TX; 5685 break; 5686 case HWRM_RING_ALLOC_RX: 5687 case HWRM_RING_ALLOC_AGG: 5688 db->db_key32 = DB_KEY_RX; 5689 break; 5690 case HWRM_RING_ALLOC_CMPL: 5691 db->db_key32 = DB_KEY_CP; 5692 break; 5693 } 5694 } 5695 } 5696 5697 static int bnxt_hwrm_ring_alloc(struct bnxt *bp) 5698 { 5699 bool agg_rings = !!(bp->flags & BNXT_FLAG_AGG_RINGS); 5700 int i, rc = 0; 5701 u32 type; 5702 5703 if (bp->flags & BNXT_FLAG_CHIP_P5) 5704 type = HWRM_RING_ALLOC_NQ; 5705 else 5706 type = HWRM_RING_ALLOC_CMPL; 5707 for (i = 0; i < bp->cp_nr_rings; i++) { 5708 struct bnxt_napi *bnapi = bp->bnapi[i]; 5709 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 5710 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 5711 u32 map_idx = ring->map_idx; 5712 unsigned int vector; 5713 5714 vector = bp->irq_tbl[map_idx].vector; 5715 disable_irq_nosync(vector); 5716 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 5717 if (rc) { 5718 enable_irq(vector); 5719 goto err_out; 5720 } 5721 bnxt_set_db(bp, &cpr->cp_db, type, map_idx, ring->fw_ring_id); 5722 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons); 5723 enable_irq(vector); 5724 bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id; 5725 5726 if (!i) { 5727 rc = bnxt_hwrm_set_async_event_cr(bp, ring->fw_ring_id); 5728 if (rc) 5729 netdev_warn(bp->dev, "Failed to set async event completion ring.\n"); 5730 } 5731 } 5732 5733 type = HWRM_RING_ALLOC_TX; 5734 for (i = 0; i < bp->tx_nr_rings; i++) { 5735 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 5736 struct bnxt_ring_struct *ring; 5737 u32 map_idx; 5738 5739 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5740 struct bnxt_napi *bnapi = txr->bnapi; 5741 struct bnxt_cp_ring_info *cpr, *cpr2; 5742 u32 type2 = HWRM_RING_ALLOC_CMPL; 5743 5744 cpr = &bnapi->cp_ring; 5745 cpr2 = cpr->cp_ring_arr[BNXT_TX_HDL]; 5746 ring = &cpr2->cp_ring_struct; 5747 ring->handle = BNXT_TX_HDL; 5748 map_idx = bnapi->index; 5749 rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx); 5750 if (rc) 5751 goto err_out; 5752 bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx, 5753 ring->fw_ring_id); 5754 bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons); 5755 } 5756 ring = &txr->tx_ring_struct; 5757 map_idx = i; 5758 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 5759 if (rc) 5760 goto err_out; 5761 bnxt_set_db(bp, &txr->tx_db, type, map_idx, ring->fw_ring_id); 5762 } 5763 5764 type = HWRM_RING_ALLOC_RX; 5765 for (i = 0; i < bp->rx_nr_rings; i++) { 5766 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 5767 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct; 5768 struct bnxt_napi *bnapi = rxr->bnapi; 5769 u32 map_idx = bnapi->index; 5770 5771 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 5772 if (rc) 5773 goto err_out; 5774 bnxt_set_db(bp, &rxr->rx_db, type, map_idx, ring->fw_ring_id); 5775 /* If we have agg rings, post agg buffers first. */ 5776 if (!agg_rings) 5777 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 5778 bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id; 5779 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5780 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 5781 u32 type2 = HWRM_RING_ALLOC_CMPL; 5782 struct bnxt_cp_ring_info *cpr2; 5783 5784 cpr2 = cpr->cp_ring_arr[BNXT_RX_HDL]; 5785 ring = &cpr2->cp_ring_struct; 5786 ring->handle = BNXT_RX_HDL; 5787 rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx); 5788 if (rc) 5789 goto err_out; 5790 bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx, 5791 ring->fw_ring_id); 5792 bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons); 5793 } 5794 } 5795 5796 if (agg_rings) { 5797 type = HWRM_RING_ALLOC_AGG; 5798 for (i = 0; i < bp->rx_nr_rings; i++) { 5799 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 5800 struct bnxt_ring_struct *ring = 5801 &rxr->rx_agg_ring_struct; 5802 u32 grp_idx = ring->grp_idx; 5803 u32 map_idx = grp_idx + bp->rx_nr_rings; 5804 5805 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 5806 if (rc) 5807 goto err_out; 5808 5809 bnxt_set_db(bp, &rxr->rx_agg_db, type, map_idx, 5810 ring->fw_ring_id); 5811 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); 5812 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 5813 bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id; 5814 } 5815 } 5816 err_out: 5817 return rc; 5818 } 5819 5820 static int hwrm_ring_free_send_msg(struct bnxt *bp, 5821 struct bnxt_ring_struct *ring, 5822 u32 ring_type, int cmpl_ring_id) 5823 { 5824 int rc; 5825 struct hwrm_ring_free_input req = {0}; 5826 struct hwrm_ring_free_output *resp = bp->hwrm_cmd_resp_addr; 5827 u16 error_code; 5828 5829 if (BNXT_NO_FW_ACCESS(bp)) 5830 return 0; 5831 5832 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_FREE, cmpl_ring_id, -1); 5833 req.ring_type = ring_type; 5834 req.ring_id = cpu_to_le16(ring->fw_ring_id); 5835 5836 mutex_lock(&bp->hwrm_cmd_lock); 5837 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5838 error_code = le16_to_cpu(resp->error_code); 5839 mutex_unlock(&bp->hwrm_cmd_lock); 5840 5841 if (rc || error_code) { 5842 netdev_err(bp->dev, "hwrm_ring_free type %d failed. rc:%x err:%x\n", 5843 ring_type, rc, error_code); 5844 return -EIO; 5845 } 5846 return 0; 5847 } 5848 5849 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path) 5850 { 5851 u32 type; 5852 int i; 5853 5854 if (!bp->bnapi) 5855 return; 5856 5857 for (i = 0; i < bp->tx_nr_rings; i++) { 5858 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 5859 struct bnxt_ring_struct *ring = &txr->tx_ring_struct; 5860 5861 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 5862 u32 cmpl_ring_id = bnxt_cp_ring_for_tx(bp, txr); 5863 5864 hwrm_ring_free_send_msg(bp, ring, 5865 RING_FREE_REQ_RING_TYPE_TX, 5866 close_path ? cmpl_ring_id : 5867 INVALID_HW_RING_ID); 5868 ring->fw_ring_id = INVALID_HW_RING_ID; 5869 } 5870 } 5871 5872 for (i = 0; i < bp->rx_nr_rings; i++) { 5873 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 5874 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct; 5875 u32 grp_idx = rxr->bnapi->index; 5876 5877 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 5878 u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr); 5879 5880 hwrm_ring_free_send_msg(bp, ring, 5881 RING_FREE_REQ_RING_TYPE_RX, 5882 close_path ? cmpl_ring_id : 5883 INVALID_HW_RING_ID); 5884 ring->fw_ring_id = INVALID_HW_RING_ID; 5885 bp->grp_info[grp_idx].rx_fw_ring_id = 5886 INVALID_HW_RING_ID; 5887 } 5888 } 5889 5890 if (bp->flags & BNXT_FLAG_CHIP_P5) 5891 type = RING_FREE_REQ_RING_TYPE_RX_AGG; 5892 else 5893 type = RING_FREE_REQ_RING_TYPE_RX; 5894 for (i = 0; i < bp->rx_nr_rings; i++) { 5895 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 5896 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct; 5897 u32 grp_idx = rxr->bnapi->index; 5898 5899 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 5900 u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr); 5901 5902 hwrm_ring_free_send_msg(bp, ring, type, 5903 close_path ? cmpl_ring_id : 5904 INVALID_HW_RING_ID); 5905 ring->fw_ring_id = INVALID_HW_RING_ID; 5906 bp->grp_info[grp_idx].agg_fw_ring_id = 5907 INVALID_HW_RING_ID; 5908 } 5909 } 5910 5911 /* The completion rings are about to be freed. After that the 5912 * IRQ doorbell will not work anymore. So we need to disable 5913 * IRQ here. 5914 */ 5915 bnxt_disable_int_sync(bp); 5916 5917 if (bp->flags & BNXT_FLAG_CHIP_P5) 5918 type = RING_FREE_REQ_RING_TYPE_NQ; 5919 else 5920 type = RING_FREE_REQ_RING_TYPE_L2_CMPL; 5921 for (i = 0; i < bp->cp_nr_rings; i++) { 5922 struct bnxt_napi *bnapi = bp->bnapi[i]; 5923 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 5924 struct bnxt_ring_struct *ring; 5925 int j; 5926 5927 for (j = 0; j < 2; j++) { 5928 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j]; 5929 5930 if (cpr2) { 5931 ring = &cpr2->cp_ring_struct; 5932 if (ring->fw_ring_id == INVALID_HW_RING_ID) 5933 continue; 5934 hwrm_ring_free_send_msg(bp, ring, 5935 RING_FREE_REQ_RING_TYPE_L2_CMPL, 5936 INVALID_HW_RING_ID); 5937 ring->fw_ring_id = INVALID_HW_RING_ID; 5938 } 5939 } 5940 ring = &cpr->cp_ring_struct; 5941 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 5942 hwrm_ring_free_send_msg(bp, ring, type, 5943 INVALID_HW_RING_ID); 5944 ring->fw_ring_id = INVALID_HW_RING_ID; 5945 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID; 5946 } 5947 } 5948 } 5949 5950 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max, 5951 bool shared); 5952 5953 static int bnxt_hwrm_get_rings(struct bnxt *bp) 5954 { 5955 struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 5956 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 5957 struct hwrm_func_qcfg_input req = {0}; 5958 int rc; 5959 5960 if (bp->hwrm_spec_code < 0x10601) 5961 return 0; 5962 5963 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1); 5964 req.fid = cpu_to_le16(0xffff); 5965 mutex_lock(&bp->hwrm_cmd_lock); 5966 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5967 if (rc) { 5968 mutex_unlock(&bp->hwrm_cmd_lock); 5969 return rc; 5970 } 5971 5972 hw_resc->resv_tx_rings = le16_to_cpu(resp->alloc_tx_rings); 5973 if (BNXT_NEW_RM(bp)) { 5974 u16 cp, stats; 5975 5976 hw_resc->resv_rx_rings = le16_to_cpu(resp->alloc_rx_rings); 5977 hw_resc->resv_hw_ring_grps = 5978 le32_to_cpu(resp->alloc_hw_ring_grps); 5979 hw_resc->resv_vnics = le16_to_cpu(resp->alloc_vnics); 5980 cp = le16_to_cpu(resp->alloc_cmpl_rings); 5981 stats = le16_to_cpu(resp->alloc_stat_ctx); 5982 hw_resc->resv_irqs = cp; 5983 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5984 int rx = hw_resc->resv_rx_rings; 5985 int tx = hw_resc->resv_tx_rings; 5986 5987 if (bp->flags & BNXT_FLAG_AGG_RINGS) 5988 rx >>= 1; 5989 if (cp < (rx + tx)) { 5990 bnxt_trim_rings(bp, &rx, &tx, cp, false); 5991 if (bp->flags & BNXT_FLAG_AGG_RINGS) 5992 rx <<= 1; 5993 hw_resc->resv_rx_rings = rx; 5994 hw_resc->resv_tx_rings = tx; 5995 } 5996 hw_resc->resv_irqs = le16_to_cpu(resp->alloc_msix); 5997 hw_resc->resv_hw_ring_grps = rx; 5998 } 5999 hw_resc->resv_cp_rings = cp; 6000 hw_resc->resv_stat_ctxs = stats; 6001 } 6002 mutex_unlock(&bp->hwrm_cmd_lock); 6003 return 0; 6004 } 6005 6006 /* Caller must hold bp->hwrm_cmd_lock */ 6007 int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings) 6008 { 6009 struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 6010 struct hwrm_func_qcfg_input req = {0}; 6011 int rc; 6012 6013 if (bp->hwrm_spec_code < 0x10601) 6014 return 0; 6015 6016 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1); 6017 req.fid = cpu_to_le16(fid); 6018 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6019 if (!rc) 6020 *tx_rings = le16_to_cpu(resp->alloc_tx_rings); 6021 6022 return rc; 6023 } 6024 6025 static bool bnxt_rfs_supported(struct bnxt *bp); 6026 6027 static void 6028 __bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, struct hwrm_func_cfg_input *req, 6029 int tx_rings, int rx_rings, int ring_grps, 6030 int cp_rings, int stats, int vnics) 6031 { 6032 u32 enables = 0; 6033 6034 bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_CFG, -1, -1); 6035 req->fid = cpu_to_le16(0xffff); 6036 enables |= tx_rings ? FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS : 0; 6037 req->num_tx_rings = cpu_to_le16(tx_rings); 6038 if (BNXT_NEW_RM(bp)) { 6039 enables |= rx_rings ? FUNC_CFG_REQ_ENABLES_NUM_RX_RINGS : 0; 6040 enables |= stats ? FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0; 6041 if (bp->flags & BNXT_FLAG_CHIP_P5) { 6042 enables |= cp_rings ? FUNC_CFG_REQ_ENABLES_NUM_MSIX : 0; 6043 enables |= tx_rings + ring_grps ? 6044 FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0; 6045 enables |= rx_rings ? 6046 FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0; 6047 } else { 6048 enables |= cp_rings ? 6049 FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0; 6050 enables |= ring_grps ? 6051 FUNC_CFG_REQ_ENABLES_NUM_HW_RING_GRPS | 6052 FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0; 6053 } 6054 enables |= vnics ? FUNC_CFG_REQ_ENABLES_NUM_VNICS : 0; 6055 6056 req->num_rx_rings = cpu_to_le16(rx_rings); 6057 if (bp->flags & BNXT_FLAG_CHIP_P5) { 6058 req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps); 6059 req->num_msix = cpu_to_le16(cp_rings); 6060 req->num_rsscos_ctxs = 6061 cpu_to_le16(DIV_ROUND_UP(ring_grps, 64)); 6062 } else { 6063 req->num_cmpl_rings = cpu_to_le16(cp_rings); 6064 req->num_hw_ring_grps = cpu_to_le16(ring_grps); 6065 req->num_rsscos_ctxs = cpu_to_le16(1); 6066 if (!(bp->flags & BNXT_FLAG_NEW_RSS_CAP) && 6067 bnxt_rfs_supported(bp)) 6068 req->num_rsscos_ctxs = 6069 cpu_to_le16(ring_grps + 1); 6070 } 6071 req->num_stat_ctxs = cpu_to_le16(stats); 6072 req->num_vnics = cpu_to_le16(vnics); 6073 } 6074 req->enables = cpu_to_le32(enables); 6075 } 6076 6077 static void 6078 __bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, 6079 struct hwrm_func_vf_cfg_input *req, int tx_rings, 6080 int rx_rings, int ring_grps, int cp_rings, 6081 int stats, int vnics) 6082 { 6083 u32 enables = 0; 6084 6085 bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_VF_CFG, -1, -1); 6086 enables |= tx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_TX_RINGS : 0; 6087 enables |= rx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_RX_RINGS | 6088 FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0; 6089 enables |= stats ? FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0; 6090 if (bp->flags & BNXT_FLAG_CHIP_P5) { 6091 enables |= tx_rings + ring_grps ? 6092 FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0; 6093 } else { 6094 enables |= cp_rings ? 6095 FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0; 6096 enables |= ring_grps ? 6097 FUNC_VF_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0; 6098 } 6099 enables |= vnics ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0; 6100 enables |= FUNC_VF_CFG_REQ_ENABLES_NUM_L2_CTXS; 6101 6102 req->num_l2_ctxs = cpu_to_le16(BNXT_VF_MAX_L2_CTX); 6103 req->num_tx_rings = cpu_to_le16(tx_rings); 6104 req->num_rx_rings = cpu_to_le16(rx_rings); 6105 if (bp->flags & BNXT_FLAG_CHIP_P5) { 6106 req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps); 6107 req->num_rsscos_ctxs = cpu_to_le16(DIV_ROUND_UP(ring_grps, 64)); 6108 } else { 6109 req->num_cmpl_rings = cpu_to_le16(cp_rings); 6110 req->num_hw_ring_grps = cpu_to_le16(ring_grps); 6111 req->num_rsscos_ctxs = cpu_to_le16(BNXT_VF_MAX_RSS_CTX); 6112 } 6113 req->num_stat_ctxs = cpu_to_le16(stats); 6114 req->num_vnics = cpu_to_le16(vnics); 6115 6116 req->enables = cpu_to_le32(enables); 6117 } 6118 6119 static int 6120 bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6121 int ring_grps, int cp_rings, int stats, int vnics) 6122 { 6123 struct hwrm_func_cfg_input req = {0}; 6124 int rc; 6125 6126 __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps, 6127 cp_rings, stats, vnics); 6128 if (!req.enables) 6129 return 0; 6130 6131 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6132 if (rc) 6133 return rc; 6134 6135 if (bp->hwrm_spec_code < 0x10601) 6136 bp->hw_resc.resv_tx_rings = tx_rings; 6137 6138 return bnxt_hwrm_get_rings(bp); 6139 } 6140 6141 static int 6142 bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6143 int ring_grps, int cp_rings, int stats, int vnics) 6144 { 6145 struct hwrm_func_vf_cfg_input req = {0}; 6146 int rc; 6147 6148 if (!BNXT_NEW_RM(bp)) { 6149 bp->hw_resc.resv_tx_rings = tx_rings; 6150 return 0; 6151 } 6152 6153 __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps, 6154 cp_rings, stats, vnics); 6155 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6156 if (rc) 6157 return rc; 6158 6159 return bnxt_hwrm_get_rings(bp); 6160 } 6161 6162 static int bnxt_hwrm_reserve_rings(struct bnxt *bp, int tx, int rx, int grp, 6163 int cp, int stat, int vnic) 6164 { 6165 if (BNXT_PF(bp)) 6166 return bnxt_hwrm_reserve_pf_rings(bp, tx, rx, grp, cp, stat, 6167 vnic); 6168 else 6169 return bnxt_hwrm_reserve_vf_rings(bp, tx, rx, grp, cp, stat, 6170 vnic); 6171 } 6172 6173 int bnxt_nq_rings_in_use(struct bnxt *bp) 6174 { 6175 int cp = bp->cp_nr_rings; 6176 int ulp_msix, ulp_base; 6177 6178 ulp_msix = bnxt_get_ulp_msix_num(bp); 6179 if (ulp_msix) { 6180 ulp_base = bnxt_get_ulp_msix_base(bp); 6181 cp += ulp_msix; 6182 if ((ulp_base + ulp_msix) > cp) 6183 cp = ulp_base + ulp_msix; 6184 } 6185 return cp; 6186 } 6187 6188 static int bnxt_cp_rings_in_use(struct bnxt *bp) 6189 { 6190 int cp; 6191 6192 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 6193 return bnxt_nq_rings_in_use(bp); 6194 6195 cp = bp->tx_nr_rings + bp->rx_nr_rings; 6196 return cp; 6197 } 6198 6199 static int bnxt_get_func_stat_ctxs(struct bnxt *bp) 6200 { 6201 int ulp_stat = bnxt_get_ulp_stat_ctxs(bp); 6202 int cp = bp->cp_nr_rings; 6203 6204 if (!ulp_stat) 6205 return cp; 6206 6207 if (bnxt_nq_rings_in_use(bp) > cp + bnxt_get_ulp_msix_num(bp)) 6208 return bnxt_get_ulp_msix_base(bp) + ulp_stat; 6209 6210 return cp + ulp_stat; 6211 } 6212 6213 /* Check if a default RSS map needs to be setup. This function is only 6214 * used on older firmware that does not require reserving RX rings. 6215 */ 6216 static void bnxt_check_rss_tbl_no_rmgr(struct bnxt *bp) 6217 { 6218 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 6219 6220 /* The RSS map is valid for RX rings set to resv_rx_rings */ 6221 if (hw_resc->resv_rx_rings != bp->rx_nr_rings) { 6222 hw_resc->resv_rx_rings = bp->rx_nr_rings; 6223 if (!netif_is_rxfh_configured(bp->dev)) 6224 bnxt_set_dflt_rss_indir_tbl(bp); 6225 } 6226 } 6227 6228 static bool bnxt_need_reserve_rings(struct bnxt *bp) 6229 { 6230 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 6231 int cp = bnxt_cp_rings_in_use(bp); 6232 int nq = bnxt_nq_rings_in_use(bp); 6233 int rx = bp->rx_nr_rings, stat; 6234 int vnic = 1, grp = rx; 6235 6236 if (hw_resc->resv_tx_rings != bp->tx_nr_rings && 6237 bp->hwrm_spec_code >= 0x10601) 6238 return true; 6239 6240 /* Old firmware does not need RX ring reservations but we still 6241 * need to setup a default RSS map when needed. With new firmware 6242 * we go through RX ring reservations first and then set up the 6243 * RSS map for the successfully reserved RX rings when needed. 6244 */ 6245 if (!BNXT_NEW_RM(bp)) { 6246 bnxt_check_rss_tbl_no_rmgr(bp); 6247 return false; 6248 } 6249 if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5)) 6250 vnic = rx + 1; 6251 if (bp->flags & BNXT_FLAG_AGG_RINGS) 6252 rx <<= 1; 6253 stat = bnxt_get_func_stat_ctxs(bp); 6254 if (hw_resc->resv_rx_rings != rx || hw_resc->resv_cp_rings != cp || 6255 hw_resc->resv_vnics != vnic || hw_resc->resv_stat_ctxs != stat || 6256 (hw_resc->resv_hw_ring_grps != grp && 6257 !(bp->flags & BNXT_FLAG_CHIP_P5))) 6258 return true; 6259 if ((bp->flags & BNXT_FLAG_CHIP_P5) && BNXT_PF(bp) && 6260 hw_resc->resv_irqs != nq) 6261 return true; 6262 return false; 6263 } 6264 6265 static int __bnxt_reserve_rings(struct bnxt *bp) 6266 { 6267 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 6268 int cp = bnxt_nq_rings_in_use(bp); 6269 int tx = bp->tx_nr_rings; 6270 int rx = bp->rx_nr_rings; 6271 int grp, rx_rings, rc; 6272 int vnic = 1, stat; 6273 bool sh = false; 6274 6275 if (!bnxt_need_reserve_rings(bp)) 6276 return 0; 6277 6278 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 6279 sh = true; 6280 if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5)) 6281 vnic = rx + 1; 6282 if (bp->flags & BNXT_FLAG_AGG_RINGS) 6283 rx <<= 1; 6284 grp = bp->rx_nr_rings; 6285 stat = bnxt_get_func_stat_ctxs(bp); 6286 6287 rc = bnxt_hwrm_reserve_rings(bp, tx, rx, grp, cp, stat, vnic); 6288 if (rc) 6289 return rc; 6290 6291 tx = hw_resc->resv_tx_rings; 6292 if (BNXT_NEW_RM(bp)) { 6293 rx = hw_resc->resv_rx_rings; 6294 cp = hw_resc->resv_irqs; 6295 grp = hw_resc->resv_hw_ring_grps; 6296 vnic = hw_resc->resv_vnics; 6297 stat = hw_resc->resv_stat_ctxs; 6298 } 6299 6300 rx_rings = rx; 6301 if (bp->flags & BNXT_FLAG_AGG_RINGS) { 6302 if (rx >= 2) { 6303 rx_rings = rx >> 1; 6304 } else { 6305 if (netif_running(bp->dev)) 6306 return -ENOMEM; 6307 6308 bp->flags &= ~BNXT_FLAG_AGG_RINGS; 6309 bp->flags |= BNXT_FLAG_NO_AGG_RINGS; 6310 bp->dev->hw_features &= ~NETIF_F_LRO; 6311 bp->dev->features &= ~NETIF_F_LRO; 6312 bnxt_set_ring_params(bp); 6313 } 6314 } 6315 rx_rings = min_t(int, rx_rings, grp); 6316 cp = min_t(int, cp, bp->cp_nr_rings); 6317 if (stat > bnxt_get_ulp_stat_ctxs(bp)) 6318 stat -= bnxt_get_ulp_stat_ctxs(bp); 6319 cp = min_t(int, cp, stat); 6320 rc = bnxt_trim_rings(bp, &rx_rings, &tx, cp, sh); 6321 if (bp->flags & BNXT_FLAG_AGG_RINGS) 6322 rx = rx_rings << 1; 6323 cp = sh ? max_t(int, tx, rx_rings) : tx + rx_rings; 6324 bp->tx_nr_rings = tx; 6325 6326 /* If we cannot reserve all the RX rings, reset the RSS map only 6327 * if absolutely necessary 6328 */ 6329 if (rx_rings != bp->rx_nr_rings) { 6330 netdev_warn(bp->dev, "Able to reserve only %d out of %d requested RX rings\n", 6331 rx_rings, bp->rx_nr_rings); 6332 if ((bp->dev->priv_flags & IFF_RXFH_CONFIGURED) && 6333 (bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings) != 6334 bnxt_get_nr_rss_ctxs(bp, rx_rings) || 6335 bnxt_get_max_rss_ring(bp) >= rx_rings)) { 6336 netdev_warn(bp->dev, "RSS table entries reverting to default\n"); 6337 bp->dev->priv_flags &= ~IFF_RXFH_CONFIGURED; 6338 } 6339 } 6340 bp->rx_nr_rings = rx_rings; 6341 bp->cp_nr_rings = cp; 6342 6343 if (!tx || !rx || !cp || !grp || !vnic || !stat) 6344 return -ENOMEM; 6345 6346 if (!netif_is_rxfh_configured(bp->dev)) 6347 bnxt_set_dflt_rss_indir_tbl(bp); 6348 6349 return rc; 6350 } 6351 6352 static int bnxt_hwrm_check_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6353 int ring_grps, int cp_rings, int stats, 6354 int vnics) 6355 { 6356 struct hwrm_func_vf_cfg_input req = {0}; 6357 u32 flags; 6358 6359 if (!BNXT_NEW_RM(bp)) 6360 return 0; 6361 6362 __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps, 6363 cp_rings, stats, vnics); 6364 flags = FUNC_VF_CFG_REQ_FLAGS_TX_ASSETS_TEST | 6365 FUNC_VF_CFG_REQ_FLAGS_RX_ASSETS_TEST | 6366 FUNC_VF_CFG_REQ_FLAGS_CMPL_ASSETS_TEST | 6367 FUNC_VF_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST | 6368 FUNC_VF_CFG_REQ_FLAGS_VNIC_ASSETS_TEST | 6369 FUNC_VF_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST; 6370 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 6371 flags |= FUNC_VF_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST; 6372 6373 req.flags = cpu_to_le32(flags); 6374 return hwrm_send_message_silent(bp, &req, sizeof(req), 6375 HWRM_CMD_TIMEOUT); 6376 } 6377 6378 static int bnxt_hwrm_check_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6379 int ring_grps, int cp_rings, int stats, 6380 int vnics) 6381 { 6382 struct hwrm_func_cfg_input req = {0}; 6383 u32 flags; 6384 6385 __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps, 6386 cp_rings, stats, vnics); 6387 flags = FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST; 6388 if (BNXT_NEW_RM(bp)) { 6389 flags |= FUNC_CFG_REQ_FLAGS_RX_ASSETS_TEST | 6390 FUNC_CFG_REQ_FLAGS_CMPL_ASSETS_TEST | 6391 FUNC_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST | 6392 FUNC_CFG_REQ_FLAGS_VNIC_ASSETS_TEST; 6393 if (bp->flags & BNXT_FLAG_CHIP_P5) 6394 flags |= FUNC_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST | 6395 FUNC_CFG_REQ_FLAGS_NQ_ASSETS_TEST; 6396 else 6397 flags |= FUNC_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST; 6398 } 6399 6400 req.flags = cpu_to_le32(flags); 6401 return hwrm_send_message_silent(bp, &req, sizeof(req), 6402 HWRM_CMD_TIMEOUT); 6403 } 6404 6405 static int bnxt_hwrm_check_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6406 int ring_grps, int cp_rings, int stats, 6407 int vnics) 6408 { 6409 if (bp->hwrm_spec_code < 0x10801) 6410 return 0; 6411 6412 if (BNXT_PF(bp)) 6413 return bnxt_hwrm_check_pf_rings(bp, tx_rings, rx_rings, 6414 ring_grps, cp_rings, stats, 6415 vnics); 6416 6417 return bnxt_hwrm_check_vf_rings(bp, tx_rings, rx_rings, ring_grps, 6418 cp_rings, stats, vnics); 6419 } 6420 6421 static void bnxt_hwrm_coal_params_qcaps(struct bnxt *bp) 6422 { 6423 struct hwrm_ring_aggint_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 6424 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 6425 struct hwrm_ring_aggint_qcaps_input req = {0}; 6426 int rc; 6427 6428 coal_cap->cmpl_params = BNXT_LEGACY_COAL_CMPL_PARAMS; 6429 coal_cap->num_cmpl_dma_aggr_max = 63; 6430 coal_cap->num_cmpl_dma_aggr_during_int_max = 63; 6431 coal_cap->cmpl_aggr_dma_tmr_max = 65535; 6432 coal_cap->cmpl_aggr_dma_tmr_during_int_max = 65535; 6433 coal_cap->int_lat_tmr_min_max = 65535; 6434 coal_cap->int_lat_tmr_max_max = 65535; 6435 coal_cap->num_cmpl_aggr_int_max = 65535; 6436 coal_cap->timer_units = 80; 6437 6438 if (bp->hwrm_spec_code < 0x10902) 6439 return; 6440 6441 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_AGGINT_QCAPS, -1, -1); 6442 mutex_lock(&bp->hwrm_cmd_lock); 6443 rc = _hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6444 if (!rc) { 6445 coal_cap->cmpl_params = le32_to_cpu(resp->cmpl_params); 6446 coal_cap->nq_params = le32_to_cpu(resp->nq_params); 6447 coal_cap->num_cmpl_dma_aggr_max = 6448 le16_to_cpu(resp->num_cmpl_dma_aggr_max); 6449 coal_cap->num_cmpl_dma_aggr_during_int_max = 6450 le16_to_cpu(resp->num_cmpl_dma_aggr_during_int_max); 6451 coal_cap->cmpl_aggr_dma_tmr_max = 6452 le16_to_cpu(resp->cmpl_aggr_dma_tmr_max); 6453 coal_cap->cmpl_aggr_dma_tmr_during_int_max = 6454 le16_to_cpu(resp->cmpl_aggr_dma_tmr_during_int_max); 6455 coal_cap->int_lat_tmr_min_max = 6456 le16_to_cpu(resp->int_lat_tmr_min_max); 6457 coal_cap->int_lat_tmr_max_max = 6458 le16_to_cpu(resp->int_lat_tmr_max_max); 6459 coal_cap->num_cmpl_aggr_int_max = 6460 le16_to_cpu(resp->num_cmpl_aggr_int_max); 6461 coal_cap->timer_units = le16_to_cpu(resp->timer_units); 6462 } 6463 mutex_unlock(&bp->hwrm_cmd_lock); 6464 } 6465 6466 static u16 bnxt_usec_to_coal_tmr(struct bnxt *bp, u16 usec) 6467 { 6468 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 6469 6470 return usec * 1000 / coal_cap->timer_units; 6471 } 6472 6473 static void bnxt_hwrm_set_coal_params(struct bnxt *bp, 6474 struct bnxt_coal *hw_coal, 6475 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req) 6476 { 6477 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 6478 u32 cmpl_params = coal_cap->cmpl_params; 6479 u16 val, tmr, max, flags = 0; 6480 6481 max = hw_coal->bufs_per_record * 128; 6482 if (hw_coal->budget) 6483 max = hw_coal->bufs_per_record * hw_coal->budget; 6484 max = min_t(u16, max, coal_cap->num_cmpl_aggr_int_max); 6485 6486 val = clamp_t(u16, hw_coal->coal_bufs, 1, max); 6487 req->num_cmpl_aggr_int = cpu_to_le16(val); 6488 6489 val = min_t(u16, val, coal_cap->num_cmpl_dma_aggr_max); 6490 req->num_cmpl_dma_aggr = cpu_to_le16(val); 6491 6492 val = clamp_t(u16, hw_coal->coal_bufs_irq, 1, 6493 coal_cap->num_cmpl_dma_aggr_during_int_max); 6494 req->num_cmpl_dma_aggr_during_int = cpu_to_le16(val); 6495 6496 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks); 6497 tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_max_max); 6498 req->int_lat_tmr_max = cpu_to_le16(tmr); 6499 6500 /* min timer set to 1/2 of interrupt timer */ 6501 if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_INT_LAT_TMR_MIN) { 6502 val = tmr / 2; 6503 val = clamp_t(u16, val, 1, coal_cap->int_lat_tmr_min_max); 6504 req->int_lat_tmr_min = cpu_to_le16(val); 6505 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE); 6506 } 6507 6508 /* buf timer set to 1/4 of interrupt timer */ 6509 val = clamp_t(u16, tmr / 4, 1, coal_cap->cmpl_aggr_dma_tmr_max); 6510 req->cmpl_aggr_dma_tmr = cpu_to_le16(val); 6511 6512 if (cmpl_params & 6513 RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_NUM_CMPL_DMA_AGGR_DURING_INT) { 6514 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks_irq); 6515 val = clamp_t(u16, tmr, 1, 6516 coal_cap->cmpl_aggr_dma_tmr_during_int_max); 6517 req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(val); 6518 req->enables |= 6519 cpu_to_le16(BNXT_COAL_CMPL_AGGR_TMR_DURING_INT_ENABLE); 6520 } 6521 6522 if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_TIMER_RESET) 6523 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET; 6524 if ((cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_RING_IDLE) && 6525 hw_coal->idle_thresh && hw_coal->coal_ticks < hw_coal->idle_thresh) 6526 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE; 6527 req->flags = cpu_to_le16(flags); 6528 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_ENABLES); 6529 } 6530 6531 /* Caller holds bp->hwrm_cmd_lock */ 6532 static int __bnxt_hwrm_set_coal_nq(struct bnxt *bp, struct bnxt_napi *bnapi, 6533 struct bnxt_coal *hw_coal) 6534 { 6535 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req = {0}; 6536 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 6537 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 6538 u32 nq_params = coal_cap->nq_params; 6539 u16 tmr; 6540 6541 if (!(nq_params & RING_AGGINT_QCAPS_RESP_NQ_PARAMS_INT_LAT_TMR_MIN)) 6542 return 0; 6543 6544 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, 6545 -1, -1); 6546 req.ring_id = cpu_to_le16(cpr->cp_ring_struct.fw_ring_id); 6547 req.flags = 6548 cpu_to_le16(RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_IS_NQ); 6549 6550 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks) / 2; 6551 tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_min_max); 6552 req.int_lat_tmr_min = cpu_to_le16(tmr); 6553 req.enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE); 6554 return _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6555 } 6556 6557 int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi) 6558 { 6559 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0}; 6560 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 6561 struct bnxt_coal coal; 6562 6563 /* Tick values in micro seconds. 6564 * 1 coal_buf x bufs_per_record = 1 completion record. 6565 */ 6566 memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal)); 6567 6568 coal.coal_ticks = cpr->rx_ring_coal.coal_ticks; 6569 coal.coal_bufs = cpr->rx_ring_coal.coal_bufs; 6570 6571 if (!bnapi->rx_ring) 6572 return -ENODEV; 6573 6574 bnxt_hwrm_cmd_hdr_init(bp, &req_rx, 6575 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1); 6576 6577 bnxt_hwrm_set_coal_params(bp, &coal, &req_rx); 6578 6579 req_rx.ring_id = cpu_to_le16(bnxt_cp_ring_for_rx(bp, bnapi->rx_ring)); 6580 6581 return hwrm_send_message(bp, &req_rx, sizeof(req_rx), 6582 HWRM_CMD_TIMEOUT); 6583 } 6584 6585 int bnxt_hwrm_set_coal(struct bnxt *bp) 6586 { 6587 int i, rc = 0; 6588 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0}, 6589 req_tx = {0}, *req; 6590 6591 bnxt_hwrm_cmd_hdr_init(bp, &req_rx, 6592 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1); 6593 bnxt_hwrm_cmd_hdr_init(bp, &req_tx, 6594 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1); 6595 6596 bnxt_hwrm_set_coal_params(bp, &bp->rx_coal, &req_rx); 6597 bnxt_hwrm_set_coal_params(bp, &bp->tx_coal, &req_tx); 6598 6599 mutex_lock(&bp->hwrm_cmd_lock); 6600 for (i = 0; i < bp->cp_nr_rings; i++) { 6601 struct bnxt_napi *bnapi = bp->bnapi[i]; 6602 struct bnxt_coal *hw_coal; 6603 u16 ring_id; 6604 6605 req = &req_rx; 6606 if (!bnapi->rx_ring) { 6607 ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring); 6608 req = &req_tx; 6609 } else { 6610 ring_id = bnxt_cp_ring_for_rx(bp, bnapi->rx_ring); 6611 } 6612 req->ring_id = cpu_to_le16(ring_id); 6613 6614 rc = _hwrm_send_message(bp, req, sizeof(*req), 6615 HWRM_CMD_TIMEOUT); 6616 if (rc) 6617 break; 6618 6619 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 6620 continue; 6621 6622 if (bnapi->rx_ring && bnapi->tx_ring) { 6623 req = &req_tx; 6624 ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring); 6625 req->ring_id = cpu_to_le16(ring_id); 6626 rc = _hwrm_send_message(bp, req, sizeof(*req), 6627 HWRM_CMD_TIMEOUT); 6628 if (rc) 6629 break; 6630 } 6631 if (bnapi->rx_ring) 6632 hw_coal = &bp->rx_coal; 6633 else 6634 hw_coal = &bp->tx_coal; 6635 __bnxt_hwrm_set_coal_nq(bp, bnapi, hw_coal); 6636 } 6637 mutex_unlock(&bp->hwrm_cmd_lock); 6638 return rc; 6639 } 6640 6641 static void bnxt_hwrm_stat_ctx_free(struct bnxt *bp) 6642 { 6643 struct hwrm_stat_ctx_clr_stats_input req0 = {0}; 6644 struct hwrm_stat_ctx_free_input req = {0}; 6645 int i; 6646 6647 if (!bp->bnapi) 6648 return; 6649 6650 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 6651 return; 6652 6653 bnxt_hwrm_cmd_hdr_init(bp, &req0, HWRM_STAT_CTX_CLR_STATS, -1, -1); 6654 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_FREE, -1, -1); 6655 6656 mutex_lock(&bp->hwrm_cmd_lock); 6657 for (i = 0; i < bp->cp_nr_rings; i++) { 6658 struct bnxt_napi *bnapi = bp->bnapi[i]; 6659 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 6660 6661 if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) { 6662 req.stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id); 6663 if (BNXT_FW_MAJ(bp) <= 20) { 6664 req0.stat_ctx_id = req.stat_ctx_id; 6665 _hwrm_send_message(bp, &req0, sizeof(req0), 6666 HWRM_CMD_TIMEOUT); 6667 } 6668 _hwrm_send_message(bp, &req, sizeof(req), 6669 HWRM_CMD_TIMEOUT); 6670 6671 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID; 6672 } 6673 } 6674 mutex_unlock(&bp->hwrm_cmd_lock); 6675 } 6676 6677 static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp) 6678 { 6679 int rc = 0, i; 6680 struct hwrm_stat_ctx_alloc_input req = {0}; 6681 struct hwrm_stat_ctx_alloc_output *resp = bp->hwrm_cmd_resp_addr; 6682 6683 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 6684 return 0; 6685 6686 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_ALLOC, -1, -1); 6687 6688 req.stats_dma_length = cpu_to_le16(bp->hw_ring_stats_size); 6689 req.update_period_ms = cpu_to_le32(bp->stats_coal_ticks / 1000); 6690 6691 mutex_lock(&bp->hwrm_cmd_lock); 6692 for (i = 0; i < bp->cp_nr_rings; i++) { 6693 struct bnxt_napi *bnapi = bp->bnapi[i]; 6694 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 6695 6696 req.stats_dma_addr = cpu_to_le64(cpr->stats.hw_stats_map); 6697 6698 rc = _hwrm_send_message(bp, &req, sizeof(req), 6699 HWRM_CMD_TIMEOUT); 6700 if (rc) 6701 break; 6702 6703 cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id); 6704 6705 bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id; 6706 } 6707 mutex_unlock(&bp->hwrm_cmd_lock); 6708 return rc; 6709 } 6710 6711 static int bnxt_hwrm_func_qcfg(struct bnxt *bp) 6712 { 6713 struct hwrm_func_qcfg_input req = {0}; 6714 struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 6715 u32 min_db_offset = 0; 6716 u16 flags; 6717 int rc; 6718 6719 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1); 6720 req.fid = cpu_to_le16(0xffff); 6721 mutex_lock(&bp->hwrm_cmd_lock); 6722 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6723 if (rc) 6724 goto func_qcfg_exit; 6725 6726 #ifdef CONFIG_BNXT_SRIOV 6727 if (BNXT_VF(bp)) { 6728 struct bnxt_vf_info *vf = &bp->vf; 6729 6730 vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK; 6731 } else { 6732 bp->pf.registered_vfs = le16_to_cpu(resp->registered_vfs); 6733 } 6734 #endif 6735 flags = le16_to_cpu(resp->flags); 6736 if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED | 6737 FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED)) { 6738 bp->fw_cap |= BNXT_FW_CAP_LLDP_AGENT; 6739 if (flags & FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED) 6740 bp->fw_cap |= BNXT_FW_CAP_DCBX_AGENT; 6741 } 6742 if (BNXT_PF(bp) && (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST)) 6743 bp->flags |= BNXT_FLAG_MULTI_HOST; 6744 if (flags & FUNC_QCFG_RESP_FLAGS_RING_MONITOR_ENABLED) 6745 bp->fw_cap |= BNXT_FW_CAP_RING_MONITOR; 6746 6747 switch (resp->port_partition_type) { 6748 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0: 6749 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5: 6750 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0: 6751 bp->port_partition_type = resp->port_partition_type; 6752 break; 6753 } 6754 if (bp->hwrm_spec_code < 0x10707 || 6755 resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEB) 6756 bp->br_mode = BRIDGE_MODE_VEB; 6757 else if (resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEPA) 6758 bp->br_mode = BRIDGE_MODE_VEPA; 6759 else 6760 bp->br_mode = BRIDGE_MODE_UNDEF; 6761 6762 bp->max_mtu = le16_to_cpu(resp->max_mtu_configured); 6763 if (!bp->max_mtu) 6764 bp->max_mtu = BNXT_MAX_MTU; 6765 6766 if (bp->db_size) 6767 goto func_qcfg_exit; 6768 6769 if (bp->flags & BNXT_FLAG_CHIP_P5) { 6770 if (BNXT_PF(bp)) 6771 min_db_offset = DB_PF_OFFSET_P5; 6772 else 6773 min_db_offset = DB_VF_OFFSET_P5; 6774 } 6775 bp->db_size = PAGE_ALIGN(le16_to_cpu(resp->l2_doorbell_bar_size_kb) * 6776 1024); 6777 if (!bp->db_size || bp->db_size > pci_resource_len(bp->pdev, 2) || 6778 bp->db_size <= min_db_offset) 6779 bp->db_size = pci_resource_len(bp->pdev, 2); 6780 6781 func_qcfg_exit: 6782 mutex_unlock(&bp->hwrm_cmd_lock); 6783 return rc; 6784 } 6785 6786 static void bnxt_init_ctx_initializer(struct bnxt_ctx_mem_info *ctx, 6787 struct hwrm_func_backing_store_qcaps_output *resp) 6788 { 6789 struct bnxt_mem_init *mem_init; 6790 u16 init_mask; 6791 u8 init_val; 6792 u8 *offset; 6793 int i; 6794 6795 init_val = resp->ctx_kind_initializer; 6796 init_mask = le16_to_cpu(resp->ctx_init_mask); 6797 offset = &resp->qp_init_offset; 6798 mem_init = &ctx->mem_init[BNXT_CTX_MEM_INIT_QP]; 6799 for (i = 0; i < BNXT_CTX_MEM_INIT_MAX; i++, mem_init++, offset++) { 6800 mem_init->init_val = init_val; 6801 mem_init->offset = BNXT_MEM_INVALID_OFFSET; 6802 if (!init_mask) 6803 continue; 6804 if (i == BNXT_CTX_MEM_INIT_STAT) 6805 offset = &resp->stat_init_offset; 6806 if (init_mask & (1 << i)) 6807 mem_init->offset = *offset * 4; 6808 else 6809 mem_init->init_val = 0; 6810 } 6811 ctx->mem_init[BNXT_CTX_MEM_INIT_QP].size = ctx->qp_entry_size; 6812 ctx->mem_init[BNXT_CTX_MEM_INIT_SRQ].size = ctx->srq_entry_size; 6813 ctx->mem_init[BNXT_CTX_MEM_INIT_CQ].size = ctx->cq_entry_size; 6814 ctx->mem_init[BNXT_CTX_MEM_INIT_VNIC].size = ctx->vnic_entry_size; 6815 ctx->mem_init[BNXT_CTX_MEM_INIT_STAT].size = ctx->stat_entry_size; 6816 ctx->mem_init[BNXT_CTX_MEM_INIT_MRAV].size = ctx->mrav_entry_size; 6817 } 6818 6819 static int bnxt_hwrm_func_backing_store_qcaps(struct bnxt *bp) 6820 { 6821 struct hwrm_func_backing_store_qcaps_input req = {0}; 6822 struct hwrm_func_backing_store_qcaps_output *resp = 6823 bp->hwrm_cmd_resp_addr; 6824 int rc; 6825 6826 if (bp->hwrm_spec_code < 0x10902 || BNXT_VF(bp) || bp->ctx) 6827 return 0; 6828 6829 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_BACKING_STORE_QCAPS, -1, -1); 6830 mutex_lock(&bp->hwrm_cmd_lock); 6831 rc = _hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6832 if (!rc) { 6833 struct bnxt_ctx_pg_info *ctx_pg; 6834 struct bnxt_ctx_mem_info *ctx; 6835 int i, tqm_rings; 6836 6837 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 6838 if (!ctx) { 6839 rc = -ENOMEM; 6840 goto ctx_err; 6841 } 6842 ctx->qp_max_entries = le32_to_cpu(resp->qp_max_entries); 6843 ctx->qp_min_qp1_entries = le16_to_cpu(resp->qp_min_qp1_entries); 6844 ctx->qp_max_l2_entries = le16_to_cpu(resp->qp_max_l2_entries); 6845 ctx->qp_entry_size = le16_to_cpu(resp->qp_entry_size); 6846 ctx->srq_max_l2_entries = le16_to_cpu(resp->srq_max_l2_entries); 6847 ctx->srq_max_entries = le32_to_cpu(resp->srq_max_entries); 6848 ctx->srq_entry_size = le16_to_cpu(resp->srq_entry_size); 6849 ctx->cq_max_l2_entries = le16_to_cpu(resp->cq_max_l2_entries); 6850 ctx->cq_max_entries = le32_to_cpu(resp->cq_max_entries); 6851 ctx->cq_entry_size = le16_to_cpu(resp->cq_entry_size); 6852 ctx->vnic_max_vnic_entries = 6853 le16_to_cpu(resp->vnic_max_vnic_entries); 6854 ctx->vnic_max_ring_table_entries = 6855 le16_to_cpu(resp->vnic_max_ring_table_entries); 6856 ctx->vnic_entry_size = le16_to_cpu(resp->vnic_entry_size); 6857 ctx->stat_max_entries = le32_to_cpu(resp->stat_max_entries); 6858 ctx->stat_entry_size = le16_to_cpu(resp->stat_entry_size); 6859 ctx->tqm_entry_size = le16_to_cpu(resp->tqm_entry_size); 6860 ctx->tqm_min_entries_per_ring = 6861 le32_to_cpu(resp->tqm_min_entries_per_ring); 6862 ctx->tqm_max_entries_per_ring = 6863 le32_to_cpu(resp->tqm_max_entries_per_ring); 6864 ctx->tqm_entries_multiple = resp->tqm_entries_multiple; 6865 if (!ctx->tqm_entries_multiple) 6866 ctx->tqm_entries_multiple = 1; 6867 ctx->mrav_max_entries = le32_to_cpu(resp->mrav_max_entries); 6868 ctx->mrav_entry_size = le16_to_cpu(resp->mrav_entry_size); 6869 ctx->mrav_num_entries_units = 6870 le16_to_cpu(resp->mrav_num_entries_units); 6871 ctx->tim_entry_size = le16_to_cpu(resp->tim_entry_size); 6872 ctx->tim_max_entries = le32_to_cpu(resp->tim_max_entries); 6873 6874 bnxt_init_ctx_initializer(ctx, resp); 6875 6876 ctx->tqm_fp_rings_count = resp->tqm_fp_rings_count; 6877 if (!ctx->tqm_fp_rings_count) 6878 ctx->tqm_fp_rings_count = bp->max_q; 6879 else if (ctx->tqm_fp_rings_count > BNXT_MAX_TQM_FP_RINGS) 6880 ctx->tqm_fp_rings_count = BNXT_MAX_TQM_FP_RINGS; 6881 6882 tqm_rings = ctx->tqm_fp_rings_count + BNXT_MAX_TQM_SP_RINGS; 6883 ctx_pg = kcalloc(tqm_rings, sizeof(*ctx_pg), GFP_KERNEL); 6884 if (!ctx_pg) { 6885 kfree(ctx); 6886 rc = -ENOMEM; 6887 goto ctx_err; 6888 } 6889 for (i = 0; i < tqm_rings; i++, ctx_pg++) 6890 ctx->tqm_mem[i] = ctx_pg; 6891 bp->ctx = ctx; 6892 } else { 6893 rc = 0; 6894 } 6895 ctx_err: 6896 mutex_unlock(&bp->hwrm_cmd_lock); 6897 return rc; 6898 } 6899 6900 static void bnxt_hwrm_set_pg_attr(struct bnxt_ring_mem_info *rmem, u8 *pg_attr, 6901 __le64 *pg_dir) 6902 { 6903 u8 pg_size = 0; 6904 6905 if (!rmem->nr_pages) 6906 return; 6907 6908 if (BNXT_PAGE_SHIFT == 13) 6909 pg_size = 1 << 4; 6910 else if (BNXT_PAGE_SIZE == 16) 6911 pg_size = 2 << 4; 6912 6913 *pg_attr = pg_size; 6914 if (rmem->depth >= 1) { 6915 if (rmem->depth == 2) 6916 *pg_attr |= 2; 6917 else 6918 *pg_attr |= 1; 6919 *pg_dir = cpu_to_le64(rmem->pg_tbl_map); 6920 } else { 6921 *pg_dir = cpu_to_le64(rmem->dma_arr[0]); 6922 } 6923 } 6924 6925 #define FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES \ 6926 (FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP | \ 6927 FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ | \ 6928 FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ | \ 6929 FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC | \ 6930 FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) 6931 6932 static int bnxt_hwrm_func_backing_store_cfg(struct bnxt *bp, u32 enables) 6933 { 6934 struct hwrm_func_backing_store_cfg_input req = {0}; 6935 struct bnxt_ctx_mem_info *ctx = bp->ctx; 6936 struct bnxt_ctx_pg_info *ctx_pg; 6937 u32 req_len = sizeof(req); 6938 __le32 *num_entries; 6939 __le64 *pg_dir; 6940 u32 flags = 0; 6941 u8 *pg_attr; 6942 u32 ena; 6943 int i; 6944 6945 if (!ctx) 6946 return 0; 6947 6948 if (req_len > bp->hwrm_max_ext_req_len) 6949 req_len = BNXT_BACKING_STORE_CFG_LEGACY_LEN; 6950 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_BACKING_STORE_CFG, -1, -1); 6951 req.enables = cpu_to_le32(enables); 6952 6953 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP) { 6954 ctx_pg = &ctx->qp_mem; 6955 req.qp_num_entries = cpu_to_le32(ctx_pg->entries); 6956 req.qp_num_qp1_entries = cpu_to_le16(ctx->qp_min_qp1_entries); 6957 req.qp_num_l2_entries = cpu_to_le16(ctx->qp_max_l2_entries); 6958 req.qp_entry_size = cpu_to_le16(ctx->qp_entry_size); 6959 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 6960 &req.qpc_pg_size_qpc_lvl, 6961 &req.qpc_page_dir); 6962 } 6963 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ) { 6964 ctx_pg = &ctx->srq_mem; 6965 req.srq_num_entries = cpu_to_le32(ctx_pg->entries); 6966 req.srq_num_l2_entries = cpu_to_le16(ctx->srq_max_l2_entries); 6967 req.srq_entry_size = cpu_to_le16(ctx->srq_entry_size); 6968 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 6969 &req.srq_pg_size_srq_lvl, 6970 &req.srq_page_dir); 6971 } 6972 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ) { 6973 ctx_pg = &ctx->cq_mem; 6974 req.cq_num_entries = cpu_to_le32(ctx_pg->entries); 6975 req.cq_num_l2_entries = cpu_to_le16(ctx->cq_max_l2_entries); 6976 req.cq_entry_size = cpu_to_le16(ctx->cq_entry_size); 6977 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, &req.cq_pg_size_cq_lvl, 6978 &req.cq_page_dir); 6979 } 6980 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC) { 6981 ctx_pg = &ctx->vnic_mem; 6982 req.vnic_num_vnic_entries = 6983 cpu_to_le16(ctx->vnic_max_vnic_entries); 6984 req.vnic_num_ring_table_entries = 6985 cpu_to_le16(ctx->vnic_max_ring_table_entries); 6986 req.vnic_entry_size = cpu_to_le16(ctx->vnic_entry_size); 6987 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 6988 &req.vnic_pg_size_vnic_lvl, 6989 &req.vnic_page_dir); 6990 } 6991 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) { 6992 ctx_pg = &ctx->stat_mem; 6993 req.stat_num_entries = cpu_to_le32(ctx->stat_max_entries); 6994 req.stat_entry_size = cpu_to_le16(ctx->stat_entry_size); 6995 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 6996 &req.stat_pg_size_stat_lvl, 6997 &req.stat_page_dir); 6998 } 6999 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV) { 7000 ctx_pg = &ctx->mrav_mem; 7001 req.mrav_num_entries = cpu_to_le32(ctx_pg->entries); 7002 if (ctx->mrav_num_entries_units) 7003 flags |= 7004 FUNC_BACKING_STORE_CFG_REQ_FLAGS_MRAV_RESERVATION_SPLIT; 7005 req.mrav_entry_size = cpu_to_le16(ctx->mrav_entry_size); 7006 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 7007 &req.mrav_pg_size_mrav_lvl, 7008 &req.mrav_page_dir); 7009 } 7010 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM) { 7011 ctx_pg = &ctx->tim_mem; 7012 req.tim_num_entries = cpu_to_le32(ctx_pg->entries); 7013 req.tim_entry_size = cpu_to_le16(ctx->tim_entry_size); 7014 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 7015 &req.tim_pg_size_tim_lvl, 7016 &req.tim_page_dir); 7017 } 7018 for (i = 0, num_entries = &req.tqm_sp_num_entries, 7019 pg_attr = &req.tqm_sp_pg_size_tqm_sp_lvl, 7020 pg_dir = &req.tqm_sp_page_dir, 7021 ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP; 7022 i < BNXT_MAX_TQM_RINGS; 7023 i++, num_entries++, pg_attr++, pg_dir++, ena <<= 1) { 7024 if (!(enables & ena)) 7025 continue; 7026 7027 req.tqm_entry_size = cpu_to_le16(ctx->tqm_entry_size); 7028 ctx_pg = ctx->tqm_mem[i]; 7029 *num_entries = cpu_to_le32(ctx_pg->entries); 7030 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, pg_attr, pg_dir); 7031 } 7032 req.flags = cpu_to_le32(flags); 7033 return hwrm_send_message(bp, &req, req_len, HWRM_CMD_TIMEOUT); 7034 } 7035 7036 static int bnxt_alloc_ctx_mem_blk(struct bnxt *bp, 7037 struct bnxt_ctx_pg_info *ctx_pg) 7038 { 7039 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem; 7040 7041 rmem->page_size = BNXT_PAGE_SIZE; 7042 rmem->pg_arr = ctx_pg->ctx_pg_arr; 7043 rmem->dma_arr = ctx_pg->ctx_dma_arr; 7044 rmem->flags = BNXT_RMEM_VALID_PTE_FLAG; 7045 if (rmem->depth >= 1) 7046 rmem->flags |= BNXT_RMEM_USE_FULL_PAGE_FLAG; 7047 return bnxt_alloc_ring(bp, rmem); 7048 } 7049 7050 static int bnxt_alloc_ctx_pg_tbls(struct bnxt *bp, 7051 struct bnxt_ctx_pg_info *ctx_pg, u32 mem_size, 7052 u8 depth, struct bnxt_mem_init *mem_init) 7053 { 7054 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem; 7055 int rc; 7056 7057 if (!mem_size) 7058 return -EINVAL; 7059 7060 ctx_pg->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE); 7061 if (ctx_pg->nr_pages > MAX_CTX_TOTAL_PAGES) { 7062 ctx_pg->nr_pages = 0; 7063 return -EINVAL; 7064 } 7065 if (ctx_pg->nr_pages > MAX_CTX_PAGES || depth > 1) { 7066 int nr_tbls, i; 7067 7068 rmem->depth = 2; 7069 ctx_pg->ctx_pg_tbl = kcalloc(MAX_CTX_PAGES, sizeof(ctx_pg), 7070 GFP_KERNEL); 7071 if (!ctx_pg->ctx_pg_tbl) 7072 return -ENOMEM; 7073 nr_tbls = DIV_ROUND_UP(ctx_pg->nr_pages, MAX_CTX_PAGES); 7074 rmem->nr_pages = nr_tbls; 7075 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg); 7076 if (rc) 7077 return rc; 7078 for (i = 0; i < nr_tbls; i++) { 7079 struct bnxt_ctx_pg_info *pg_tbl; 7080 7081 pg_tbl = kzalloc(sizeof(*pg_tbl), GFP_KERNEL); 7082 if (!pg_tbl) 7083 return -ENOMEM; 7084 ctx_pg->ctx_pg_tbl[i] = pg_tbl; 7085 rmem = &pg_tbl->ring_mem; 7086 rmem->pg_tbl = ctx_pg->ctx_pg_arr[i]; 7087 rmem->pg_tbl_map = ctx_pg->ctx_dma_arr[i]; 7088 rmem->depth = 1; 7089 rmem->nr_pages = MAX_CTX_PAGES; 7090 rmem->mem_init = mem_init; 7091 if (i == (nr_tbls - 1)) { 7092 int rem = ctx_pg->nr_pages % MAX_CTX_PAGES; 7093 7094 if (rem) 7095 rmem->nr_pages = rem; 7096 } 7097 rc = bnxt_alloc_ctx_mem_blk(bp, pg_tbl); 7098 if (rc) 7099 break; 7100 } 7101 } else { 7102 rmem->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE); 7103 if (rmem->nr_pages > 1 || depth) 7104 rmem->depth = 1; 7105 rmem->mem_init = mem_init; 7106 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg); 7107 } 7108 return rc; 7109 } 7110 7111 static void bnxt_free_ctx_pg_tbls(struct bnxt *bp, 7112 struct bnxt_ctx_pg_info *ctx_pg) 7113 { 7114 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem; 7115 7116 if (rmem->depth > 1 || ctx_pg->nr_pages > MAX_CTX_PAGES || 7117 ctx_pg->ctx_pg_tbl) { 7118 int i, nr_tbls = rmem->nr_pages; 7119 7120 for (i = 0; i < nr_tbls; i++) { 7121 struct bnxt_ctx_pg_info *pg_tbl; 7122 struct bnxt_ring_mem_info *rmem2; 7123 7124 pg_tbl = ctx_pg->ctx_pg_tbl[i]; 7125 if (!pg_tbl) 7126 continue; 7127 rmem2 = &pg_tbl->ring_mem; 7128 bnxt_free_ring(bp, rmem2); 7129 ctx_pg->ctx_pg_arr[i] = NULL; 7130 kfree(pg_tbl); 7131 ctx_pg->ctx_pg_tbl[i] = NULL; 7132 } 7133 kfree(ctx_pg->ctx_pg_tbl); 7134 ctx_pg->ctx_pg_tbl = NULL; 7135 } 7136 bnxt_free_ring(bp, rmem); 7137 ctx_pg->nr_pages = 0; 7138 } 7139 7140 static void bnxt_free_ctx_mem(struct bnxt *bp) 7141 { 7142 struct bnxt_ctx_mem_info *ctx = bp->ctx; 7143 int i; 7144 7145 if (!ctx) 7146 return; 7147 7148 if (ctx->tqm_mem[0]) { 7149 for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++) 7150 bnxt_free_ctx_pg_tbls(bp, ctx->tqm_mem[i]); 7151 kfree(ctx->tqm_mem[0]); 7152 ctx->tqm_mem[0] = NULL; 7153 } 7154 7155 bnxt_free_ctx_pg_tbls(bp, &ctx->tim_mem); 7156 bnxt_free_ctx_pg_tbls(bp, &ctx->mrav_mem); 7157 bnxt_free_ctx_pg_tbls(bp, &ctx->stat_mem); 7158 bnxt_free_ctx_pg_tbls(bp, &ctx->vnic_mem); 7159 bnxt_free_ctx_pg_tbls(bp, &ctx->cq_mem); 7160 bnxt_free_ctx_pg_tbls(bp, &ctx->srq_mem); 7161 bnxt_free_ctx_pg_tbls(bp, &ctx->qp_mem); 7162 ctx->flags &= ~BNXT_CTX_FLAG_INITED; 7163 } 7164 7165 static int bnxt_alloc_ctx_mem(struct bnxt *bp) 7166 { 7167 struct bnxt_ctx_pg_info *ctx_pg; 7168 struct bnxt_ctx_mem_info *ctx; 7169 struct bnxt_mem_init *init; 7170 u32 mem_size, ena, entries; 7171 u32 entries_sp, min; 7172 u32 num_mr, num_ah; 7173 u32 extra_srqs = 0; 7174 u32 extra_qps = 0; 7175 u8 pg_lvl = 1; 7176 int i, rc; 7177 7178 rc = bnxt_hwrm_func_backing_store_qcaps(bp); 7179 if (rc) { 7180 netdev_err(bp->dev, "Failed querying context mem capability, rc = %d.\n", 7181 rc); 7182 return rc; 7183 } 7184 ctx = bp->ctx; 7185 if (!ctx || (ctx->flags & BNXT_CTX_FLAG_INITED)) 7186 return 0; 7187 7188 if ((bp->flags & BNXT_FLAG_ROCE_CAP) && !is_kdump_kernel()) { 7189 pg_lvl = 2; 7190 extra_qps = 65536; 7191 extra_srqs = 8192; 7192 } 7193 7194 ctx_pg = &ctx->qp_mem; 7195 ctx_pg->entries = ctx->qp_min_qp1_entries + ctx->qp_max_l2_entries + 7196 extra_qps; 7197 if (ctx->qp_entry_size) { 7198 mem_size = ctx->qp_entry_size * ctx_pg->entries; 7199 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_QP]; 7200 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, init); 7201 if (rc) 7202 return rc; 7203 } 7204 7205 ctx_pg = &ctx->srq_mem; 7206 ctx_pg->entries = ctx->srq_max_l2_entries + extra_srqs; 7207 if (ctx->srq_entry_size) { 7208 mem_size = ctx->srq_entry_size * ctx_pg->entries; 7209 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_SRQ]; 7210 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, init); 7211 if (rc) 7212 return rc; 7213 } 7214 7215 ctx_pg = &ctx->cq_mem; 7216 ctx_pg->entries = ctx->cq_max_l2_entries + extra_qps * 2; 7217 if (ctx->cq_entry_size) { 7218 mem_size = ctx->cq_entry_size * ctx_pg->entries; 7219 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_CQ]; 7220 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, init); 7221 if (rc) 7222 return rc; 7223 } 7224 7225 ctx_pg = &ctx->vnic_mem; 7226 ctx_pg->entries = ctx->vnic_max_vnic_entries + 7227 ctx->vnic_max_ring_table_entries; 7228 if (ctx->vnic_entry_size) { 7229 mem_size = ctx->vnic_entry_size * ctx_pg->entries; 7230 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_VNIC]; 7231 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, init); 7232 if (rc) 7233 return rc; 7234 } 7235 7236 ctx_pg = &ctx->stat_mem; 7237 ctx_pg->entries = ctx->stat_max_entries; 7238 if (ctx->stat_entry_size) { 7239 mem_size = ctx->stat_entry_size * ctx_pg->entries; 7240 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_STAT]; 7241 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, init); 7242 if (rc) 7243 return rc; 7244 } 7245 7246 ena = 0; 7247 if (!(bp->flags & BNXT_FLAG_ROCE_CAP)) 7248 goto skip_rdma; 7249 7250 ctx_pg = &ctx->mrav_mem; 7251 /* 128K extra is needed to accommodate static AH context 7252 * allocation by f/w. 7253 */ 7254 num_mr = 1024 * 256; 7255 num_ah = 1024 * 128; 7256 ctx_pg->entries = num_mr + num_ah; 7257 if (ctx->mrav_entry_size) { 7258 mem_size = ctx->mrav_entry_size * ctx_pg->entries; 7259 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_MRAV]; 7260 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 2, init); 7261 if (rc) 7262 return rc; 7263 } 7264 ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV; 7265 if (ctx->mrav_num_entries_units) 7266 ctx_pg->entries = 7267 ((num_mr / ctx->mrav_num_entries_units) << 16) | 7268 (num_ah / ctx->mrav_num_entries_units); 7269 7270 ctx_pg = &ctx->tim_mem; 7271 ctx_pg->entries = ctx->qp_mem.entries; 7272 if (ctx->tim_entry_size) { 7273 mem_size = ctx->tim_entry_size * ctx_pg->entries; 7274 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, NULL); 7275 if (rc) 7276 return rc; 7277 } 7278 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM; 7279 7280 skip_rdma: 7281 min = ctx->tqm_min_entries_per_ring; 7282 entries_sp = ctx->vnic_max_vnic_entries + ctx->qp_max_l2_entries + 7283 2 * (extra_qps + ctx->qp_min_qp1_entries) + min; 7284 entries_sp = roundup(entries_sp, ctx->tqm_entries_multiple); 7285 entries = ctx->qp_max_l2_entries + extra_qps + ctx->qp_min_qp1_entries; 7286 entries = roundup(entries, ctx->tqm_entries_multiple); 7287 entries = clamp_t(u32, entries, min, ctx->tqm_max_entries_per_ring); 7288 for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++) { 7289 ctx_pg = ctx->tqm_mem[i]; 7290 ctx_pg->entries = i ? entries : entries_sp; 7291 if (ctx->tqm_entry_size) { 7292 mem_size = ctx->tqm_entry_size * ctx_pg->entries; 7293 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, 7294 NULL); 7295 if (rc) 7296 return rc; 7297 } 7298 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP << i; 7299 } 7300 ena |= FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES; 7301 rc = bnxt_hwrm_func_backing_store_cfg(bp, ena); 7302 if (rc) { 7303 netdev_err(bp->dev, "Failed configuring context mem, rc = %d.\n", 7304 rc); 7305 return rc; 7306 } 7307 ctx->flags |= BNXT_CTX_FLAG_INITED; 7308 return 0; 7309 } 7310 7311 int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp, bool all) 7312 { 7313 struct hwrm_func_resource_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 7314 struct hwrm_func_resource_qcaps_input req = {0}; 7315 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 7316 int rc; 7317 7318 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESOURCE_QCAPS, -1, -1); 7319 req.fid = cpu_to_le16(0xffff); 7320 7321 mutex_lock(&bp->hwrm_cmd_lock); 7322 rc = _hwrm_send_message_silent(bp, &req, sizeof(req), 7323 HWRM_CMD_TIMEOUT); 7324 if (rc) 7325 goto hwrm_func_resc_qcaps_exit; 7326 7327 hw_resc->max_tx_sch_inputs = le16_to_cpu(resp->max_tx_scheduler_inputs); 7328 if (!all) 7329 goto hwrm_func_resc_qcaps_exit; 7330 7331 hw_resc->min_rsscos_ctxs = le16_to_cpu(resp->min_rsscos_ctx); 7332 hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx); 7333 hw_resc->min_cp_rings = le16_to_cpu(resp->min_cmpl_rings); 7334 hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings); 7335 hw_resc->min_tx_rings = le16_to_cpu(resp->min_tx_rings); 7336 hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings); 7337 hw_resc->min_rx_rings = le16_to_cpu(resp->min_rx_rings); 7338 hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings); 7339 hw_resc->min_hw_ring_grps = le16_to_cpu(resp->min_hw_ring_grps); 7340 hw_resc->max_hw_ring_grps = le16_to_cpu(resp->max_hw_ring_grps); 7341 hw_resc->min_l2_ctxs = le16_to_cpu(resp->min_l2_ctxs); 7342 hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs); 7343 hw_resc->min_vnics = le16_to_cpu(resp->min_vnics); 7344 hw_resc->max_vnics = le16_to_cpu(resp->max_vnics); 7345 hw_resc->min_stat_ctxs = le16_to_cpu(resp->min_stat_ctx); 7346 hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx); 7347 7348 if (bp->flags & BNXT_FLAG_CHIP_P5) { 7349 u16 max_msix = le16_to_cpu(resp->max_msix); 7350 7351 hw_resc->max_nqs = max_msix; 7352 hw_resc->max_hw_ring_grps = hw_resc->max_rx_rings; 7353 } 7354 7355 if (BNXT_PF(bp)) { 7356 struct bnxt_pf_info *pf = &bp->pf; 7357 7358 pf->vf_resv_strategy = 7359 le16_to_cpu(resp->vf_reservation_strategy); 7360 if (pf->vf_resv_strategy > BNXT_VF_RESV_STRATEGY_MINIMAL_STATIC) 7361 pf->vf_resv_strategy = BNXT_VF_RESV_STRATEGY_MAXIMAL; 7362 } 7363 hwrm_func_resc_qcaps_exit: 7364 mutex_unlock(&bp->hwrm_cmd_lock); 7365 return rc; 7366 } 7367 7368 static int __bnxt_hwrm_func_qcaps(struct bnxt *bp) 7369 { 7370 int rc = 0; 7371 struct hwrm_func_qcaps_input req = {0}; 7372 struct hwrm_func_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 7373 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 7374 u32 flags, flags_ext; 7375 7376 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCAPS, -1, -1); 7377 req.fid = cpu_to_le16(0xffff); 7378 7379 mutex_lock(&bp->hwrm_cmd_lock); 7380 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 7381 if (rc) 7382 goto hwrm_func_qcaps_exit; 7383 7384 flags = le32_to_cpu(resp->flags); 7385 if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED) 7386 bp->flags |= BNXT_FLAG_ROCEV1_CAP; 7387 if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED) 7388 bp->flags |= BNXT_FLAG_ROCEV2_CAP; 7389 if (flags & FUNC_QCAPS_RESP_FLAGS_PCIE_STATS_SUPPORTED) 7390 bp->fw_cap |= BNXT_FW_CAP_PCIE_STATS_SUPPORTED; 7391 if (flags & FUNC_QCAPS_RESP_FLAGS_HOT_RESET_CAPABLE) 7392 bp->fw_cap |= BNXT_FW_CAP_HOT_RESET; 7393 if (flags & FUNC_QCAPS_RESP_FLAGS_EXT_STATS_SUPPORTED) 7394 bp->fw_cap |= BNXT_FW_CAP_EXT_STATS_SUPPORTED; 7395 if (flags & FUNC_QCAPS_RESP_FLAGS_ERROR_RECOVERY_CAPABLE) 7396 bp->fw_cap |= BNXT_FW_CAP_ERROR_RECOVERY; 7397 if (flags & FUNC_QCAPS_RESP_FLAGS_ERR_RECOVER_RELOAD) 7398 bp->fw_cap |= BNXT_FW_CAP_ERR_RECOVER_RELOAD; 7399 if (!(flags & FUNC_QCAPS_RESP_FLAGS_VLAN_ACCELERATION_TX_DISABLED)) 7400 bp->fw_cap |= BNXT_FW_CAP_VLAN_TX_INSERT; 7401 7402 flags_ext = le32_to_cpu(resp->flags_ext); 7403 if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_EXT_HW_STATS_SUPPORTED) 7404 bp->fw_cap |= BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED; 7405 7406 bp->tx_push_thresh = 0; 7407 if ((flags & FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED) && 7408 BNXT_FW_MAJ(bp) > 217) 7409 bp->tx_push_thresh = BNXT_TX_PUSH_THRESH; 7410 7411 hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx); 7412 hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings); 7413 hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings); 7414 hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings); 7415 hw_resc->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps); 7416 if (!hw_resc->max_hw_ring_grps) 7417 hw_resc->max_hw_ring_grps = hw_resc->max_tx_rings; 7418 hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs); 7419 hw_resc->max_vnics = le16_to_cpu(resp->max_vnics); 7420 hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx); 7421 7422 if (BNXT_PF(bp)) { 7423 struct bnxt_pf_info *pf = &bp->pf; 7424 7425 pf->fw_fid = le16_to_cpu(resp->fid); 7426 pf->port_id = le16_to_cpu(resp->port_id); 7427 memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN); 7428 pf->first_vf_id = le16_to_cpu(resp->first_vf_id); 7429 pf->max_vfs = le16_to_cpu(resp->max_vfs); 7430 pf->max_encap_records = le32_to_cpu(resp->max_encap_records); 7431 pf->max_decap_records = le32_to_cpu(resp->max_decap_records); 7432 pf->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows); 7433 pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows); 7434 pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows); 7435 pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows); 7436 bp->flags &= ~BNXT_FLAG_WOL_CAP; 7437 if (flags & FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED) 7438 bp->flags |= BNXT_FLAG_WOL_CAP; 7439 } else { 7440 #ifdef CONFIG_BNXT_SRIOV 7441 struct bnxt_vf_info *vf = &bp->vf; 7442 7443 vf->fw_fid = le16_to_cpu(resp->fid); 7444 memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN); 7445 #endif 7446 } 7447 7448 hwrm_func_qcaps_exit: 7449 mutex_unlock(&bp->hwrm_cmd_lock); 7450 return rc; 7451 } 7452 7453 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp); 7454 7455 static int bnxt_hwrm_func_qcaps(struct bnxt *bp) 7456 { 7457 int rc; 7458 7459 rc = __bnxt_hwrm_func_qcaps(bp); 7460 if (rc) 7461 return rc; 7462 rc = bnxt_hwrm_queue_qportcfg(bp); 7463 if (rc) { 7464 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %d\n", rc); 7465 return rc; 7466 } 7467 if (bp->hwrm_spec_code >= 0x10803) { 7468 rc = bnxt_alloc_ctx_mem(bp); 7469 if (rc) 7470 return rc; 7471 rc = bnxt_hwrm_func_resc_qcaps(bp, true); 7472 if (!rc) 7473 bp->fw_cap |= BNXT_FW_CAP_NEW_RM; 7474 } 7475 return 0; 7476 } 7477 7478 static int bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(struct bnxt *bp) 7479 { 7480 struct hwrm_cfa_adv_flow_mgnt_qcaps_input req = {0}; 7481 struct hwrm_cfa_adv_flow_mgnt_qcaps_output *resp; 7482 int rc = 0; 7483 u32 flags; 7484 7485 if (!(bp->fw_cap & BNXT_FW_CAP_CFA_ADV_FLOW)) 7486 return 0; 7487 7488 resp = bp->hwrm_cmd_resp_addr; 7489 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_ADV_FLOW_MGNT_QCAPS, -1, -1); 7490 7491 mutex_lock(&bp->hwrm_cmd_lock); 7492 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 7493 if (rc) 7494 goto hwrm_cfa_adv_qcaps_exit; 7495 7496 flags = le32_to_cpu(resp->flags); 7497 if (flags & 7498 CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_V2_SUPPORTED) 7499 bp->fw_cap |= BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2; 7500 7501 hwrm_cfa_adv_qcaps_exit: 7502 mutex_unlock(&bp->hwrm_cmd_lock); 7503 return rc; 7504 } 7505 7506 static int __bnxt_alloc_fw_health(struct bnxt *bp) 7507 { 7508 if (bp->fw_health) 7509 return 0; 7510 7511 bp->fw_health = kzalloc(sizeof(*bp->fw_health), GFP_KERNEL); 7512 if (!bp->fw_health) 7513 return -ENOMEM; 7514 7515 return 0; 7516 } 7517 7518 static int bnxt_alloc_fw_health(struct bnxt *bp) 7519 { 7520 int rc; 7521 7522 if (!(bp->fw_cap & BNXT_FW_CAP_HOT_RESET) && 7523 !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) 7524 return 0; 7525 7526 rc = __bnxt_alloc_fw_health(bp); 7527 if (rc) { 7528 bp->fw_cap &= ~BNXT_FW_CAP_HOT_RESET; 7529 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY; 7530 return rc; 7531 } 7532 7533 return 0; 7534 } 7535 7536 static void __bnxt_map_fw_health_reg(struct bnxt *bp, u32 reg) 7537 { 7538 writel(reg & BNXT_GRC_BASE_MASK, bp->bar0 + 7539 BNXT_GRCPF_REG_WINDOW_BASE_OUT + 7540 BNXT_FW_HEALTH_WIN_MAP_OFF); 7541 } 7542 7543 static void bnxt_try_map_fw_health_reg(struct bnxt *bp) 7544 { 7545 void __iomem *hs; 7546 u32 status_loc; 7547 u32 reg_type; 7548 u32 sig; 7549 7550 __bnxt_map_fw_health_reg(bp, HCOMM_STATUS_STRUCT_LOC); 7551 hs = bp->bar0 + BNXT_FW_HEALTH_WIN_OFF(HCOMM_STATUS_STRUCT_LOC); 7552 7553 sig = readl(hs + offsetof(struct hcomm_status, sig_ver)); 7554 if ((sig & HCOMM_STATUS_SIGNATURE_MASK) != HCOMM_STATUS_SIGNATURE_VAL) { 7555 if (!bp->chip_num) { 7556 __bnxt_map_fw_health_reg(bp, BNXT_GRC_REG_BASE); 7557 bp->chip_num = readl(bp->bar0 + 7558 BNXT_FW_HEALTH_WIN_BASE + 7559 BNXT_GRC_REG_CHIP_NUM); 7560 } 7561 if (!BNXT_CHIP_P5(bp)) { 7562 if (bp->fw_health) 7563 bp->fw_health->status_reliable = false; 7564 return; 7565 } 7566 status_loc = BNXT_GRC_REG_STATUS_P5 | 7567 BNXT_FW_HEALTH_REG_TYPE_BAR0; 7568 } else { 7569 status_loc = readl(hs + offsetof(struct hcomm_status, 7570 fw_status_loc)); 7571 } 7572 7573 if (__bnxt_alloc_fw_health(bp)) { 7574 netdev_warn(bp->dev, "no memory for firmware status checks\n"); 7575 return; 7576 } 7577 7578 bp->fw_health->regs[BNXT_FW_HEALTH_REG] = status_loc; 7579 reg_type = BNXT_FW_HEALTH_REG_TYPE(status_loc); 7580 if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC) { 7581 __bnxt_map_fw_health_reg(bp, status_loc); 7582 bp->fw_health->mapped_regs[BNXT_FW_HEALTH_REG] = 7583 BNXT_FW_HEALTH_WIN_OFF(status_loc); 7584 } 7585 7586 bp->fw_health->status_reliable = true; 7587 } 7588 7589 static int bnxt_map_fw_health_regs(struct bnxt *bp) 7590 { 7591 struct bnxt_fw_health *fw_health = bp->fw_health; 7592 u32 reg_base = 0xffffffff; 7593 int i; 7594 7595 /* Only pre-map the monitoring GRC registers using window 3 */ 7596 for (i = 0; i < 4; i++) { 7597 u32 reg = fw_health->regs[i]; 7598 7599 if (BNXT_FW_HEALTH_REG_TYPE(reg) != BNXT_FW_HEALTH_REG_TYPE_GRC) 7600 continue; 7601 if (reg_base == 0xffffffff) 7602 reg_base = reg & BNXT_GRC_BASE_MASK; 7603 if ((reg & BNXT_GRC_BASE_MASK) != reg_base) 7604 return -ERANGE; 7605 fw_health->mapped_regs[i] = BNXT_FW_HEALTH_WIN_OFF(reg); 7606 } 7607 if (reg_base == 0xffffffff) 7608 return 0; 7609 7610 __bnxt_map_fw_health_reg(bp, reg_base); 7611 return 0; 7612 } 7613 7614 static int bnxt_hwrm_error_recovery_qcfg(struct bnxt *bp) 7615 { 7616 struct hwrm_error_recovery_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 7617 struct bnxt_fw_health *fw_health = bp->fw_health; 7618 struct hwrm_error_recovery_qcfg_input req = {0}; 7619 int rc, i; 7620 7621 if (!(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) 7622 return 0; 7623 7624 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_ERROR_RECOVERY_QCFG, -1, -1); 7625 mutex_lock(&bp->hwrm_cmd_lock); 7626 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 7627 if (rc) 7628 goto err_recovery_out; 7629 fw_health->flags = le32_to_cpu(resp->flags); 7630 if ((fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) && 7631 !(bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL)) { 7632 rc = -EINVAL; 7633 goto err_recovery_out; 7634 } 7635 fw_health->polling_dsecs = le32_to_cpu(resp->driver_polling_freq); 7636 fw_health->master_func_wait_dsecs = 7637 le32_to_cpu(resp->master_func_wait_period); 7638 fw_health->normal_func_wait_dsecs = 7639 le32_to_cpu(resp->normal_func_wait_period); 7640 fw_health->post_reset_wait_dsecs = 7641 le32_to_cpu(resp->master_func_wait_period_after_reset); 7642 fw_health->post_reset_max_wait_dsecs = 7643 le32_to_cpu(resp->max_bailout_time_after_reset); 7644 fw_health->regs[BNXT_FW_HEALTH_REG] = 7645 le32_to_cpu(resp->fw_health_status_reg); 7646 fw_health->regs[BNXT_FW_HEARTBEAT_REG] = 7647 le32_to_cpu(resp->fw_heartbeat_reg); 7648 fw_health->regs[BNXT_FW_RESET_CNT_REG] = 7649 le32_to_cpu(resp->fw_reset_cnt_reg); 7650 fw_health->regs[BNXT_FW_RESET_INPROG_REG] = 7651 le32_to_cpu(resp->reset_inprogress_reg); 7652 fw_health->fw_reset_inprog_reg_mask = 7653 le32_to_cpu(resp->reset_inprogress_reg_mask); 7654 fw_health->fw_reset_seq_cnt = resp->reg_array_cnt; 7655 if (fw_health->fw_reset_seq_cnt >= 16) { 7656 rc = -EINVAL; 7657 goto err_recovery_out; 7658 } 7659 for (i = 0; i < fw_health->fw_reset_seq_cnt; i++) { 7660 fw_health->fw_reset_seq_regs[i] = 7661 le32_to_cpu(resp->reset_reg[i]); 7662 fw_health->fw_reset_seq_vals[i] = 7663 le32_to_cpu(resp->reset_reg_val[i]); 7664 fw_health->fw_reset_seq_delay_msec[i] = 7665 resp->delay_after_reset[i]; 7666 } 7667 err_recovery_out: 7668 mutex_unlock(&bp->hwrm_cmd_lock); 7669 if (!rc) 7670 rc = bnxt_map_fw_health_regs(bp); 7671 if (rc) 7672 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY; 7673 return rc; 7674 } 7675 7676 static int bnxt_hwrm_func_reset(struct bnxt *bp) 7677 { 7678 struct hwrm_func_reset_input req = {0}; 7679 7680 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESET, -1, -1); 7681 req.enables = 0; 7682 7683 return hwrm_send_message(bp, &req, sizeof(req), HWRM_RESET_TIMEOUT); 7684 } 7685 7686 static void bnxt_nvm_cfg_ver_get(struct bnxt *bp) 7687 { 7688 struct hwrm_nvm_get_dev_info_output nvm_info; 7689 7690 if (!bnxt_hwrm_nvm_get_dev_info(bp, &nvm_info)) 7691 snprintf(bp->nvm_cfg_ver, FW_VER_STR_LEN, "%d.%d.%d", 7692 nvm_info.nvm_cfg_ver_maj, nvm_info.nvm_cfg_ver_min, 7693 nvm_info.nvm_cfg_ver_upd); 7694 } 7695 7696 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp) 7697 { 7698 int rc = 0; 7699 struct hwrm_queue_qportcfg_input req = {0}; 7700 struct hwrm_queue_qportcfg_output *resp = bp->hwrm_cmd_resp_addr; 7701 u8 i, j, *qptr; 7702 bool no_rdma; 7703 7704 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_QUEUE_QPORTCFG, -1, -1); 7705 7706 mutex_lock(&bp->hwrm_cmd_lock); 7707 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 7708 if (rc) 7709 goto qportcfg_exit; 7710 7711 if (!resp->max_configurable_queues) { 7712 rc = -EINVAL; 7713 goto qportcfg_exit; 7714 } 7715 bp->max_tc = resp->max_configurable_queues; 7716 bp->max_lltc = resp->max_configurable_lossless_queues; 7717 if (bp->max_tc > BNXT_MAX_QUEUE) 7718 bp->max_tc = BNXT_MAX_QUEUE; 7719 7720 no_rdma = !(bp->flags & BNXT_FLAG_ROCE_CAP); 7721 qptr = &resp->queue_id0; 7722 for (i = 0, j = 0; i < bp->max_tc; i++) { 7723 bp->q_info[j].queue_id = *qptr; 7724 bp->q_ids[i] = *qptr++; 7725 bp->q_info[j].queue_profile = *qptr++; 7726 bp->tc_to_qidx[j] = j; 7727 if (!BNXT_CNPQ(bp->q_info[j].queue_profile) || 7728 (no_rdma && BNXT_PF(bp))) 7729 j++; 7730 } 7731 bp->max_q = bp->max_tc; 7732 bp->max_tc = max_t(u8, j, 1); 7733 7734 if (resp->queue_cfg_info & QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG) 7735 bp->max_tc = 1; 7736 7737 if (bp->max_lltc > bp->max_tc) 7738 bp->max_lltc = bp->max_tc; 7739 7740 qportcfg_exit: 7741 mutex_unlock(&bp->hwrm_cmd_lock); 7742 return rc; 7743 } 7744 7745 static int __bnxt_hwrm_ver_get(struct bnxt *bp, bool silent) 7746 { 7747 struct hwrm_ver_get_input req = {0}; 7748 int rc; 7749 7750 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VER_GET, -1, -1); 7751 req.hwrm_intf_maj = HWRM_VERSION_MAJOR; 7752 req.hwrm_intf_min = HWRM_VERSION_MINOR; 7753 req.hwrm_intf_upd = HWRM_VERSION_UPDATE; 7754 7755 rc = bnxt_hwrm_do_send_msg(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT, 7756 silent); 7757 return rc; 7758 } 7759 7760 static int bnxt_hwrm_ver_get(struct bnxt *bp) 7761 { 7762 struct hwrm_ver_get_output *resp = bp->hwrm_cmd_resp_addr; 7763 u16 fw_maj, fw_min, fw_bld, fw_rsv; 7764 u32 dev_caps_cfg, hwrm_ver; 7765 int rc, len; 7766 7767 bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN; 7768 mutex_lock(&bp->hwrm_cmd_lock); 7769 rc = __bnxt_hwrm_ver_get(bp, false); 7770 if (rc) 7771 goto hwrm_ver_get_exit; 7772 7773 memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output)); 7774 7775 bp->hwrm_spec_code = resp->hwrm_intf_maj_8b << 16 | 7776 resp->hwrm_intf_min_8b << 8 | 7777 resp->hwrm_intf_upd_8b; 7778 if (resp->hwrm_intf_maj_8b < 1) { 7779 netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n", 7780 resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b, 7781 resp->hwrm_intf_upd_8b); 7782 netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n"); 7783 } 7784 7785 hwrm_ver = HWRM_VERSION_MAJOR << 16 | HWRM_VERSION_MINOR << 8 | 7786 HWRM_VERSION_UPDATE; 7787 7788 if (bp->hwrm_spec_code > hwrm_ver) 7789 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d", 7790 HWRM_VERSION_MAJOR, HWRM_VERSION_MINOR, 7791 HWRM_VERSION_UPDATE); 7792 else 7793 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d", 7794 resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b, 7795 resp->hwrm_intf_upd_8b); 7796 7797 fw_maj = le16_to_cpu(resp->hwrm_fw_major); 7798 if (bp->hwrm_spec_code > 0x10803 && fw_maj) { 7799 fw_min = le16_to_cpu(resp->hwrm_fw_minor); 7800 fw_bld = le16_to_cpu(resp->hwrm_fw_build); 7801 fw_rsv = le16_to_cpu(resp->hwrm_fw_patch); 7802 len = FW_VER_STR_LEN; 7803 } else { 7804 fw_maj = resp->hwrm_fw_maj_8b; 7805 fw_min = resp->hwrm_fw_min_8b; 7806 fw_bld = resp->hwrm_fw_bld_8b; 7807 fw_rsv = resp->hwrm_fw_rsvd_8b; 7808 len = BC_HWRM_STR_LEN; 7809 } 7810 bp->fw_ver_code = BNXT_FW_VER_CODE(fw_maj, fw_min, fw_bld, fw_rsv); 7811 snprintf(bp->fw_ver_str, len, "%d.%d.%d.%d", fw_maj, fw_min, fw_bld, 7812 fw_rsv); 7813 7814 if (strlen(resp->active_pkg_name)) { 7815 int fw_ver_len = strlen(bp->fw_ver_str); 7816 7817 snprintf(bp->fw_ver_str + fw_ver_len, 7818 FW_VER_STR_LEN - fw_ver_len - 1, "/pkg %s", 7819 resp->active_pkg_name); 7820 bp->fw_cap |= BNXT_FW_CAP_PKG_VER; 7821 } 7822 7823 bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout); 7824 if (!bp->hwrm_cmd_timeout) 7825 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT; 7826 7827 if (resp->hwrm_intf_maj_8b >= 1) { 7828 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len); 7829 bp->hwrm_max_ext_req_len = le16_to_cpu(resp->max_ext_req_len); 7830 } 7831 if (bp->hwrm_max_ext_req_len < HWRM_MAX_REQ_LEN) 7832 bp->hwrm_max_ext_req_len = HWRM_MAX_REQ_LEN; 7833 7834 bp->chip_num = le16_to_cpu(resp->chip_num); 7835 bp->chip_rev = resp->chip_rev; 7836 if (bp->chip_num == CHIP_NUM_58700 && !resp->chip_rev && 7837 !resp->chip_metal) 7838 bp->flags |= BNXT_FLAG_CHIP_NITRO_A0; 7839 7840 dev_caps_cfg = le32_to_cpu(resp->dev_caps_cfg); 7841 if ((dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) && 7842 (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED)) 7843 bp->fw_cap |= BNXT_FW_CAP_SHORT_CMD; 7844 7845 if (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_KONG_MB_CHNL_SUPPORTED) 7846 bp->fw_cap |= BNXT_FW_CAP_KONG_MB_CHNL; 7847 7848 if (dev_caps_cfg & 7849 VER_GET_RESP_DEV_CAPS_CFG_FLOW_HANDLE_64BIT_SUPPORTED) 7850 bp->fw_cap |= BNXT_FW_CAP_OVS_64BIT_HANDLE; 7851 7852 if (dev_caps_cfg & 7853 VER_GET_RESP_DEV_CAPS_CFG_TRUSTED_VF_SUPPORTED) 7854 bp->fw_cap |= BNXT_FW_CAP_TRUSTED_VF; 7855 7856 if (dev_caps_cfg & 7857 VER_GET_RESP_DEV_CAPS_CFG_CFA_ADV_FLOW_MGNT_SUPPORTED) 7858 bp->fw_cap |= BNXT_FW_CAP_CFA_ADV_FLOW; 7859 7860 hwrm_ver_get_exit: 7861 mutex_unlock(&bp->hwrm_cmd_lock); 7862 return rc; 7863 } 7864 7865 int bnxt_hwrm_fw_set_time(struct bnxt *bp) 7866 { 7867 struct hwrm_fw_set_time_input req = {0}; 7868 struct tm tm; 7869 time64_t now = ktime_get_real_seconds(); 7870 7871 if ((BNXT_VF(bp) && bp->hwrm_spec_code < 0x10901) || 7872 bp->hwrm_spec_code < 0x10400) 7873 return -EOPNOTSUPP; 7874 7875 time64_to_tm(now, 0, &tm); 7876 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FW_SET_TIME, -1, -1); 7877 req.year = cpu_to_le16(1900 + tm.tm_year); 7878 req.month = 1 + tm.tm_mon; 7879 req.day = tm.tm_mday; 7880 req.hour = tm.tm_hour; 7881 req.minute = tm.tm_min; 7882 req.second = tm.tm_sec; 7883 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 7884 } 7885 7886 static void bnxt_add_one_ctr(u64 hw, u64 *sw, u64 mask) 7887 { 7888 u64 sw_tmp; 7889 7890 hw &= mask; 7891 sw_tmp = (*sw & ~mask) | hw; 7892 if (hw < (*sw & mask)) 7893 sw_tmp += mask + 1; 7894 WRITE_ONCE(*sw, sw_tmp); 7895 } 7896 7897 static void __bnxt_accumulate_stats(__le64 *hw_stats, u64 *sw_stats, u64 *masks, 7898 int count, bool ignore_zero) 7899 { 7900 int i; 7901 7902 for (i = 0; i < count; i++) { 7903 u64 hw = le64_to_cpu(READ_ONCE(hw_stats[i])); 7904 7905 if (ignore_zero && !hw) 7906 continue; 7907 7908 if (masks[i] == -1ULL) 7909 sw_stats[i] = hw; 7910 else 7911 bnxt_add_one_ctr(hw, &sw_stats[i], masks[i]); 7912 } 7913 } 7914 7915 static void bnxt_accumulate_stats(struct bnxt_stats_mem *stats) 7916 { 7917 if (!stats->hw_stats) 7918 return; 7919 7920 __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats, 7921 stats->hw_masks, stats->len / 8, false); 7922 } 7923 7924 static void bnxt_accumulate_all_stats(struct bnxt *bp) 7925 { 7926 struct bnxt_stats_mem *ring0_stats; 7927 bool ignore_zero = false; 7928 int i; 7929 7930 /* Chip bug. Counter intermittently becomes 0. */ 7931 if (bp->flags & BNXT_FLAG_CHIP_P5) 7932 ignore_zero = true; 7933 7934 for (i = 0; i < bp->cp_nr_rings; i++) { 7935 struct bnxt_napi *bnapi = bp->bnapi[i]; 7936 struct bnxt_cp_ring_info *cpr; 7937 struct bnxt_stats_mem *stats; 7938 7939 cpr = &bnapi->cp_ring; 7940 stats = &cpr->stats; 7941 if (!i) 7942 ring0_stats = stats; 7943 __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats, 7944 ring0_stats->hw_masks, 7945 ring0_stats->len / 8, ignore_zero); 7946 } 7947 if (bp->flags & BNXT_FLAG_PORT_STATS) { 7948 struct bnxt_stats_mem *stats = &bp->port_stats; 7949 __le64 *hw_stats = stats->hw_stats; 7950 u64 *sw_stats = stats->sw_stats; 7951 u64 *masks = stats->hw_masks; 7952 int cnt; 7953 7954 cnt = sizeof(struct rx_port_stats) / 8; 7955 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false); 7956 7957 hw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 7958 sw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 7959 masks += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 7960 cnt = sizeof(struct tx_port_stats) / 8; 7961 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false); 7962 } 7963 if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) { 7964 bnxt_accumulate_stats(&bp->rx_port_stats_ext); 7965 bnxt_accumulate_stats(&bp->tx_port_stats_ext); 7966 } 7967 } 7968 7969 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags) 7970 { 7971 struct bnxt_pf_info *pf = &bp->pf; 7972 struct hwrm_port_qstats_input req = {0}; 7973 7974 if (!(bp->flags & BNXT_FLAG_PORT_STATS)) 7975 return 0; 7976 7977 if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED)) 7978 return -EOPNOTSUPP; 7979 7980 req.flags = flags; 7981 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS, -1, -1); 7982 req.port_id = cpu_to_le16(pf->port_id); 7983 req.tx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map + 7984 BNXT_TX_PORT_STATS_BYTE_OFFSET); 7985 req.rx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map); 7986 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 7987 } 7988 7989 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags) 7990 { 7991 struct hwrm_port_qstats_ext_output *resp = bp->hwrm_cmd_resp_addr; 7992 struct hwrm_queue_pri2cos_qcfg_input req2 = {0}; 7993 struct hwrm_port_qstats_ext_input req = {0}; 7994 struct bnxt_pf_info *pf = &bp->pf; 7995 u32 tx_stat_size; 7996 int rc; 7997 7998 if (!(bp->flags & BNXT_FLAG_PORT_STATS_EXT)) 7999 return 0; 8000 8001 if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED)) 8002 return -EOPNOTSUPP; 8003 8004 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS_EXT, -1, -1); 8005 req.flags = flags; 8006 req.port_id = cpu_to_le16(pf->port_id); 8007 req.rx_stat_size = cpu_to_le16(sizeof(struct rx_port_stats_ext)); 8008 req.rx_stat_host_addr = cpu_to_le64(bp->rx_port_stats_ext.hw_stats_map); 8009 tx_stat_size = bp->tx_port_stats_ext.hw_stats ? 8010 sizeof(struct tx_port_stats_ext) : 0; 8011 req.tx_stat_size = cpu_to_le16(tx_stat_size); 8012 req.tx_stat_host_addr = cpu_to_le64(bp->tx_port_stats_ext.hw_stats_map); 8013 mutex_lock(&bp->hwrm_cmd_lock); 8014 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 8015 if (!rc) { 8016 bp->fw_rx_stats_ext_size = le16_to_cpu(resp->rx_stat_size) / 8; 8017 bp->fw_tx_stats_ext_size = tx_stat_size ? 8018 le16_to_cpu(resp->tx_stat_size) / 8 : 0; 8019 } else { 8020 bp->fw_rx_stats_ext_size = 0; 8021 bp->fw_tx_stats_ext_size = 0; 8022 } 8023 if (flags) 8024 goto qstats_done; 8025 8026 if (bp->fw_tx_stats_ext_size <= 8027 offsetof(struct tx_port_stats_ext, pfc_pri0_tx_duration_us) / 8) { 8028 mutex_unlock(&bp->hwrm_cmd_lock); 8029 bp->pri2cos_valid = 0; 8030 return rc; 8031 } 8032 8033 bnxt_hwrm_cmd_hdr_init(bp, &req2, HWRM_QUEUE_PRI2COS_QCFG, -1, -1); 8034 req2.flags = cpu_to_le32(QUEUE_PRI2COS_QCFG_REQ_FLAGS_IVLAN); 8035 8036 rc = _hwrm_send_message(bp, &req2, sizeof(req2), HWRM_CMD_TIMEOUT); 8037 if (!rc) { 8038 struct hwrm_queue_pri2cos_qcfg_output *resp2; 8039 u8 *pri2cos; 8040 int i, j; 8041 8042 resp2 = bp->hwrm_cmd_resp_addr; 8043 pri2cos = &resp2->pri0_cos_queue_id; 8044 for (i = 0; i < 8; i++) { 8045 u8 queue_id = pri2cos[i]; 8046 u8 queue_idx; 8047 8048 /* Per port queue IDs start from 0, 10, 20, etc */ 8049 queue_idx = queue_id % 10; 8050 if (queue_idx > BNXT_MAX_QUEUE) { 8051 bp->pri2cos_valid = false; 8052 goto qstats_done; 8053 } 8054 for (j = 0; j < bp->max_q; j++) { 8055 if (bp->q_ids[j] == queue_id) 8056 bp->pri2cos_idx[i] = queue_idx; 8057 } 8058 } 8059 bp->pri2cos_valid = 1; 8060 } 8061 qstats_done: 8062 mutex_unlock(&bp->hwrm_cmd_lock); 8063 return rc; 8064 } 8065 8066 static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp) 8067 { 8068 if (bp->vxlan_fw_dst_port_id != INVALID_HW_RING_ID) 8069 bnxt_hwrm_tunnel_dst_port_free( 8070 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN); 8071 if (bp->nge_fw_dst_port_id != INVALID_HW_RING_ID) 8072 bnxt_hwrm_tunnel_dst_port_free( 8073 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE); 8074 } 8075 8076 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa) 8077 { 8078 int rc, i; 8079 u32 tpa_flags = 0; 8080 8081 if (set_tpa) 8082 tpa_flags = bp->flags & BNXT_FLAG_TPA; 8083 else if (BNXT_NO_FW_ACCESS(bp)) 8084 return 0; 8085 for (i = 0; i < bp->nr_vnics; i++) { 8086 rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags); 8087 if (rc) { 8088 netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n", 8089 i, rc); 8090 return rc; 8091 } 8092 } 8093 return 0; 8094 } 8095 8096 static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp) 8097 { 8098 int i; 8099 8100 for (i = 0; i < bp->nr_vnics; i++) 8101 bnxt_hwrm_vnic_set_rss(bp, i, false); 8102 } 8103 8104 static void bnxt_clear_vnic(struct bnxt *bp) 8105 { 8106 if (!bp->vnic_info) 8107 return; 8108 8109 bnxt_hwrm_clear_vnic_filter(bp); 8110 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) { 8111 /* clear all RSS setting before free vnic ctx */ 8112 bnxt_hwrm_clear_vnic_rss(bp); 8113 bnxt_hwrm_vnic_ctx_free(bp); 8114 } 8115 /* before free the vnic, undo the vnic tpa settings */ 8116 if (bp->flags & BNXT_FLAG_TPA) 8117 bnxt_set_tpa(bp, false); 8118 bnxt_hwrm_vnic_free(bp); 8119 if (bp->flags & BNXT_FLAG_CHIP_P5) 8120 bnxt_hwrm_vnic_ctx_free(bp); 8121 } 8122 8123 static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path, 8124 bool irq_re_init) 8125 { 8126 bnxt_clear_vnic(bp); 8127 bnxt_hwrm_ring_free(bp, close_path); 8128 bnxt_hwrm_ring_grp_free(bp); 8129 if (irq_re_init) { 8130 bnxt_hwrm_stat_ctx_free(bp); 8131 bnxt_hwrm_free_tunnel_ports(bp); 8132 } 8133 } 8134 8135 static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode) 8136 { 8137 struct hwrm_func_cfg_input req = {0}; 8138 8139 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1); 8140 req.fid = cpu_to_le16(0xffff); 8141 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_EVB_MODE); 8142 if (br_mode == BRIDGE_MODE_VEB) 8143 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEB; 8144 else if (br_mode == BRIDGE_MODE_VEPA) 8145 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEPA; 8146 else 8147 return -EINVAL; 8148 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 8149 } 8150 8151 static int bnxt_hwrm_set_cache_line_size(struct bnxt *bp, int size) 8152 { 8153 struct hwrm_func_cfg_input req = {0}; 8154 8155 if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10803) 8156 return 0; 8157 8158 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1); 8159 req.fid = cpu_to_le16(0xffff); 8160 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_CACHE_LINESIZE); 8161 req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_64; 8162 if (size == 128) 8163 req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_128; 8164 8165 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 8166 } 8167 8168 static int __bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id) 8169 { 8170 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 8171 int rc; 8172 8173 if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) 8174 goto skip_rss_ctx; 8175 8176 /* allocate context for vnic */ 8177 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 0); 8178 if (rc) { 8179 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n", 8180 vnic_id, rc); 8181 goto vnic_setup_err; 8182 } 8183 bp->rsscos_nr_ctxs++; 8184 8185 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 8186 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 1); 8187 if (rc) { 8188 netdev_err(bp->dev, "hwrm vnic %d cos ctx alloc failure rc: %x\n", 8189 vnic_id, rc); 8190 goto vnic_setup_err; 8191 } 8192 bp->rsscos_nr_ctxs++; 8193 } 8194 8195 skip_rss_ctx: 8196 /* configure default vnic, ring grp */ 8197 rc = bnxt_hwrm_vnic_cfg(bp, vnic_id); 8198 if (rc) { 8199 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n", 8200 vnic_id, rc); 8201 goto vnic_setup_err; 8202 } 8203 8204 /* Enable RSS hashing on vnic */ 8205 rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true); 8206 if (rc) { 8207 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n", 8208 vnic_id, rc); 8209 goto vnic_setup_err; 8210 } 8211 8212 if (bp->flags & BNXT_FLAG_AGG_RINGS) { 8213 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id); 8214 if (rc) { 8215 netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n", 8216 vnic_id, rc); 8217 } 8218 } 8219 8220 vnic_setup_err: 8221 return rc; 8222 } 8223 8224 static int __bnxt_setup_vnic_p5(struct bnxt *bp, u16 vnic_id) 8225 { 8226 int rc, i, nr_ctxs; 8227 8228 nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings); 8229 for (i = 0; i < nr_ctxs; i++) { 8230 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, i); 8231 if (rc) { 8232 netdev_err(bp->dev, "hwrm vnic %d ctx %d alloc failure rc: %x\n", 8233 vnic_id, i, rc); 8234 break; 8235 } 8236 bp->rsscos_nr_ctxs++; 8237 } 8238 if (i < nr_ctxs) 8239 return -ENOMEM; 8240 8241 rc = bnxt_hwrm_vnic_set_rss_p5(bp, vnic_id, true); 8242 if (rc) { 8243 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %d\n", 8244 vnic_id, rc); 8245 return rc; 8246 } 8247 rc = bnxt_hwrm_vnic_cfg(bp, vnic_id); 8248 if (rc) { 8249 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n", 8250 vnic_id, rc); 8251 return rc; 8252 } 8253 if (bp->flags & BNXT_FLAG_AGG_RINGS) { 8254 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id); 8255 if (rc) { 8256 netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n", 8257 vnic_id, rc); 8258 } 8259 } 8260 return rc; 8261 } 8262 8263 static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id) 8264 { 8265 if (bp->flags & BNXT_FLAG_CHIP_P5) 8266 return __bnxt_setup_vnic_p5(bp, vnic_id); 8267 else 8268 return __bnxt_setup_vnic(bp, vnic_id); 8269 } 8270 8271 static int bnxt_alloc_rfs_vnics(struct bnxt *bp) 8272 { 8273 #ifdef CONFIG_RFS_ACCEL 8274 int i, rc = 0; 8275 8276 if (bp->flags & BNXT_FLAG_CHIP_P5) 8277 return 0; 8278 8279 for (i = 0; i < bp->rx_nr_rings; i++) { 8280 struct bnxt_vnic_info *vnic; 8281 u16 vnic_id = i + 1; 8282 u16 ring_id = i; 8283 8284 if (vnic_id >= bp->nr_vnics) 8285 break; 8286 8287 vnic = &bp->vnic_info[vnic_id]; 8288 vnic->flags |= BNXT_VNIC_RFS_FLAG; 8289 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP) 8290 vnic->flags |= BNXT_VNIC_RFS_NEW_RSS_FLAG; 8291 rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, ring_id, 1); 8292 if (rc) { 8293 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n", 8294 vnic_id, rc); 8295 break; 8296 } 8297 rc = bnxt_setup_vnic(bp, vnic_id); 8298 if (rc) 8299 break; 8300 } 8301 return rc; 8302 #else 8303 return 0; 8304 #endif 8305 } 8306 8307 /* Allow PF and VF with default VLAN to be in promiscuous mode */ 8308 static bool bnxt_promisc_ok(struct bnxt *bp) 8309 { 8310 #ifdef CONFIG_BNXT_SRIOV 8311 if (BNXT_VF(bp) && !bp->vf.vlan) 8312 return false; 8313 #endif 8314 return true; 8315 } 8316 8317 static int bnxt_setup_nitroa0_vnic(struct bnxt *bp) 8318 { 8319 unsigned int rc = 0; 8320 8321 rc = bnxt_hwrm_vnic_alloc(bp, 1, bp->rx_nr_rings - 1, 1); 8322 if (rc) { 8323 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n", 8324 rc); 8325 return rc; 8326 } 8327 8328 rc = bnxt_hwrm_vnic_cfg(bp, 1); 8329 if (rc) { 8330 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n", 8331 rc); 8332 return rc; 8333 } 8334 return rc; 8335 } 8336 8337 static int bnxt_cfg_rx_mode(struct bnxt *); 8338 static bool bnxt_mc_list_updated(struct bnxt *, u32 *); 8339 8340 static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init) 8341 { 8342 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 8343 int rc = 0; 8344 unsigned int rx_nr_rings = bp->rx_nr_rings; 8345 8346 if (irq_re_init) { 8347 rc = bnxt_hwrm_stat_ctx_alloc(bp); 8348 if (rc) { 8349 netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n", 8350 rc); 8351 goto err_out; 8352 } 8353 } 8354 8355 rc = bnxt_hwrm_ring_alloc(bp); 8356 if (rc) { 8357 netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc); 8358 goto err_out; 8359 } 8360 8361 rc = bnxt_hwrm_ring_grp_alloc(bp); 8362 if (rc) { 8363 netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc); 8364 goto err_out; 8365 } 8366 8367 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 8368 rx_nr_rings--; 8369 8370 /* default vnic 0 */ 8371 rc = bnxt_hwrm_vnic_alloc(bp, 0, 0, rx_nr_rings); 8372 if (rc) { 8373 netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc); 8374 goto err_out; 8375 } 8376 8377 rc = bnxt_setup_vnic(bp, 0); 8378 if (rc) 8379 goto err_out; 8380 8381 if (bp->flags & BNXT_FLAG_RFS) { 8382 rc = bnxt_alloc_rfs_vnics(bp); 8383 if (rc) 8384 goto err_out; 8385 } 8386 8387 if (bp->flags & BNXT_FLAG_TPA) { 8388 rc = bnxt_set_tpa(bp, true); 8389 if (rc) 8390 goto err_out; 8391 } 8392 8393 if (BNXT_VF(bp)) 8394 bnxt_update_vf_mac(bp); 8395 8396 /* Filter for default vnic 0 */ 8397 rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr); 8398 if (rc) { 8399 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc); 8400 goto err_out; 8401 } 8402 vnic->uc_filter_count = 1; 8403 8404 vnic->rx_mask = 0; 8405 if (bp->dev->flags & IFF_BROADCAST) 8406 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST; 8407 8408 if ((bp->dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp)) 8409 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 8410 8411 if (bp->dev->flags & IFF_ALLMULTI) { 8412 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 8413 vnic->mc_list_count = 0; 8414 } else { 8415 u32 mask = 0; 8416 8417 bnxt_mc_list_updated(bp, &mask); 8418 vnic->rx_mask |= mask; 8419 } 8420 8421 rc = bnxt_cfg_rx_mode(bp); 8422 if (rc) 8423 goto err_out; 8424 8425 rc = bnxt_hwrm_set_coal(bp); 8426 if (rc) 8427 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n", 8428 rc); 8429 8430 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 8431 rc = bnxt_setup_nitroa0_vnic(bp); 8432 if (rc) 8433 netdev_err(bp->dev, "Special vnic setup failure for NS2 A0 rc: %x\n", 8434 rc); 8435 } 8436 8437 if (BNXT_VF(bp)) { 8438 bnxt_hwrm_func_qcfg(bp); 8439 netdev_update_features(bp->dev); 8440 } 8441 8442 return 0; 8443 8444 err_out: 8445 bnxt_hwrm_resource_free(bp, 0, true); 8446 8447 return rc; 8448 } 8449 8450 static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init) 8451 { 8452 bnxt_hwrm_resource_free(bp, 1, irq_re_init); 8453 return 0; 8454 } 8455 8456 static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init) 8457 { 8458 bnxt_init_cp_rings(bp); 8459 bnxt_init_rx_rings(bp); 8460 bnxt_init_tx_rings(bp); 8461 bnxt_init_ring_grps(bp, irq_re_init); 8462 bnxt_init_vnics(bp); 8463 8464 return bnxt_init_chip(bp, irq_re_init); 8465 } 8466 8467 static int bnxt_set_real_num_queues(struct bnxt *bp) 8468 { 8469 int rc; 8470 struct net_device *dev = bp->dev; 8471 8472 rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings - 8473 bp->tx_nr_rings_xdp); 8474 if (rc) 8475 return rc; 8476 8477 rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings); 8478 if (rc) 8479 return rc; 8480 8481 #ifdef CONFIG_RFS_ACCEL 8482 if (bp->flags & BNXT_FLAG_RFS) 8483 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings); 8484 #endif 8485 8486 return rc; 8487 } 8488 8489 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max, 8490 bool shared) 8491 { 8492 int _rx = *rx, _tx = *tx; 8493 8494 if (shared) { 8495 *rx = min_t(int, _rx, max); 8496 *tx = min_t(int, _tx, max); 8497 } else { 8498 if (max < 2) 8499 return -ENOMEM; 8500 8501 while (_rx + _tx > max) { 8502 if (_rx > _tx && _rx > 1) 8503 _rx--; 8504 else if (_tx > 1) 8505 _tx--; 8506 } 8507 *rx = _rx; 8508 *tx = _tx; 8509 } 8510 return 0; 8511 } 8512 8513 static void bnxt_setup_msix(struct bnxt *bp) 8514 { 8515 const int len = sizeof(bp->irq_tbl[0].name); 8516 struct net_device *dev = bp->dev; 8517 int tcs, i; 8518 8519 tcs = netdev_get_num_tc(dev); 8520 if (tcs) { 8521 int i, off, count; 8522 8523 for (i = 0; i < tcs; i++) { 8524 count = bp->tx_nr_rings_per_tc; 8525 off = i * count; 8526 netdev_set_tc_queue(dev, i, count, off); 8527 } 8528 } 8529 8530 for (i = 0; i < bp->cp_nr_rings; i++) { 8531 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 8532 char *attr; 8533 8534 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 8535 attr = "TxRx"; 8536 else if (i < bp->rx_nr_rings) 8537 attr = "rx"; 8538 else 8539 attr = "tx"; 8540 8541 snprintf(bp->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name, 8542 attr, i); 8543 bp->irq_tbl[map_idx].handler = bnxt_msix; 8544 } 8545 } 8546 8547 static void bnxt_setup_inta(struct bnxt *bp) 8548 { 8549 const int len = sizeof(bp->irq_tbl[0].name); 8550 8551 if (netdev_get_num_tc(bp->dev)) 8552 netdev_reset_tc(bp->dev); 8553 8554 snprintf(bp->irq_tbl[0].name, len, "%s-%s-%d", bp->dev->name, "TxRx", 8555 0); 8556 bp->irq_tbl[0].handler = bnxt_inta; 8557 } 8558 8559 static int bnxt_init_int_mode(struct bnxt *bp); 8560 8561 static int bnxt_setup_int_mode(struct bnxt *bp) 8562 { 8563 int rc; 8564 8565 if (!bp->irq_tbl) { 8566 rc = bnxt_init_int_mode(bp); 8567 if (rc || !bp->irq_tbl) 8568 return rc ?: -ENODEV; 8569 } 8570 8571 if (bp->flags & BNXT_FLAG_USING_MSIX) 8572 bnxt_setup_msix(bp); 8573 else 8574 bnxt_setup_inta(bp); 8575 8576 rc = bnxt_set_real_num_queues(bp); 8577 return rc; 8578 } 8579 8580 #ifdef CONFIG_RFS_ACCEL 8581 static unsigned int bnxt_get_max_func_rss_ctxs(struct bnxt *bp) 8582 { 8583 return bp->hw_resc.max_rsscos_ctxs; 8584 } 8585 8586 static unsigned int bnxt_get_max_func_vnics(struct bnxt *bp) 8587 { 8588 return bp->hw_resc.max_vnics; 8589 } 8590 #endif 8591 8592 unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp) 8593 { 8594 return bp->hw_resc.max_stat_ctxs; 8595 } 8596 8597 unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp) 8598 { 8599 return bp->hw_resc.max_cp_rings; 8600 } 8601 8602 static unsigned int bnxt_get_max_func_cp_rings_for_en(struct bnxt *bp) 8603 { 8604 unsigned int cp = bp->hw_resc.max_cp_rings; 8605 8606 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 8607 cp -= bnxt_get_ulp_msix_num(bp); 8608 8609 return cp; 8610 } 8611 8612 static unsigned int bnxt_get_max_func_irqs(struct bnxt *bp) 8613 { 8614 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 8615 8616 if (bp->flags & BNXT_FLAG_CHIP_P5) 8617 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_nqs); 8618 8619 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_cp_rings); 8620 } 8621 8622 static void bnxt_set_max_func_irqs(struct bnxt *bp, unsigned int max_irqs) 8623 { 8624 bp->hw_resc.max_irqs = max_irqs; 8625 } 8626 8627 unsigned int bnxt_get_avail_cp_rings_for_en(struct bnxt *bp) 8628 { 8629 unsigned int cp; 8630 8631 cp = bnxt_get_max_func_cp_rings_for_en(bp); 8632 if (bp->flags & BNXT_FLAG_CHIP_P5) 8633 return cp - bp->rx_nr_rings - bp->tx_nr_rings; 8634 else 8635 return cp - bp->cp_nr_rings; 8636 } 8637 8638 unsigned int bnxt_get_avail_stat_ctxs_for_en(struct bnxt *bp) 8639 { 8640 return bnxt_get_max_func_stat_ctxs(bp) - bnxt_get_func_stat_ctxs(bp); 8641 } 8642 8643 int bnxt_get_avail_msix(struct bnxt *bp, int num) 8644 { 8645 int max_cp = bnxt_get_max_func_cp_rings(bp); 8646 int max_irq = bnxt_get_max_func_irqs(bp); 8647 int total_req = bp->cp_nr_rings + num; 8648 int max_idx, avail_msix; 8649 8650 max_idx = bp->total_irqs; 8651 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 8652 max_idx = min_t(int, bp->total_irqs, max_cp); 8653 avail_msix = max_idx - bp->cp_nr_rings; 8654 if (!BNXT_NEW_RM(bp) || avail_msix >= num) 8655 return avail_msix; 8656 8657 if (max_irq < total_req) { 8658 num = max_irq - bp->cp_nr_rings; 8659 if (num <= 0) 8660 return 0; 8661 } 8662 return num; 8663 } 8664 8665 static int bnxt_get_num_msix(struct bnxt *bp) 8666 { 8667 if (!BNXT_NEW_RM(bp)) 8668 return bnxt_get_max_func_irqs(bp); 8669 8670 return bnxt_nq_rings_in_use(bp); 8671 } 8672 8673 static int bnxt_init_msix(struct bnxt *bp) 8674 { 8675 int i, total_vecs, max, rc = 0, min = 1, ulp_msix; 8676 struct msix_entry *msix_ent; 8677 8678 total_vecs = bnxt_get_num_msix(bp); 8679 max = bnxt_get_max_func_irqs(bp); 8680 if (total_vecs > max) 8681 total_vecs = max; 8682 8683 if (!total_vecs) 8684 return 0; 8685 8686 msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL); 8687 if (!msix_ent) 8688 return -ENOMEM; 8689 8690 for (i = 0; i < total_vecs; i++) { 8691 msix_ent[i].entry = i; 8692 msix_ent[i].vector = 0; 8693 } 8694 8695 if (!(bp->flags & BNXT_FLAG_SHARED_RINGS)) 8696 min = 2; 8697 8698 total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs); 8699 ulp_msix = bnxt_get_ulp_msix_num(bp); 8700 if (total_vecs < 0 || total_vecs < ulp_msix) { 8701 rc = -ENODEV; 8702 goto msix_setup_exit; 8703 } 8704 8705 bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL); 8706 if (bp->irq_tbl) { 8707 for (i = 0; i < total_vecs; i++) 8708 bp->irq_tbl[i].vector = msix_ent[i].vector; 8709 8710 bp->total_irqs = total_vecs; 8711 /* Trim rings based upon num of vectors allocated */ 8712 rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings, 8713 total_vecs - ulp_msix, min == 1); 8714 if (rc) 8715 goto msix_setup_exit; 8716 8717 bp->cp_nr_rings = (min == 1) ? 8718 max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) : 8719 bp->tx_nr_rings + bp->rx_nr_rings; 8720 8721 } else { 8722 rc = -ENOMEM; 8723 goto msix_setup_exit; 8724 } 8725 bp->flags |= BNXT_FLAG_USING_MSIX; 8726 kfree(msix_ent); 8727 return 0; 8728 8729 msix_setup_exit: 8730 netdev_err(bp->dev, "bnxt_init_msix err: %x\n", rc); 8731 kfree(bp->irq_tbl); 8732 bp->irq_tbl = NULL; 8733 pci_disable_msix(bp->pdev); 8734 kfree(msix_ent); 8735 return rc; 8736 } 8737 8738 static int bnxt_init_inta(struct bnxt *bp) 8739 { 8740 bp->irq_tbl = kzalloc(sizeof(struct bnxt_irq), GFP_KERNEL); 8741 if (!bp->irq_tbl) 8742 return -ENOMEM; 8743 8744 bp->total_irqs = 1; 8745 bp->rx_nr_rings = 1; 8746 bp->tx_nr_rings = 1; 8747 bp->cp_nr_rings = 1; 8748 bp->flags |= BNXT_FLAG_SHARED_RINGS; 8749 bp->irq_tbl[0].vector = bp->pdev->irq; 8750 return 0; 8751 } 8752 8753 static int bnxt_init_int_mode(struct bnxt *bp) 8754 { 8755 int rc = -ENODEV; 8756 8757 if (bp->flags & BNXT_FLAG_MSIX_CAP) 8758 rc = bnxt_init_msix(bp); 8759 8760 if (!(bp->flags & BNXT_FLAG_USING_MSIX) && BNXT_PF(bp)) { 8761 /* fallback to INTA */ 8762 rc = bnxt_init_inta(bp); 8763 } 8764 return rc; 8765 } 8766 8767 static void bnxt_clear_int_mode(struct bnxt *bp) 8768 { 8769 if (bp->flags & BNXT_FLAG_USING_MSIX) 8770 pci_disable_msix(bp->pdev); 8771 8772 kfree(bp->irq_tbl); 8773 bp->irq_tbl = NULL; 8774 bp->flags &= ~BNXT_FLAG_USING_MSIX; 8775 } 8776 8777 int bnxt_reserve_rings(struct bnxt *bp, bool irq_re_init) 8778 { 8779 int tcs = netdev_get_num_tc(bp->dev); 8780 bool irq_cleared = false; 8781 int rc; 8782 8783 if (!bnxt_need_reserve_rings(bp)) 8784 return 0; 8785 8786 if (irq_re_init && BNXT_NEW_RM(bp) && 8787 bnxt_get_num_msix(bp) != bp->total_irqs) { 8788 bnxt_ulp_irq_stop(bp); 8789 bnxt_clear_int_mode(bp); 8790 irq_cleared = true; 8791 } 8792 rc = __bnxt_reserve_rings(bp); 8793 if (irq_cleared) { 8794 if (!rc) 8795 rc = bnxt_init_int_mode(bp); 8796 bnxt_ulp_irq_restart(bp, rc); 8797 } 8798 if (rc) { 8799 netdev_err(bp->dev, "ring reservation/IRQ init failure rc: %d\n", rc); 8800 return rc; 8801 } 8802 if (tcs && (bp->tx_nr_rings_per_tc * tcs != bp->tx_nr_rings)) { 8803 netdev_err(bp->dev, "tx ring reservation failure\n"); 8804 netdev_reset_tc(bp->dev); 8805 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 8806 return -ENOMEM; 8807 } 8808 return 0; 8809 } 8810 8811 static void bnxt_free_irq(struct bnxt *bp) 8812 { 8813 struct bnxt_irq *irq; 8814 int i; 8815 8816 #ifdef CONFIG_RFS_ACCEL 8817 free_irq_cpu_rmap(bp->dev->rx_cpu_rmap); 8818 bp->dev->rx_cpu_rmap = NULL; 8819 #endif 8820 if (!bp->irq_tbl || !bp->bnapi) 8821 return; 8822 8823 for (i = 0; i < bp->cp_nr_rings; i++) { 8824 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 8825 8826 irq = &bp->irq_tbl[map_idx]; 8827 if (irq->requested) { 8828 if (irq->have_cpumask) { 8829 irq_set_affinity_hint(irq->vector, NULL); 8830 free_cpumask_var(irq->cpu_mask); 8831 irq->have_cpumask = 0; 8832 } 8833 free_irq(irq->vector, bp->bnapi[i]); 8834 } 8835 8836 irq->requested = 0; 8837 } 8838 } 8839 8840 static int bnxt_request_irq(struct bnxt *bp) 8841 { 8842 int i, j, rc = 0; 8843 unsigned long flags = 0; 8844 #ifdef CONFIG_RFS_ACCEL 8845 struct cpu_rmap *rmap; 8846 #endif 8847 8848 rc = bnxt_setup_int_mode(bp); 8849 if (rc) { 8850 netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n", 8851 rc); 8852 return rc; 8853 } 8854 #ifdef CONFIG_RFS_ACCEL 8855 rmap = bp->dev->rx_cpu_rmap; 8856 #endif 8857 if (!(bp->flags & BNXT_FLAG_USING_MSIX)) 8858 flags = IRQF_SHARED; 8859 8860 for (i = 0, j = 0; i < bp->cp_nr_rings; i++) { 8861 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 8862 struct bnxt_irq *irq = &bp->irq_tbl[map_idx]; 8863 8864 #ifdef CONFIG_RFS_ACCEL 8865 if (rmap && bp->bnapi[i]->rx_ring) { 8866 rc = irq_cpu_rmap_add(rmap, irq->vector); 8867 if (rc) 8868 netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n", 8869 j); 8870 j++; 8871 } 8872 #endif 8873 rc = request_irq(irq->vector, irq->handler, flags, irq->name, 8874 bp->bnapi[i]); 8875 if (rc) 8876 break; 8877 8878 irq->requested = 1; 8879 8880 if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) { 8881 int numa_node = dev_to_node(&bp->pdev->dev); 8882 8883 irq->have_cpumask = 1; 8884 cpumask_set_cpu(cpumask_local_spread(i, numa_node), 8885 irq->cpu_mask); 8886 rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask); 8887 if (rc) { 8888 netdev_warn(bp->dev, 8889 "Set affinity failed, IRQ = %d\n", 8890 irq->vector); 8891 break; 8892 } 8893 } 8894 } 8895 return rc; 8896 } 8897 8898 static void bnxt_del_napi(struct bnxt *bp) 8899 { 8900 int i; 8901 8902 if (!bp->bnapi) 8903 return; 8904 8905 for (i = 0; i < bp->cp_nr_rings; i++) { 8906 struct bnxt_napi *bnapi = bp->bnapi[i]; 8907 8908 __netif_napi_del(&bnapi->napi); 8909 } 8910 /* We called __netif_napi_del(), we need 8911 * to respect an RCU grace period before freeing napi structures. 8912 */ 8913 synchronize_net(); 8914 } 8915 8916 static void bnxt_init_napi(struct bnxt *bp) 8917 { 8918 int i; 8919 unsigned int cp_nr_rings = bp->cp_nr_rings; 8920 struct bnxt_napi *bnapi; 8921 8922 if (bp->flags & BNXT_FLAG_USING_MSIX) { 8923 int (*poll_fn)(struct napi_struct *, int) = bnxt_poll; 8924 8925 if (bp->flags & BNXT_FLAG_CHIP_P5) 8926 poll_fn = bnxt_poll_p5; 8927 else if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 8928 cp_nr_rings--; 8929 for (i = 0; i < cp_nr_rings; i++) { 8930 bnapi = bp->bnapi[i]; 8931 netif_napi_add(bp->dev, &bnapi->napi, poll_fn, 64); 8932 } 8933 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 8934 bnapi = bp->bnapi[cp_nr_rings]; 8935 netif_napi_add(bp->dev, &bnapi->napi, 8936 bnxt_poll_nitroa0, 64); 8937 } 8938 } else { 8939 bnapi = bp->bnapi[0]; 8940 netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll, 64); 8941 } 8942 } 8943 8944 static void bnxt_disable_napi(struct bnxt *bp) 8945 { 8946 int i; 8947 8948 if (!bp->bnapi || 8949 test_and_set_bit(BNXT_STATE_NAPI_DISABLED, &bp->state)) 8950 return; 8951 8952 for (i = 0; i < bp->cp_nr_rings; i++) { 8953 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring; 8954 8955 if (bp->bnapi[i]->rx_ring) 8956 cancel_work_sync(&cpr->dim.work); 8957 8958 napi_disable(&bp->bnapi[i]->napi); 8959 } 8960 } 8961 8962 static void bnxt_enable_napi(struct bnxt *bp) 8963 { 8964 int i; 8965 8966 clear_bit(BNXT_STATE_NAPI_DISABLED, &bp->state); 8967 for (i = 0; i < bp->cp_nr_rings; i++) { 8968 struct bnxt_napi *bnapi = bp->bnapi[i]; 8969 struct bnxt_cp_ring_info *cpr; 8970 8971 cpr = &bnapi->cp_ring; 8972 if (bnapi->in_reset) 8973 cpr->sw_stats.rx.rx_resets++; 8974 bnapi->in_reset = false; 8975 8976 if (bnapi->rx_ring) { 8977 INIT_WORK(&cpr->dim.work, bnxt_dim_work); 8978 cpr->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; 8979 } 8980 napi_enable(&bnapi->napi); 8981 } 8982 } 8983 8984 void bnxt_tx_disable(struct bnxt *bp) 8985 { 8986 int i; 8987 struct bnxt_tx_ring_info *txr; 8988 8989 if (bp->tx_ring) { 8990 for (i = 0; i < bp->tx_nr_rings; i++) { 8991 txr = &bp->tx_ring[i]; 8992 txr->dev_state = BNXT_DEV_STATE_CLOSING; 8993 } 8994 } 8995 /* Drop carrier first to prevent TX timeout */ 8996 netif_carrier_off(bp->dev); 8997 /* Stop all TX queues */ 8998 netif_tx_disable(bp->dev); 8999 } 9000 9001 void bnxt_tx_enable(struct bnxt *bp) 9002 { 9003 int i; 9004 struct bnxt_tx_ring_info *txr; 9005 9006 for (i = 0; i < bp->tx_nr_rings; i++) { 9007 txr = &bp->tx_ring[i]; 9008 txr->dev_state = 0; 9009 } 9010 netif_tx_wake_all_queues(bp->dev); 9011 if (bp->link_info.link_up) 9012 netif_carrier_on(bp->dev); 9013 } 9014 9015 static char *bnxt_report_fec(struct bnxt_link_info *link_info) 9016 { 9017 u8 active_fec = link_info->active_fec_sig_mode & 9018 PORT_PHY_QCFG_RESP_ACTIVE_FEC_MASK; 9019 9020 switch (active_fec) { 9021 default: 9022 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_NONE_ACTIVE: 9023 return "None"; 9024 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE74_ACTIVE: 9025 return "Clause 74 BaseR"; 9026 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE91_ACTIVE: 9027 return "Clause 91 RS(528,514)"; 9028 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_1XN_ACTIVE: 9029 return "Clause 91 RS544_1XN"; 9030 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_IEEE_ACTIVE: 9031 return "Clause 91 RS(544,514)"; 9032 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_1XN_ACTIVE: 9033 return "Clause 91 RS272_1XN"; 9034 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_IEEE_ACTIVE: 9035 return "Clause 91 RS(272,257)"; 9036 } 9037 } 9038 9039 static void bnxt_report_link(struct bnxt *bp) 9040 { 9041 if (bp->link_info.link_up) { 9042 const char *duplex; 9043 const char *flow_ctrl; 9044 u32 speed; 9045 u16 fec; 9046 9047 netif_carrier_on(bp->dev); 9048 speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed); 9049 if (speed == SPEED_UNKNOWN) { 9050 netdev_info(bp->dev, "NIC Link is Up, speed unknown\n"); 9051 return; 9052 } 9053 if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL) 9054 duplex = "full"; 9055 else 9056 duplex = "half"; 9057 if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH) 9058 flow_ctrl = "ON - receive & transmit"; 9059 else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX) 9060 flow_ctrl = "ON - transmit"; 9061 else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX) 9062 flow_ctrl = "ON - receive"; 9063 else 9064 flow_ctrl = "none"; 9065 netdev_info(bp->dev, "NIC Link is Up, %u Mbps %s duplex, Flow control: %s\n", 9066 speed, duplex, flow_ctrl); 9067 if (bp->flags & BNXT_FLAG_EEE_CAP) 9068 netdev_info(bp->dev, "EEE is %s\n", 9069 bp->eee.eee_active ? "active" : 9070 "not active"); 9071 fec = bp->link_info.fec_cfg; 9072 if (!(fec & PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED)) 9073 netdev_info(bp->dev, "FEC autoneg %s encoding: %s\n", 9074 (fec & BNXT_FEC_AUTONEG) ? "on" : "off", 9075 bnxt_report_fec(&bp->link_info)); 9076 } else { 9077 netif_carrier_off(bp->dev); 9078 netdev_err(bp->dev, "NIC Link is Down\n"); 9079 } 9080 } 9081 9082 static bool bnxt_phy_qcaps_no_speed(struct hwrm_port_phy_qcaps_output *resp) 9083 { 9084 if (!resp->supported_speeds_auto_mode && 9085 !resp->supported_speeds_force_mode && 9086 !resp->supported_pam4_speeds_auto_mode && 9087 !resp->supported_pam4_speeds_force_mode) 9088 return true; 9089 return false; 9090 } 9091 9092 static int bnxt_hwrm_phy_qcaps(struct bnxt *bp) 9093 { 9094 int rc = 0; 9095 struct hwrm_port_phy_qcaps_input req = {0}; 9096 struct hwrm_port_phy_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 9097 struct bnxt_link_info *link_info = &bp->link_info; 9098 9099 bp->flags &= ~BNXT_FLAG_EEE_CAP; 9100 if (bp->test_info) 9101 bp->test_info->flags &= ~(BNXT_TEST_FL_EXT_LPBK | 9102 BNXT_TEST_FL_AN_PHY_LPBK); 9103 if (bp->hwrm_spec_code < 0x10201) 9104 return 0; 9105 9106 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCAPS, -1, -1); 9107 9108 mutex_lock(&bp->hwrm_cmd_lock); 9109 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 9110 if (rc) 9111 goto hwrm_phy_qcaps_exit; 9112 9113 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) { 9114 struct ethtool_eee *eee = &bp->eee; 9115 u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode); 9116 9117 bp->flags |= BNXT_FLAG_EEE_CAP; 9118 eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0); 9119 bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) & 9120 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK; 9121 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) & 9122 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK; 9123 } 9124 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EXTERNAL_LPBK_SUPPORTED) { 9125 if (bp->test_info) 9126 bp->test_info->flags |= BNXT_TEST_FL_EXT_LPBK; 9127 } 9128 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_AUTONEG_LPBK_SUPPORTED) { 9129 if (bp->test_info) 9130 bp->test_info->flags |= BNXT_TEST_FL_AN_PHY_LPBK; 9131 } 9132 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_SHARED_PHY_CFG_SUPPORTED) { 9133 if (BNXT_PF(bp)) 9134 bp->fw_cap |= BNXT_FW_CAP_SHARED_PORT_CFG; 9135 } 9136 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_CUMULATIVE_COUNTERS_ON_RESET) 9137 bp->fw_cap |= BNXT_FW_CAP_PORT_STATS_NO_RESET; 9138 9139 if (bp->hwrm_spec_code >= 0x10a01) { 9140 if (bnxt_phy_qcaps_no_speed(resp)) { 9141 link_info->phy_state = BNXT_PHY_STATE_DISABLED; 9142 netdev_warn(bp->dev, "Ethernet link disabled\n"); 9143 } else if (link_info->phy_state == BNXT_PHY_STATE_DISABLED) { 9144 link_info->phy_state = BNXT_PHY_STATE_ENABLED; 9145 netdev_info(bp->dev, "Ethernet link enabled\n"); 9146 /* Phy re-enabled, reprobe the speeds */ 9147 link_info->support_auto_speeds = 0; 9148 link_info->support_pam4_auto_speeds = 0; 9149 } 9150 } 9151 if (resp->supported_speeds_auto_mode) 9152 link_info->support_auto_speeds = 9153 le16_to_cpu(resp->supported_speeds_auto_mode); 9154 if (resp->supported_pam4_speeds_auto_mode) 9155 link_info->support_pam4_auto_speeds = 9156 le16_to_cpu(resp->supported_pam4_speeds_auto_mode); 9157 9158 bp->port_count = resp->port_cnt; 9159 9160 hwrm_phy_qcaps_exit: 9161 mutex_unlock(&bp->hwrm_cmd_lock); 9162 return rc; 9163 } 9164 9165 static bool bnxt_support_dropped(u16 advertising, u16 supported) 9166 { 9167 u16 diff = advertising ^ supported; 9168 9169 return ((supported | diff) != supported); 9170 } 9171 9172 int bnxt_update_link(struct bnxt *bp, bool chng_link_state) 9173 { 9174 int rc = 0; 9175 struct bnxt_link_info *link_info = &bp->link_info; 9176 struct hwrm_port_phy_qcfg_input req = {0}; 9177 struct hwrm_port_phy_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 9178 u8 link_up = link_info->link_up; 9179 bool support_changed = false; 9180 9181 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCFG, -1, -1); 9182 9183 mutex_lock(&bp->hwrm_cmd_lock); 9184 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 9185 if (rc) { 9186 mutex_unlock(&bp->hwrm_cmd_lock); 9187 return rc; 9188 } 9189 9190 memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp)); 9191 link_info->phy_link_status = resp->link; 9192 link_info->duplex = resp->duplex_cfg; 9193 if (bp->hwrm_spec_code >= 0x10800) 9194 link_info->duplex = resp->duplex_state; 9195 link_info->pause = resp->pause; 9196 link_info->auto_mode = resp->auto_mode; 9197 link_info->auto_pause_setting = resp->auto_pause; 9198 link_info->lp_pause = resp->link_partner_adv_pause; 9199 link_info->force_pause_setting = resp->force_pause; 9200 link_info->duplex_setting = resp->duplex_cfg; 9201 if (link_info->phy_link_status == BNXT_LINK_LINK) 9202 link_info->link_speed = le16_to_cpu(resp->link_speed); 9203 else 9204 link_info->link_speed = 0; 9205 link_info->force_link_speed = le16_to_cpu(resp->force_link_speed); 9206 link_info->force_pam4_link_speed = 9207 le16_to_cpu(resp->force_pam4_link_speed); 9208 link_info->support_speeds = le16_to_cpu(resp->support_speeds); 9209 link_info->support_pam4_speeds = le16_to_cpu(resp->support_pam4_speeds); 9210 link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask); 9211 link_info->auto_pam4_link_speeds = 9212 le16_to_cpu(resp->auto_pam4_link_speed_mask); 9213 link_info->lp_auto_link_speeds = 9214 le16_to_cpu(resp->link_partner_adv_speeds); 9215 link_info->lp_auto_pam4_link_speeds = 9216 resp->link_partner_pam4_adv_speeds; 9217 link_info->preemphasis = le32_to_cpu(resp->preemphasis); 9218 link_info->phy_ver[0] = resp->phy_maj; 9219 link_info->phy_ver[1] = resp->phy_min; 9220 link_info->phy_ver[2] = resp->phy_bld; 9221 link_info->media_type = resp->media_type; 9222 link_info->phy_type = resp->phy_type; 9223 link_info->transceiver = resp->xcvr_pkg_type; 9224 link_info->phy_addr = resp->eee_config_phy_addr & 9225 PORT_PHY_QCFG_RESP_PHY_ADDR_MASK; 9226 link_info->module_status = resp->module_status; 9227 9228 if (bp->flags & BNXT_FLAG_EEE_CAP) { 9229 struct ethtool_eee *eee = &bp->eee; 9230 u16 fw_speeds; 9231 9232 eee->eee_active = 0; 9233 if (resp->eee_config_phy_addr & 9234 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) { 9235 eee->eee_active = 1; 9236 fw_speeds = le16_to_cpu( 9237 resp->link_partner_adv_eee_link_speed_mask); 9238 eee->lp_advertised = 9239 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0); 9240 } 9241 9242 /* Pull initial EEE config */ 9243 if (!chng_link_state) { 9244 if (resp->eee_config_phy_addr & 9245 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED) 9246 eee->eee_enabled = 1; 9247 9248 fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask); 9249 eee->advertised = 9250 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0); 9251 9252 if (resp->eee_config_phy_addr & 9253 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) { 9254 __le32 tmr; 9255 9256 eee->tx_lpi_enabled = 1; 9257 tmr = resp->xcvr_identifier_type_tx_lpi_timer; 9258 eee->tx_lpi_timer = le32_to_cpu(tmr) & 9259 PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK; 9260 } 9261 } 9262 } 9263 9264 link_info->fec_cfg = PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED; 9265 if (bp->hwrm_spec_code >= 0x10504) { 9266 link_info->fec_cfg = le16_to_cpu(resp->fec_cfg); 9267 link_info->active_fec_sig_mode = resp->active_fec_signal_mode; 9268 } 9269 /* TODO: need to add more logic to report VF link */ 9270 if (chng_link_state) { 9271 if (link_info->phy_link_status == BNXT_LINK_LINK) 9272 link_info->link_up = 1; 9273 else 9274 link_info->link_up = 0; 9275 if (link_up != link_info->link_up) 9276 bnxt_report_link(bp); 9277 } else { 9278 /* alwasy link down if not require to update link state */ 9279 link_info->link_up = 0; 9280 } 9281 mutex_unlock(&bp->hwrm_cmd_lock); 9282 9283 if (!BNXT_PHY_CFG_ABLE(bp)) 9284 return 0; 9285 9286 /* Check if any advertised speeds are no longer supported. The caller 9287 * holds the link_lock mutex, so we can modify link_info settings. 9288 */ 9289 if (bnxt_support_dropped(link_info->advertising, 9290 link_info->support_auto_speeds)) { 9291 link_info->advertising = link_info->support_auto_speeds; 9292 support_changed = true; 9293 } 9294 if (bnxt_support_dropped(link_info->advertising_pam4, 9295 link_info->support_pam4_auto_speeds)) { 9296 link_info->advertising_pam4 = link_info->support_pam4_auto_speeds; 9297 support_changed = true; 9298 } 9299 if (support_changed && (link_info->autoneg & BNXT_AUTONEG_SPEED)) 9300 bnxt_hwrm_set_link_setting(bp, true, false); 9301 return 0; 9302 } 9303 9304 static void bnxt_get_port_module_status(struct bnxt *bp) 9305 { 9306 struct bnxt_link_info *link_info = &bp->link_info; 9307 struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp; 9308 u8 module_status; 9309 9310 if (bnxt_update_link(bp, true)) 9311 return; 9312 9313 module_status = link_info->module_status; 9314 switch (module_status) { 9315 case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX: 9316 case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN: 9317 case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG: 9318 netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n", 9319 bp->pf.port_id); 9320 if (bp->hwrm_spec_code >= 0x10201) { 9321 netdev_warn(bp->dev, "Module part number %s\n", 9322 resp->phy_vendor_partnumber); 9323 } 9324 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX) 9325 netdev_warn(bp->dev, "TX is disabled\n"); 9326 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN) 9327 netdev_warn(bp->dev, "SFP+ module is shutdown\n"); 9328 } 9329 } 9330 9331 static void 9332 bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req) 9333 { 9334 if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) { 9335 if (bp->hwrm_spec_code >= 0x10201) 9336 req->auto_pause = 9337 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE; 9338 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX) 9339 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX; 9340 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX) 9341 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX; 9342 req->enables |= 9343 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE); 9344 } else { 9345 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX) 9346 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX; 9347 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX) 9348 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX; 9349 req->enables |= 9350 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE); 9351 if (bp->hwrm_spec_code >= 0x10201) { 9352 req->auto_pause = req->force_pause; 9353 req->enables |= cpu_to_le32( 9354 PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE); 9355 } 9356 } 9357 } 9358 9359 static void bnxt_hwrm_set_link_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req) 9360 { 9361 if (bp->link_info.autoneg & BNXT_AUTONEG_SPEED) { 9362 req->auto_mode |= PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK; 9363 if (bp->link_info.advertising) { 9364 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK); 9365 req->auto_link_speed_mask = cpu_to_le16(bp->link_info.advertising); 9366 } 9367 if (bp->link_info.advertising_pam4) { 9368 req->enables |= 9369 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAM4_LINK_SPEED_MASK); 9370 req->auto_link_pam4_speed_mask = 9371 cpu_to_le16(bp->link_info.advertising_pam4); 9372 } 9373 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE); 9374 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG); 9375 } else { 9376 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE); 9377 if (bp->link_info.req_signal_mode == BNXT_SIG_MODE_PAM4) { 9378 req->force_pam4_link_speed = cpu_to_le16(bp->link_info.req_link_speed); 9379 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAM4_LINK_SPEED); 9380 } else { 9381 req->force_link_speed = cpu_to_le16(bp->link_info.req_link_speed); 9382 } 9383 } 9384 9385 /* tell chimp that the setting takes effect immediately */ 9386 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY); 9387 } 9388 9389 int bnxt_hwrm_set_pause(struct bnxt *bp) 9390 { 9391 struct hwrm_port_phy_cfg_input req = {0}; 9392 int rc; 9393 9394 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1); 9395 bnxt_hwrm_set_pause_common(bp, &req); 9396 9397 if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) || 9398 bp->link_info.force_link_chng) 9399 bnxt_hwrm_set_link_common(bp, &req); 9400 9401 mutex_lock(&bp->hwrm_cmd_lock); 9402 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 9403 if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) { 9404 /* since changing of pause setting doesn't trigger any link 9405 * change event, the driver needs to update the current pause 9406 * result upon successfully return of the phy_cfg command 9407 */ 9408 bp->link_info.pause = 9409 bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl; 9410 bp->link_info.auto_pause_setting = 0; 9411 if (!bp->link_info.force_link_chng) 9412 bnxt_report_link(bp); 9413 } 9414 bp->link_info.force_link_chng = false; 9415 mutex_unlock(&bp->hwrm_cmd_lock); 9416 return rc; 9417 } 9418 9419 static void bnxt_hwrm_set_eee(struct bnxt *bp, 9420 struct hwrm_port_phy_cfg_input *req) 9421 { 9422 struct ethtool_eee *eee = &bp->eee; 9423 9424 if (eee->eee_enabled) { 9425 u16 eee_speeds; 9426 u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE; 9427 9428 if (eee->tx_lpi_enabled) 9429 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE; 9430 else 9431 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE; 9432 9433 req->flags |= cpu_to_le32(flags); 9434 eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised); 9435 req->eee_link_speed_mask = cpu_to_le16(eee_speeds); 9436 req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer); 9437 } else { 9438 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE); 9439 } 9440 } 9441 9442 int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee) 9443 { 9444 struct hwrm_port_phy_cfg_input req = {0}; 9445 9446 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1); 9447 if (set_pause) 9448 bnxt_hwrm_set_pause_common(bp, &req); 9449 9450 bnxt_hwrm_set_link_common(bp, &req); 9451 9452 if (set_eee) 9453 bnxt_hwrm_set_eee(bp, &req); 9454 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 9455 } 9456 9457 static int bnxt_hwrm_shutdown_link(struct bnxt *bp) 9458 { 9459 struct hwrm_port_phy_cfg_input req = {0}; 9460 9461 if (!BNXT_SINGLE_PF(bp)) 9462 return 0; 9463 9464 if (pci_num_vf(bp->pdev)) 9465 return 0; 9466 9467 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1); 9468 req.flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN); 9469 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 9470 } 9471 9472 static int bnxt_fw_init_one(struct bnxt *bp); 9473 9474 static int bnxt_fw_reset_via_optee(struct bnxt *bp) 9475 { 9476 #ifdef CONFIG_TEE_BNXT_FW 9477 int rc = tee_bnxt_fw_load(); 9478 9479 if (rc) 9480 netdev_err(bp->dev, "Failed FW reset via OP-TEE, rc=%d\n", rc); 9481 9482 return rc; 9483 #else 9484 netdev_err(bp->dev, "OP-TEE not supported\n"); 9485 return -ENODEV; 9486 #endif 9487 } 9488 9489 static int bnxt_try_recover_fw(struct bnxt *bp) 9490 { 9491 if (bp->fw_health && bp->fw_health->status_reliable) { 9492 int retry = 0, rc; 9493 u32 sts; 9494 9495 mutex_lock(&bp->hwrm_cmd_lock); 9496 do { 9497 rc = __bnxt_hwrm_ver_get(bp, true); 9498 sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG); 9499 if (!sts || !BNXT_FW_IS_BOOTING(sts)) 9500 break; 9501 retry++; 9502 } while (rc == -EBUSY && retry < BNXT_FW_RETRY); 9503 mutex_unlock(&bp->hwrm_cmd_lock); 9504 9505 if (!BNXT_FW_IS_HEALTHY(sts)) { 9506 netdev_err(bp->dev, 9507 "Firmware not responding, status: 0x%x\n", 9508 sts); 9509 rc = -ENODEV; 9510 } 9511 if (sts & FW_STATUS_REG_CRASHED_NO_MASTER) { 9512 netdev_warn(bp->dev, "Firmware recover via OP-TEE requested\n"); 9513 return bnxt_fw_reset_via_optee(bp); 9514 } 9515 return rc; 9516 } 9517 9518 return -ENODEV; 9519 } 9520 9521 static int bnxt_hwrm_if_change(struct bnxt *bp, bool up) 9522 { 9523 struct hwrm_func_drv_if_change_output *resp = bp->hwrm_cmd_resp_addr; 9524 struct hwrm_func_drv_if_change_input req = {0}; 9525 bool fw_reset = !bp->irq_tbl; 9526 bool resc_reinit = false; 9527 int rc, retry = 0; 9528 u32 flags = 0; 9529 9530 if (!(bp->fw_cap & BNXT_FW_CAP_IF_CHANGE)) 9531 return 0; 9532 9533 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_IF_CHANGE, -1, -1); 9534 if (up) 9535 req.flags = cpu_to_le32(FUNC_DRV_IF_CHANGE_REQ_FLAGS_UP); 9536 mutex_lock(&bp->hwrm_cmd_lock); 9537 while (retry < BNXT_FW_IF_RETRY) { 9538 rc = _hwrm_send_message(bp, &req, sizeof(req), 9539 HWRM_CMD_TIMEOUT); 9540 if (rc != -EAGAIN) 9541 break; 9542 9543 msleep(50); 9544 retry++; 9545 } 9546 if (!rc) 9547 flags = le32_to_cpu(resp->flags); 9548 mutex_unlock(&bp->hwrm_cmd_lock); 9549 9550 if (rc == -EAGAIN) 9551 return rc; 9552 if (rc && up) { 9553 rc = bnxt_try_recover_fw(bp); 9554 fw_reset = true; 9555 } 9556 if (rc) 9557 return rc; 9558 9559 if (!up) 9560 return 0; 9561 9562 if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_RESC_CHANGE) 9563 resc_reinit = true; 9564 if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_HOT_FW_RESET_DONE) 9565 fw_reset = true; 9566 9567 if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state) && !fw_reset) { 9568 netdev_err(bp->dev, "RESET_DONE not set during FW reset.\n"); 9569 set_bit(BNXT_STATE_ABORT_ERR, &bp->state); 9570 return -ENODEV; 9571 } 9572 if (resc_reinit || fw_reset) { 9573 if (fw_reset) { 9574 if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 9575 bnxt_ulp_stop(bp); 9576 bnxt_free_ctx_mem(bp); 9577 kfree(bp->ctx); 9578 bp->ctx = NULL; 9579 bnxt_dcb_free(bp); 9580 rc = bnxt_fw_init_one(bp); 9581 if (rc) { 9582 set_bit(BNXT_STATE_ABORT_ERR, &bp->state); 9583 return rc; 9584 } 9585 bnxt_clear_int_mode(bp); 9586 rc = bnxt_init_int_mode(bp); 9587 if (rc) { 9588 netdev_err(bp->dev, "init int mode failed\n"); 9589 return rc; 9590 } 9591 set_bit(BNXT_STATE_FW_RESET_DET, &bp->state); 9592 } 9593 if (BNXT_NEW_RM(bp)) { 9594 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 9595 9596 rc = bnxt_hwrm_func_resc_qcaps(bp, true); 9597 hw_resc->resv_cp_rings = 0; 9598 hw_resc->resv_stat_ctxs = 0; 9599 hw_resc->resv_irqs = 0; 9600 hw_resc->resv_tx_rings = 0; 9601 hw_resc->resv_rx_rings = 0; 9602 hw_resc->resv_hw_ring_grps = 0; 9603 hw_resc->resv_vnics = 0; 9604 if (!fw_reset) { 9605 bp->tx_nr_rings = 0; 9606 bp->rx_nr_rings = 0; 9607 } 9608 } 9609 } 9610 return 0; 9611 } 9612 9613 static int bnxt_hwrm_port_led_qcaps(struct bnxt *bp) 9614 { 9615 struct hwrm_port_led_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 9616 struct hwrm_port_led_qcaps_input req = {0}; 9617 struct bnxt_pf_info *pf = &bp->pf; 9618 int rc; 9619 9620 bp->num_leds = 0; 9621 if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10601) 9622 return 0; 9623 9624 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_LED_QCAPS, -1, -1); 9625 req.port_id = cpu_to_le16(pf->port_id); 9626 mutex_lock(&bp->hwrm_cmd_lock); 9627 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 9628 if (rc) { 9629 mutex_unlock(&bp->hwrm_cmd_lock); 9630 return rc; 9631 } 9632 if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) { 9633 int i; 9634 9635 bp->num_leds = resp->num_leds; 9636 memcpy(bp->leds, &resp->led0_id, sizeof(bp->leds[0]) * 9637 bp->num_leds); 9638 for (i = 0; i < bp->num_leds; i++) { 9639 struct bnxt_led_info *led = &bp->leds[i]; 9640 __le16 caps = led->led_state_caps; 9641 9642 if (!led->led_group_id || 9643 !BNXT_LED_ALT_BLINK_CAP(caps)) { 9644 bp->num_leds = 0; 9645 break; 9646 } 9647 } 9648 } 9649 mutex_unlock(&bp->hwrm_cmd_lock); 9650 return 0; 9651 } 9652 9653 int bnxt_hwrm_alloc_wol_fltr(struct bnxt *bp) 9654 { 9655 struct hwrm_wol_filter_alloc_input req = {0}; 9656 struct hwrm_wol_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr; 9657 int rc; 9658 9659 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_ALLOC, -1, -1); 9660 req.port_id = cpu_to_le16(bp->pf.port_id); 9661 req.wol_type = WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT; 9662 req.enables = cpu_to_le32(WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS); 9663 memcpy(req.mac_address, bp->dev->dev_addr, ETH_ALEN); 9664 mutex_lock(&bp->hwrm_cmd_lock); 9665 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 9666 if (!rc) 9667 bp->wol_filter_id = resp->wol_filter_id; 9668 mutex_unlock(&bp->hwrm_cmd_lock); 9669 return rc; 9670 } 9671 9672 int bnxt_hwrm_free_wol_fltr(struct bnxt *bp) 9673 { 9674 struct hwrm_wol_filter_free_input req = {0}; 9675 9676 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_FREE, -1, -1); 9677 req.port_id = cpu_to_le16(bp->pf.port_id); 9678 req.enables = cpu_to_le32(WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID); 9679 req.wol_filter_id = bp->wol_filter_id; 9680 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 9681 } 9682 9683 static u16 bnxt_hwrm_get_wol_fltrs(struct bnxt *bp, u16 handle) 9684 { 9685 struct hwrm_wol_filter_qcfg_input req = {0}; 9686 struct hwrm_wol_filter_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 9687 u16 next_handle = 0; 9688 int rc; 9689 9690 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_QCFG, -1, -1); 9691 req.port_id = cpu_to_le16(bp->pf.port_id); 9692 req.handle = cpu_to_le16(handle); 9693 mutex_lock(&bp->hwrm_cmd_lock); 9694 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 9695 if (!rc) { 9696 next_handle = le16_to_cpu(resp->next_handle); 9697 if (next_handle != 0) { 9698 if (resp->wol_type == 9699 WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT) { 9700 bp->wol = 1; 9701 bp->wol_filter_id = resp->wol_filter_id; 9702 } 9703 } 9704 } 9705 mutex_unlock(&bp->hwrm_cmd_lock); 9706 return next_handle; 9707 } 9708 9709 static void bnxt_get_wol_settings(struct bnxt *bp) 9710 { 9711 u16 handle = 0; 9712 9713 bp->wol = 0; 9714 if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_WOL_CAP)) 9715 return; 9716 9717 do { 9718 handle = bnxt_hwrm_get_wol_fltrs(bp, handle); 9719 } while (handle && handle != 0xffff); 9720 } 9721 9722 #ifdef CONFIG_BNXT_HWMON 9723 static ssize_t bnxt_show_temp(struct device *dev, 9724 struct device_attribute *devattr, char *buf) 9725 { 9726 struct hwrm_temp_monitor_query_input req = {0}; 9727 struct hwrm_temp_monitor_query_output *resp; 9728 struct bnxt *bp = dev_get_drvdata(dev); 9729 u32 len = 0; 9730 int rc; 9731 9732 resp = bp->hwrm_cmd_resp_addr; 9733 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TEMP_MONITOR_QUERY, -1, -1); 9734 mutex_lock(&bp->hwrm_cmd_lock); 9735 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 9736 if (!rc) 9737 len = sprintf(buf, "%u\n", resp->temp * 1000); /* display millidegree */ 9738 mutex_unlock(&bp->hwrm_cmd_lock); 9739 if (rc) 9740 return rc; 9741 return len; 9742 } 9743 static SENSOR_DEVICE_ATTR(temp1_input, 0444, bnxt_show_temp, NULL, 0); 9744 9745 static struct attribute *bnxt_attrs[] = { 9746 &sensor_dev_attr_temp1_input.dev_attr.attr, 9747 NULL 9748 }; 9749 ATTRIBUTE_GROUPS(bnxt); 9750 9751 static void bnxt_hwmon_close(struct bnxt *bp) 9752 { 9753 if (bp->hwmon_dev) { 9754 hwmon_device_unregister(bp->hwmon_dev); 9755 bp->hwmon_dev = NULL; 9756 } 9757 } 9758 9759 static void bnxt_hwmon_open(struct bnxt *bp) 9760 { 9761 struct hwrm_temp_monitor_query_input req = {0}; 9762 struct pci_dev *pdev = bp->pdev; 9763 int rc; 9764 9765 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TEMP_MONITOR_QUERY, -1, -1); 9766 rc = hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 9767 if (rc == -EACCES || rc == -EOPNOTSUPP) { 9768 bnxt_hwmon_close(bp); 9769 return; 9770 } 9771 9772 if (bp->hwmon_dev) 9773 return; 9774 9775 bp->hwmon_dev = hwmon_device_register_with_groups(&pdev->dev, 9776 DRV_MODULE_NAME, bp, 9777 bnxt_groups); 9778 if (IS_ERR(bp->hwmon_dev)) { 9779 bp->hwmon_dev = NULL; 9780 dev_warn(&pdev->dev, "Cannot register hwmon device\n"); 9781 } 9782 } 9783 #else 9784 static void bnxt_hwmon_close(struct bnxt *bp) 9785 { 9786 } 9787 9788 static void bnxt_hwmon_open(struct bnxt *bp) 9789 { 9790 } 9791 #endif 9792 9793 static bool bnxt_eee_config_ok(struct bnxt *bp) 9794 { 9795 struct ethtool_eee *eee = &bp->eee; 9796 struct bnxt_link_info *link_info = &bp->link_info; 9797 9798 if (!(bp->flags & BNXT_FLAG_EEE_CAP)) 9799 return true; 9800 9801 if (eee->eee_enabled) { 9802 u32 advertising = 9803 _bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0); 9804 9805 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) { 9806 eee->eee_enabled = 0; 9807 return false; 9808 } 9809 if (eee->advertised & ~advertising) { 9810 eee->advertised = advertising & eee->supported; 9811 return false; 9812 } 9813 } 9814 return true; 9815 } 9816 9817 static int bnxt_update_phy_setting(struct bnxt *bp) 9818 { 9819 int rc; 9820 bool update_link = false; 9821 bool update_pause = false; 9822 bool update_eee = false; 9823 struct bnxt_link_info *link_info = &bp->link_info; 9824 9825 rc = bnxt_update_link(bp, true); 9826 if (rc) { 9827 netdev_err(bp->dev, "failed to update link (rc: %x)\n", 9828 rc); 9829 return rc; 9830 } 9831 if (!BNXT_SINGLE_PF(bp)) 9832 return 0; 9833 9834 if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) && 9835 (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) != 9836 link_info->req_flow_ctrl) 9837 update_pause = true; 9838 if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) && 9839 link_info->force_pause_setting != link_info->req_flow_ctrl) 9840 update_pause = true; 9841 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) { 9842 if (BNXT_AUTO_MODE(link_info->auto_mode)) 9843 update_link = true; 9844 if (link_info->req_signal_mode == BNXT_SIG_MODE_NRZ && 9845 link_info->req_link_speed != link_info->force_link_speed) 9846 update_link = true; 9847 else if (link_info->req_signal_mode == BNXT_SIG_MODE_PAM4 && 9848 link_info->req_link_speed != link_info->force_pam4_link_speed) 9849 update_link = true; 9850 if (link_info->req_duplex != link_info->duplex_setting) 9851 update_link = true; 9852 } else { 9853 if (link_info->auto_mode == BNXT_LINK_AUTO_NONE) 9854 update_link = true; 9855 if (link_info->advertising != link_info->auto_link_speeds || 9856 link_info->advertising_pam4 != link_info->auto_pam4_link_speeds) 9857 update_link = true; 9858 } 9859 9860 /* The last close may have shutdown the link, so need to call 9861 * PHY_CFG to bring it back up. 9862 */ 9863 if (!bp->link_info.link_up) 9864 update_link = true; 9865 9866 if (!bnxt_eee_config_ok(bp)) 9867 update_eee = true; 9868 9869 if (update_link) 9870 rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee); 9871 else if (update_pause) 9872 rc = bnxt_hwrm_set_pause(bp); 9873 if (rc) { 9874 netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n", 9875 rc); 9876 return rc; 9877 } 9878 9879 return rc; 9880 } 9881 9882 /* Common routine to pre-map certain register block to different GRC window. 9883 * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows 9884 * in PF and 3 windows in VF that can be customized to map in different 9885 * register blocks. 9886 */ 9887 static void bnxt_preset_reg_win(struct bnxt *bp) 9888 { 9889 if (BNXT_PF(bp)) { 9890 /* CAG registers map to GRC window #4 */ 9891 writel(BNXT_CAG_REG_BASE, 9892 bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12); 9893 } 9894 } 9895 9896 static int bnxt_init_dflt_ring_mode(struct bnxt *bp); 9897 9898 static int bnxt_reinit_after_abort(struct bnxt *bp) 9899 { 9900 int rc; 9901 9902 if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 9903 return -EBUSY; 9904 9905 if (bp->dev->reg_state == NETREG_UNREGISTERED) 9906 return -ENODEV; 9907 9908 rc = bnxt_fw_init_one(bp); 9909 if (!rc) { 9910 bnxt_clear_int_mode(bp); 9911 rc = bnxt_init_int_mode(bp); 9912 if (!rc) { 9913 clear_bit(BNXT_STATE_ABORT_ERR, &bp->state); 9914 set_bit(BNXT_STATE_FW_RESET_DET, &bp->state); 9915 } 9916 } 9917 return rc; 9918 } 9919 9920 static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) 9921 { 9922 int rc = 0; 9923 9924 bnxt_preset_reg_win(bp); 9925 netif_carrier_off(bp->dev); 9926 if (irq_re_init) { 9927 /* Reserve rings now if none were reserved at driver probe. */ 9928 rc = bnxt_init_dflt_ring_mode(bp); 9929 if (rc) { 9930 netdev_err(bp->dev, "Failed to reserve default rings at open\n"); 9931 return rc; 9932 } 9933 } 9934 rc = bnxt_reserve_rings(bp, irq_re_init); 9935 if (rc) 9936 return rc; 9937 if ((bp->flags & BNXT_FLAG_RFS) && 9938 !(bp->flags & BNXT_FLAG_USING_MSIX)) { 9939 /* disable RFS if falling back to INTA */ 9940 bp->dev->hw_features &= ~NETIF_F_NTUPLE; 9941 bp->flags &= ~BNXT_FLAG_RFS; 9942 } 9943 9944 rc = bnxt_alloc_mem(bp, irq_re_init); 9945 if (rc) { 9946 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc); 9947 goto open_err_free_mem; 9948 } 9949 9950 if (irq_re_init) { 9951 bnxt_init_napi(bp); 9952 rc = bnxt_request_irq(bp); 9953 if (rc) { 9954 netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc); 9955 goto open_err_irq; 9956 } 9957 } 9958 9959 rc = bnxt_init_nic(bp, irq_re_init); 9960 if (rc) { 9961 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc); 9962 goto open_err_irq; 9963 } 9964 9965 bnxt_enable_napi(bp); 9966 bnxt_debug_dev_init(bp); 9967 9968 if (link_re_init) { 9969 mutex_lock(&bp->link_lock); 9970 rc = bnxt_update_phy_setting(bp); 9971 mutex_unlock(&bp->link_lock); 9972 if (rc) { 9973 netdev_warn(bp->dev, "failed to update phy settings\n"); 9974 if (BNXT_SINGLE_PF(bp)) { 9975 bp->link_info.phy_retry = true; 9976 bp->link_info.phy_retry_expires = 9977 jiffies + 5 * HZ; 9978 } 9979 } 9980 } 9981 9982 if (irq_re_init) 9983 udp_tunnel_nic_reset_ntf(bp->dev); 9984 9985 set_bit(BNXT_STATE_OPEN, &bp->state); 9986 bnxt_enable_int(bp); 9987 /* Enable TX queues */ 9988 bnxt_tx_enable(bp); 9989 mod_timer(&bp->timer, jiffies + bp->current_interval); 9990 /* Poll link status and check for SFP+ module status */ 9991 bnxt_get_port_module_status(bp); 9992 9993 /* VF-reps may need to be re-opened after the PF is re-opened */ 9994 if (BNXT_PF(bp)) 9995 bnxt_vf_reps_open(bp); 9996 return 0; 9997 9998 open_err_irq: 9999 bnxt_del_napi(bp); 10000 10001 open_err_free_mem: 10002 bnxt_free_skbs(bp); 10003 bnxt_free_irq(bp); 10004 bnxt_free_mem(bp, true); 10005 return rc; 10006 } 10007 10008 /* rtnl_lock held */ 10009 int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) 10010 { 10011 int rc = 0; 10012 10013 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) 10014 rc = -EIO; 10015 if (!rc) 10016 rc = __bnxt_open_nic(bp, irq_re_init, link_re_init); 10017 if (rc) { 10018 netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc); 10019 dev_close(bp->dev); 10020 } 10021 return rc; 10022 } 10023 10024 /* rtnl_lock held, open the NIC half way by allocating all resources, but 10025 * NAPI, IRQ, and TX are not enabled. This is mainly used for offline 10026 * self tests. 10027 */ 10028 int bnxt_half_open_nic(struct bnxt *bp) 10029 { 10030 int rc = 0; 10031 10032 rc = bnxt_alloc_mem(bp, false); 10033 if (rc) { 10034 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc); 10035 goto half_open_err; 10036 } 10037 rc = bnxt_init_nic(bp, false); 10038 if (rc) { 10039 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc); 10040 goto half_open_err; 10041 } 10042 return 0; 10043 10044 half_open_err: 10045 bnxt_free_skbs(bp); 10046 bnxt_free_mem(bp, false); 10047 dev_close(bp->dev); 10048 return rc; 10049 } 10050 10051 /* rtnl_lock held, this call can only be made after a previous successful 10052 * call to bnxt_half_open_nic(). 10053 */ 10054 void bnxt_half_close_nic(struct bnxt *bp) 10055 { 10056 bnxt_hwrm_resource_free(bp, false, false); 10057 bnxt_free_skbs(bp); 10058 bnxt_free_mem(bp, false); 10059 } 10060 10061 static void bnxt_reenable_sriov(struct bnxt *bp) 10062 { 10063 if (BNXT_PF(bp)) { 10064 struct bnxt_pf_info *pf = &bp->pf; 10065 int n = pf->active_vfs; 10066 10067 if (n) 10068 bnxt_cfg_hw_sriov(bp, &n, true); 10069 } 10070 } 10071 10072 static int bnxt_open(struct net_device *dev) 10073 { 10074 struct bnxt *bp = netdev_priv(dev); 10075 int rc; 10076 10077 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) { 10078 rc = bnxt_reinit_after_abort(bp); 10079 if (rc) { 10080 if (rc == -EBUSY) 10081 netdev_err(bp->dev, "A previous firmware reset has not completed, aborting\n"); 10082 else 10083 netdev_err(bp->dev, "Failed to reinitialize after aborted firmware reset\n"); 10084 return -ENODEV; 10085 } 10086 } 10087 10088 rc = bnxt_hwrm_if_change(bp, true); 10089 if (rc) 10090 return rc; 10091 rc = __bnxt_open_nic(bp, true, true); 10092 if (rc) { 10093 bnxt_hwrm_if_change(bp, false); 10094 } else { 10095 if (test_and_clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state)) { 10096 if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) { 10097 bnxt_ulp_start(bp, 0); 10098 bnxt_reenable_sriov(bp); 10099 } 10100 } 10101 bnxt_hwmon_open(bp); 10102 } 10103 10104 return rc; 10105 } 10106 10107 static bool bnxt_drv_busy(struct bnxt *bp) 10108 { 10109 return (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state) || 10110 test_bit(BNXT_STATE_READ_STATS, &bp->state)); 10111 } 10112 10113 static void bnxt_get_ring_stats(struct bnxt *bp, 10114 struct rtnl_link_stats64 *stats); 10115 10116 static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init, 10117 bool link_re_init) 10118 { 10119 /* Close the VF-reps before closing PF */ 10120 if (BNXT_PF(bp)) 10121 bnxt_vf_reps_close(bp); 10122 10123 /* Change device state to avoid TX queue wake up's */ 10124 bnxt_tx_disable(bp); 10125 10126 clear_bit(BNXT_STATE_OPEN, &bp->state); 10127 smp_mb__after_atomic(); 10128 while (bnxt_drv_busy(bp)) 10129 msleep(20); 10130 10131 /* Flush rings and and disable interrupts */ 10132 bnxt_shutdown_nic(bp, irq_re_init); 10133 10134 /* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */ 10135 10136 bnxt_debug_dev_exit(bp); 10137 bnxt_disable_napi(bp); 10138 del_timer_sync(&bp->timer); 10139 bnxt_free_skbs(bp); 10140 10141 /* Save ring stats before shutdown */ 10142 if (bp->bnapi && irq_re_init) 10143 bnxt_get_ring_stats(bp, &bp->net_stats_prev); 10144 if (irq_re_init) { 10145 bnxt_free_irq(bp); 10146 bnxt_del_napi(bp); 10147 } 10148 bnxt_free_mem(bp, irq_re_init); 10149 } 10150 10151 int bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) 10152 { 10153 int rc = 0; 10154 10155 if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) { 10156 /* If we get here, it means firmware reset is in progress 10157 * while we are trying to close. We can safely proceed with 10158 * the close because we are holding rtnl_lock(). Some firmware 10159 * messages may fail as we proceed to close. We set the 10160 * ABORT_ERR flag here so that the FW reset thread will later 10161 * abort when it gets the rtnl_lock() and sees the flag. 10162 */ 10163 netdev_warn(bp->dev, "FW reset in progress during close, FW reset will be aborted\n"); 10164 set_bit(BNXT_STATE_ABORT_ERR, &bp->state); 10165 } 10166 10167 #ifdef CONFIG_BNXT_SRIOV 10168 if (bp->sriov_cfg) { 10169 rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait, 10170 !bp->sriov_cfg, 10171 BNXT_SRIOV_CFG_WAIT_TMO); 10172 if (rc) 10173 netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete!\n"); 10174 } 10175 #endif 10176 __bnxt_close_nic(bp, irq_re_init, link_re_init); 10177 return rc; 10178 } 10179 10180 static int bnxt_close(struct net_device *dev) 10181 { 10182 struct bnxt *bp = netdev_priv(dev); 10183 10184 bnxt_hwmon_close(bp); 10185 bnxt_close_nic(bp, true, true); 10186 bnxt_hwrm_shutdown_link(bp); 10187 bnxt_hwrm_if_change(bp, false); 10188 return 0; 10189 } 10190 10191 static int bnxt_hwrm_port_phy_read(struct bnxt *bp, u16 phy_addr, u16 reg, 10192 u16 *val) 10193 { 10194 struct hwrm_port_phy_mdio_read_output *resp = bp->hwrm_cmd_resp_addr; 10195 struct hwrm_port_phy_mdio_read_input req = {0}; 10196 int rc; 10197 10198 if (bp->hwrm_spec_code < 0x10a00) 10199 return -EOPNOTSUPP; 10200 10201 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_MDIO_READ, -1, -1); 10202 req.port_id = cpu_to_le16(bp->pf.port_id); 10203 req.phy_addr = phy_addr; 10204 req.reg_addr = cpu_to_le16(reg & 0x1f); 10205 if (mdio_phy_id_is_c45(phy_addr)) { 10206 req.cl45_mdio = 1; 10207 req.phy_addr = mdio_phy_id_prtad(phy_addr); 10208 req.dev_addr = mdio_phy_id_devad(phy_addr); 10209 req.reg_addr = cpu_to_le16(reg); 10210 } 10211 10212 mutex_lock(&bp->hwrm_cmd_lock); 10213 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 10214 if (!rc) 10215 *val = le16_to_cpu(resp->reg_data); 10216 mutex_unlock(&bp->hwrm_cmd_lock); 10217 return rc; 10218 } 10219 10220 static int bnxt_hwrm_port_phy_write(struct bnxt *bp, u16 phy_addr, u16 reg, 10221 u16 val) 10222 { 10223 struct hwrm_port_phy_mdio_write_input req = {0}; 10224 10225 if (bp->hwrm_spec_code < 0x10a00) 10226 return -EOPNOTSUPP; 10227 10228 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_MDIO_WRITE, -1, -1); 10229 req.port_id = cpu_to_le16(bp->pf.port_id); 10230 req.phy_addr = phy_addr; 10231 req.reg_addr = cpu_to_le16(reg & 0x1f); 10232 if (mdio_phy_id_is_c45(phy_addr)) { 10233 req.cl45_mdio = 1; 10234 req.phy_addr = mdio_phy_id_prtad(phy_addr); 10235 req.dev_addr = mdio_phy_id_devad(phy_addr); 10236 req.reg_addr = cpu_to_le16(reg); 10237 } 10238 req.reg_data = cpu_to_le16(val); 10239 10240 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 10241 } 10242 10243 /* rtnl_lock held */ 10244 static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) 10245 { 10246 struct mii_ioctl_data *mdio = if_mii(ifr); 10247 struct bnxt *bp = netdev_priv(dev); 10248 int rc; 10249 10250 switch (cmd) { 10251 case SIOCGMIIPHY: 10252 mdio->phy_id = bp->link_info.phy_addr; 10253 10254 fallthrough; 10255 case SIOCGMIIREG: { 10256 u16 mii_regval = 0; 10257 10258 if (!netif_running(dev)) 10259 return -EAGAIN; 10260 10261 rc = bnxt_hwrm_port_phy_read(bp, mdio->phy_id, mdio->reg_num, 10262 &mii_regval); 10263 mdio->val_out = mii_regval; 10264 return rc; 10265 } 10266 10267 case SIOCSMIIREG: 10268 if (!netif_running(dev)) 10269 return -EAGAIN; 10270 10271 return bnxt_hwrm_port_phy_write(bp, mdio->phy_id, mdio->reg_num, 10272 mdio->val_in); 10273 10274 default: 10275 /* do nothing */ 10276 break; 10277 } 10278 return -EOPNOTSUPP; 10279 } 10280 10281 static void bnxt_get_ring_stats(struct bnxt *bp, 10282 struct rtnl_link_stats64 *stats) 10283 { 10284 int i; 10285 10286 for (i = 0; i < bp->cp_nr_rings; i++) { 10287 struct bnxt_napi *bnapi = bp->bnapi[i]; 10288 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 10289 u64 *sw = cpr->stats.sw_stats; 10290 10291 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_ucast_pkts); 10292 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts); 10293 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_bcast_pkts); 10294 10295 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_ucast_pkts); 10296 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_mcast_pkts); 10297 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_bcast_pkts); 10298 10299 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_ucast_bytes); 10300 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_mcast_bytes); 10301 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_bcast_bytes); 10302 10303 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_ucast_bytes); 10304 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_mcast_bytes); 10305 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_bcast_bytes); 10306 10307 stats->rx_missed_errors += 10308 BNXT_GET_RING_STATS64(sw, rx_discard_pkts); 10309 10310 stats->multicast += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts); 10311 10312 stats->tx_dropped += BNXT_GET_RING_STATS64(sw, tx_error_pkts); 10313 } 10314 } 10315 10316 static void bnxt_add_prev_stats(struct bnxt *bp, 10317 struct rtnl_link_stats64 *stats) 10318 { 10319 struct rtnl_link_stats64 *prev_stats = &bp->net_stats_prev; 10320 10321 stats->rx_packets += prev_stats->rx_packets; 10322 stats->tx_packets += prev_stats->tx_packets; 10323 stats->rx_bytes += prev_stats->rx_bytes; 10324 stats->tx_bytes += prev_stats->tx_bytes; 10325 stats->rx_missed_errors += prev_stats->rx_missed_errors; 10326 stats->multicast += prev_stats->multicast; 10327 stats->tx_dropped += prev_stats->tx_dropped; 10328 } 10329 10330 static void 10331 bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) 10332 { 10333 struct bnxt *bp = netdev_priv(dev); 10334 10335 set_bit(BNXT_STATE_READ_STATS, &bp->state); 10336 /* Make sure bnxt_close_nic() sees that we are reading stats before 10337 * we check the BNXT_STATE_OPEN flag. 10338 */ 10339 smp_mb__after_atomic(); 10340 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 10341 clear_bit(BNXT_STATE_READ_STATS, &bp->state); 10342 *stats = bp->net_stats_prev; 10343 return; 10344 } 10345 10346 bnxt_get_ring_stats(bp, stats); 10347 bnxt_add_prev_stats(bp, stats); 10348 10349 if (bp->flags & BNXT_FLAG_PORT_STATS) { 10350 u64 *rx = bp->port_stats.sw_stats; 10351 u64 *tx = bp->port_stats.sw_stats + 10352 BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 10353 10354 stats->rx_crc_errors = 10355 BNXT_GET_RX_PORT_STATS64(rx, rx_fcs_err_frames); 10356 stats->rx_frame_errors = 10357 BNXT_GET_RX_PORT_STATS64(rx, rx_align_err_frames); 10358 stats->rx_length_errors = 10359 BNXT_GET_RX_PORT_STATS64(rx, rx_undrsz_frames) + 10360 BNXT_GET_RX_PORT_STATS64(rx, rx_ovrsz_frames) + 10361 BNXT_GET_RX_PORT_STATS64(rx, rx_runt_frames); 10362 stats->rx_errors = 10363 BNXT_GET_RX_PORT_STATS64(rx, rx_false_carrier_frames) + 10364 BNXT_GET_RX_PORT_STATS64(rx, rx_jbr_frames); 10365 stats->collisions = 10366 BNXT_GET_TX_PORT_STATS64(tx, tx_total_collisions); 10367 stats->tx_fifo_errors = 10368 BNXT_GET_TX_PORT_STATS64(tx, tx_fifo_underruns); 10369 stats->tx_errors = BNXT_GET_TX_PORT_STATS64(tx, tx_err); 10370 } 10371 clear_bit(BNXT_STATE_READ_STATS, &bp->state); 10372 } 10373 10374 static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask) 10375 { 10376 struct net_device *dev = bp->dev; 10377 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 10378 struct netdev_hw_addr *ha; 10379 u8 *haddr; 10380 int mc_count = 0; 10381 bool update = false; 10382 int off = 0; 10383 10384 netdev_for_each_mc_addr(ha, dev) { 10385 if (mc_count >= BNXT_MAX_MC_ADDRS) { 10386 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 10387 vnic->mc_list_count = 0; 10388 return false; 10389 } 10390 haddr = ha->addr; 10391 if (!ether_addr_equal(haddr, vnic->mc_list + off)) { 10392 memcpy(vnic->mc_list + off, haddr, ETH_ALEN); 10393 update = true; 10394 } 10395 off += ETH_ALEN; 10396 mc_count++; 10397 } 10398 if (mc_count) 10399 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST; 10400 10401 if (mc_count != vnic->mc_list_count) { 10402 vnic->mc_list_count = mc_count; 10403 update = true; 10404 } 10405 return update; 10406 } 10407 10408 static bool bnxt_uc_list_updated(struct bnxt *bp) 10409 { 10410 struct net_device *dev = bp->dev; 10411 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 10412 struct netdev_hw_addr *ha; 10413 int off = 0; 10414 10415 if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1)) 10416 return true; 10417 10418 netdev_for_each_uc_addr(ha, dev) { 10419 if (!ether_addr_equal(ha->addr, vnic->uc_list + off)) 10420 return true; 10421 10422 off += ETH_ALEN; 10423 } 10424 return false; 10425 } 10426 10427 static void bnxt_set_rx_mode(struct net_device *dev) 10428 { 10429 struct bnxt *bp = netdev_priv(dev); 10430 struct bnxt_vnic_info *vnic; 10431 bool mc_update = false; 10432 bool uc_update; 10433 u32 mask; 10434 10435 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) 10436 return; 10437 10438 vnic = &bp->vnic_info[0]; 10439 mask = vnic->rx_mask; 10440 mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS | 10441 CFA_L2_SET_RX_MASK_REQ_MASK_MCAST | 10442 CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST | 10443 CFA_L2_SET_RX_MASK_REQ_MASK_BCAST); 10444 10445 if ((dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp)) 10446 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 10447 10448 uc_update = bnxt_uc_list_updated(bp); 10449 10450 if (dev->flags & IFF_BROADCAST) 10451 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST; 10452 if (dev->flags & IFF_ALLMULTI) { 10453 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 10454 vnic->mc_list_count = 0; 10455 } else { 10456 mc_update = bnxt_mc_list_updated(bp, &mask); 10457 } 10458 10459 if (mask != vnic->rx_mask || uc_update || mc_update) { 10460 vnic->rx_mask = mask; 10461 10462 set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event); 10463 bnxt_queue_sp_work(bp); 10464 } 10465 } 10466 10467 static int bnxt_cfg_rx_mode(struct bnxt *bp) 10468 { 10469 struct net_device *dev = bp->dev; 10470 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 10471 struct netdev_hw_addr *ha; 10472 int i, off = 0, rc; 10473 bool uc_update; 10474 10475 netif_addr_lock_bh(dev); 10476 uc_update = bnxt_uc_list_updated(bp); 10477 netif_addr_unlock_bh(dev); 10478 10479 if (!uc_update) 10480 goto skip_uc; 10481 10482 mutex_lock(&bp->hwrm_cmd_lock); 10483 for (i = 1; i < vnic->uc_filter_count; i++) { 10484 struct hwrm_cfa_l2_filter_free_input req = {0}; 10485 10486 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_FREE, -1, 10487 -1); 10488 10489 req.l2_filter_id = vnic->fw_l2_filter_id[i]; 10490 10491 rc = _hwrm_send_message(bp, &req, sizeof(req), 10492 HWRM_CMD_TIMEOUT); 10493 } 10494 mutex_unlock(&bp->hwrm_cmd_lock); 10495 10496 vnic->uc_filter_count = 1; 10497 10498 netif_addr_lock_bh(dev); 10499 if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) { 10500 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 10501 } else { 10502 netdev_for_each_uc_addr(ha, dev) { 10503 memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN); 10504 off += ETH_ALEN; 10505 vnic->uc_filter_count++; 10506 } 10507 } 10508 netif_addr_unlock_bh(dev); 10509 10510 for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) { 10511 rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off); 10512 if (rc) { 10513 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", 10514 rc); 10515 vnic->uc_filter_count = i; 10516 return rc; 10517 } 10518 } 10519 10520 skip_uc: 10521 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0); 10522 if (rc && vnic->mc_list_count) { 10523 netdev_info(bp->dev, "Failed setting MC filters rc: %d, turning on ALL_MCAST mode\n", 10524 rc); 10525 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 10526 vnic->mc_list_count = 0; 10527 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0); 10528 } 10529 if (rc) 10530 netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %d\n", 10531 rc); 10532 10533 return rc; 10534 } 10535 10536 static bool bnxt_can_reserve_rings(struct bnxt *bp) 10537 { 10538 #ifdef CONFIG_BNXT_SRIOV 10539 if (BNXT_NEW_RM(bp) && BNXT_VF(bp)) { 10540 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 10541 10542 /* No minimum rings were provisioned by the PF. Don't 10543 * reserve rings by default when device is down. 10544 */ 10545 if (hw_resc->min_tx_rings || hw_resc->resv_tx_rings) 10546 return true; 10547 10548 if (!netif_running(bp->dev)) 10549 return false; 10550 } 10551 #endif 10552 return true; 10553 } 10554 10555 /* If the chip and firmware supports RFS */ 10556 static bool bnxt_rfs_supported(struct bnxt *bp) 10557 { 10558 if (bp->flags & BNXT_FLAG_CHIP_P5) { 10559 if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2) 10560 return true; 10561 return false; 10562 } 10563 if (BNXT_PF(bp) && !BNXT_CHIP_TYPE_NITRO_A0(bp)) 10564 return true; 10565 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP) 10566 return true; 10567 return false; 10568 } 10569 10570 /* If runtime conditions support RFS */ 10571 static bool bnxt_rfs_capable(struct bnxt *bp) 10572 { 10573 #ifdef CONFIG_RFS_ACCEL 10574 int vnics, max_vnics, max_rss_ctxs; 10575 10576 if (bp->flags & BNXT_FLAG_CHIP_P5) 10577 return bnxt_rfs_supported(bp); 10578 if (!(bp->flags & BNXT_FLAG_MSIX_CAP) || !bnxt_can_reserve_rings(bp)) 10579 return false; 10580 10581 vnics = 1 + bp->rx_nr_rings; 10582 max_vnics = bnxt_get_max_func_vnics(bp); 10583 max_rss_ctxs = bnxt_get_max_func_rss_ctxs(bp); 10584 10585 /* RSS contexts not a limiting factor */ 10586 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP) 10587 max_rss_ctxs = max_vnics; 10588 if (vnics > max_vnics || vnics > max_rss_ctxs) { 10589 if (bp->rx_nr_rings > 1) 10590 netdev_warn(bp->dev, 10591 "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n", 10592 min(max_rss_ctxs - 1, max_vnics - 1)); 10593 return false; 10594 } 10595 10596 if (!BNXT_NEW_RM(bp)) 10597 return true; 10598 10599 if (vnics == bp->hw_resc.resv_vnics) 10600 return true; 10601 10602 bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, vnics); 10603 if (vnics <= bp->hw_resc.resv_vnics) 10604 return true; 10605 10606 netdev_warn(bp->dev, "Unable to reserve resources to support NTUPLE filters.\n"); 10607 bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, 1); 10608 return false; 10609 #else 10610 return false; 10611 #endif 10612 } 10613 10614 static netdev_features_t bnxt_fix_features(struct net_device *dev, 10615 netdev_features_t features) 10616 { 10617 struct bnxt *bp = netdev_priv(dev); 10618 netdev_features_t vlan_features; 10619 10620 if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp)) 10621 features &= ~NETIF_F_NTUPLE; 10622 10623 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS) 10624 features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); 10625 10626 if (!(features & NETIF_F_GRO)) 10627 features &= ~NETIF_F_GRO_HW; 10628 10629 if (features & NETIF_F_GRO_HW) 10630 features &= ~NETIF_F_LRO; 10631 10632 /* Both CTAG and STAG VLAN accelaration on the RX side have to be 10633 * turned on or off together. 10634 */ 10635 vlan_features = features & BNXT_HW_FEATURE_VLAN_ALL_RX; 10636 if (vlan_features != BNXT_HW_FEATURE_VLAN_ALL_RX) { 10637 if (dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX) 10638 features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX; 10639 else if (vlan_features) 10640 features |= BNXT_HW_FEATURE_VLAN_ALL_RX; 10641 } 10642 #ifdef CONFIG_BNXT_SRIOV 10643 if (BNXT_VF(bp) && bp->vf.vlan) 10644 features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX; 10645 #endif 10646 return features; 10647 } 10648 10649 static int bnxt_set_features(struct net_device *dev, netdev_features_t features) 10650 { 10651 struct bnxt *bp = netdev_priv(dev); 10652 u32 flags = bp->flags; 10653 u32 changes; 10654 int rc = 0; 10655 bool re_init = false; 10656 bool update_tpa = false; 10657 10658 flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS; 10659 if (features & NETIF_F_GRO_HW) 10660 flags |= BNXT_FLAG_GRO; 10661 else if (features & NETIF_F_LRO) 10662 flags |= BNXT_FLAG_LRO; 10663 10664 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS) 10665 flags &= ~BNXT_FLAG_TPA; 10666 10667 if (features & BNXT_HW_FEATURE_VLAN_ALL_RX) 10668 flags |= BNXT_FLAG_STRIP_VLAN; 10669 10670 if (features & NETIF_F_NTUPLE) 10671 flags |= BNXT_FLAG_RFS; 10672 10673 changes = flags ^ bp->flags; 10674 if (changes & BNXT_FLAG_TPA) { 10675 update_tpa = true; 10676 if ((bp->flags & BNXT_FLAG_TPA) == 0 || 10677 (flags & BNXT_FLAG_TPA) == 0 || 10678 (bp->flags & BNXT_FLAG_CHIP_P5)) 10679 re_init = true; 10680 } 10681 10682 if (changes & ~BNXT_FLAG_TPA) 10683 re_init = true; 10684 10685 if (flags != bp->flags) { 10686 u32 old_flags = bp->flags; 10687 10688 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 10689 bp->flags = flags; 10690 if (update_tpa) 10691 bnxt_set_ring_params(bp); 10692 return rc; 10693 } 10694 10695 if (re_init) { 10696 bnxt_close_nic(bp, false, false); 10697 bp->flags = flags; 10698 if (update_tpa) 10699 bnxt_set_ring_params(bp); 10700 10701 return bnxt_open_nic(bp, false, false); 10702 } 10703 if (update_tpa) { 10704 bp->flags = flags; 10705 rc = bnxt_set_tpa(bp, 10706 (flags & BNXT_FLAG_TPA) ? 10707 true : false); 10708 if (rc) 10709 bp->flags = old_flags; 10710 } 10711 } 10712 return rc; 10713 } 10714 10715 int bnxt_dbg_hwrm_rd_reg(struct bnxt *bp, u32 reg_off, u16 num_words, 10716 u32 *reg_buf) 10717 { 10718 struct hwrm_dbg_read_direct_output *resp = bp->hwrm_cmd_resp_addr; 10719 struct hwrm_dbg_read_direct_input req = {0}; 10720 __le32 *dbg_reg_buf; 10721 dma_addr_t mapping; 10722 int rc, i; 10723 10724 dbg_reg_buf = dma_alloc_coherent(&bp->pdev->dev, num_words * 4, 10725 &mapping, GFP_KERNEL); 10726 if (!dbg_reg_buf) 10727 return -ENOMEM; 10728 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_DBG_READ_DIRECT, -1, -1); 10729 req.host_dest_addr = cpu_to_le64(mapping); 10730 req.read_addr = cpu_to_le32(reg_off + CHIMP_REG_VIEW_ADDR); 10731 req.read_len32 = cpu_to_le32(num_words); 10732 mutex_lock(&bp->hwrm_cmd_lock); 10733 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 10734 if (rc || resp->error_code) { 10735 rc = -EIO; 10736 goto dbg_rd_reg_exit; 10737 } 10738 for (i = 0; i < num_words; i++) 10739 reg_buf[i] = le32_to_cpu(dbg_reg_buf[i]); 10740 10741 dbg_rd_reg_exit: 10742 mutex_unlock(&bp->hwrm_cmd_lock); 10743 dma_free_coherent(&bp->pdev->dev, num_words * 4, dbg_reg_buf, mapping); 10744 return rc; 10745 } 10746 10747 static int bnxt_dbg_hwrm_ring_info_get(struct bnxt *bp, u8 ring_type, 10748 u32 ring_id, u32 *prod, u32 *cons) 10749 { 10750 struct hwrm_dbg_ring_info_get_output *resp = bp->hwrm_cmd_resp_addr; 10751 struct hwrm_dbg_ring_info_get_input req = {0}; 10752 int rc; 10753 10754 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_DBG_RING_INFO_GET, -1, -1); 10755 req.ring_type = ring_type; 10756 req.fw_ring_id = cpu_to_le32(ring_id); 10757 mutex_lock(&bp->hwrm_cmd_lock); 10758 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 10759 if (!rc) { 10760 *prod = le32_to_cpu(resp->producer_index); 10761 *cons = le32_to_cpu(resp->consumer_index); 10762 } 10763 mutex_unlock(&bp->hwrm_cmd_lock); 10764 return rc; 10765 } 10766 10767 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi) 10768 { 10769 struct bnxt_tx_ring_info *txr = bnapi->tx_ring; 10770 int i = bnapi->index; 10771 10772 if (!txr) 10773 return; 10774 10775 netdev_info(bnapi->bp->dev, "[%d]: tx{fw_ring: %d prod: %x cons: %x}\n", 10776 i, txr->tx_ring_struct.fw_ring_id, txr->tx_prod, 10777 txr->tx_cons); 10778 } 10779 10780 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi) 10781 { 10782 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 10783 int i = bnapi->index; 10784 10785 if (!rxr) 10786 return; 10787 10788 netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n", 10789 i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod, 10790 rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod, 10791 rxr->rx_sw_agg_prod); 10792 } 10793 10794 static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi) 10795 { 10796 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 10797 int i = bnapi->index; 10798 10799 netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n", 10800 i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons); 10801 } 10802 10803 static void bnxt_dbg_dump_states(struct bnxt *bp) 10804 { 10805 int i; 10806 struct bnxt_napi *bnapi; 10807 10808 for (i = 0; i < bp->cp_nr_rings; i++) { 10809 bnapi = bp->bnapi[i]; 10810 if (netif_msg_drv(bp)) { 10811 bnxt_dump_tx_sw_state(bnapi); 10812 bnxt_dump_rx_sw_state(bnapi); 10813 bnxt_dump_cp_sw_state(bnapi); 10814 } 10815 } 10816 } 10817 10818 static int bnxt_hwrm_rx_ring_reset(struct bnxt *bp, int ring_nr) 10819 { 10820 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr]; 10821 struct hwrm_ring_reset_input req = {0}; 10822 struct bnxt_napi *bnapi = rxr->bnapi; 10823 struct bnxt_cp_ring_info *cpr; 10824 u16 cp_ring_id; 10825 10826 cpr = &bnapi->cp_ring; 10827 cp_ring_id = cpr->cp_ring_struct.fw_ring_id; 10828 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_RESET, cp_ring_id, -1); 10829 req.ring_type = RING_RESET_REQ_RING_TYPE_RX_RING_GRP; 10830 req.ring_id = cpu_to_le16(bp->grp_info[bnapi->index].fw_grp_id); 10831 return hwrm_send_message_silent(bp, &req, sizeof(req), 10832 HWRM_CMD_TIMEOUT); 10833 } 10834 10835 static void bnxt_reset_task(struct bnxt *bp, bool silent) 10836 { 10837 if (!silent) 10838 bnxt_dbg_dump_states(bp); 10839 if (netif_running(bp->dev)) { 10840 int rc; 10841 10842 if (silent) { 10843 bnxt_close_nic(bp, false, false); 10844 bnxt_open_nic(bp, false, false); 10845 } else { 10846 bnxt_ulp_stop(bp); 10847 bnxt_close_nic(bp, true, false); 10848 rc = bnxt_open_nic(bp, true, false); 10849 bnxt_ulp_start(bp, rc); 10850 } 10851 } 10852 } 10853 10854 static void bnxt_tx_timeout(struct net_device *dev, unsigned int txqueue) 10855 { 10856 struct bnxt *bp = netdev_priv(dev); 10857 10858 netdev_err(bp->dev, "TX timeout detected, starting reset task!\n"); 10859 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event); 10860 bnxt_queue_sp_work(bp); 10861 } 10862 10863 static void bnxt_fw_health_check(struct bnxt *bp) 10864 { 10865 struct bnxt_fw_health *fw_health = bp->fw_health; 10866 u32 val; 10867 10868 if (!fw_health->enabled || test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 10869 return; 10870 10871 if (fw_health->tmr_counter) { 10872 fw_health->tmr_counter--; 10873 return; 10874 } 10875 10876 val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG); 10877 if (val == fw_health->last_fw_heartbeat) 10878 goto fw_reset; 10879 10880 fw_health->last_fw_heartbeat = val; 10881 10882 val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG); 10883 if (val != fw_health->last_fw_reset_cnt) 10884 goto fw_reset; 10885 10886 fw_health->tmr_counter = fw_health->tmr_multiplier; 10887 return; 10888 10889 fw_reset: 10890 set_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event); 10891 bnxt_queue_sp_work(bp); 10892 } 10893 10894 static void bnxt_timer(struct timer_list *t) 10895 { 10896 struct bnxt *bp = from_timer(bp, t, timer); 10897 struct net_device *dev = bp->dev; 10898 10899 if (!netif_running(dev) || !test_bit(BNXT_STATE_OPEN, &bp->state)) 10900 return; 10901 10902 if (atomic_read(&bp->intr_sem) != 0) 10903 goto bnxt_restart_timer; 10904 10905 if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) 10906 bnxt_fw_health_check(bp); 10907 10908 if (bp->link_info.link_up && bp->stats_coal_ticks) { 10909 set_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event); 10910 bnxt_queue_sp_work(bp); 10911 } 10912 10913 if (bnxt_tc_flower_enabled(bp)) { 10914 set_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event); 10915 bnxt_queue_sp_work(bp); 10916 } 10917 10918 #ifdef CONFIG_RFS_ACCEL 10919 if ((bp->flags & BNXT_FLAG_RFS) && bp->ntp_fltr_count) { 10920 set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event); 10921 bnxt_queue_sp_work(bp); 10922 } 10923 #endif /*CONFIG_RFS_ACCEL*/ 10924 10925 if (bp->link_info.phy_retry) { 10926 if (time_after(jiffies, bp->link_info.phy_retry_expires)) { 10927 bp->link_info.phy_retry = false; 10928 netdev_warn(bp->dev, "failed to update phy settings after maximum retries.\n"); 10929 } else { 10930 set_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event); 10931 bnxt_queue_sp_work(bp); 10932 } 10933 } 10934 10935 if ((bp->flags & BNXT_FLAG_CHIP_P5) && !bp->chip_rev && 10936 netif_carrier_ok(dev)) { 10937 set_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event); 10938 bnxt_queue_sp_work(bp); 10939 } 10940 bnxt_restart_timer: 10941 mod_timer(&bp->timer, jiffies + bp->current_interval); 10942 } 10943 10944 static void bnxt_rtnl_lock_sp(struct bnxt *bp) 10945 { 10946 /* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK 10947 * set. If the device is being closed, bnxt_close() may be holding 10948 * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear. So we 10949 * must clear BNXT_STATE_IN_SP_TASK before holding rtnl(). 10950 */ 10951 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 10952 rtnl_lock(); 10953 } 10954 10955 static void bnxt_rtnl_unlock_sp(struct bnxt *bp) 10956 { 10957 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 10958 rtnl_unlock(); 10959 } 10960 10961 /* Only called from bnxt_sp_task() */ 10962 static void bnxt_reset(struct bnxt *bp, bool silent) 10963 { 10964 bnxt_rtnl_lock_sp(bp); 10965 if (test_bit(BNXT_STATE_OPEN, &bp->state)) 10966 bnxt_reset_task(bp, silent); 10967 bnxt_rtnl_unlock_sp(bp); 10968 } 10969 10970 /* Only called from bnxt_sp_task() */ 10971 static void bnxt_rx_ring_reset(struct bnxt *bp) 10972 { 10973 int i; 10974 10975 bnxt_rtnl_lock_sp(bp); 10976 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 10977 bnxt_rtnl_unlock_sp(bp); 10978 return; 10979 } 10980 /* Disable and flush TPA before resetting the RX ring */ 10981 if (bp->flags & BNXT_FLAG_TPA) 10982 bnxt_set_tpa(bp, false); 10983 for (i = 0; i < bp->rx_nr_rings; i++) { 10984 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 10985 struct bnxt_cp_ring_info *cpr; 10986 int rc; 10987 10988 if (!rxr->bnapi->in_reset) 10989 continue; 10990 10991 rc = bnxt_hwrm_rx_ring_reset(bp, i); 10992 if (rc) { 10993 if (rc == -EINVAL || rc == -EOPNOTSUPP) 10994 netdev_info_once(bp->dev, "RX ring reset not supported by firmware, falling back to global reset\n"); 10995 else 10996 netdev_warn(bp->dev, "RX ring reset failed, rc = %d, falling back to global reset\n", 10997 rc); 10998 bnxt_reset_task(bp, true); 10999 break; 11000 } 11001 bnxt_free_one_rx_ring_skbs(bp, i); 11002 rxr->rx_prod = 0; 11003 rxr->rx_agg_prod = 0; 11004 rxr->rx_sw_agg_prod = 0; 11005 rxr->rx_next_cons = 0; 11006 rxr->bnapi->in_reset = false; 11007 bnxt_alloc_one_rx_ring(bp, i); 11008 cpr = &rxr->bnapi->cp_ring; 11009 cpr->sw_stats.rx.rx_resets++; 11010 if (bp->flags & BNXT_FLAG_AGG_RINGS) 11011 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); 11012 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 11013 } 11014 if (bp->flags & BNXT_FLAG_TPA) 11015 bnxt_set_tpa(bp, true); 11016 bnxt_rtnl_unlock_sp(bp); 11017 } 11018 11019 static void bnxt_fw_reset_close(struct bnxt *bp) 11020 { 11021 bnxt_ulp_stop(bp); 11022 /* When firmware is in fatal state, quiesce device and disable 11023 * bus master to prevent any potential bad DMAs before freeing 11024 * kernel memory. 11025 */ 11026 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) { 11027 u16 val = 0; 11028 11029 pci_read_config_word(bp->pdev, PCI_SUBSYSTEM_ID, &val); 11030 if (val == 0xffff) 11031 bp->fw_reset_min_dsecs = 0; 11032 bnxt_tx_disable(bp); 11033 bnxt_disable_napi(bp); 11034 bnxt_disable_int_sync(bp); 11035 bnxt_free_irq(bp); 11036 bnxt_clear_int_mode(bp); 11037 pci_disable_device(bp->pdev); 11038 } 11039 __bnxt_close_nic(bp, true, false); 11040 bnxt_clear_int_mode(bp); 11041 bnxt_hwrm_func_drv_unrgtr(bp); 11042 if (pci_is_enabled(bp->pdev)) 11043 pci_disable_device(bp->pdev); 11044 bnxt_free_ctx_mem(bp); 11045 kfree(bp->ctx); 11046 bp->ctx = NULL; 11047 } 11048 11049 static bool is_bnxt_fw_ok(struct bnxt *bp) 11050 { 11051 struct bnxt_fw_health *fw_health = bp->fw_health; 11052 bool no_heartbeat = false, has_reset = false; 11053 u32 val; 11054 11055 val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG); 11056 if (val == fw_health->last_fw_heartbeat) 11057 no_heartbeat = true; 11058 11059 val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG); 11060 if (val != fw_health->last_fw_reset_cnt) 11061 has_reset = true; 11062 11063 if (!no_heartbeat && has_reset) 11064 return true; 11065 11066 return false; 11067 } 11068 11069 /* rtnl_lock is acquired before calling this function */ 11070 static void bnxt_force_fw_reset(struct bnxt *bp) 11071 { 11072 struct bnxt_fw_health *fw_health = bp->fw_health; 11073 u32 wait_dsecs; 11074 11075 if (!test_bit(BNXT_STATE_OPEN, &bp->state) || 11076 test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 11077 return; 11078 11079 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 11080 bnxt_fw_reset_close(bp); 11081 wait_dsecs = fw_health->master_func_wait_dsecs; 11082 if (fw_health->master) { 11083 if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) 11084 wait_dsecs = 0; 11085 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW; 11086 } else { 11087 bp->fw_reset_timestamp = jiffies + wait_dsecs * HZ / 10; 11088 wait_dsecs = fw_health->normal_func_wait_dsecs; 11089 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 11090 } 11091 11092 bp->fw_reset_min_dsecs = fw_health->post_reset_wait_dsecs; 11093 bp->fw_reset_max_dsecs = fw_health->post_reset_max_wait_dsecs; 11094 bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10); 11095 } 11096 11097 void bnxt_fw_exception(struct bnxt *bp) 11098 { 11099 netdev_warn(bp->dev, "Detected firmware fatal condition, initiating reset\n"); 11100 set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state); 11101 bnxt_rtnl_lock_sp(bp); 11102 bnxt_force_fw_reset(bp); 11103 bnxt_rtnl_unlock_sp(bp); 11104 } 11105 11106 /* Returns the number of registered VFs, or 1 if VF configuration is pending, or 11107 * < 0 on error. 11108 */ 11109 static int bnxt_get_registered_vfs(struct bnxt *bp) 11110 { 11111 #ifdef CONFIG_BNXT_SRIOV 11112 int rc; 11113 11114 if (!BNXT_PF(bp)) 11115 return 0; 11116 11117 rc = bnxt_hwrm_func_qcfg(bp); 11118 if (rc) { 11119 netdev_err(bp->dev, "func_qcfg cmd failed, rc = %d\n", rc); 11120 return rc; 11121 } 11122 if (bp->pf.registered_vfs) 11123 return bp->pf.registered_vfs; 11124 if (bp->sriov_cfg) 11125 return 1; 11126 #endif 11127 return 0; 11128 } 11129 11130 void bnxt_fw_reset(struct bnxt *bp) 11131 { 11132 bnxt_rtnl_lock_sp(bp); 11133 if (test_bit(BNXT_STATE_OPEN, &bp->state) && 11134 !test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) { 11135 int n = 0, tmo; 11136 11137 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 11138 if (bp->pf.active_vfs && 11139 !test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) 11140 n = bnxt_get_registered_vfs(bp); 11141 if (n < 0) { 11142 netdev_err(bp->dev, "Firmware reset aborted, rc = %d\n", 11143 n); 11144 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 11145 dev_close(bp->dev); 11146 goto fw_reset_exit; 11147 } else if (n > 0) { 11148 u16 vf_tmo_dsecs = n * 10; 11149 11150 if (bp->fw_reset_max_dsecs < vf_tmo_dsecs) 11151 bp->fw_reset_max_dsecs = vf_tmo_dsecs; 11152 bp->fw_reset_state = 11153 BNXT_FW_RESET_STATE_POLL_VF; 11154 bnxt_queue_fw_reset_work(bp, HZ / 10); 11155 goto fw_reset_exit; 11156 } 11157 bnxt_fw_reset_close(bp); 11158 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) { 11159 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN; 11160 tmo = HZ / 10; 11161 } else { 11162 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 11163 tmo = bp->fw_reset_min_dsecs * HZ / 10; 11164 } 11165 bnxt_queue_fw_reset_work(bp, tmo); 11166 } 11167 fw_reset_exit: 11168 bnxt_rtnl_unlock_sp(bp); 11169 } 11170 11171 static void bnxt_chk_missed_irq(struct bnxt *bp) 11172 { 11173 int i; 11174 11175 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 11176 return; 11177 11178 for (i = 0; i < bp->cp_nr_rings; i++) { 11179 struct bnxt_napi *bnapi = bp->bnapi[i]; 11180 struct bnxt_cp_ring_info *cpr; 11181 u32 fw_ring_id; 11182 int j; 11183 11184 if (!bnapi) 11185 continue; 11186 11187 cpr = &bnapi->cp_ring; 11188 for (j = 0; j < 2; j++) { 11189 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j]; 11190 u32 val[2]; 11191 11192 if (!cpr2 || cpr2->has_more_work || 11193 !bnxt_has_work(bp, cpr2)) 11194 continue; 11195 11196 if (cpr2->cp_raw_cons != cpr2->last_cp_raw_cons) { 11197 cpr2->last_cp_raw_cons = cpr2->cp_raw_cons; 11198 continue; 11199 } 11200 fw_ring_id = cpr2->cp_ring_struct.fw_ring_id; 11201 bnxt_dbg_hwrm_ring_info_get(bp, 11202 DBG_RING_INFO_GET_REQ_RING_TYPE_L2_CMPL, 11203 fw_ring_id, &val[0], &val[1]); 11204 cpr->sw_stats.cmn.missed_irqs++; 11205 } 11206 } 11207 } 11208 11209 static void bnxt_cfg_ntp_filters(struct bnxt *); 11210 11211 static void bnxt_init_ethtool_link_settings(struct bnxt *bp) 11212 { 11213 struct bnxt_link_info *link_info = &bp->link_info; 11214 11215 if (BNXT_AUTO_MODE(link_info->auto_mode)) { 11216 link_info->autoneg = BNXT_AUTONEG_SPEED; 11217 if (bp->hwrm_spec_code >= 0x10201) { 11218 if (link_info->auto_pause_setting & 11219 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE) 11220 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL; 11221 } else { 11222 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL; 11223 } 11224 link_info->advertising = link_info->auto_link_speeds; 11225 link_info->advertising_pam4 = link_info->auto_pam4_link_speeds; 11226 } else { 11227 link_info->req_link_speed = link_info->force_link_speed; 11228 link_info->req_signal_mode = BNXT_SIG_MODE_NRZ; 11229 if (link_info->force_pam4_link_speed) { 11230 link_info->req_link_speed = 11231 link_info->force_pam4_link_speed; 11232 link_info->req_signal_mode = BNXT_SIG_MODE_PAM4; 11233 } 11234 link_info->req_duplex = link_info->duplex_setting; 11235 } 11236 if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) 11237 link_info->req_flow_ctrl = 11238 link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH; 11239 else 11240 link_info->req_flow_ctrl = link_info->force_pause_setting; 11241 } 11242 11243 static void bnxt_fw_echo_reply(struct bnxt *bp) 11244 { 11245 struct bnxt_fw_health *fw_health = bp->fw_health; 11246 struct hwrm_func_echo_response_input req = {0}; 11247 11248 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_ECHO_RESPONSE, -1, -1); 11249 req.event_data1 = cpu_to_le32(fw_health->echo_req_data1); 11250 req.event_data2 = cpu_to_le32(fw_health->echo_req_data2); 11251 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 11252 } 11253 11254 static void bnxt_sp_task(struct work_struct *work) 11255 { 11256 struct bnxt *bp = container_of(work, struct bnxt, sp_task); 11257 11258 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 11259 smp_mb__after_atomic(); 11260 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 11261 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 11262 return; 11263 } 11264 11265 if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event)) 11266 bnxt_cfg_rx_mode(bp); 11267 11268 if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event)) 11269 bnxt_cfg_ntp_filters(bp); 11270 if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event)) 11271 bnxt_hwrm_exec_fwd_req(bp); 11272 if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event)) { 11273 bnxt_hwrm_port_qstats(bp, 0); 11274 bnxt_hwrm_port_qstats_ext(bp, 0); 11275 bnxt_accumulate_all_stats(bp); 11276 } 11277 11278 if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) { 11279 int rc; 11280 11281 mutex_lock(&bp->link_lock); 11282 if (test_and_clear_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, 11283 &bp->sp_event)) 11284 bnxt_hwrm_phy_qcaps(bp); 11285 11286 rc = bnxt_update_link(bp, true); 11287 if (rc) 11288 netdev_err(bp->dev, "SP task can't update link (rc: %x)\n", 11289 rc); 11290 11291 if (test_and_clear_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT, 11292 &bp->sp_event)) 11293 bnxt_init_ethtool_link_settings(bp); 11294 mutex_unlock(&bp->link_lock); 11295 } 11296 if (test_and_clear_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event)) { 11297 int rc; 11298 11299 mutex_lock(&bp->link_lock); 11300 rc = bnxt_update_phy_setting(bp); 11301 mutex_unlock(&bp->link_lock); 11302 if (rc) { 11303 netdev_warn(bp->dev, "update phy settings retry failed\n"); 11304 } else { 11305 bp->link_info.phy_retry = false; 11306 netdev_info(bp->dev, "update phy settings retry succeeded\n"); 11307 } 11308 } 11309 if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event)) { 11310 mutex_lock(&bp->link_lock); 11311 bnxt_get_port_module_status(bp); 11312 mutex_unlock(&bp->link_lock); 11313 } 11314 11315 if (test_and_clear_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event)) 11316 bnxt_tc_flow_stats_work(bp); 11317 11318 if (test_and_clear_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event)) 11319 bnxt_chk_missed_irq(bp); 11320 11321 if (test_and_clear_bit(BNXT_FW_ECHO_REQUEST_SP_EVENT, &bp->sp_event)) 11322 bnxt_fw_echo_reply(bp); 11323 11324 /* These functions below will clear BNXT_STATE_IN_SP_TASK. They 11325 * must be the last functions to be called before exiting. 11326 */ 11327 if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event)) 11328 bnxt_reset(bp, false); 11329 11330 if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event)) 11331 bnxt_reset(bp, true); 11332 11333 if (test_and_clear_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event)) 11334 bnxt_rx_ring_reset(bp); 11335 11336 if (test_and_clear_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event)) 11337 bnxt_devlink_health_report(bp, BNXT_FW_RESET_NOTIFY_SP_EVENT); 11338 11339 if (test_and_clear_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event)) { 11340 if (!is_bnxt_fw_ok(bp)) 11341 bnxt_devlink_health_report(bp, 11342 BNXT_FW_EXCEPTION_SP_EVENT); 11343 } 11344 11345 smp_mb__before_atomic(); 11346 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 11347 } 11348 11349 /* Under rtnl_lock */ 11350 int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs, 11351 int tx_xdp) 11352 { 11353 int max_rx, max_tx, tx_sets = 1; 11354 int tx_rings_needed, stats; 11355 int rx_rings = rx; 11356 int cp, vnics, rc; 11357 11358 if (tcs) 11359 tx_sets = tcs; 11360 11361 rc = bnxt_get_max_rings(bp, &max_rx, &max_tx, sh); 11362 if (rc) 11363 return rc; 11364 11365 if (max_rx < rx) 11366 return -ENOMEM; 11367 11368 tx_rings_needed = tx * tx_sets + tx_xdp; 11369 if (max_tx < tx_rings_needed) 11370 return -ENOMEM; 11371 11372 vnics = 1; 11373 if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS) 11374 vnics += rx_rings; 11375 11376 if (bp->flags & BNXT_FLAG_AGG_RINGS) 11377 rx_rings <<= 1; 11378 cp = sh ? max_t(int, tx_rings_needed, rx) : tx_rings_needed + rx; 11379 stats = cp; 11380 if (BNXT_NEW_RM(bp)) { 11381 cp += bnxt_get_ulp_msix_num(bp); 11382 stats += bnxt_get_ulp_stat_ctxs(bp); 11383 } 11384 return bnxt_hwrm_check_rings(bp, tx_rings_needed, rx_rings, rx, cp, 11385 stats, vnics); 11386 } 11387 11388 static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev) 11389 { 11390 if (bp->bar2) { 11391 pci_iounmap(pdev, bp->bar2); 11392 bp->bar2 = NULL; 11393 } 11394 11395 if (bp->bar1) { 11396 pci_iounmap(pdev, bp->bar1); 11397 bp->bar1 = NULL; 11398 } 11399 11400 if (bp->bar0) { 11401 pci_iounmap(pdev, bp->bar0); 11402 bp->bar0 = NULL; 11403 } 11404 } 11405 11406 static void bnxt_cleanup_pci(struct bnxt *bp) 11407 { 11408 bnxt_unmap_bars(bp, bp->pdev); 11409 pci_release_regions(bp->pdev); 11410 if (pci_is_enabled(bp->pdev)) 11411 pci_disable_device(bp->pdev); 11412 } 11413 11414 static void bnxt_init_dflt_coal(struct bnxt *bp) 11415 { 11416 struct bnxt_coal *coal; 11417 11418 /* Tick values in micro seconds. 11419 * 1 coal_buf x bufs_per_record = 1 completion record. 11420 */ 11421 coal = &bp->rx_coal; 11422 coal->coal_ticks = 10; 11423 coal->coal_bufs = 30; 11424 coal->coal_ticks_irq = 1; 11425 coal->coal_bufs_irq = 2; 11426 coal->idle_thresh = 50; 11427 coal->bufs_per_record = 2; 11428 coal->budget = 64; /* NAPI budget */ 11429 11430 coal = &bp->tx_coal; 11431 coal->coal_ticks = 28; 11432 coal->coal_bufs = 30; 11433 coal->coal_ticks_irq = 2; 11434 coal->coal_bufs_irq = 2; 11435 coal->bufs_per_record = 1; 11436 11437 bp->stats_coal_ticks = BNXT_DEF_STATS_COAL_TICKS; 11438 } 11439 11440 static int bnxt_fw_init_one_p1(struct bnxt *bp) 11441 { 11442 int rc; 11443 11444 bp->fw_cap = 0; 11445 rc = bnxt_hwrm_ver_get(bp); 11446 bnxt_try_map_fw_health_reg(bp); 11447 if (rc) { 11448 rc = bnxt_try_recover_fw(bp); 11449 if (rc) 11450 return rc; 11451 rc = bnxt_hwrm_ver_get(bp); 11452 if (rc) 11453 return rc; 11454 } 11455 11456 if (bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL) { 11457 rc = bnxt_alloc_kong_hwrm_resources(bp); 11458 if (rc) 11459 bp->fw_cap &= ~BNXT_FW_CAP_KONG_MB_CHNL; 11460 } 11461 11462 if ((bp->fw_cap & BNXT_FW_CAP_SHORT_CMD) || 11463 bp->hwrm_max_ext_req_len > BNXT_HWRM_MAX_REQ_LEN) { 11464 rc = bnxt_alloc_hwrm_short_cmd_req(bp); 11465 if (rc) 11466 return rc; 11467 } 11468 bnxt_nvm_cfg_ver_get(bp); 11469 11470 rc = bnxt_hwrm_func_reset(bp); 11471 if (rc) 11472 return -ENODEV; 11473 11474 bnxt_hwrm_fw_set_time(bp); 11475 return 0; 11476 } 11477 11478 static int bnxt_fw_init_one_p2(struct bnxt *bp) 11479 { 11480 int rc; 11481 11482 /* Get the MAX capabilities for this function */ 11483 rc = bnxt_hwrm_func_qcaps(bp); 11484 if (rc) { 11485 netdev_err(bp->dev, "hwrm query capability failure rc: %x\n", 11486 rc); 11487 return -ENODEV; 11488 } 11489 11490 rc = bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(bp); 11491 if (rc) 11492 netdev_warn(bp->dev, "hwrm query adv flow mgnt failure rc: %d\n", 11493 rc); 11494 11495 if (bnxt_alloc_fw_health(bp)) { 11496 netdev_warn(bp->dev, "no memory for firmware error recovery\n"); 11497 } else { 11498 rc = bnxt_hwrm_error_recovery_qcfg(bp); 11499 if (rc) 11500 netdev_warn(bp->dev, "hwrm query error recovery failure rc: %d\n", 11501 rc); 11502 } 11503 11504 rc = bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false); 11505 if (rc) 11506 return -ENODEV; 11507 11508 bnxt_hwrm_func_qcfg(bp); 11509 bnxt_hwrm_vnic_qcaps(bp); 11510 bnxt_hwrm_port_led_qcaps(bp); 11511 bnxt_ethtool_init(bp); 11512 bnxt_dcb_init(bp); 11513 return 0; 11514 } 11515 11516 static void bnxt_set_dflt_rss_hash_type(struct bnxt *bp) 11517 { 11518 bp->flags &= ~BNXT_FLAG_UDP_RSS_CAP; 11519 bp->rss_hash_cfg = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 | 11520 VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 | 11521 VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 | 11522 VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6; 11523 if (BNXT_CHIP_P4_PLUS(bp) && bp->hwrm_spec_code >= 0x10501) { 11524 bp->flags |= BNXT_FLAG_UDP_RSS_CAP; 11525 bp->rss_hash_cfg |= VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 | 11526 VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6; 11527 } 11528 } 11529 11530 static void bnxt_set_dflt_rfs(struct bnxt *bp) 11531 { 11532 struct net_device *dev = bp->dev; 11533 11534 dev->hw_features &= ~NETIF_F_NTUPLE; 11535 dev->features &= ~NETIF_F_NTUPLE; 11536 bp->flags &= ~BNXT_FLAG_RFS; 11537 if (bnxt_rfs_supported(bp)) { 11538 dev->hw_features |= NETIF_F_NTUPLE; 11539 if (bnxt_rfs_capable(bp)) { 11540 bp->flags |= BNXT_FLAG_RFS; 11541 dev->features |= NETIF_F_NTUPLE; 11542 } 11543 } 11544 } 11545 11546 static void bnxt_fw_init_one_p3(struct bnxt *bp) 11547 { 11548 struct pci_dev *pdev = bp->pdev; 11549 11550 bnxt_set_dflt_rss_hash_type(bp); 11551 bnxt_set_dflt_rfs(bp); 11552 11553 bnxt_get_wol_settings(bp); 11554 if (bp->flags & BNXT_FLAG_WOL_CAP) 11555 device_set_wakeup_enable(&pdev->dev, bp->wol); 11556 else 11557 device_set_wakeup_capable(&pdev->dev, false); 11558 11559 bnxt_hwrm_set_cache_line_size(bp, cache_line_size()); 11560 bnxt_hwrm_coal_params_qcaps(bp); 11561 } 11562 11563 static int bnxt_fw_init_one(struct bnxt *bp) 11564 { 11565 int rc; 11566 11567 rc = bnxt_fw_init_one_p1(bp); 11568 if (rc) { 11569 netdev_err(bp->dev, "Firmware init phase 1 failed\n"); 11570 return rc; 11571 } 11572 rc = bnxt_fw_init_one_p2(bp); 11573 if (rc) { 11574 netdev_err(bp->dev, "Firmware init phase 2 failed\n"); 11575 return rc; 11576 } 11577 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, false); 11578 if (rc) 11579 return rc; 11580 11581 /* In case fw capabilities have changed, destroy the unneeded 11582 * reporters and create newly capable ones. 11583 */ 11584 bnxt_dl_fw_reporters_destroy(bp, false); 11585 bnxt_dl_fw_reporters_create(bp); 11586 bnxt_fw_init_one_p3(bp); 11587 return 0; 11588 } 11589 11590 static void bnxt_fw_reset_writel(struct bnxt *bp, int reg_idx) 11591 { 11592 struct bnxt_fw_health *fw_health = bp->fw_health; 11593 u32 reg = fw_health->fw_reset_seq_regs[reg_idx]; 11594 u32 val = fw_health->fw_reset_seq_vals[reg_idx]; 11595 u32 reg_type, reg_off, delay_msecs; 11596 11597 delay_msecs = fw_health->fw_reset_seq_delay_msec[reg_idx]; 11598 reg_type = BNXT_FW_HEALTH_REG_TYPE(reg); 11599 reg_off = BNXT_FW_HEALTH_REG_OFF(reg); 11600 switch (reg_type) { 11601 case BNXT_FW_HEALTH_REG_TYPE_CFG: 11602 pci_write_config_dword(bp->pdev, reg_off, val); 11603 break; 11604 case BNXT_FW_HEALTH_REG_TYPE_GRC: 11605 writel(reg_off & BNXT_GRC_BASE_MASK, 11606 bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4); 11607 reg_off = (reg_off & BNXT_GRC_OFFSET_MASK) + 0x2000; 11608 fallthrough; 11609 case BNXT_FW_HEALTH_REG_TYPE_BAR0: 11610 writel(val, bp->bar0 + reg_off); 11611 break; 11612 case BNXT_FW_HEALTH_REG_TYPE_BAR1: 11613 writel(val, bp->bar1 + reg_off); 11614 break; 11615 } 11616 if (delay_msecs) { 11617 pci_read_config_dword(bp->pdev, 0, &val); 11618 msleep(delay_msecs); 11619 } 11620 } 11621 11622 static void bnxt_reset_all(struct bnxt *bp) 11623 { 11624 struct bnxt_fw_health *fw_health = bp->fw_health; 11625 int i, rc; 11626 11627 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) { 11628 bnxt_fw_reset_via_optee(bp); 11629 bp->fw_reset_timestamp = jiffies; 11630 return; 11631 } 11632 11633 if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_HOST) { 11634 for (i = 0; i < fw_health->fw_reset_seq_cnt; i++) 11635 bnxt_fw_reset_writel(bp, i); 11636 } else if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) { 11637 struct hwrm_fw_reset_input req = {0}; 11638 11639 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FW_RESET, -1, -1); 11640 req.resp_addr = cpu_to_le64(bp->hwrm_cmd_kong_resp_dma_addr); 11641 req.embedded_proc_type = FW_RESET_REQ_EMBEDDED_PROC_TYPE_CHIP; 11642 req.selfrst_status = FW_RESET_REQ_SELFRST_STATUS_SELFRSTASAP; 11643 req.flags = FW_RESET_REQ_FLAGS_RESET_GRACEFUL; 11644 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 11645 if (rc) 11646 netdev_warn(bp->dev, "Unable to reset FW rc=%d\n", rc); 11647 } 11648 bp->fw_reset_timestamp = jiffies; 11649 } 11650 11651 static bool bnxt_fw_reset_timeout(struct bnxt *bp) 11652 { 11653 return time_after(jiffies, bp->fw_reset_timestamp + 11654 (bp->fw_reset_max_dsecs * HZ / 10)); 11655 } 11656 11657 static void bnxt_fw_reset_task(struct work_struct *work) 11658 { 11659 struct bnxt *bp = container_of(work, struct bnxt, fw_reset_task.work); 11660 int rc; 11661 11662 if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) { 11663 netdev_err(bp->dev, "bnxt_fw_reset_task() called when not in fw reset mode!\n"); 11664 return; 11665 } 11666 11667 switch (bp->fw_reset_state) { 11668 case BNXT_FW_RESET_STATE_POLL_VF: { 11669 int n = bnxt_get_registered_vfs(bp); 11670 int tmo; 11671 11672 if (n < 0) { 11673 netdev_err(bp->dev, "Firmware reset aborted, subsequent func_qcfg cmd failed, rc = %d, %d msecs since reset timestamp\n", 11674 n, jiffies_to_msecs(jiffies - 11675 bp->fw_reset_timestamp)); 11676 goto fw_reset_abort; 11677 } else if (n > 0) { 11678 if (bnxt_fw_reset_timeout(bp)) { 11679 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 11680 bp->fw_reset_state = 0; 11681 netdev_err(bp->dev, "Firmware reset aborted, bnxt_get_registered_vfs() returns %d\n", 11682 n); 11683 return; 11684 } 11685 bnxt_queue_fw_reset_work(bp, HZ / 10); 11686 return; 11687 } 11688 bp->fw_reset_timestamp = jiffies; 11689 rtnl_lock(); 11690 bnxt_fw_reset_close(bp); 11691 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) { 11692 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN; 11693 tmo = HZ / 10; 11694 } else { 11695 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 11696 tmo = bp->fw_reset_min_dsecs * HZ / 10; 11697 } 11698 rtnl_unlock(); 11699 bnxt_queue_fw_reset_work(bp, tmo); 11700 return; 11701 } 11702 case BNXT_FW_RESET_STATE_POLL_FW_DOWN: { 11703 u32 val; 11704 11705 val = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG); 11706 if (!(val & BNXT_FW_STATUS_SHUTDOWN) && 11707 !bnxt_fw_reset_timeout(bp)) { 11708 bnxt_queue_fw_reset_work(bp, HZ / 5); 11709 return; 11710 } 11711 11712 if (!bp->fw_health->master) { 11713 u32 wait_dsecs = bp->fw_health->normal_func_wait_dsecs; 11714 11715 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 11716 bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10); 11717 return; 11718 } 11719 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW; 11720 } 11721 fallthrough; 11722 case BNXT_FW_RESET_STATE_RESET_FW: 11723 bnxt_reset_all(bp); 11724 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 11725 bnxt_queue_fw_reset_work(bp, bp->fw_reset_min_dsecs * HZ / 10); 11726 return; 11727 case BNXT_FW_RESET_STATE_ENABLE_DEV: 11728 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) { 11729 u32 val; 11730 11731 if (!bp->fw_reset_min_dsecs) { 11732 u16 val; 11733 11734 pci_read_config_word(bp->pdev, PCI_SUBSYSTEM_ID, 11735 &val); 11736 if (val == 0xffff) { 11737 if (bnxt_fw_reset_timeout(bp)) { 11738 netdev_err(bp->dev, "Firmware reset aborted, PCI config space invalid\n"); 11739 goto fw_reset_abort; 11740 } 11741 bnxt_queue_fw_reset_work(bp, HZ / 1000); 11742 return; 11743 } 11744 } 11745 val = bnxt_fw_health_readl(bp, 11746 BNXT_FW_RESET_INPROG_REG); 11747 if (val) 11748 netdev_warn(bp->dev, "FW reset inprog %x after min wait time.\n", 11749 val); 11750 } 11751 clear_bit(BNXT_STATE_FW_FATAL_COND, &bp->state); 11752 if (pci_enable_device(bp->pdev)) { 11753 netdev_err(bp->dev, "Cannot re-enable PCI device\n"); 11754 goto fw_reset_abort; 11755 } 11756 pci_set_master(bp->pdev); 11757 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW; 11758 fallthrough; 11759 case BNXT_FW_RESET_STATE_POLL_FW: 11760 bp->hwrm_cmd_timeout = SHORT_HWRM_CMD_TIMEOUT; 11761 rc = __bnxt_hwrm_ver_get(bp, true); 11762 if (rc) { 11763 if (bnxt_fw_reset_timeout(bp)) { 11764 netdev_err(bp->dev, "Firmware reset aborted\n"); 11765 goto fw_reset_abort_status; 11766 } 11767 bnxt_queue_fw_reset_work(bp, HZ / 5); 11768 return; 11769 } 11770 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT; 11771 bp->fw_reset_state = BNXT_FW_RESET_STATE_OPENING; 11772 fallthrough; 11773 case BNXT_FW_RESET_STATE_OPENING: 11774 while (!rtnl_trylock()) { 11775 bnxt_queue_fw_reset_work(bp, HZ / 10); 11776 return; 11777 } 11778 rc = bnxt_open(bp->dev); 11779 if (rc) { 11780 netdev_err(bp->dev, "bnxt_open_nic() failed\n"); 11781 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 11782 dev_close(bp->dev); 11783 } 11784 11785 bp->fw_reset_state = 0; 11786 /* Make sure fw_reset_state is 0 before clearing the flag */ 11787 smp_mb__before_atomic(); 11788 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 11789 bnxt_ulp_start(bp, rc); 11790 if (!rc) 11791 bnxt_reenable_sriov(bp); 11792 bnxt_dl_health_recovery_done(bp); 11793 bnxt_dl_health_status_update(bp, true); 11794 rtnl_unlock(); 11795 break; 11796 } 11797 return; 11798 11799 fw_reset_abort_status: 11800 if (bp->fw_health->status_reliable || 11801 (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) { 11802 u32 sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG); 11803 11804 netdev_err(bp->dev, "fw_health_status 0x%x\n", sts); 11805 } 11806 fw_reset_abort: 11807 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 11808 if (bp->fw_reset_state != BNXT_FW_RESET_STATE_POLL_VF) 11809 bnxt_dl_health_status_update(bp, false); 11810 bp->fw_reset_state = 0; 11811 rtnl_lock(); 11812 dev_close(bp->dev); 11813 rtnl_unlock(); 11814 } 11815 11816 static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev) 11817 { 11818 int rc; 11819 struct bnxt *bp = netdev_priv(dev); 11820 11821 SET_NETDEV_DEV(dev, &pdev->dev); 11822 11823 /* enable device (incl. PCI PM wakeup), and bus-mastering */ 11824 rc = pci_enable_device(pdev); 11825 if (rc) { 11826 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n"); 11827 goto init_err; 11828 } 11829 11830 if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) { 11831 dev_err(&pdev->dev, 11832 "Cannot find PCI device base address, aborting\n"); 11833 rc = -ENODEV; 11834 goto init_err_disable; 11835 } 11836 11837 rc = pci_request_regions(pdev, DRV_MODULE_NAME); 11838 if (rc) { 11839 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n"); 11840 goto init_err_disable; 11841 } 11842 11843 if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 && 11844 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) { 11845 dev_err(&pdev->dev, "System does not support DMA, aborting\n"); 11846 rc = -EIO; 11847 goto init_err_release; 11848 } 11849 11850 pci_set_master(pdev); 11851 11852 bp->dev = dev; 11853 bp->pdev = pdev; 11854 11855 /* Doorbell BAR bp->bar1 is mapped after bnxt_fw_init_one_p2() 11856 * determines the BAR size. 11857 */ 11858 bp->bar0 = pci_ioremap_bar(pdev, 0); 11859 if (!bp->bar0) { 11860 dev_err(&pdev->dev, "Cannot map device registers, aborting\n"); 11861 rc = -ENOMEM; 11862 goto init_err_release; 11863 } 11864 11865 bp->bar2 = pci_ioremap_bar(pdev, 4); 11866 if (!bp->bar2) { 11867 dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n"); 11868 rc = -ENOMEM; 11869 goto init_err_release; 11870 } 11871 11872 pci_enable_pcie_error_reporting(pdev); 11873 11874 INIT_WORK(&bp->sp_task, bnxt_sp_task); 11875 INIT_DELAYED_WORK(&bp->fw_reset_task, bnxt_fw_reset_task); 11876 11877 spin_lock_init(&bp->ntp_fltr_lock); 11878 #if BITS_PER_LONG == 32 11879 spin_lock_init(&bp->db_lock); 11880 #endif 11881 11882 bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE; 11883 bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE; 11884 11885 bnxt_init_dflt_coal(bp); 11886 11887 timer_setup(&bp->timer, bnxt_timer, 0); 11888 bp->current_interval = BNXT_TIMER_INTERVAL; 11889 11890 bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID; 11891 bp->nge_fw_dst_port_id = INVALID_HW_RING_ID; 11892 11893 clear_bit(BNXT_STATE_OPEN, &bp->state); 11894 return 0; 11895 11896 init_err_release: 11897 bnxt_unmap_bars(bp, pdev); 11898 pci_release_regions(pdev); 11899 11900 init_err_disable: 11901 pci_disable_device(pdev); 11902 11903 init_err: 11904 return rc; 11905 } 11906 11907 /* rtnl_lock held */ 11908 static int bnxt_change_mac_addr(struct net_device *dev, void *p) 11909 { 11910 struct sockaddr *addr = p; 11911 struct bnxt *bp = netdev_priv(dev); 11912 int rc = 0; 11913 11914 if (!is_valid_ether_addr(addr->sa_data)) 11915 return -EADDRNOTAVAIL; 11916 11917 if (ether_addr_equal(addr->sa_data, dev->dev_addr)) 11918 return 0; 11919 11920 rc = bnxt_approve_mac(bp, addr->sa_data, true); 11921 if (rc) 11922 return rc; 11923 11924 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len); 11925 if (netif_running(dev)) { 11926 bnxt_close_nic(bp, false, false); 11927 rc = bnxt_open_nic(bp, false, false); 11928 } 11929 11930 return rc; 11931 } 11932 11933 /* rtnl_lock held */ 11934 static int bnxt_change_mtu(struct net_device *dev, int new_mtu) 11935 { 11936 struct bnxt *bp = netdev_priv(dev); 11937 11938 if (netif_running(dev)) 11939 bnxt_close_nic(bp, true, false); 11940 11941 dev->mtu = new_mtu; 11942 bnxt_set_ring_params(bp); 11943 11944 if (netif_running(dev)) 11945 return bnxt_open_nic(bp, true, false); 11946 11947 return 0; 11948 } 11949 11950 int bnxt_setup_mq_tc(struct net_device *dev, u8 tc) 11951 { 11952 struct bnxt *bp = netdev_priv(dev); 11953 bool sh = false; 11954 int rc; 11955 11956 if (tc > bp->max_tc) { 11957 netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n", 11958 tc, bp->max_tc); 11959 return -EINVAL; 11960 } 11961 11962 if (netdev_get_num_tc(dev) == tc) 11963 return 0; 11964 11965 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 11966 sh = true; 11967 11968 rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings, 11969 sh, tc, bp->tx_nr_rings_xdp); 11970 if (rc) 11971 return rc; 11972 11973 /* Needs to close the device and do hw resource re-allocations */ 11974 if (netif_running(bp->dev)) 11975 bnxt_close_nic(bp, true, false); 11976 11977 if (tc) { 11978 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc; 11979 netdev_set_num_tc(dev, tc); 11980 } else { 11981 bp->tx_nr_rings = bp->tx_nr_rings_per_tc; 11982 netdev_reset_tc(dev); 11983 } 11984 bp->tx_nr_rings += bp->tx_nr_rings_xdp; 11985 bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) : 11986 bp->tx_nr_rings + bp->rx_nr_rings; 11987 11988 if (netif_running(bp->dev)) 11989 return bnxt_open_nic(bp, true, false); 11990 11991 return 0; 11992 } 11993 11994 static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 11995 void *cb_priv) 11996 { 11997 struct bnxt *bp = cb_priv; 11998 11999 if (!bnxt_tc_flower_enabled(bp) || 12000 !tc_cls_can_offload_and_chain0(bp->dev, type_data)) 12001 return -EOPNOTSUPP; 12002 12003 switch (type) { 12004 case TC_SETUP_CLSFLOWER: 12005 return bnxt_tc_setup_flower(bp, bp->pf.fw_fid, type_data); 12006 default: 12007 return -EOPNOTSUPP; 12008 } 12009 } 12010 12011 LIST_HEAD(bnxt_block_cb_list); 12012 12013 static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type, 12014 void *type_data) 12015 { 12016 struct bnxt *bp = netdev_priv(dev); 12017 12018 switch (type) { 12019 case TC_SETUP_BLOCK: 12020 return flow_block_cb_setup_simple(type_data, 12021 &bnxt_block_cb_list, 12022 bnxt_setup_tc_block_cb, 12023 bp, bp, true); 12024 case TC_SETUP_QDISC_MQPRIO: { 12025 struct tc_mqprio_qopt *mqprio = type_data; 12026 12027 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 12028 12029 return bnxt_setup_mq_tc(dev, mqprio->num_tc); 12030 } 12031 default: 12032 return -EOPNOTSUPP; 12033 } 12034 } 12035 12036 #ifdef CONFIG_RFS_ACCEL 12037 static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1, 12038 struct bnxt_ntuple_filter *f2) 12039 { 12040 struct flow_keys *keys1 = &f1->fkeys; 12041 struct flow_keys *keys2 = &f2->fkeys; 12042 12043 if (keys1->basic.n_proto != keys2->basic.n_proto || 12044 keys1->basic.ip_proto != keys2->basic.ip_proto) 12045 return false; 12046 12047 if (keys1->basic.n_proto == htons(ETH_P_IP)) { 12048 if (keys1->addrs.v4addrs.src != keys2->addrs.v4addrs.src || 12049 keys1->addrs.v4addrs.dst != keys2->addrs.v4addrs.dst) 12050 return false; 12051 } else { 12052 if (memcmp(&keys1->addrs.v6addrs.src, &keys2->addrs.v6addrs.src, 12053 sizeof(keys1->addrs.v6addrs.src)) || 12054 memcmp(&keys1->addrs.v6addrs.dst, &keys2->addrs.v6addrs.dst, 12055 sizeof(keys1->addrs.v6addrs.dst))) 12056 return false; 12057 } 12058 12059 if (keys1->ports.ports == keys2->ports.ports && 12060 keys1->control.flags == keys2->control.flags && 12061 ether_addr_equal(f1->src_mac_addr, f2->src_mac_addr) && 12062 ether_addr_equal(f1->dst_mac_addr, f2->dst_mac_addr)) 12063 return true; 12064 12065 return false; 12066 } 12067 12068 static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb, 12069 u16 rxq_index, u32 flow_id) 12070 { 12071 struct bnxt *bp = netdev_priv(dev); 12072 struct bnxt_ntuple_filter *fltr, *new_fltr; 12073 struct flow_keys *fkeys; 12074 struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb); 12075 int rc = 0, idx, bit_id, l2_idx = 0; 12076 struct hlist_head *head; 12077 u32 flags; 12078 12079 if (!ether_addr_equal(dev->dev_addr, eth->h_dest)) { 12080 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 12081 int off = 0, j; 12082 12083 netif_addr_lock_bh(dev); 12084 for (j = 0; j < vnic->uc_filter_count; j++, off += ETH_ALEN) { 12085 if (ether_addr_equal(eth->h_dest, 12086 vnic->uc_list + off)) { 12087 l2_idx = j + 1; 12088 break; 12089 } 12090 } 12091 netif_addr_unlock_bh(dev); 12092 if (!l2_idx) 12093 return -EINVAL; 12094 } 12095 new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC); 12096 if (!new_fltr) 12097 return -ENOMEM; 12098 12099 fkeys = &new_fltr->fkeys; 12100 if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) { 12101 rc = -EPROTONOSUPPORT; 12102 goto err_free; 12103 } 12104 12105 if ((fkeys->basic.n_proto != htons(ETH_P_IP) && 12106 fkeys->basic.n_proto != htons(ETH_P_IPV6)) || 12107 ((fkeys->basic.ip_proto != IPPROTO_TCP) && 12108 (fkeys->basic.ip_proto != IPPROTO_UDP))) { 12109 rc = -EPROTONOSUPPORT; 12110 goto err_free; 12111 } 12112 if (fkeys->basic.n_proto == htons(ETH_P_IPV6) && 12113 bp->hwrm_spec_code < 0x10601) { 12114 rc = -EPROTONOSUPPORT; 12115 goto err_free; 12116 } 12117 flags = fkeys->control.flags; 12118 if (((flags & FLOW_DIS_ENCAPSULATION) && 12119 bp->hwrm_spec_code < 0x10601) || (flags & FLOW_DIS_IS_FRAGMENT)) { 12120 rc = -EPROTONOSUPPORT; 12121 goto err_free; 12122 } 12123 12124 memcpy(new_fltr->dst_mac_addr, eth->h_dest, ETH_ALEN); 12125 memcpy(new_fltr->src_mac_addr, eth->h_source, ETH_ALEN); 12126 12127 idx = skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK; 12128 head = &bp->ntp_fltr_hash_tbl[idx]; 12129 rcu_read_lock(); 12130 hlist_for_each_entry_rcu(fltr, head, hash) { 12131 if (bnxt_fltr_match(fltr, new_fltr)) { 12132 rcu_read_unlock(); 12133 rc = 0; 12134 goto err_free; 12135 } 12136 } 12137 rcu_read_unlock(); 12138 12139 spin_lock_bh(&bp->ntp_fltr_lock); 12140 bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap, 12141 BNXT_NTP_FLTR_MAX_FLTR, 0); 12142 if (bit_id < 0) { 12143 spin_unlock_bh(&bp->ntp_fltr_lock); 12144 rc = -ENOMEM; 12145 goto err_free; 12146 } 12147 12148 new_fltr->sw_id = (u16)bit_id; 12149 new_fltr->flow_id = flow_id; 12150 new_fltr->l2_fltr_idx = l2_idx; 12151 new_fltr->rxq = rxq_index; 12152 hlist_add_head_rcu(&new_fltr->hash, head); 12153 bp->ntp_fltr_count++; 12154 spin_unlock_bh(&bp->ntp_fltr_lock); 12155 12156 set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event); 12157 bnxt_queue_sp_work(bp); 12158 12159 return new_fltr->sw_id; 12160 12161 err_free: 12162 kfree(new_fltr); 12163 return rc; 12164 } 12165 12166 static void bnxt_cfg_ntp_filters(struct bnxt *bp) 12167 { 12168 int i; 12169 12170 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) { 12171 struct hlist_head *head; 12172 struct hlist_node *tmp; 12173 struct bnxt_ntuple_filter *fltr; 12174 int rc; 12175 12176 head = &bp->ntp_fltr_hash_tbl[i]; 12177 hlist_for_each_entry_safe(fltr, tmp, head, hash) { 12178 bool del = false; 12179 12180 if (test_bit(BNXT_FLTR_VALID, &fltr->state)) { 12181 if (rps_may_expire_flow(bp->dev, fltr->rxq, 12182 fltr->flow_id, 12183 fltr->sw_id)) { 12184 bnxt_hwrm_cfa_ntuple_filter_free(bp, 12185 fltr); 12186 del = true; 12187 } 12188 } else { 12189 rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp, 12190 fltr); 12191 if (rc) 12192 del = true; 12193 else 12194 set_bit(BNXT_FLTR_VALID, &fltr->state); 12195 } 12196 12197 if (del) { 12198 spin_lock_bh(&bp->ntp_fltr_lock); 12199 hlist_del_rcu(&fltr->hash); 12200 bp->ntp_fltr_count--; 12201 spin_unlock_bh(&bp->ntp_fltr_lock); 12202 synchronize_rcu(); 12203 clear_bit(fltr->sw_id, bp->ntp_fltr_bmap); 12204 kfree(fltr); 12205 } 12206 } 12207 } 12208 if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event)) 12209 netdev_info(bp->dev, "Receive PF driver unload event!\n"); 12210 } 12211 12212 #else 12213 12214 static void bnxt_cfg_ntp_filters(struct bnxt *bp) 12215 { 12216 } 12217 12218 #endif /* CONFIG_RFS_ACCEL */ 12219 12220 static int bnxt_udp_tunnel_sync(struct net_device *netdev, unsigned int table) 12221 { 12222 struct bnxt *bp = netdev_priv(netdev); 12223 struct udp_tunnel_info ti; 12224 unsigned int cmd; 12225 12226 udp_tunnel_nic_get_port(netdev, table, 0, &ti); 12227 if (ti.type == UDP_TUNNEL_TYPE_VXLAN) 12228 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN; 12229 else 12230 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE; 12231 12232 if (ti.port) 12233 return bnxt_hwrm_tunnel_dst_port_alloc(bp, ti.port, cmd); 12234 12235 return bnxt_hwrm_tunnel_dst_port_free(bp, cmd); 12236 } 12237 12238 static const struct udp_tunnel_nic_info bnxt_udp_tunnels = { 12239 .sync_table = bnxt_udp_tunnel_sync, 12240 .flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP | 12241 UDP_TUNNEL_NIC_INFO_OPEN_ONLY, 12242 .tables = { 12243 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN, }, 12244 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_GENEVE, }, 12245 }, 12246 }; 12247 12248 static int bnxt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 12249 struct net_device *dev, u32 filter_mask, 12250 int nlflags) 12251 { 12252 struct bnxt *bp = netdev_priv(dev); 12253 12254 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bp->br_mode, 0, 0, 12255 nlflags, filter_mask, NULL); 12256 } 12257 12258 static int bnxt_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh, 12259 u16 flags, struct netlink_ext_ack *extack) 12260 { 12261 struct bnxt *bp = netdev_priv(dev); 12262 struct nlattr *attr, *br_spec; 12263 int rem, rc = 0; 12264 12265 if (bp->hwrm_spec_code < 0x10708 || !BNXT_SINGLE_PF(bp)) 12266 return -EOPNOTSUPP; 12267 12268 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 12269 if (!br_spec) 12270 return -EINVAL; 12271 12272 nla_for_each_nested(attr, br_spec, rem) { 12273 u16 mode; 12274 12275 if (nla_type(attr) != IFLA_BRIDGE_MODE) 12276 continue; 12277 12278 if (nla_len(attr) < sizeof(mode)) 12279 return -EINVAL; 12280 12281 mode = nla_get_u16(attr); 12282 if (mode == bp->br_mode) 12283 break; 12284 12285 rc = bnxt_hwrm_set_br_mode(bp, mode); 12286 if (!rc) 12287 bp->br_mode = mode; 12288 break; 12289 } 12290 return rc; 12291 } 12292 12293 int bnxt_get_port_parent_id(struct net_device *dev, 12294 struct netdev_phys_item_id *ppid) 12295 { 12296 struct bnxt *bp = netdev_priv(dev); 12297 12298 if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV) 12299 return -EOPNOTSUPP; 12300 12301 /* The PF and it's VF-reps only support the switchdev framework */ 12302 if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_DSN_VALID)) 12303 return -EOPNOTSUPP; 12304 12305 ppid->id_len = sizeof(bp->dsn); 12306 memcpy(ppid->id, bp->dsn, ppid->id_len); 12307 12308 return 0; 12309 } 12310 12311 static struct devlink_port *bnxt_get_devlink_port(struct net_device *dev) 12312 { 12313 struct bnxt *bp = netdev_priv(dev); 12314 12315 return &bp->dl_port; 12316 } 12317 12318 static const struct net_device_ops bnxt_netdev_ops = { 12319 .ndo_open = bnxt_open, 12320 .ndo_start_xmit = bnxt_start_xmit, 12321 .ndo_stop = bnxt_close, 12322 .ndo_get_stats64 = bnxt_get_stats64, 12323 .ndo_set_rx_mode = bnxt_set_rx_mode, 12324 .ndo_do_ioctl = bnxt_ioctl, 12325 .ndo_validate_addr = eth_validate_addr, 12326 .ndo_set_mac_address = bnxt_change_mac_addr, 12327 .ndo_change_mtu = bnxt_change_mtu, 12328 .ndo_fix_features = bnxt_fix_features, 12329 .ndo_set_features = bnxt_set_features, 12330 .ndo_tx_timeout = bnxt_tx_timeout, 12331 #ifdef CONFIG_BNXT_SRIOV 12332 .ndo_get_vf_config = bnxt_get_vf_config, 12333 .ndo_set_vf_mac = bnxt_set_vf_mac, 12334 .ndo_set_vf_vlan = bnxt_set_vf_vlan, 12335 .ndo_set_vf_rate = bnxt_set_vf_bw, 12336 .ndo_set_vf_link_state = bnxt_set_vf_link_state, 12337 .ndo_set_vf_spoofchk = bnxt_set_vf_spoofchk, 12338 .ndo_set_vf_trust = bnxt_set_vf_trust, 12339 #endif 12340 .ndo_setup_tc = bnxt_setup_tc, 12341 #ifdef CONFIG_RFS_ACCEL 12342 .ndo_rx_flow_steer = bnxt_rx_flow_steer, 12343 #endif 12344 .ndo_bpf = bnxt_xdp, 12345 .ndo_xdp_xmit = bnxt_xdp_xmit, 12346 .ndo_bridge_getlink = bnxt_bridge_getlink, 12347 .ndo_bridge_setlink = bnxt_bridge_setlink, 12348 .ndo_get_devlink_port = bnxt_get_devlink_port, 12349 }; 12350 12351 static void bnxt_remove_one(struct pci_dev *pdev) 12352 { 12353 struct net_device *dev = pci_get_drvdata(pdev); 12354 struct bnxt *bp = netdev_priv(dev); 12355 12356 if (BNXT_PF(bp)) 12357 bnxt_sriov_disable(bp); 12358 12359 if (BNXT_PF(bp)) 12360 devlink_port_type_clear(&bp->dl_port); 12361 pci_disable_pcie_error_reporting(pdev); 12362 unregister_netdev(dev); 12363 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 12364 /* Flush any pending tasks */ 12365 cancel_work_sync(&bp->sp_task); 12366 cancel_delayed_work_sync(&bp->fw_reset_task); 12367 bp->sp_event = 0; 12368 12369 bnxt_dl_fw_reporters_destroy(bp, true); 12370 bnxt_dl_unregister(bp); 12371 bnxt_shutdown_tc(bp); 12372 12373 bnxt_clear_int_mode(bp); 12374 bnxt_hwrm_func_drv_unrgtr(bp); 12375 bnxt_free_hwrm_resources(bp); 12376 bnxt_free_hwrm_short_cmd_req(bp); 12377 bnxt_ethtool_free(bp); 12378 bnxt_dcb_free(bp); 12379 kfree(bp->edev); 12380 bp->edev = NULL; 12381 kfree(bp->fw_health); 12382 bp->fw_health = NULL; 12383 bnxt_cleanup_pci(bp); 12384 bnxt_free_ctx_mem(bp); 12385 kfree(bp->ctx); 12386 bp->ctx = NULL; 12387 kfree(bp->rss_indir_tbl); 12388 bp->rss_indir_tbl = NULL; 12389 bnxt_free_port_stats(bp); 12390 free_netdev(dev); 12391 } 12392 12393 static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt) 12394 { 12395 int rc = 0; 12396 struct bnxt_link_info *link_info = &bp->link_info; 12397 12398 rc = bnxt_hwrm_phy_qcaps(bp); 12399 if (rc) { 12400 netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n", 12401 rc); 12402 return rc; 12403 } 12404 if (!fw_dflt) 12405 return 0; 12406 12407 rc = bnxt_update_link(bp, false); 12408 if (rc) { 12409 netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n", 12410 rc); 12411 return rc; 12412 } 12413 12414 /* Older firmware does not have supported_auto_speeds, so assume 12415 * that all supported speeds can be autonegotiated. 12416 */ 12417 if (link_info->auto_link_speeds && !link_info->support_auto_speeds) 12418 link_info->support_auto_speeds = link_info->support_speeds; 12419 12420 bnxt_init_ethtool_link_settings(bp); 12421 return 0; 12422 } 12423 12424 static int bnxt_get_max_irq(struct pci_dev *pdev) 12425 { 12426 u16 ctrl; 12427 12428 if (!pdev->msix_cap) 12429 return 1; 12430 12431 pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl); 12432 return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1; 12433 } 12434 12435 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, 12436 int *max_cp) 12437 { 12438 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 12439 int max_ring_grps = 0, max_irq; 12440 12441 *max_tx = hw_resc->max_tx_rings; 12442 *max_rx = hw_resc->max_rx_rings; 12443 *max_cp = bnxt_get_max_func_cp_rings_for_en(bp); 12444 max_irq = min_t(int, bnxt_get_max_func_irqs(bp) - 12445 bnxt_get_ulp_msix_num(bp), 12446 hw_resc->max_stat_ctxs - bnxt_get_ulp_stat_ctxs(bp)); 12447 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 12448 *max_cp = min_t(int, *max_cp, max_irq); 12449 max_ring_grps = hw_resc->max_hw_ring_grps; 12450 if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) { 12451 *max_cp -= 1; 12452 *max_rx -= 2; 12453 } 12454 if (bp->flags & BNXT_FLAG_AGG_RINGS) 12455 *max_rx >>= 1; 12456 if (bp->flags & BNXT_FLAG_CHIP_P5) { 12457 bnxt_trim_rings(bp, max_rx, max_tx, *max_cp, false); 12458 /* On P5 chips, max_cp output param should be available NQs */ 12459 *max_cp = max_irq; 12460 } 12461 *max_rx = min_t(int, *max_rx, max_ring_grps); 12462 } 12463 12464 int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared) 12465 { 12466 int rx, tx, cp; 12467 12468 _bnxt_get_max_rings(bp, &rx, &tx, &cp); 12469 *max_rx = rx; 12470 *max_tx = tx; 12471 if (!rx || !tx || !cp) 12472 return -ENOMEM; 12473 12474 return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared); 12475 } 12476 12477 static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx, 12478 bool shared) 12479 { 12480 int rc; 12481 12482 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared); 12483 if (rc && (bp->flags & BNXT_FLAG_AGG_RINGS)) { 12484 /* Not enough rings, try disabling agg rings. */ 12485 bp->flags &= ~BNXT_FLAG_AGG_RINGS; 12486 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared); 12487 if (rc) { 12488 /* set BNXT_FLAG_AGG_RINGS back for consistency */ 12489 bp->flags |= BNXT_FLAG_AGG_RINGS; 12490 return rc; 12491 } 12492 bp->flags |= BNXT_FLAG_NO_AGG_RINGS; 12493 bp->dev->hw_features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); 12494 bp->dev->features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); 12495 bnxt_set_ring_params(bp); 12496 } 12497 12498 if (bp->flags & BNXT_FLAG_ROCE_CAP) { 12499 int max_cp, max_stat, max_irq; 12500 12501 /* Reserve minimum resources for RoCE */ 12502 max_cp = bnxt_get_max_func_cp_rings(bp); 12503 max_stat = bnxt_get_max_func_stat_ctxs(bp); 12504 max_irq = bnxt_get_max_func_irqs(bp); 12505 if (max_cp <= BNXT_MIN_ROCE_CP_RINGS || 12506 max_irq <= BNXT_MIN_ROCE_CP_RINGS || 12507 max_stat <= BNXT_MIN_ROCE_STAT_CTXS) 12508 return 0; 12509 12510 max_cp -= BNXT_MIN_ROCE_CP_RINGS; 12511 max_irq -= BNXT_MIN_ROCE_CP_RINGS; 12512 max_stat -= BNXT_MIN_ROCE_STAT_CTXS; 12513 max_cp = min_t(int, max_cp, max_irq); 12514 max_cp = min_t(int, max_cp, max_stat); 12515 rc = bnxt_trim_rings(bp, max_rx, max_tx, max_cp, shared); 12516 if (rc) 12517 rc = 0; 12518 } 12519 return rc; 12520 } 12521 12522 /* In initial default shared ring setting, each shared ring must have a 12523 * RX/TX ring pair. 12524 */ 12525 static void bnxt_trim_dflt_sh_rings(struct bnxt *bp) 12526 { 12527 bp->cp_nr_rings = min_t(int, bp->tx_nr_rings_per_tc, bp->rx_nr_rings); 12528 bp->rx_nr_rings = bp->cp_nr_rings; 12529 bp->tx_nr_rings_per_tc = bp->cp_nr_rings; 12530 bp->tx_nr_rings = bp->tx_nr_rings_per_tc; 12531 } 12532 12533 static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh) 12534 { 12535 int dflt_rings, max_rx_rings, max_tx_rings, rc; 12536 12537 if (!bnxt_can_reserve_rings(bp)) 12538 return 0; 12539 12540 if (sh) 12541 bp->flags |= BNXT_FLAG_SHARED_RINGS; 12542 dflt_rings = is_kdump_kernel() ? 1 : netif_get_num_default_rss_queues(); 12543 /* Reduce default rings on multi-port cards so that total default 12544 * rings do not exceed CPU count. 12545 */ 12546 if (bp->port_count > 1) { 12547 int max_rings = 12548 max_t(int, num_online_cpus() / bp->port_count, 1); 12549 12550 dflt_rings = min_t(int, dflt_rings, max_rings); 12551 } 12552 rc = bnxt_get_dflt_rings(bp, &max_rx_rings, &max_tx_rings, sh); 12553 if (rc) 12554 return rc; 12555 bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings); 12556 bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings); 12557 if (sh) 12558 bnxt_trim_dflt_sh_rings(bp); 12559 else 12560 bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings; 12561 bp->tx_nr_rings = bp->tx_nr_rings_per_tc; 12562 12563 rc = __bnxt_reserve_rings(bp); 12564 if (rc) 12565 netdev_warn(bp->dev, "Unable to reserve tx rings\n"); 12566 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 12567 if (sh) 12568 bnxt_trim_dflt_sh_rings(bp); 12569 12570 /* Rings may have been trimmed, re-reserve the trimmed rings. */ 12571 if (bnxt_need_reserve_rings(bp)) { 12572 rc = __bnxt_reserve_rings(bp); 12573 if (rc) 12574 netdev_warn(bp->dev, "2nd rings reservation failed.\n"); 12575 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 12576 } 12577 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 12578 bp->rx_nr_rings++; 12579 bp->cp_nr_rings++; 12580 } 12581 if (rc) { 12582 bp->tx_nr_rings = 0; 12583 bp->rx_nr_rings = 0; 12584 } 12585 return rc; 12586 } 12587 12588 static int bnxt_init_dflt_ring_mode(struct bnxt *bp) 12589 { 12590 int rc; 12591 12592 if (bp->tx_nr_rings) 12593 return 0; 12594 12595 bnxt_ulp_irq_stop(bp); 12596 bnxt_clear_int_mode(bp); 12597 rc = bnxt_set_dflt_rings(bp, true); 12598 if (rc) { 12599 netdev_err(bp->dev, "Not enough rings available.\n"); 12600 goto init_dflt_ring_err; 12601 } 12602 rc = bnxt_init_int_mode(bp); 12603 if (rc) 12604 goto init_dflt_ring_err; 12605 12606 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 12607 if (bnxt_rfs_supported(bp) && bnxt_rfs_capable(bp)) { 12608 bp->flags |= BNXT_FLAG_RFS; 12609 bp->dev->features |= NETIF_F_NTUPLE; 12610 } 12611 init_dflt_ring_err: 12612 bnxt_ulp_irq_restart(bp, rc); 12613 return rc; 12614 } 12615 12616 int bnxt_restore_pf_fw_resources(struct bnxt *bp) 12617 { 12618 int rc; 12619 12620 ASSERT_RTNL(); 12621 bnxt_hwrm_func_qcaps(bp); 12622 12623 if (netif_running(bp->dev)) 12624 __bnxt_close_nic(bp, true, false); 12625 12626 bnxt_ulp_irq_stop(bp); 12627 bnxt_clear_int_mode(bp); 12628 rc = bnxt_init_int_mode(bp); 12629 bnxt_ulp_irq_restart(bp, rc); 12630 12631 if (netif_running(bp->dev)) { 12632 if (rc) 12633 dev_close(bp->dev); 12634 else 12635 rc = bnxt_open_nic(bp, true, false); 12636 } 12637 12638 return rc; 12639 } 12640 12641 static int bnxt_init_mac_addr(struct bnxt *bp) 12642 { 12643 int rc = 0; 12644 12645 if (BNXT_PF(bp)) { 12646 memcpy(bp->dev->dev_addr, bp->pf.mac_addr, ETH_ALEN); 12647 } else { 12648 #ifdef CONFIG_BNXT_SRIOV 12649 struct bnxt_vf_info *vf = &bp->vf; 12650 bool strict_approval = true; 12651 12652 if (is_valid_ether_addr(vf->mac_addr)) { 12653 /* overwrite netdev dev_addr with admin VF MAC */ 12654 memcpy(bp->dev->dev_addr, vf->mac_addr, ETH_ALEN); 12655 /* Older PF driver or firmware may not approve this 12656 * correctly. 12657 */ 12658 strict_approval = false; 12659 } else { 12660 eth_hw_addr_random(bp->dev); 12661 } 12662 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, strict_approval); 12663 #endif 12664 } 12665 return rc; 12666 } 12667 12668 #define BNXT_VPD_LEN 512 12669 static void bnxt_vpd_read_info(struct bnxt *bp) 12670 { 12671 struct pci_dev *pdev = bp->pdev; 12672 int i, len, pos, ro_size, size; 12673 ssize_t vpd_size; 12674 u8 *vpd_data; 12675 12676 vpd_data = kmalloc(BNXT_VPD_LEN, GFP_KERNEL); 12677 if (!vpd_data) 12678 return; 12679 12680 vpd_size = pci_read_vpd(pdev, 0, BNXT_VPD_LEN, vpd_data); 12681 if (vpd_size <= 0) { 12682 netdev_err(bp->dev, "Unable to read VPD\n"); 12683 goto exit; 12684 } 12685 12686 i = pci_vpd_find_tag(vpd_data, 0, vpd_size, PCI_VPD_LRDT_RO_DATA); 12687 if (i < 0) { 12688 netdev_err(bp->dev, "VPD READ-Only not found\n"); 12689 goto exit; 12690 } 12691 12692 ro_size = pci_vpd_lrdt_size(&vpd_data[i]); 12693 i += PCI_VPD_LRDT_TAG_SIZE; 12694 if (i + ro_size > vpd_size) 12695 goto exit; 12696 12697 pos = pci_vpd_find_info_keyword(vpd_data, i, ro_size, 12698 PCI_VPD_RO_KEYWORD_PARTNO); 12699 if (pos < 0) 12700 goto read_sn; 12701 12702 len = pci_vpd_info_field_size(&vpd_data[pos]); 12703 pos += PCI_VPD_INFO_FLD_HDR_SIZE; 12704 if (len + pos > vpd_size) 12705 goto read_sn; 12706 12707 size = min(len, BNXT_VPD_FLD_LEN - 1); 12708 memcpy(bp->board_partno, &vpd_data[pos], size); 12709 12710 read_sn: 12711 pos = pci_vpd_find_info_keyword(vpd_data, i, ro_size, 12712 PCI_VPD_RO_KEYWORD_SERIALNO); 12713 if (pos < 0) 12714 goto exit; 12715 12716 len = pci_vpd_info_field_size(&vpd_data[pos]); 12717 pos += PCI_VPD_INFO_FLD_HDR_SIZE; 12718 if (len + pos > vpd_size) 12719 goto exit; 12720 12721 size = min(len, BNXT_VPD_FLD_LEN - 1); 12722 memcpy(bp->board_serialno, &vpd_data[pos], size); 12723 exit: 12724 kfree(vpd_data); 12725 } 12726 12727 static int bnxt_pcie_dsn_get(struct bnxt *bp, u8 dsn[]) 12728 { 12729 struct pci_dev *pdev = bp->pdev; 12730 u64 qword; 12731 12732 qword = pci_get_dsn(pdev); 12733 if (!qword) { 12734 netdev_info(bp->dev, "Unable to read adapter's DSN\n"); 12735 return -EOPNOTSUPP; 12736 } 12737 12738 put_unaligned_le64(qword, dsn); 12739 12740 bp->flags |= BNXT_FLAG_DSN_VALID; 12741 return 0; 12742 } 12743 12744 static int bnxt_map_db_bar(struct bnxt *bp) 12745 { 12746 if (!bp->db_size) 12747 return -ENODEV; 12748 bp->bar1 = pci_iomap(bp->pdev, 2, bp->db_size); 12749 if (!bp->bar1) 12750 return -ENOMEM; 12751 return 0; 12752 } 12753 12754 static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) 12755 { 12756 struct net_device *dev; 12757 struct bnxt *bp; 12758 int rc, max_irqs; 12759 12760 if (pci_is_bridge(pdev)) 12761 return -ENODEV; 12762 12763 /* Clear any pending DMA transactions from crash kernel 12764 * while loading driver in capture kernel. 12765 */ 12766 if (is_kdump_kernel()) { 12767 pci_clear_master(pdev); 12768 pcie_flr(pdev); 12769 } 12770 12771 max_irqs = bnxt_get_max_irq(pdev); 12772 dev = alloc_etherdev_mq(sizeof(*bp), max_irqs); 12773 if (!dev) 12774 return -ENOMEM; 12775 12776 bp = netdev_priv(dev); 12777 bp->msg_enable = BNXT_DEF_MSG_ENABLE; 12778 bnxt_set_max_func_irqs(bp, max_irqs); 12779 12780 if (bnxt_vf_pciid(ent->driver_data)) 12781 bp->flags |= BNXT_FLAG_VF; 12782 12783 if (pdev->msix_cap) 12784 bp->flags |= BNXT_FLAG_MSIX_CAP; 12785 12786 rc = bnxt_init_board(pdev, dev); 12787 if (rc < 0) 12788 goto init_err_free; 12789 12790 dev->netdev_ops = &bnxt_netdev_ops; 12791 dev->watchdog_timeo = BNXT_TX_TIMEOUT; 12792 dev->ethtool_ops = &bnxt_ethtool_ops; 12793 pci_set_drvdata(pdev, dev); 12794 12795 rc = bnxt_alloc_hwrm_resources(bp); 12796 if (rc) 12797 goto init_err_pci_clean; 12798 12799 mutex_init(&bp->hwrm_cmd_lock); 12800 mutex_init(&bp->link_lock); 12801 12802 rc = bnxt_fw_init_one_p1(bp); 12803 if (rc) 12804 goto init_err_pci_clean; 12805 12806 if (BNXT_PF(bp)) 12807 bnxt_vpd_read_info(bp); 12808 12809 if (BNXT_CHIP_P5(bp)) { 12810 bp->flags |= BNXT_FLAG_CHIP_P5; 12811 if (BNXT_CHIP_SR2(bp)) 12812 bp->flags |= BNXT_FLAG_CHIP_SR2; 12813 } 12814 12815 rc = bnxt_alloc_rss_indir_tbl(bp); 12816 if (rc) 12817 goto init_err_pci_clean; 12818 12819 rc = bnxt_fw_init_one_p2(bp); 12820 if (rc) 12821 goto init_err_pci_clean; 12822 12823 rc = bnxt_map_db_bar(bp); 12824 if (rc) { 12825 dev_err(&pdev->dev, "Cannot map doorbell BAR rc = %d, aborting\n", 12826 rc); 12827 goto init_err_pci_clean; 12828 } 12829 12830 dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG | 12831 NETIF_F_TSO | NETIF_F_TSO6 | 12832 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE | 12833 NETIF_F_GSO_IPXIP4 | 12834 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM | 12835 NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH | 12836 NETIF_F_RXCSUM | NETIF_F_GRO; 12837 12838 if (BNXT_SUPPORTS_TPA(bp)) 12839 dev->hw_features |= NETIF_F_LRO; 12840 12841 dev->hw_enc_features = 12842 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG | 12843 NETIF_F_TSO | NETIF_F_TSO6 | 12844 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE | 12845 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM | 12846 NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL; 12847 dev->udp_tunnel_nic_info = &bnxt_udp_tunnels; 12848 12849 dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM | 12850 NETIF_F_GSO_GRE_CSUM; 12851 dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA; 12852 if (bp->fw_cap & BNXT_FW_CAP_VLAN_RX_STRIP) 12853 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_RX; 12854 if (bp->fw_cap & BNXT_FW_CAP_VLAN_TX_INSERT) 12855 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_TX; 12856 if (BNXT_SUPPORTS_TPA(bp)) 12857 dev->hw_features |= NETIF_F_GRO_HW; 12858 dev->features |= dev->hw_features | NETIF_F_HIGHDMA; 12859 if (dev->features & NETIF_F_GRO_HW) 12860 dev->features &= ~NETIF_F_LRO; 12861 dev->priv_flags |= IFF_UNICAST_FLT; 12862 12863 #ifdef CONFIG_BNXT_SRIOV 12864 init_waitqueue_head(&bp->sriov_cfg_wait); 12865 mutex_init(&bp->sriov_lock); 12866 #endif 12867 if (BNXT_SUPPORTS_TPA(bp)) { 12868 bp->gro_func = bnxt_gro_func_5730x; 12869 if (BNXT_CHIP_P4(bp)) 12870 bp->gro_func = bnxt_gro_func_5731x; 12871 else if (BNXT_CHIP_P5(bp)) 12872 bp->gro_func = bnxt_gro_func_5750x; 12873 } 12874 if (!BNXT_CHIP_P4_PLUS(bp)) 12875 bp->flags |= BNXT_FLAG_DOUBLE_DB; 12876 12877 bp->ulp_probe = bnxt_ulp_probe; 12878 12879 rc = bnxt_init_mac_addr(bp); 12880 if (rc) { 12881 dev_err(&pdev->dev, "Unable to initialize mac address.\n"); 12882 rc = -EADDRNOTAVAIL; 12883 goto init_err_pci_clean; 12884 } 12885 12886 if (BNXT_PF(bp)) { 12887 /* Read the adapter's DSN to use as the eswitch switch_id */ 12888 rc = bnxt_pcie_dsn_get(bp, bp->dsn); 12889 } 12890 12891 /* MTU range: 60 - FW defined max */ 12892 dev->min_mtu = ETH_ZLEN; 12893 dev->max_mtu = bp->max_mtu; 12894 12895 rc = bnxt_probe_phy(bp, true); 12896 if (rc) 12897 goto init_err_pci_clean; 12898 12899 bnxt_set_rx_skb_mode(bp, false); 12900 bnxt_set_tpa_flags(bp); 12901 bnxt_set_ring_params(bp); 12902 rc = bnxt_set_dflt_rings(bp, true); 12903 if (rc) { 12904 netdev_err(bp->dev, "Not enough rings available.\n"); 12905 rc = -ENOMEM; 12906 goto init_err_pci_clean; 12907 } 12908 12909 bnxt_fw_init_one_p3(bp); 12910 12911 if (dev->hw_features & BNXT_HW_FEATURE_VLAN_ALL_RX) 12912 bp->flags |= BNXT_FLAG_STRIP_VLAN; 12913 12914 rc = bnxt_init_int_mode(bp); 12915 if (rc) 12916 goto init_err_pci_clean; 12917 12918 /* No TC has been set yet and rings may have been trimmed due to 12919 * limited MSIX, so we re-initialize the TX rings per TC. 12920 */ 12921 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 12922 12923 if (BNXT_PF(bp)) { 12924 if (!bnxt_pf_wq) { 12925 bnxt_pf_wq = 12926 create_singlethread_workqueue("bnxt_pf_wq"); 12927 if (!bnxt_pf_wq) { 12928 dev_err(&pdev->dev, "Unable to create workqueue.\n"); 12929 rc = -ENOMEM; 12930 goto init_err_pci_clean; 12931 } 12932 } 12933 rc = bnxt_init_tc(bp); 12934 if (rc) 12935 netdev_err(dev, "Failed to initialize TC flower offload, err = %d.\n", 12936 rc); 12937 } 12938 12939 bnxt_dl_register(bp); 12940 12941 rc = register_netdev(dev); 12942 if (rc) 12943 goto init_err_cleanup; 12944 12945 if (BNXT_PF(bp)) 12946 devlink_port_type_eth_set(&bp->dl_port, bp->dev); 12947 bnxt_dl_fw_reporters_create(bp); 12948 12949 netdev_info(dev, "%s found at mem %lx, node addr %pM\n", 12950 board_info[ent->driver_data].name, 12951 (long)pci_resource_start(pdev, 0), dev->dev_addr); 12952 pcie_print_link_status(pdev); 12953 12954 pci_save_state(pdev); 12955 return 0; 12956 12957 init_err_cleanup: 12958 bnxt_dl_unregister(bp); 12959 bnxt_shutdown_tc(bp); 12960 bnxt_clear_int_mode(bp); 12961 12962 init_err_pci_clean: 12963 bnxt_hwrm_func_drv_unrgtr(bp); 12964 bnxt_free_hwrm_short_cmd_req(bp); 12965 bnxt_free_hwrm_resources(bp); 12966 kfree(bp->fw_health); 12967 bp->fw_health = NULL; 12968 bnxt_cleanup_pci(bp); 12969 bnxt_free_ctx_mem(bp); 12970 kfree(bp->ctx); 12971 bp->ctx = NULL; 12972 kfree(bp->rss_indir_tbl); 12973 bp->rss_indir_tbl = NULL; 12974 12975 init_err_free: 12976 free_netdev(dev); 12977 return rc; 12978 } 12979 12980 static void bnxt_shutdown(struct pci_dev *pdev) 12981 { 12982 struct net_device *dev = pci_get_drvdata(pdev); 12983 struct bnxt *bp; 12984 12985 if (!dev) 12986 return; 12987 12988 rtnl_lock(); 12989 bp = netdev_priv(dev); 12990 if (!bp) 12991 goto shutdown_exit; 12992 12993 if (netif_running(dev)) 12994 dev_close(dev); 12995 12996 bnxt_ulp_shutdown(bp); 12997 bnxt_clear_int_mode(bp); 12998 pci_disable_device(pdev); 12999 13000 if (system_state == SYSTEM_POWER_OFF) { 13001 pci_wake_from_d3(pdev, bp->wol); 13002 pci_set_power_state(pdev, PCI_D3hot); 13003 } 13004 13005 shutdown_exit: 13006 rtnl_unlock(); 13007 } 13008 13009 #ifdef CONFIG_PM_SLEEP 13010 static int bnxt_suspend(struct device *device) 13011 { 13012 struct net_device *dev = dev_get_drvdata(device); 13013 struct bnxt *bp = netdev_priv(dev); 13014 int rc = 0; 13015 13016 rtnl_lock(); 13017 bnxt_ulp_stop(bp); 13018 if (netif_running(dev)) { 13019 netif_device_detach(dev); 13020 rc = bnxt_close(dev); 13021 } 13022 bnxt_hwrm_func_drv_unrgtr(bp); 13023 pci_disable_device(bp->pdev); 13024 bnxt_free_ctx_mem(bp); 13025 kfree(bp->ctx); 13026 bp->ctx = NULL; 13027 rtnl_unlock(); 13028 return rc; 13029 } 13030 13031 static int bnxt_resume(struct device *device) 13032 { 13033 struct net_device *dev = dev_get_drvdata(device); 13034 struct bnxt *bp = netdev_priv(dev); 13035 int rc = 0; 13036 13037 rtnl_lock(); 13038 rc = pci_enable_device(bp->pdev); 13039 if (rc) { 13040 netdev_err(dev, "Cannot re-enable PCI device during resume, err = %d\n", 13041 rc); 13042 goto resume_exit; 13043 } 13044 pci_set_master(bp->pdev); 13045 if (bnxt_hwrm_ver_get(bp)) { 13046 rc = -ENODEV; 13047 goto resume_exit; 13048 } 13049 rc = bnxt_hwrm_func_reset(bp); 13050 if (rc) { 13051 rc = -EBUSY; 13052 goto resume_exit; 13053 } 13054 13055 rc = bnxt_hwrm_func_qcaps(bp); 13056 if (rc) 13057 goto resume_exit; 13058 13059 if (bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false)) { 13060 rc = -ENODEV; 13061 goto resume_exit; 13062 } 13063 13064 bnxt_get_wol_settings(bp); 13065 if (netif_running(dev)) { 13066 rc = bnxt_open(dev); 13067 if (!rc) 13068 netif_device_attach(dev); 13069 } 13070 13071 resume_exit: 13072 bnxt_ulp_start(bp, rc); 13073 if (!rc) 13074 bnxt_reenable_sriov(bp); 13075 rtnl_unlock(); 13076 return rc; 13077 } 13078 13079 static SIMPLE_DEV_PM_OPS(bnxt_pm_ops, bnxt_suspend, bnxt_resume); 13080 #define BNXT_PM_OPS (&bnxt_pm_ops) 13081 13082 #else 13083 13084 #define BNXT_PM_OPS NULL 13085 13086 #endif /* CONFIG_PM_SLEEP */ 13087 13088 /** 13089 * bnxt_io_error_detected - called when PCI error is detected 13090 * @pdev: Pointer to PCI device 13091 * @state: The current pci connection state 13092 * 13093 * This function is called after a PCI bus error affecting 13094 * this device has been detected. 13095 */ 13096 static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev, 13097 pci_channel_state_t state) 13098 { 13099 struct net_device *netdev = pci_get_drvdata(pdev); 13100 struct bnxt *bp = netdev_priv(netdev); 13101 13102 netdev_info(netdev, "PCI I/O error detected\n"); 13103 13104 rtnl_lock(); 13105 netif_device_detach(netdev); 13106 13107 bnxt_ulp_stop(bp); 13108 13109 if (state == pci_channel_io_perm_failure) { 13110 rtnl_unlock(); 13111 return PCI_ERS_RESULT_DISCONNECT; 13112 } 13113 13114 if (state == pci_channel_io_frozen) 13115 set_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, &bp->state); 13116 13117 if (netif_running(netdev)) 13118 bnxt_close(netdev); 13119 13120 pci_disable_device(pdev); 13121 bnxt_free_ctx_mem(bp); 13122 kfree(bp->ctx); 13123 bp->ctx = NULL; 13124 rtnl_unlock(); 13125 13126 /* Request a slot slot reset. */ 13127 return PCI_ERS_RESULT_NEED_RESET; 13128 } 13129 13130 /** 13131 * bnxt_io_slot_reset - called after the pci bus has been reset. 13132 * @pdev: Pointer to PCI device 13133 * 13134 * Restart the card from scratch, as if from a cold-boot. 13135 * At this point, the card has exprienced a hard reset, 13136 * followed by fixups by BIOS, and has its config space 13137 * set up identically to what it was at cold boot. 13138 */ 13139 static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev) 13140 { 13141 pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT; 13142 struct net_device *netdev = pci_get_drvdata(pdev); 13143 struct bnxt *bp = netdev_priv(netdev); 13144 int err = 0, off; 13145 13146 netdev_info(bp->dev, "PCI Slot Reset\n"); 13147 13148 rtnl_lock(); 13149 13150 if (pci_enable_device(pdev)) { 13151 dev_err(&pdev->dev, 13152 "Cannot re-enable PCI device after reset.\n"); 13153 } else { 13154 pci_set_master(pdev); 13155 /* Upon fatal error, our device internal logic that latches to 13156 * BAR value is getting reset and will restore only upon 13157 * rewritting the BARs. 13158 * 13159 * As pci_restore_state() does not re-write the BARs if the 13160 * value is same as saved value earlier, driver needs to 13161 * write the BARs to 0 to force restore, in case of fatal error. 13162 */ 13163 if (test_and_clear_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, 13164 &bp->state)) { 13165 for (off = PCI_BASE_ADDRESS_0; 13166 off <= PCI_BASE_ADDRESS_5; off += 4) 13167 pci_write_config_dword(bp->pdev, off, 0); 13168 } 13169 pci_restore_state(pdev); 13170 pci_save_state(pdev); 13171 13172 err = bnxt_hwrm_func_reset(bp); 13173 if (!err) 13174 result = PCI_ERS_RESULT_RECOVERED; 13175 } 13176 13177 rtnl_unlock(); 13178 13179 return result; 13180 } 13181 13182 /** 13183 * bnxt_io_resume - called when traffic can start flowing again. 13184 * @pdev: Pointer to PCI device 13185 * 13186 * This callback is called when the error recovery driver tells 13187 * us that its OK to resume normal operation. 13188 */ 13189 static void bnxt_io_resume(struct pci_dev *pdev) 13190 { 13191 struct net_device *netdev = pci_get_drvdata(pdev); 13192 struct bnxt *bp = netdev_priv(netdev); 13193 int err; 13194 13195 netdev_info(bp->dev, "PCI Slot Resume\n"); 13196 rtnl_lock(); 13197 13198 err = bnxt_hwrm_func_qcaps(bp); 13199 if (!err && netif_running(netdev)) 13200 err = bnxt_open(netdev); 13201 13202 bnxt_ulp_start(bp, err); 13203 if (!err) { 13204 bnxt_reenable_sriov(bp); 13205 netif_device_attach(netdev); 13206 } 13207 13208 rtnl_unlock(); 13209 } 13210 13211 static const struct pci_error_handlers bnxt_err_handler = { 13212 .error_detected = bnxt_io_error_detected, 13213 .slot_reset = bnxt_io_slot_reset, 13214 .resume = bnxt_io_resume 13215 }; 13216 13217 static struct pci_driver bnxt_pci_driver = { 13218 .name = DRV_MODULE_NAME, 13219 .id_table = bnxt_pci_tbl, 13220 .probe = bnxt_init_one, 13221 .remove = bnxt_remove_one, 13222 .shutdown = bnxt_shutdown, 13223 .driver.pm = BNXT_PM_OPS, 13224 .err_handler = &bnxt_err_handler, 13225 #if defined(CONFIG_BNXT_SRIOV) 13226 .sriov_configure = bnxt_sriov_configure, 13227 #endif 13228 }; 13229 13230 static int __init bnxt_init(void) 13231 { 13232 bnxt_debug_init(); 13233 return pci_register_driver(&bnxt_pci_driver); 13234 } 13235 13236 static void __exit bnxt_exit(void) 13237 { 13238 pci_unregister_driver(&bnxt_pci_driver); 13239 if (bnxt_pf_wq) 13240 destroy_workqueue(bnxt_pf_wq); 13241 bnxt_debug_exit(); 13242 } 13243 13244 module_init(bnxt_init); 13245 module_exit(bnxt_exit); 13246