1 /* QLogic qede NIC Driver 2 * Copyright (c) 2015-2017 QLogic Corporation 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and /or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 #ifndef _QEDE_H_ 33 #define _QEDE_H_ 34 #include <linux/compiler.h> 35 #include <linux/version.h> 36 #include <linux/workqueue.h> 37 #include <linux/netdevice.h> 38 #include <linux/interrupt.h> 39 #include <linux/bitmap.h> 40 #include <linux/kernel.h> 41 #include <linux/mutex.h> 42 #include <linux/bpf.h> 43 #include <net/xdp.h> 44 #include <linux/qed/qede_rdma.h> 45 #include <linux/io.h> 46 #ifdef CONFIG_RFS_ACCEL 47 #include <linux/cpu_rmap.h> 48 #endif 49 #include <linux/qed/common_hsi.h> 50 #include <linux/qed/eth_common.h> 51 #include <linux/qed/qed_if.h> 52 #include <linux/qed/qed_chain.h> 53 #include <linux/qed/qed_eth_if.h> 54 55 #include <net/pkt_cls.h> 56 #include <net/tc_act/tc_gact.h> 57 58 #define QEDE_MAJOR_VERSION 8 59 #define QEDE_MINOR_VERSION 37 60 #define QEDE_REVISION_VERSION 0 61 #define QEDE_ENGINEERING_VERSION 20 62 #define DRV_MODULE_VERSION __stringify(QEDE_MAJOR_VERSION) "." \ 63 __stringify(QEDE_MINOR_VERSION) "." \ 64 __stringify(QEDE_REVISION_VERSION) "." \ 65 __stringify(QEDE_ENGINEERING_VERSION) 66 67 #define DRV_MODULE_SYM qede 68 69 struct qede_stats_common { 70 u64 no_buff_discards; 71 u64 packet_too_big_discard; 72 u64 ttl0_discard; 73 u64 rx_ucast_bytes; 74 u64 rx_mcast_bytes; 75 u64 rx_bcast_bytes; 76 u64 rx_ucast_pkts; 77 u64 rx_mcast_pkts; 78 u64 rx_bcast_pkts; 79 u64 mftag_filter_discards; 80 u64 mac_filter_discards; 81 u64 gft_filter_drop; 82 u64 tx_ucast_bytes; 83 u64 tx_mcast_bytes; 84 u64 tx_bcast_bytes; 85 u64 tx_ucast_pkts; 86 u64 tx_mcast_pkts; 87 u64 tx_bcast_pkts; 88 u64 tx_err_drop_pkts; 89 u64 coalesced_pkts; 90 u64 coalesced_events; 91 u64 coalesced_aborts_num; 92 u64 non_coalesced_pkts; 93 u64 coalesced_bytes; 94 u64 link_change_count; 95 96 /* port */ 97 u64 rx_64_byte_packets; 98 u64 rx_65_to_127_byte_packets; 99 u64 rx_128_to_255_byte_packets; 100 u64 rx_256_to_511_byte_packets; 101 u64 rx_512_to_1023_byte_packets; 102 u64 rx_1024_to_1518_byte_packets; 103 u64 rx_crc_errors; 104 u64 rx_mac_crtl_frames; 105 u64 rx_pause_frames; 106 u64 rx_pfc_frames; 107 u64 rx_align_errors; 108 u64 rx_carrier_errors; 109 u64 rx_oversize_packets; 110 u64 rx_jabbers; 111 u64 rx_undersize_packets; 112 u64 rx_fragments; 113 u64 tx_64_byte_packets; 114 u64 tx_65_to_127_byte_packets; 115 u64 tx_128_to_255_byte_packets; 116 u64 tx_256_to_511_byte_packets; 117 u64 tx_512_to_1023_byte_packets; 118 u64 tx_1024_to_1518_byte_packets; 119 u64 tx_pause_frames; 120 u64 tx_pfc_frames; 121 u64 brb_truncates; 122 u64 brb_discards; 123 u64 tx_mac_ctrl_frames; 124 }; 125 126 struct qede_stats_bb { 127 u64 rx_1519_to_1522_byte_packets; 128 u64 rx_1519_to_2047_byte_packets; 129 u64 rx_2048_to_4095_byte_packets; 130 u64 rx_4096_to_9216_byte_packets; 131 u64 rx_9217_to_16383_byte_packets; 132 u64 tx_1519_to_2047_byte_packets; 133 u64 tx_2048_to_4095_byte_packets; 134 u64 tx_4096_to_9216_byte_packets; 135 u64 tx_9217_to_16383_byte_packets; 136 u64 tx_lpi_entry_count; 137 u64 tx_total_collisions; 138 }; 139 140 struct qede_stats_ah { 141 u64 rx_1519_to_max_byte_packets; 142 u64 tx_1519_to_max_byte_packets; 143 }; 144 145 struct qede_stats { 146 struct qede_stats_common common; 147 148 union { 149 struct qede_stats_bb bb; 150 struct qede_stats_ah ah; 151 }; 152 }; 153 154 struct qede_vlan { 155 struct list_head list; 156 u16 vid; 157 bool configured; 158 }; 159 160 struct qede_rdma_dev { 161 struct qedr_dev *qedr_dev; 162 struct list_head entry; 163 struct list_head rdma_event_list; 164 struct workqueue_struct *rdma_wq; 165 bool exp_recovery; 166 }; 167 168 struct qede_ptp; 169 170 #define QEDE_RFS_MAX_FLTR 256 171 172 enum qede_flags_bit { 173 QEDE_FLAGS_IS_VF = 0, 174 QEDE_FLAGS_LINK_REQUESTED, 175 QEDE_FLAGS_PTP_TX_IN_PRORGESS, 176 QEDE_FLAGS_TX_TIMESTAMPING_EN 177 }; 178 179 struct qede_dev { 180 struct qed_dev *cdev; 181 struct net_device *ndev; 182 struct pci_dev *pdev; 183 184 u32 dp_module; 185 u8 dp_level; 186 187 unsigned long flags; 188 #define IS_VF(edev) (test_bit(QEDE_FLAGS_IS_VF, &(edev)->flags)) 189 190 const struct qed_eth_ops *ops; 191 struct qede_ptp *ptp; 192 193 struct qed_dev_eth_info dev_info; 194 #define QEDE_MAX_RSS_CNT(edev) ((edev)->dev_info.num_queues) 195 #define QEDE_MAX_TSS_CNT(edev) ((edev)->dev_info.num_queues) 196 #define QEDE_IS_BB(edev) \ 197 ((edev)->dev_info.common.dev_type == QED_DEV_TYPE_BB) 198 #define QEDE_IS_AH(edev) \ 199 ((edev)->dev_info.common.dev_type == QED_DEV_TYPE_AH) 200 201 struct qede_fastpath *fp_array; 202 u8 req_num_tx; 203 u8 fp_num_tx; 204 u8 req_num_rx; 205 u8 fp_num_rx; 206 u16 req_queues; 207 u16 num_queues; 208 #define QEDE_QUEUE_CNT(edev) ((edev)->num_queues) 209 #define QEDE_RSS_COUNT(edev) ((edev)->num_queues - (edev)->fp_num_tx) 210 #define QEDE_RX_QUEUE_IDX(edev, i) (i) 211 #define QEDE_TSS_COUNT(edev) ((edev)->num_queues - (edev)->fp_num_rx) 212 213 struct qed_int_info int_info; 214 215 /* Smaller private varaiant of the RTNL lock */ 216 struct mutex qede_lock; 217 u32 state; /* Protected by qede_lock */ 218 u16 rx_buf_size; 219 u32 rx_copybreak; 220 221 /* L2 header size + 2*VLANs (8 bytes) + LLC SNAP (8 bytes) */ 222 #define ETH_OVERHEAD (ETH_HLEN + 8 + 8) 223 /* Max supported alignment is 256 (8 shift) 224 * minimal alignment shift 6 is optimal for 57xxx HW performance 225 */ 226 #define QEDE_RX_ALIGN_SHIFT max(6, min(8, L1_CACHE_SHIFT)) 227 /* We assume skb_build() uses sizeof(struct skb_shared_info) bytes 228 * at the end of skb->data, to avoid wasting a full cache line. 229 * This reduces memory use (skb->truesize). 230 */ 231 #define QEDE_FW_RX_ALIGN_END \ 232 max_t(u64, 1UL << QEDE_RX_ALIGN_SHIFT, \ 233 SKB_DATA_ALIGN(sizeof(struct skb_shared_info))) 234 235 struct qede_stats stats; 236 #define QEDE_RSS_INDIR_INITED BIT(0) 237 #define QEDE_RSS_KEY_INITED BIT(1) 238 #define QEDE_RSS_CAPS_INITED BIT(2) 239 u32 rss_params_inited; /* bit-field to track initialized rss params */ 240 u16 rss_ind_table[128]; 241 u32 rss_key[10]; 242 u8 rss_caps; 243 244 u16 q_num_rx_buffers; /* Must be a power of two */ 245 u16 q_num_tx_buffers; /* Must be a power of two */ 246 247 bool gro_disable; 248 struct list_head vlan_list; 249 u16 configured_vlans; 250 u16 non_configured_vlans; 251 bool accept_any_vlan; 252 struct delayed_work sp_task; 253 unsigned long sp_flags; 254 u16 vxlan_dst_port; 255 u16 geneve_dst_port; 256 257 struct qede_arfs *arfs; 258 bool wol_enabled; 259 260 struct qede_rdma_dev rdma_info; 261 262 struct bpf_prog *xdp_prog; 263 }; 264 265 enum QEDE_STATE { 266 QEDE_STATE_CLOSED, 267 QEDE_STATE_OPEN, 268 QEDE_STATE_RECOVERY, 269 }; 270 271 #define HILO_U64(hi, lo) ((((u64)(hi)) << 32) + (lo)) 272 273 #define MAX_NUM_TC 8 274 #define MAX_NUM_PRI 8 275 276 /* The driver supports the new build_skb() API: 277 * RX ring buffer contains pointer to kmalloc() data only, 278 * skb are built only after the frame was DMA-ed. 279 */ 280 struct sw_rx_data { 281 struct page *data; 282 dma_addr_t mapping; 283 unsigned int page_offset; 284 }; 285 286 enum qede_agg_state { 287 QEDE_AGG_STATE_NONE = 0, 288 QEDE_AGG_STATE_START = 1, 289 QEDE_AGG_STATE_ERROR = 2 290 }; 291 292 struct qede_agg_info { 293 /* rx_buf is a data buffer that can be placed / consumed from rx bd 294 * chain. It has two purposes: We will preallocate the data buffer 295 * for each aggregation when we open the interface and will place this 296 * buffer on the rx-bd-ring when we receive TPA_START. We don't want 297 * to be in a state where allocation fails, as we can't reuse the 298 * consumer buffer in the rx-chain since FW may still be writing to it 299 * (since header needs to be modified for TPA). 300 * The second purpose is to keep a pointer to the bd buffer during 301 * aggregation. 302 */ 303 struct sw_rx_data buffer; 304 struct sk_buff *skb; 305 306 /* We need some structs from the start cookie until termination */ 307 u16 vlan_tag; 308 309 bool tpa_start_fail; 310 u8 state; 311 u8 frag_id; 312 313 u8 tunnel_type; 314 }; 315 316 struct qede_rx_queue { 317 __le16 *hw_cons_ptr; 318 void __iomem *hw_rxq_prod_addr; 319 320 /* Required for the allocation of replacement buffers */ 321 struct device *dev; 322 323 struct bpf_prog *xdp_prog; 324 325 u16 sw_rx_cons; 326 u16 sw_rx_prod; 327 328 u16 filled_buffers; 329 u8 data_direction; 330 u8 rxq_id; 331 332 /* Used once per each NAPI run */ 333 u16 num_rx_buffers; 334 335 u16 rx_headroom; 336 337 u32 rx_buf_size; 338 u32 rx_buf_seg_size; 339 340 struct sw_rx_data *sw_rx_ring; 341 struct qed_chain rx_bd_ring; 342 struct qed_chain rx_comp_ring ____cacheline_aligned; 343 344 /* GRO */ 345 struct qede_agg_info tpa_info[ETH_TPA_MAX_AGGS_NUM]; 346 347 /* Used once per each NAPI run */ 348 u64 rcv_pkts; 349 350 u64 rx_hw_errors; 351 u64 rx_alloc_errors; 352 u64 rx_ip_frags; 353 354 u64 xdp_no_pass; 355 356 void *handle; 357 struct xdp_rxq_info xdp_rxq; 358 }; 359 360 union db_prod { 361 struct eth_db_data data; 362 u32 raw; 363 }; 364 365 struct sw_tx_bd { 366 struct sk_buff *skb; 367 u8 flags; 368 /* Set on the first BD descriptor when there is a split BD */ 369 #define QEDE_TSO_SPLIT_BD BIT(0) 370 }; 371 372 struct sw_tx_xdp { 373 struct page *page; 374 dma_addr_t mapping; 375 }; 376 377 struct qede_tx_queue { 378 u8 is_xdp; 379 bool is_legacy; 380 u16 sw_tx_cons; 381 u16 sw_tx_prod; 382 u16 num_tx_buffers; /* Slowpath only */ 383 384 u64 xmit_pkts; 385 u64 stopped_cnt; 386 u64 tx_mem_alloc_err; 387 388 __le16 *hw_cons_ptr; 389 390 /* Needed for the mapping of packets */ 391 struct device *dev; 392 393 void __iomem *doorbell_addr; 394 union db_prod tx_db; 395 int index; /* Slowpath only */ 396 #define QEDE_TXQ_XDP_TO_IDX(edev, txq) ((txq)->index - \ 397 QEDE_MAX_TSS_CNT(edev)) 398 #define QEDE_TXQ_IDX_TO_XDP(edev, idx) ((idx) + QEDE_MAX_TSS_CNT(edev)) 399 #define QEDE_NDEV_TXQ_ID_TO_FP_ID(edev, idx) ((edev)->fp_num_rx + \ 400 ((idx) % QEDE_TSS_COUNT(edev))) 401 #define QEDE_NDEV_TXQ_ID_TO_TXQ_COS(edev, idx) ((idx) / QEDE_TSS_COUNT(edev)) 402 #define QEDE_TXQ_TO_NDEV_TXQ_ID(edev, txq) ((QEDE_TSS_COUNT(edev) * \ 403 (txq)->cos) + (txq)->index) 404 #define QEDE_NDEV_TXQ_ID_TO_TXQ(edev, idx) \ 405 (&((edev)->fp_array[QEDE_NDEV_TXQ_ID_TO_FP_ID(edev, idx)].txq \ 406 [QEDE_NDEV_TXQ_ID_TO_TXQ_COS(edev, idx)])) 407 #define QEDE_FP_TC0_TXQ(fp) (&((fp)->txq[0])) 408 409 /* Regular Tx requires skb + metadata for release purpose, 410 * while XDP requires the pages and the mapped address. 411 */ 412 union { 413 struct sw_tx_bd *skbs; 414 struct sw_tx_xdp *xdp; 415 } sw_tx_ring; 416 417 struct qed_chain tx_pbl; 418 419 /* Slowpath; Should be kept in end [unless missing padding] */ 420 void *handle; 421 u16 cos; 422 u16 ndev_txq_id; 423 }; 424 425 #define BD_UNMAP_ADDR(bd) HILO_U64(le32_to_cpu((bd)->addr.hi), \ 426 le32_to_cpu((bd)->addr.lo)) 427 #define BD_SET_UNMAP_ADDR_LEN(bd, maddr, len) \ 428 do { \ 429 (bd)->addr.hi = cpu_to_le32(upper_32_bits(maddr)); \ 430 (bd)->addr.lo = cpu_to_le32(lower_32_bits(maddr)); \ 431 (bd)->nbytes = cpu_to_le16(len); \ 432 } while (0) 433 #define BD_UNMAP_LEN(bd) (le16_to_cpu((bd)->nbytes)) 434 435 struct qede_fastpath { 436 struct qede_dev *edev; 437 #define QEDE_FASTPATH_TX BIT(0) 438 #define QEDE_FASTPATH_RX BIT(1) 439 #define QEDE_FASTPATH_XDP BIT(2) 440 #define QEDE_FASTPATH_COMBINED (QEDE_FASTPATH_TX | QEDE_FASTPATH_RX) 441 u8 type; 442 u8 id; 443 u8 xdp_xmit; 444 struct napi_struct napi; 445 struct qed_sb_info *sb_info; 446 struct qede_rx_queue *rxq; 447 struct qede_tx_queue *txq; 448 struct qede_tx_queue *xdp_tx; 449 450 #define VEC_NAME_SIZE (FIELD_SIZEOF(struct net_device, name) + 8) 451 char name[VEC_NAME_SIZE]; 452 }; 453 454 /* Debug print definitions */ 455 #define DP_NAME(edev) ((edev)->ndev->name) 456 457 #define XMIT_PLAIN 0 458 #define XMIT_L4_CSUM BIT(0) 459 #define XMIT_LSO BIT(1) 460 #define XMIT_ENC BIT(2) 461 #define XMIT_ENC_GSO_L4_CSUM BIT(3) 462 463 #define QEDE_CSUM_ERROR BIT(0) 464 #define QEDE_CSUM_UNNECESSARY BIT(1) 465 #define QEDE_TUNN_CSUM_UNNECESSARY BIT(2) 466 467 #define QEDE_SP_RECOVERY 0 468 #define QEDE_SP_RX_MODE 1 469 470 #ifdef CONFIG_RFS_ACCEL 471 int qede_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb, 472 u16 rxq_index, u32 flow_id); 473 #define QEDE_SP_ARFS_CONFIG 4 474 #define QEDE_SP_TASK_POLL_DELAY (5 * HZ) 475 #endif 476 477 void qede_process_arfs_filters(struct qede_dev *edev, bool free_fltr); 478 void qede_poll_for_freeing_arfs_filters(struct qede_dev *edev); 479 void qede_arfs_filter_op(void *dev, void *filter, u8 fw_rc); 480 void qede_free_arfs(struct qede_dev *edev); 481 int qede_alloc_arfs(struct qede_dev *edev); 482 int qede_add_cls_rule(struct qede_dev *edev, struct ethtool_rxnfc *info); 483 int qede_delete_flow_filter(struct qede_dev *edev, u64 cookie); 484 int qede_get_cls_rule_entry(struct qede_dev *edev, struct ethtool_rxnfc *cmd); 485 int qede_get_cls_rule_all(struct qede_dev *edev, struct ethtool_rxnfc *info, 486 u32 *rule_locs); 487 int qede_get_arfs_filter_count(struct qede_dev *edev); 488 489 struct qede_reload_args { 490 void (*func)(struct qede_dev *edev, struct qede_reload_args *args); 491 union { 492 netdev_features_t features; 493 struct bpf_prog *new_prog; 494 u16 mtu; 495 } u; 496 }; 497 498 /* Datapath functions definition */ 499 netdev_tx_t qede_start_xmit(struct sk_buff *skb, struct net_device *ndev); 500 u16 qede_select_queue(struct net_device *dev, struct sk_buff *skb, 501 struct net_device *sb_dev); 502 netdev_features_t qede_features_check(struct sk_buff *skb, 503 struct net_device *dev, 504 netdev_features_t features); 505 void qede_tx_log_print(struct qede_dev *edev, struct qede_fastpath *fp); 506 int qede_alloc_rx_buffer(struct qede_rx_queue *rxq, bool allow_lazy); 507 int qede_free_tx_pkt(struct qede_dev *edev, 508 struct qede_tx_queue *txq, int *len); 509 int qede_poll(struct napi_struct *napi, int budget); 510 irqreturn_t qede_msix_fp_int(int irq, void *fp_cookie); 511 512 /* Filtering function definitions */ 513 void qede_force_mac(void *dev, u8 *mac, bool forced); 514 void qede_udp_ports_update(void *dev, u16 vxlan_port, u16 geneve_port); 515 int qede_set_mac_addr(struct net_device *ndev, void *p); 516 517 int qede_vlan_rx_add_vid(struct net_device *dev, __be16 proto, u16 vid); 518 int qede_vlan_rx_kill_vid(struct net_device *dev, __be16 proto, u16 vid); 519 void qede_vlan_mark_nonconfigured(struct qede_dev *edev); 520 int qede_configure_vlan_filters(struct qede_dev *edev); 521 522 netdev_features_t qede_fix_features(struct net_device *dev, 523 netdev_features_t features); 524 int qede_set_features(struct net_device *dev, netdev_features_t features); 525 void qede_set_rx_mode(struct net_device *ndev); 526 void qede_config_rx_mode(struct net_device *ndev); 527 void qede_fill_rss_params(struct qede_dev *edev, 528 struct qed_update_vport_rss_params *rss, u8 *update); 529 530 void qede_udp_tunnel_add(struct net_device *dev, struct udp_tunnel_info *ti); 531 void qede_udp_tunnel_del(struct net_device *dev, struct udp_tunnel_info *ti); 532 533 int qede_xdp(struct net_device *dev, struct netdev_bpf *xdp); 534 535 #ifdef CONFIG_DCB 536 void qede_set_dcbnl_ops(struct net_device *ndev); 537 #endif 538 539 void qede_config_debug(uint debug, u32 *p_dp_module, u8 *p_dp_level); 540 void qede_set_ethtool_ops(struct net_device *netdev); 541 void qede_reload(struct qede_dev *edev, 542 struct qede_reload_args *args, bool is_locked); 543 int qede_change_mtu(struct net_device *dev, int new_mtu); 544 void qede_fill_by_demand_stats(struct qede_dev *edev); 545 void __qede_lock(struct qede_dev *edev); 546 void __qede_unlock(struct qede_dev *edev); 547 bool qede_has_rx_work(struct qede_rx_queue *rxq); 548 int qede_txq_has_work(struct qede_tx_queue *txq); 549 void qede_recycle_rx_bd_ring(struct qede_rx_queue *rxq, u8 count); 550 void qede_update_rx_prod(struct qede_dev *edev, struct qede_rx_queue *rxq); 551 int qede_add_tc_flower_fltr(struct qede_dev *edev, __be16 proto, 552 struct tc_cls_flower_offload *f); 553 554 #define RX_RING_SIZE_POW 13 555 #define RX_RING_SIZE ((u16)BIT(RX_RING_SIZE_POW)) 556 #define NUM_RX_BDS_MAX (RX_RING_SIZE - 1) 557 #define NUM_RX_BDS_MIN 128 558 #define NUM_RX_BDS_DEF ((u16)BIT(10) - 1) 559 560 #define TX_RING_SIZE_POW 13 561 #define TX_RING_SIZE ((u16)BIT(TX_RING_SIZE_POW)) 562 #define NUM_TX_BDS_MAX (TX_RING_SIZE - 1) 563 #define NUM_TX_BDS_MIN 128 564 #define NUM_TX_BDS_DEF NUM_TX_BDS_MAX 565 566 #define QEDE_MIN_PKT_LEN 64 567 #define QEDE_RX_HDR_SIZE 256 568 #define QEDE_MAX_JUMBO_PACKET_SIZE 9600 569 #define for_each_queue(i) for (i = 0; i < edev->num_queues; i++) 570 #define for_each_cos_in_txq(edev, var) \ 571 for ((var) = 0; (var) < (edev)->dev_info.num_tc; (var)++) 572 573 #endif /* _QEDE_H_ */ 574