12025cf9eSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 2c3e79bafSAviad Krawczyk /* 3c3e79bafSAviad Krawczyk * Huawei HiNIC PCI Express Linux driver 4c3e79bafSAviad Krawczyk * Copyright(c) 2017 Huawei Technologies Co., Ltd 5c3e79bafSAviad Krawczyk */ 6c3e79bafSAviad Krawczyk 7e2585ea7SAviad Krawczyk #include <linux/kernel.h> 8e2585ea7SAviad Krawczyk #include <linux/types.h> 9e2585ea7SAviad Krawczyk #include <linux/errno.h> 10e2585ea7SAviad Krawczyk #include <linux/pci.h> 11e2585ea7SAviad Krawczyk #include <linux/device.h> 12c3e79bafSAviad Krawczyk #include <linux/netdevice.h> 13e2585ea7SAviad Krawczyk #include <linux/etherdevice.h> 14c3e79bafSAviad Krawczyk #include <linux/u64_stats_sync.h> 15e2585ea7SAviad Krawczyk #include <linux/slab.h> 16e2585ea7SAviad Krawczyk #include <linux/interrupt.h> 17e2585ea7SAviad Krawczyk #include <linux/skbuff.h> 18e2585ea7SAviad Krawczyk #include <linux/dma-mapping.h> 19e2585ea7SAviad Krawczyk #include <linux/prefetch.h> 20352f58b0SAviad Krawczyk #include <linux/cpumask.h> 21aebd17b7SXue Chaojing #include <linux/if_vlan.h> 22e2585ea7SAviad Krawczyk #include <asm/barrier.h> 23c3e79bafSAviad Krawczyk 24e2585ea7SAviad Krawczyk #include "hinic_common.h" 25e2585ea7SAviad Krawczyk #include "hinic_hw_if.h" 26e2585ea7SAviad Krawczyk #include "hinic_hw_wqe.h" 27e2585ea7SAviad Krawczyk #include "hinic_hw_wq.h" 28c3e79bafSAviad Krawczyk #include "hinic_hw_qp.h" 29e2585ea7SAviad Krawczyk #include "hinic_hw_dev.h" 30c3e79bafSAviad Krawczyk #include "hinic_rx.h" 31e2585ea7SAviad Krawczyk #include "hinic_dev.h" 32e2585ea7SAviad Krawczyk 33e2585ea7SAviad Krawczyk #define RX_IRQ_NO_PENDING 0 34e2585ea7SAviad Krawczyk #define RX_IRQ_NO_COALESC 0 35e2585ea7SAviad Krawczyk #define RX_IRQ_NO_LLI_TIMER 0 36e2585ea7SAviad Krawczyk #define RX_IRQ_NO_CREDIT 0 37e2585ea7SAviad Krawczyk #define RX_IRQ_NO_RESEND_TIMER 0 38e1a76515SXue Chaojing #define HINIC_RX_BUFFER_WRITE 16 39c3e79bafSAviad Krawczyk 401e007181SXue Chaojing #define HINIC_RX_IPV6_PKT 7 411e007181SXue Chaojing #define LRO_PKT_HDR_LEN_IPV4 66 421e007181SXue Chaojing #define LRO_PKT_HDR_LEN_IPV6 86 431e007181SXue Chaojing #define LRO_REPLENISH_THLD 256 441e007181SXue Chaojing 451e007181SXue Chaojing #define LRO_PKT_HDR_LEN(cqe) \ 461e007181SXue Chaojing (HINIC_GET_RX_PKT_TYPE(be32_to_cpu((cqe)->offload_type)) == \ 471e007181SXue Chaojing HINIC_RX_IPV6_PKT ? LRO_PKT_HDR_LEN_IPV6 : LRO_PKT_HDR_LEN_IPV4) 481e007181SXue Chaojing 49c3e79bafSAviad Krawczyk /** 50c3e79bafSAviad Krawczyk * hinic_rxq_clean_stats - Clean the statistics of specific queue 51c3e79bafSAviad Krawczyk * @rxq: Logical Rx Queue 52c3e79bafSAviad Krawczyk **/ 53c3e79bafSAviad Krawczyk void hinic_rxq_clean_stats(struct hinic_rxq *rxq) 54c3e79bafSAviad Krawczyk { 55c3e79bafSAviad Krawczyk struct hinic_rxq_stats *rxq_stats = &rxq->rxq_stats; 56c3e79bafSAviad Krawczyk 57c3e79bafSAviad Krawczyk u64_stats_update_begin(&rxq_stats->syncp); 58c3e79bafSAviad Krawczyk rxq_stats->pkts = 0; 59c3e79bafSAviad Krawczyk rxq_stats->bytes = 0; 60e54fbbdfSXue Chaojing rxq_stats->errors = 0; 61e54fbbdfSXue Chaojing rxq_stats->csum_errors = 0; 62e54fbbdfSXue Chaojing rxq_stats->other_errors = 0; 63c3e79bafSAviad Krawczyk u64_stats_update_end(&rxq_stats->syncp); 64c3e79bafSAviad Krawczyk } 65c3e79bafSAviad Krawczyk 66c3e79bafSAviad Krawczyk /** 67edd384f6SAviad Krawczyk * hinic_rxq_get_stats - get statistics of Rx Queue 68edd384f6SAviad Krawczyk * @rxq: Logical Rx Queue 69edd384f6SAviad Krawczyk * @stats: return updated stats here 70edd384f6SAviad Krawczyk **/ 71edd384f6SAviad Krawczyk void hinic_rxq_get_stats(struct hinic_rxq *rxq, struct hinic_rxq_stats *stats) 72edd384f6SAviad Krawczyk { 73edd384f6SAviad Krawczyk struct hinic_rxq_stats *rxq_stats = &rxq->rxq_stats; 74edd384f6SAviad Krawczyk unsigned int start; 75edd384f6SAviad Krawczyk 76edd384f6SAviad Krawczyk u64_stats_update_begin(&stats->syncp); 77edd384f6SAviad Krawczyk do { 78edd384f6SAviad Krawczyk start = u64_stats_fetch_begin(&rxq_stats->syncp); 79edd384f6SAviad Krawczyk stats->pkts = rxq_stats->pkts; 80edd384f6SAviad Krawczyk stats->bytes = rxq_stats->bytes; 81e54fbbdfSXue Chaojing stats->errors = rxq_stats->csum_errors + 82e54fbbdfSXue Chaojing rxq_stats->other_errors; 83e54fbbdfSXue Chaojing stats->csum_errors = rxq_stats->csum_errors; 84e54fbbdfSXue Chaojing stats->other_errors = rxq_stats->other_errors; 85edd384f6SAviad Krawczyk } while (u64_stats_fetch_retry(&rxq_stats->syncp, start)); 86edd384f6SAviad Krawczyk u64_stats_update_end(&stats->syncp); 87edd384f6SAviad Krawczyk } 88edd384f6SAviad Krawczyk 89edd384f6SAviad Krawczyk /** 90c3e79bafSAviad Krawczyk * rxq_stats_init - Initialize the statistics of specific queue 91c3e79bafSAviad Krawczyk * @rxq: Logical Rx Queue 92c3e79bafSAviad Krawczyk **/ 93c3e79bafSAviad Krawczyk static void rxq_stats_init(struct hinic_rxq *rxq) 94c3e79bafSAviad Krawczyk { 95c3e79bafSAviad Krawczyk struct hinic_rxq_stats *rxq_stats = &rxq->rxq_stats; 96c3e79bafSAviad Krawczyk 97c3e79bafSAviad Krawczyk u64_stats_init(&rxq_stats->syncp); 98c3e79bafSAviad Krawczyk hinic_rxq_clean_stats(rxq); 99c3e79bafSAviad Krawczyk } 100c3e79bafSAviad Krawczyk 1011e007181SXue Chaojing static void rx_csum(struct hinic_rxq *rxq, u32 status, 1024a61abb1SXue Chaojing struct sk_buff *skb) 1034a61abb1SXue Chaojing { 1044a61abb1SXue Chaojing struct net_device *netdev = rxq->netdev; 1054a61abb1SXue Chaojing u32 csum_err; 1064a61abb1SXue Chaojing 1074a61abb1SXue Chaojing csum_err = HINIC_RQ_CQE_STATUS_GET(status, CSUM_ERR); 1084a61abb1SXue Chaojing 1094a61abb1SXue Chaojing if (!(netdev->features & NETIF_F_RXCSUM)) 1104a61abb1SXue Chaojing return; 1114a61abb1SXue Chaojing 112e54fbbdfSXue Chaojing if (!csum_err) { 1134a61abb1SXue Chaojing skb->ip_summed = CHECKSUM_UNNECESSARY; 114e54fbbdfSXue Chaojing } else { 115e54fbbdfSXue Chaojing if (!(csum_err & (HINIC_RX_CSUM_HW_CHECK_NONE | 116e54fbbdfSXue Chaojing HINIC_RX_CSUM_IPSU_OTHER_ERR))) 117e54fbbdfSXue Chaojing rxq->rxq_stats.csum_errors++; 1184a61abb1SXue Chaojing skb->ip_summed = CHECKSUM_NONE; 1194a61abb1SXue Chaojing } 120e54fbbdfSXue Chaojing } 1213402ab54SGuangbin Huang 122c3e79bafSAviad Krawczyk /** 123e2585ea7SAviad Krawczyk * rx_alloc_skb - allocate skb and map it to dma address 124e2585ea7SAviad Krawczyk * @rxq: rx queue 125e2585ea7SAviad Krawczyk * @dma_addr: returned dma address for the skb 126e2585ea7SAviad Krawczyk * 127e2585ea7SAviad Krawczyk * Return skb 128e2585ea7SAviad Krawczyk **/ 129e2585ea7SAviad Krawczyk static struct sk_buff *rx_alloc_skb(struct hinic_rxq *rxq, 130e2585ea7SAviad Krawczyk dma_addr_t *dma_addr) 131e2585ea7SAviad Krawczyk { 132e2585ea7SAviad Krawczyk struct hinic_dev *nic_dev = netdev_priv(rxq->netdev); 133e2585ea7SAviad Krawczyk struct hinic_hwdev *hwdev = nic_dev->hwdev; 134e2585ea7SAviad Krawczyk struct hinic_hwif *hwif = hwdev->hwif; 135e2585ea7SAviad Krawczyk struct pci_dev *pdev = hwif->pdev; 136e2585ea7SAviad Krawczyk struct sk_buff *skb; 137e2585ea7SAviad Krawczyk dma_addr_t addr; 138e2585ea7SAviad Krawczyk int err; 139e2585ea7SAviad Krawczyk 140e2585ea7SAviad Krawczyk skb = netdev_alloc_skb_ip_align(rxq->netdev, rxq->rq->buf_sz); 141c199fdb8SDaode Huang if (!skb) 142e2585ea7SAviad Krawczyk return NULL; 143e2585ea7SAviad Krawczyk 144e2585ea7SAviad Krawczyk addr = dma_map_single(&pdev->dev, skb->data, rxq->rq->buf_sz, 145e2585ea7SAviad Krawczyk DMA_FROM_DEVICE); 146e2585ea7SAviad Krawczyk err = dma_mapping_error(&pdev->dev, addr); 147e2585ea7SAviad Krawczyk if (err) { 148e2585ea7SAviad Krawczyk dev_err(&pdev->dev, "Failed to map Rx DMA, err = %d\n", err); 149e2585ea7SAviad Krawczyk goto err_rx_map; 150e2585ea7SAviad Krawczyk } 151e2585ea7SAviad Krawczyk 152e2585ea7SAviad Krawczyk *dma_addr = addr; 153e2585ea7SAviad Krawczyk return skb; 154e2585ea7SAviad Krawczyk 155e2585ea7SAviad Krawczyk err_rx_map: 156e2585ea7SAviad Krawczyk dev_kfree_skb_any(skb); 157e2585ea7SAviad Krawczyk return NULL; 158e2585ea7SAviad Krawczyk } 159e2585ea7SAviad Krawczyk 160e2585ea7SAviad Krawczyk /** 161e2585ea7SAviad Krawczyk * rx_unmap_skb - unmap the dma address of the skb 162e2585ea7SAviad Krawczyk * @rxq: rx queue 163e2585ea7SAviad Krawczyk * @dma_addr: dma address of the skb 164e2585ea7SAviad Krawczyk **/ 165e2585ea7SAviad Krawczyk static void rx_unmap_skb(struct hinic_rxq *rxq, dma_addr_t dma_addr) 166e2585ea7SAviad Krawczyk { 167e2585ea7SAviad Krawczyk struct hinic_dev *nic_dev = netdev_priv(rxq->netdev); 168e2585ea7SAviad Krawczyk struct hinic_hwdev *hwdev = nic_dev->hwdev; 169e2585ea7SAviad Krawczyk struct hinic_hwif *hwif = hwdev->hwif; 170e2585ea7SAviad Krawczyk struct pci_dev *pdev = hwif->pdev; 171e2585ea7SAviad Krawczyk 172e2585ea7SAviad Krawczyk dma_unmap_single(&pdev->dev, dma_addr, rxq->rq->buf_sz, 173e2585ea7SAviad Krawczyk DMA_FROM_DEVICE); 174e2585ea7SAviad Krawczyk } 175e2585ea7SAviad Krawczyk 176e2585ea7SAviad Krawczyk /** 177e2585ea7SAviad Krawczyk * rx_free_skb - unmap and free skb 178e2585ea7SAviad Krawczyk * @rxq: rx queue 179e2585ea7SAviad Krawczyk * @skb: skb to free 180e2585ea7SAviad Krawczyk * @dma_addr: dma address of the skb 181e2585ea7SAviad Krawczyk **/ 182e2585ea7SAviad Krawczyk static void rx_free_skb(struct hinic_rxq *rxq, struct sk_buff *skb, 183e2585ea7SAviad Krawczyk dma_addr_t dma_addr) 184e2585ea7SAviad Krawczyk { 185e2585ea7SAviad Krawczyk rx_unmap_skb(rxq, dma_addr); 186e2585ea7SAviad Krawczyk dev_kfree_skb_any(skb); 187e2585ea7SAviad Krawczyk } 188e2585ea7SAviad Krawczyk 189e2585ea7SAviad Krawczyk /** 190e2585ea7SAviad Krawczyk * rx_alloc_pkts - allocate pkts in rx queue 191e2585ea7SAviad Krawczyk * @rxq: rx queue 192e2585ea7SAviad Krawczyk * 193e2585ea7SAviad Krawczyk * Return number of skbs allocated 194e2585ea7SAviad Krawczyk **/ 195e2585ea7SAviad Krawczyk static int rx_alloc_pkts(struct hinic_rxq *rxq) 196e2585ea7SAviad Krawczyk { 197e2585ea7SAviad Krawczyk struct hinic_dev *nic_dev = netdev_priv(rxq->netdev); 198e2585ea7SAviad Krawczyk struct hinic_rq_wqe *rq_wqe; 199e2585ea7SAviad Krawczyk unsigned int free_wqebbs; 200e2585ea7SAviad Krawczyk struct hinic_sge sge; 201e2585ea7SAviad Krawczyk dma_addr_t dma_addr; 202e2585ea7SAviad Krawczyk struct sk_buff *skb; 203e2585ea7SAviad Krawczyk u16 prod_idx; 204352f58b0SAviad Krawczyk int i; 205e2585ea7SAviad Krawczyk 206e2585ea7SAviad Krawczyk free_wqebbs = hinic_get_rq_free_wqebbs(rxq->rq); 207e2585ea7SAviad Krawczyk 208e2585ea7SAviad Krawczyk /* Limit the allocation chunks */ 209e2585ea7SAviad Krawczyk if (free_wqebbs > nic_dev->rx_weight) 210e2585ea7SAviad Krawczyk free_wqebbs = nic_dev->rx_weight; 211e2585ea7SAviad Krawczyk 212e2585ea7SAviad Krawczyk for (i = 0; i < free_wqebbs; i++) { 213e2585ea7SAviad Krawczyk skb = rx_alloc_skb(rxq, &dma_addr); 214c199fdb8SDaode Huang if (!skb) 215e2585ea7SAviad Krawczyk goto skb_out; 216e2585ea7SAviad Krawczyk 217e2585ea7SAviad Krawczyk hinic_set_sge(&sge, dma_addr, skb->len); 218e2585ea7SAviad Krawczyk 219e2585ea7SAviad Krawczyk rq_wqe = hinic_rq_get_wqe(rxq->rq, HINIC_RQ_WQE_SIZE, 220e2585ea7SAviad Krawczyk &prod_idx); 221e2585ea7SAviad Krawczyk if (!rq_wqe) { 222e2585ea7SAviad Krawczyk rx_free_skb(rxq, skb, dma_addr); 223e2585ea7SAviad Krawczyk goto skb_out; 224e2585ea7SAviad Krawczyk } 225e2585ea7SAviad Krawczyk 226e2585ea7SAviad Krawczyk hinic_rq_prepare_wqe(rxq->rq, prod_idx, rq_wqe, &sge); 227e2585ea7SAviad Krawczyk 228e2585ea7SAviad Krawczyk hinic_rq_write_wqe(rxq->rq, prod_idx, rq_wqe, skb); 229e2585ea7SAviad Krawczyk } 230e2585ea7SAviad Krawczyk 231e2585ea7SAviad Krawczyk skb_out: 232e2585ea7SAviad Krawczyk if (i) { 233e2585ea7SAviad Krawczyk wmb(); /* write all the wqes before update PI */ 234e2585ea7SAviad Krawczyk 235e2585ea7SAviad Krawczyk hinic_rq_update(rxq->rq, prod_idx); 236e2585ea7SAviad Krawczyk } 237e2585ea7SAviad Krawczyk 238e2585ea7SAviad Krawczyk return i; 239e2585ea7SAviad Krawczyk } 240e2585ea7SAviad Krawczyk 241e2585ea7SAviad Krawczyk /** 242e2585ea7SAviad Krawczyk * free_all_rx_skbs - free all skbs in rx queue 243e2585ea7SAviad Krawczyk * @rxq: rx queue 244e2585ea7SAviad Krawczyk **/ 245e2585ea7SAviad Krawczyk static void free_all_rx_skbs(struct hinic_rxq *rxq) 246e2585ea7SAviad Krawczyk { 247e2585ea7SAviad Krawczyk struct hinic_rq *rq = rxq->rq; 248e2585ea7SAviad Krawczyk struct hinic_hw_wqe *hw_wqe; 249e2585ea7SAviad Krawczyk struct hinic_sge sge; 250e2585ea7SAviad Krawczyk u16 ci; 251e2585ea7SAviad Krawczyk 252e2585ea7SAviad Krawczyk while ((hw_wqe = hinic_read_wqe(rq->wq, HINIC_RQ_WQE_SIZE, &ci))) { 253e2585ea7SAviad Krawczyk if (IS_ERR(hw_wqe)) 254e2585ea7SAviad Krawczyk break; 255e2585ea7SAviad Krawczyk 256e2585ea7SAviad Krawczyk hinic_rq_get_sge(rq, &hw_wqe->rq_wqe, ci, &sge); 257e2585ea7SAviad Krawczyk 258e2585ea7SAviad Krawczyk hinic_put_wqe(rq->wq, HINIC_RQ_WQE_SIZE); 259e2585ea7SAviad Krawczyk 260e2585ea7SAviad Krawczyk rx_free_skb(rxq, rq->saved_skb[ci], hinic_sge_to_dma(&sge)); 261e2585ea7SAviad Krawczyk } 262e2585ea7SAviad Krawczyk } 263e2585ea7SAviad Krawczyk 264e2585ea7SAviad Krawczyk /** 265e2585ea7SAviad Krawczyk * rx_recv_jumbo_pkt - Rx handler for jumbo pkt 266e2585ea7SAviad Krawczyk * @rxq: rx queue 267e2585ea7SAviad Krawczyk * @head_skb: the first skb in the list 268e2585ea7SAviad Krawczyk * @left_pkt_len: left size of the pkt exclude head skb 269e2585ea7SAviad Krawczyk * @ci: consumer index 270e2585ea7SAviad Krawczyk * 271e2585ea7SAviad Krawczyk * Return number of wqes that used for the left of the pkt 272e2585ea7SAviad Krawczyk **/ 273e2585ea7SAviad Krawczyk static int rx_recv_jumbo_pkt(struct hinic_rxq *rxq, struct sk_buff *head_skb, 274e2585ea7SAviad Krawczyk unsigned int left_pkt_len, u16 ci) 275e2585ea7SAviad Krawczyk { 276e2585ea7SAviad Krawczyk struct sk_buff *skb, *curr_skb = head_skb; 277e2585ea7SAviad Krawczyk struct hinic_rq_wqe *rq_wqe; 278e2585ea7SAviad Krawczyk unsigned int curr_len; 279e2585ea7SAviad Krawczyk struct hinic_sge sge; 280e2585ea7SAviad Krawczyk int num_wqes = 0; 281e2585ea7SAviad Krawczyk 282e2585ea7SAviad Krawczyk while (left_pkt_len > 0) { 283e2585ea7SAviad Krawczyk rq_wqe = hinic_rq_read_next_wqe(rxq->rq, HINIC_RQ_WQE_SIZE, 284e2585ea7SAviad Krawczyk &skb, &ci); 285e2585ea7SAviad Krawczyk 286e2585ea7SAviad Krawczyk num_wqes++; 287e2585ea7SAviad Krawczyk 288e2585ea7SAviad Krawczyk hinic_rq_get_sge(rxq->rq, rq_wqe, ci, &sge); 289e2585ea7SAviad Krawczyk 290e2585ea7SAviad Krawczyk rx_unmap_skb(rxq, hinic_sge_to_dma(&sge)); 291e2585ea7SAviad Krawczyk 292e2585ea7SAviad Krawczyk prefetch(skb->data); 293e2585ea7SAviad Krawczyk 294e2585ea7SAviad Krawczyk curr_len = (left_pkt_len > HINIC_RX_BUF_SZ) ? HINIC_RX_BUF_SZ : 295e2585ea7SAviad Krawczyk left_pkt_len; 296e2585ea7SAviad Krawczyk 297e2585ea7SAviad Krawczyk left_pkt_len -= curr_len; 298e2585ea7SAviad Krawczyk 299e2585ea7SAviad Krawczyk __skb_put(skb, curr_len); 300e2585ea7SAviad Krawczyk 301e2585ea7SAviad Krawczyk if (curr_skb == head_skb) 302e2585ea7SAviad Krawczyk skb_shinfo(head_skb)->frag_list = skb; 303e2585ea7SAviad Krawczyk else 304e2585ea7SAviad Krawczyk curr_skb->next = skb; 305e2585ea7SAviad Krawczyk 306e2585ea7SAviad Krawczyk head_skb->len += skb->len; 307e2585ea7SAviad Krawczyk head_skb->data_len += skb->len; 308e2585ea7SAviad Krawczyk head_skb->truesize += skb->truesize; 309e2585ea7SAviad Krawczyk 310e2585ea7SAviad Krawczyk curr_skb = skb; 311e2585ea7SAviad Krawczyk } 312e2585ea7SAviad Krawczyk 313e2585ea7SAviad Krawczyk return num_wqes; 314e2585ea7SAviad Krawczyk } 315e2585ea7SAviad Krawczyk 3164aa218a4SLuo bin static void hinic_copy_lp_data(struct hinic_dev *nic_dev, 3174aa218a4SLuo bin struct sk_buff *skb) 3184aa218a4SLuo bin { 3194aa218a4SLuo bin struct net_device *netdev = nic_dev->netdev; 3204aa218a4SLuo bin u8 *lb_buf = nic_dev->lb_test_rx_buf; 3214aa218a4SLuo bin int lb_len = nic_dev->lb_pkt_len; 3224aa218a4SLuo bin int pkt_offset, frag_len, i; 3234aa218a4SLuo bin void *frag_data = NULL; 3244aa218a4SLuo bin 3254aa218a4SLuo bin if (nic_dev->lb_test_rx_idx == LP_PKT_CNT) { 3264aa218a4SLuo bin nic_dev->lb_test_rx_idx = 0; 3274aa218a4SLuo bin netif_warn(nic_dev, drv, netdev, "Loopback test warning, receive too more test pkts\n"); 3284aa218a4SLuo bin } 3294aa218a4SLuo bin 3304aa218a4SLuo bin if (skb->len != nic_dev->lb_pkt_len) { 3314aa218a4SLuo bin netif_warn(nic_dev, drv, netdev, "Wrong packet length\n"); 3324aa218a4SLuo bin nic_dev->lb_test_rx_idx++; 3334aa218a4SLuo bin return; 3344aa218a4SLuo bin } 3354aa218a4SLuo bin 3364aa218a4SLuo bin pkt_offset = nic_dev->lb_test_rx_idx * lb_len; 3374aa218a4SLuo bin frag_len = (int)skb_headlen(skb); 3384aa218a4SLuo bin memcpy(lb_buf + pkt_offset, skb->data, frag_len); 3394aa218a4SLuo bin pkt_offset += frag_len; 3404aa218a4SLuo bin for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 3414aa218a4SLuo bin frag_data = skb_frag_address(&skb_shinfo(skb)->frags[i]); 3424aa218a4SLuo bin frag_len = (int)skb_frag_size(&skb_shinfo(skb)->frags[i]); 3434aa218a4SLuo bin memcpy((lb_buf + pkt_offset), frag_data, frag_len); 3444aa218a4SLuo bin pkt_offset += frag_len; 3454aa218a4SLuo bin } 3464aa218a4SLuo bin nic_dev->lb_test_rx_idx++; 3474aa218a4SLuo bin } 3484aa218a4SLuo bin 349e2585ea7SAviad Krawczyk /** 350e2585ea7SAviad Krawczyk * rxq_recv - Rx handler 351e2585ea7SAviad Krawczyk * @rxq: rx queue 352e2585ea7SAviad Krawczyk * @budget: maximum pkts to process 353e2585ea7SAviad Krawczyk * 354e2585ea7SAviad Krawczyk * Return number of pkts received 355e2585ea7SAviad Krawczyk **/ 356e2585ea7SAviad Krawczyk static int rxq_recv(struct hinic_rxq *rxq, int budget) 357e2585ea7SAviad Krawczyk { 358e2585ea7SAviad Krawczyk struct hinic_qp *qp = container_of(rxq->rq, struct hinic_qp, rq); 359aebd17b7SXue Chaojing struct net_device *netdev = rxq->netdev; 360e2585ea7SAviad Krawczyk u64 pkt_len = 0, rx_bytes = 0; 3611e007181SXue Chaojing struct hinic_rq *rq = rxq->rq; 362e2585ea7SAviad Krawczyk struct hinic_rq_wqe *rq_wqe; 3634aa218a4SLuo bin struct hinic_dev *nic_dev; 364e1a76515SXue Chaojing unsigned int free_wqebbs; 3651e007181SXue Chaojing struct hinic_rq_cqe *cqe; 366e2585ea7SAviad Krawczyk int num_wqes, pkts = 0; 367e2585ea7SAviad Krawczyk struct hinic_sge sge; 3681e007181SXue Chaojing unsigned int status; 369e2585ea7SAviad Krawczyk struct sk_buff *skb; 370aebd17b7SXue Chaojing u32 offload_type; 3711e007181SXue Chaojing u16 ci, num_lro; 3721e007181SXue Chaojing u16 num_wqe = 0; 373aebd17b7SXue Chaojing u32 vlan_len; 374aebd17b7SXue Chaojing u16 vid; 375e2585ea7SAviad Krawczyk 3764aa218a4SLuo bin nic_dev = netdev_priv(netdev); 3774aa218a4SLuo bin 378e2585ea7SAviad Krawczyk while (pkts < budget) { 379e2585ea7SAviad Krawczyk num_wqes = 0; 380e2585ea7SAviad Krawczyk 381e2585ea7SAviad Krawczyk rq_wqe = hinic_rq_read_wqe(rxq->rq, HINIC_RQ_WQE_SIZE, &skb, 382e2585ea7SAviad Krawczyk &ci); 383e2585ea7SAviad Krawczyk if (!rq_wqe) 384e2585ea7SAviad Krawczyk break; 385e2585ea7SAviad Krawczyk 38633f15da2SLuo bin /* make sure we read rx_done before packet length */ 38733f15da2SLuo bin dma_rmb(); 38833f15da2SLuo bin 3891e007181SXue Chaojing cqe = rq->cqe[ci]; 3901e007181SXue Chaojing status = be32_to_cpu(cqe->status); 391e2585ea7SAviad Krawczyk hinic_rq_get_sge(rxq->rq, rq_wqe, ci, &sge); 392e2585ea7SAviad Krawczyk 393e2585ea7SAviad Krawczyk rx_unmap_skb(rxq, hinic_sge_to_dma(&sge)); 394e2585ea7SAviad Krawczyk 3951e007181SXue Chaojing rx_csum(rxq, status, skb); 3964a61abb1SXue Chaojing 397e2585ea7SAviad Krawczyk prefetch(skb->data); 398e2585ea7SAviad Krawczyk 399e2585ea7SAviad Krawczyk pkt_len = sge.len; 400e2585ea7SAviad Krawczyk 401e2585ea7SAviad Krawczyk if (pkt_len <= HINIC_RX_BUF_SZ) { 402e2585ea7SAviad Krawczyk __skb_put(skb, pkt_len); 403e2585ea7SAviad Krawczyk } else { 404e2585ea7SAviad Krawczyk __skb_put(skb, HINIC_RX_BUF_SZ); 405e2585ea7SAviad Krawczyk num_wqes = rx_recv_jumbo_pkt(rxq, skb, pkt_len - 406e2585ea7SAviad Krawczyk HINIC_RX_BUF_SZ, ci); 407e2585ea7SAviad Krawczyk } 408e2585ea7SAviad Krawczyk 4091e007181SXue Chaojing hinic_rq_put_wqe(rq, ci, 410e2585ea7SAviad Krawczyk (num_wqes + 1) * HINIC_RQ_WQE_SIZE); 411e2585ea7SAviad Krawczyk 412aebd17b7SXue Chaojing offload_type = be32_to_cpu(cqe->offload_type); 413aebd17b7SXue Chaojing vlan_len = be32_to_cpu(cqe->len); 414aebd17b7SXue Chaojing if ((netdev->features & NETIF_F_HW_VLAN_CTAG_RX) && 415aebd17b7SXue Chaojing HINIC_GET_RX_VLAN_OFFLOAD_EN(offload_type)) { 416aebd17b7SXue Chaojing vid = HINIC_GET_RX_VLAN_TAG(vlan_len); 417aebd17b7SXue Chaojing __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid); 418aebd17b7SXue Chaojing } 419aebd17b7SXue Chaojing 4204aa218a4SLuo bin if (unlikely(nic_dev->flags & HINIC_LP_TEST)) 4214aa218a4SLuo bin hinic_copy_lp_data(nic_dev, skb); 4224aa218a4SLuo bin 423e2585ea7SAviad Krawczyk skb_record_rx_queue(skb, qp->q_id); 424e2585ea7SAviad Krawczyk skb->protocol = eth_type_trans(skb, rxq->netdev); 425e2585ea7SAviad Krawczyk 426e2585ea7SAviad Krawczyk napi_gro_receive(&rxq->napi, skb); 427e2585ea7SAviad Krawczyk 428e2585ea7SAviad Krawczyk pkts++; 429e2585ea7SAviad Krawczyk rx_bytes += pkt_len; 4301e007181SXue Chaojing 4311e007181SXue Chaojing num_lro = HINIC_GET_RX_NUM_LRO(status); 4321e007181SXue Chaojing if (num_lro) { 4331e007181SXue Chaojing rx_bytes += ((num_lro - 1) * 4341e007181SXue Chaojing LRO_PKT_HDR_LEN(cqe)); 4351e007181SXue Chaojing 4361e007181SXue Chaojing num_wqe += 4371e007181SXue Chaojing (u16)(pkt_len >> rxq->rx_buff_shift) + 4381e007181SXue Chaojing ((pkt_len & (rxq->buf_len - 1)) ? 1 : 0); 4391e007181SXue Chaojing } 4401e007181SXue Chaojing 4411e007181SXue Chaojing cqe->status = 0; 4421e007181SXue Chaojing 4431e007181SXue Chaojing if (num_wqe >= LRO_REPLENISH_THLD) 4441e007181SXue Chaojing break; 445e2585ea7SAviad Krawczyk } 446e2585ea7SAviad Krawczyk 447e1a76515SXue Chaojing free_wqebbs = hinic_get_rq_free_wqebbs(rxq->rq); 448e1a76515SXue Chaojing if (free_wqebbs > HINIC_RX_BUFFER_WRITE) 449e1a76515SXue Chaojing rx_alloc_pkts(rxq); 450e2585ea7SAviad Krawczyk 451e2585ea7SAviad Krawczyk u64_stats_update_begin(&rxq->rxq_stats.syncp); 452e2585ea7SAviad Krawczyk rxq->rxq_stats.pkts += pkts; 453e2585ea7SAviad Krawczyk rxq->rxq_stats.bytes += rx_bytes; 454e2585ea7SAviad Krawczyk u64_stats_update_end(&rxq->rxq_stats.syncp); 455e2585ea7SAviad Krawczyk 456e2585ea7SAviad Krawczyk return pkts; 457e2585ea7SAviad Krawczyk } 458e2585ea7SAviad Krawczyk 459e2585ea7SAviad Krawczyk static int rx_poll(struct napi_struct *napi, int budget) 460e2585ea7SAviad Krawczyk { 461e2585ea7SAviad Krawczyk struct hinic_rxq *rxq = container_of(napi, struct hinic_rxq, napi); 462905b464aSXue Chaojing struct hinic_dev *nic_dev = netdev_priv(rxq->netdev); 463e2585ea7SAviad Krawczyk struct hinic_rq *rq = rxq->rq; 464e2585ea7SAviad Krawczyk int pkts; 465e2585ea7SAviad Krawczyk 466e2585ea7SAviad Krawczyk pkts = rxq_recv(rxq, budget); 467e2585ea7SAviad Krawczyk if (pkts >= budget) 468e2585ea7SAviad Krawczyk return budget; 469e2585ea7SAviad Krawczyk 470e2585ea7SAviad Krawczyk napi_complete(napi); 4717dd29ee1SLuo bin 4727dd29ee1SLuo bin if (!HINIC_IS_VF(nic_dev->hwdev->hwif)) 473905b464aSXue Chaojing hinic_hwdev_set_msix_state(nic_dev->hwdev, 474905b464aSXue Chaojing rq->msix_entry, 475905b464aSXue Chaojing HINIC_MSIX_ENABLE); 476905b464aSXue Chaojing 477e2585ea7SAviad Krawczyk return pkts; 478e2585ea7SAviad Krawczyk } 479e2585ea7SAviad Krawczyk 480e2585ea7SAviad Krawczyk static void rx_add_napi(struct hinic_rxq *rxq) 481e2585ea7SAviad Krawczyk { 482e2585ea7SAviad Krawczyk struct hinic_dev *nic_dev = netdev_priv(rxq->netdev); 483e2585ea7SAviad Krawczyk 484e2585ea7SAviad Krawczyk netif_napi_add(rxq->netdev, &rxq->napi, rx_poll, nic_dev->rx_weight); 485e2585ea7SAviad Krawczyk napi_enable(&rxq->napi); 486e2585ea7SAviad Krawczyk } 487e2585ea7SAviad Krawczyk 488e2585ea7SAviad Krawczyk static void rx_del_napi(struct hinic_rxq *rxq) 489e2585ea7SAviad Krawczyk { 490e2585ea7SAviad Krawczyk napi_disable(&rxq->napi); 491e2585ea7SAviad Krawczyk netif_napi_del(&rxq->napi); 492e2585ea7SAviad Krawczyk } 493e2585ea7SAviad Krawczyk 494e2585ea7SAviad Krawczyk static irqreturn_t rx_irq(int irq, void *data) 495e2585ea7SAviad Krawczyk { 496e2585ea7SAviad Krawczyk struct hinic_rxq *rxq = (struct hinic_rxq *)data; 497e2585ea7SAviad Krawczyk struct hinic_rq *rq = rxq->rq; 498e2585ea7SAviad Krawczyk struct hinic_dev *nic_dev; 499e2585ea7SAviad Krawczyk 500e2585ea7SAviad Krawczyk /* Disable the interrupt until napi will be completed */ 501905b464aSXue Chaojing nic_dev = netdev_priv(rxq->netdev); 5027dd29ee1SLuo bin if (!HINIC_IS_VF(nic_dev->hwdev->hwif)) 503905b464aSXue Chaojing hinic_hwdev_set_msix_state(nic_dev->hwdev, 504905b464aSXue Chaojing rq->msix_entry, 505905b464aSXue Chaojing HINIC_MSIX_DISABLE); 506e2585ea7SAviad Krawczyk 507e2585ea7SAviad Krawczyk nic_dev = netdev_priv(rxq->netdev); 508e2585ea7SAviad Krawczyk hinic_hwdev_msix_cnt_set(nic_dev->hwdev, rq->msix_entry); 509e2585ea7SAviad Krawczyk 510e2585ea7SAviad Krawczyk napi_schedule(&rxq->napi); 511e2585ea7SAviad Krawczyk return IRQ_HANDLED; 512e2585ea7SAviad Krawczyk } 513e2585ea7SAviad Krawczyk 514e2585ea7SAviad Krawczyk static int rx_request_irq(struct hinic_rxq *rxq) 515e2585ea7SAviad Krawczyk { 516e2585ea7SAviad Krawczyk struct hinic_dev *nic_dev = netdev_priv(rxq->netdev); 517a0337c0dSLuo bin struct hinic_msix_config interrupt_info = {0}; 518a0337c0dSLuo bin struct hinic_intr_coal_info *intr_coal = NULL; 519e2585ea7SAviad Krawczyk struct hinic_hwdev *hwdev = nic_dev->hwdev; 520e2585ea7SAviad Krawczyk struct hinic_rq *rq = rxq->rq; 521352f58b0SAviad Krawczyk struct hinic_qp *qp; 522e2585ea7SAviad Krawczyk int err; 523e2585ea7SAviad Krawczyk 524a0337c0dSLuo bin qp = container_of(rq, struct hinic_qp, rq); 525a0337c0dSLuo bin 526e2585ea7SAviad Krawczyk rx_add_napi(rxq); 527e2585ea7SAviad Krawczyk 528e2585ea7SAviad Krawczyk hinic_hwdev_msix_set(hwdev, rq->msix_entry, 529e2585ea7SAviad Krawczyk RX_IRQ_NO_PENDING, RX_IRQ_NO_COALESC, 530e2585ea7SAviad Krawczyk RX_IRQ_NO_LLI_TIMER, RX_IRQ_NO_CREDIT, 531e2585ea7SAviad Krawczyk RX_IRQ_NO_RESEND_TIMER); 532e2585ea7SAviad Krawczyk 533a0337c0dSLuo bin intr_coal = &nic_dev->rx_intr_coalesce[qp->q_id]; 534a0337c0dSLuo bin interrupt_info.msix_index = rq->msix_entry; 535a0337c0dSLuo bin interrupt_info.coalesce_timer_cnt = intr_coal->coalesce_timer_cfg; 536a0337c0dSLuo bin interrupt_info.pending_cnt = intr_coal->pending_limt; 537a0337c0dSLuo bin interrupt_info.resend_timer_cnt = intr_coal->resend_timer_cfg; 538a0337c0dSLuo bin 539a0337c0dSLuo bin err = hinic_set_interrupt_cfg(hwdev, &interrupt_info); 540a0337c0dSLuo bin if (err) { 541a0337c0dSLuo bin netif_err(nic_dev, drv, rxq->netdev, 542a0337c0dSLuo bin "Failed to set RX interrupt coalescing attribute\n"); 543ce000c61SWei Li goto err_req_irq; 544a0337c0dSLuo bin } 545a0337c0dSLuo bin 546e2585ea7SAviad Krawczyk err = request_irq(rq->irq, rx_irq, 0, rxq->irq_name, rxq); 547ce000c61SWei Li if (err) 548ce000c61SWei Li goto err_req_irq; 549e2585ea7SAviad Krawczyk 5500bff777bSLuo bin cpumask_set_cpu(qp->q_id % num_online_cpus(), &rq->affinity_mask); 551*2d1e72f2SNitesh Narayan Lal err = irq_set_affinity_and_hint(rq->irq, &rq->affinity_mask); 552ce000c61SWei Li if (err) 553ce000c61SWei Li goto err_irq_affinity; 554ce000c61SWei Li 555ce000c61SWei Li return 0; 556ce000c61SWei Li 557ce000c61SWei Li err_irq_affinity: 558ce000c61SWei Li free_irq(rq->irq, rxq); 559ce000c61SWei Li err_req_irq: 560ce000c61SWei Li rx_del_napi(rxq); 561ce000c61SWei Li return err; 562e2585ea7SAviad Krawczyk } 563e2585ea7SAviad Krawczyk 564e2585ea7SAviad Krawczyk static void rx_free_irq(struct hinic_rxq *rxq) 565e2585ea7SAviad Krawczyk { 566e2585ea7SAviad Krawczyk struct hinic_rq *rq = rxq->rq; 567e2585ea7SAviad Krawczyk 568*2d1e72f2SNitesh Narayan Lal irq_update_affinity_hint(rq->irq, NULL); 569e2585ea7SAviad Krawczyk free_irq(rq->irq, rxq); 570e2585ea7SAviad Krawczyk rx_del_napi(rxq); 571e2585ea7SAviad Krawczyk } 572e2585ea7SAviad Krawczyk 573e2585ea7SAviad Krawczyk /** 574c3e79bafSAviad Krawczyk * hinic_init_rxq - Initialize the Rx Queue 575c3e79bafSAviad Krawczyk * @rxq: Logical Rx Queue 576c3e79bafSAviad Krawczyk * @rq: Hardware Rx Queue to connect the Logical queue with 577c3e79bafSAviad Krawczyk * @netdev: network device to connect the Logical queue with 578c3e79bafSAviad Krawczyk * 579c3e79bafSAviad Krawczyk * Return 0 - Success, negative - Failure 580c3e79bafSAviad Krawczyk **/ 581c3e79bafSAviad Krawczyk int hinic_init_rxq(struct hinic_rxq *rxq, struct hinic_rq *rq, 582c3e79bafSAviad Krawczyk struct net_device *netdev) 583c3e79bafSAviad Krawczyk { 584e2585ea7SAviad Krawczyk struct hinic_qp *qp = container_of(rq, struct hinic_qp, rq); 585930cfe0fSChristophe JAILLET int err, pkts; 586e2585ea7SAviad Krawczyk 587c3e79bafSAviad Krawczyk rxq->netdev = netdev; 588c3e79bafSAviad Krawczyk rxq->rq = rq; 5891e007181SXue Chaojing rxq->buf_len = HINIC_RX_BUF_SZ; 5901e007181SXue Chaojing rxq->rx_buff_shift = ilog2(HINIC_RX_BUF_SZ); 591c3e79bafSAviad Krawczyk 592c3e79bafSAviad Krawczyk rxq_stats_init(rxq); 593e2585ea7SAviad Krawczyk 594930cfe0fSChristophe JAILLET rxq->irq_name = devm_kasprintf(&netdev->dev, GFP_KERNEL, 595a9fd686aSLuo bin "%s_rxq%d", netdev->name, qp->q_id); 596e2585ea7SAviad Krawczyk if (!rxq->irq_name) 597e2585ea7SAviad Krawczyk return -ENOMEM; 598e2585ea7SAviad Krawczyk 599e2585ea7SAviad Krawczyk pkts = rx_alloc_pkts(rxq); 600e2585ea7SAviad Krawczyk if (!pkts) { 601e2585ea7SAviad Krawczyk err = -ENOMEM; 602e2585ea7SAviad Krawczyk goto err_rx_pkts; 603e2585ea7SAviad Krawczyk } 604e2585ea7SAviad Krawczyk 605e2585ea7SAviad Krawczyk err = rx_request_irq(rxq); 606e2585ea7SAviad Krawczyk if (err) { 607e2585ea7SAviad Krawczyk netdev_err(netdev, "Failed to request Rx irq\n"); 608e2585ea7SAviad Krawczyk goto err_req_rx_irq; 609e2585ea7SAviad Krawczyk } 610e2585ea7SAviad Krawczyk 611c3e79bafSAviad Krawczyk return 0; 612e2585ea7SAviad Krawczyk 613e2585ea7SAviad Krawczyk err_req_rx_irq: 614e2585ea7SAviad Krawczyk err_rx_pkts: 615e2585ea7SAviad Krawczyk free_all_rx_skbs(rxq); 616e2585ea7SAviad Krawczyk devm_kfree(&netdev->dev, rxq->irq_name); 617e2585ea7SAviad Krawczyk return err; 618c3e79bafSAviad Krawczyk } 619c3e79bafSAviad Krawczyk 620c3e79bafSAviad Krawczyk /** 621c3e79bafSAviad Krawczyk * hinic_clean_rxq - Clean the Rx Queue 622c3e79bafSAviad Krawczyk * @rxq: Logical Rx Queue 623c3e79bafSAviad Krawczyk **/ 624c3e79bafSAviad Krawczyk void hinic_clean_rxq(struct hinic_rxq *rxq) 625c3e79bafSAviad Krawczyk { 626e2585ea7SAviad Krawczyk struct net_device *netdev = rxq->netdev; 627e2585ea7SAviad Krawczyk 628e2585ea7SAviad Krawczyk rx_free_irq(rxq); 629e2585ea7SAviad Krawczyk 630e2585ea7SAviad Krawczyk free_all_rx_skbs(rxq); 631e2585ea7SAviad Krawczyk devm_kfree(&netdev->dev, rxq->irq_name); 632c3e79bafSAviad Krawczyk } 633