1c3e79bafSAviad Krawczyk /* 2c3e79bafSAviad Krawczyk * Huawei HiNIC PCI Express Linux driver 3c3e79bafSAviad Krawczyk * Copyright(c) 2017 Huawei Technologies Co., Ltd 4c3e79bafSAviad Krawczyk * 5c3e79bafSAviad Krawczyk * This program is free software; you can redistribute it and/or modify it 6c3e79bafSAviad Krawczyk * under the terms and conditions of the GNU General Public License, 7c3e79bafSAviad Krawczyk * version 2, as published by the Free Software Foundation. 8c3e79bafSAviad Krawczyk * 9c3e79bafSAviad Krawczyk * This program is distributed in the hope it will be useful, but WITHOUT 10c3e79bafSAviad Krawczyk * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11c3e79bafSAviad Krawczyk * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12c3e79bafSAviad Krawczyk * for more details. 13c3e79bafSAviad Krawczyk * 14c3e79bafSAviad Krawczyk */ 15c3e79bafSAviad Krawczyk 16e2585ea7SAviad Krawczyk #include <linux/kernel.h> 17e2585ea7SAviad Krawczyk #include <linux/types.h> 18e2585ea7SAviad Krawczyk #include <linux/errno.h> 19e2585ea7SAviad Krawczyk #include <linux/pci.h> 20e2585ea7SAviad Krawczyk #include <linux/device.h> 21c3e79bafSAviad Krawczyk #include <linux/netdevice.h> 22e2585ea7SAviad Krawczyk #include <linux/etherdevice.h> 23c3e79bafSAviad Krawczyk #include <linux/u64_stats_sync.h> 24e2585ea7SAviad Krawczyk #include <linux/slab.h> 25e2585ea7SAviad Krawczyk #include <linux/interrupt.h> 26e2585ea7SAviad Krawczyk #include <linux/skbuff.h> 27e2585ea7SAviad Krawczyk #include <linux/dma-mapping.h> 28e2585ea7SAviad Krawczyk #include <linux/prefetch.h> 29352f58b0SAviad Krawczyk #include <linux/cpumask.h> 30e2585ea7SAviad Krawczyk #include <asm/barrier.h> 31c3e79bafSAviad Krawczyk 32e2585ea7SAviad Krawczyk #include "hinic_common.h" 33e2585ea7SAviad Krawczyk #include "hinic_hw_if.h" 34e2585ea7SAviad Krawczyk #include "hinic_hw_wqe.h" 35e2585ea7SAviad Krawczyk #include "hinic_hw_wq.h" 36c3e79bafSAviad Krawczyk #include "hinic_hw_qp.h" 37e2585ea7SAviad Krawczyk #include "hinic_hw_dev.h" 38c3e79bafSAviad Krawczyk #include "hinic_rx.h" 39e2585ea7SAviad Krawczyk #include "hinic_dev.h" 40e2585ea7SAviad Krawczyk 41e2585ea7SAviad Krawczyk #define RX_IRQ_NO_PENDING 0 42e2585ea7SAviad Krawczyk #define RX_IRQ_NO_COALESC 0 43e2585ea7SAviad Krawczyk #define RX_IRQ_NO_LLI_TIMER 0 44e2585ea7SAviad Krawczyk #define RX_IRQ_NO_CREDIT 0 45e2585ea7SAviad Krawczyk #define RX_IRQ_NO_RESEND_TIMER 0 46e1a76515SXue Chaojing #define HINIC_RX_BUFFER_WRITE 16 47c3e79bafSAviad Krawczyk 48c3e79bafSAviad Krawczyk /** 49c3e79bafSAviad Krawczyk * hinic_rxq_clean_stats - Clean the statistics of specific queue 50c3e79bafSAviad Krawczyk * @rxq: Logical Rx Queue 51c3e79bafSAviad Krawczyk **/ 52c3e79bafSAviad Krawczyk void hinic_rxq_clean_stats(struct hinic_rxq *rxq) 53c3e79bafSAviad Krawczyk { 54c3e79bafSAviad Krawczyk struct hinic_rxq_stats *rxq_stats = &rxq->rxq_stats; 55c3e79bafSAviad Krawczyk 56c3e79bafSAviad Krawczyk u64_stats_update_begin(&rxq_stats->syncp); 57c3e79bafSAviad Krawczyk rxq_stats->pkts = 0; 58c3e79bafSAviad Krawczyk rxq_stats->bytes = 0; 59c3e79bafSAviad Krawczyk u64_stats_update_end(&rxq_stats->syncp); 60c3e79bafSAviad Krawczyk } 61c3e79bafSAviad Krawczyk 62c3e79bafSAviad Krawczyk /** 63edd384f6SAviad Krawczyk * hinic_rxq_get_stats - get statistics of Rx Queue 64edd384f6SAviad Krawczyk * @rxq: Logical Rx Queue 65edd384f6SAviad Krawczyk * @stats: return updated stats here 66edd384f6SAviad Krawczyk **/ 67edd384f6SAviad Krawczyk void hinic_rxq_get_stats(struct hinic_rxq *rxq, struct hinic_rxq_stats *stats) 68edd384f6SAviad Krawczyk { 69edd384f6SAviad Krawczyk struct hinic_rxq_stats *rxq_stats = &rxq->rxq_stats; 70edd384f6SAviad Krawczyk unsigned int start; 71edd384f6SAviad Krawczyk 72edd384f6SAviad Krawczyk u64_stats_update_begin(&stats->syncp); 73edd384f6SAviad Krawczyk do { 74edd384f6SAviad Krawczyk start = u64_stats_fetch_begin(&rxq_stats->syncp); 75edd384f6SAviad Krawczyk stats->pkts = rxq_stats->pkts; 76edd384f6SAviad Krawczyk stats->bytes = rxq_stats->bytes; 77edd384f6SAviad Krawczyk } while (u64_stats_fetch_retry(&rxq_stats->syncp, start)); 78edd384f6SAviad Krawczyk u64_stats_update_end(&stats->syncp); 79edd384f6SAviad Krawczyk } 80edd384f6SAviad Krawczyk 81edd384f6SAviad Krawczyk /** 82c3e79bafSAviad Krawczyk * rxq_stats_init - Initialize the statistics of specific queue 83c3e79bafSAviad Krawczyk * @rxq: Logical Rx Queue 84c3e79bafSAviad Krawczyk **/ 85c3e79bafSAviad Krawczyk static void rxq_stats_init(struct hinic_rxq *rxq) 86c3e79bafSAviad Krawczyk { 87c3e79bafSAviad Krawczyk struct hinic_rxq_stats *rxq_stats = &rxq->rxq_stats; 88c3e79bafSAviad Krawczyk 89c3e79bafSAviad Krawczyk u64_stats_init(&rxq_stats->syncp); 90c3e79bafSAviad Krawczyk hinic_rxq_clean_stats(rxq); 91c3e79bafSAviad Krawczyk } 92c3e79bafSAviad Krawczyk 934a61abb1SXue Chaojing static void rx_csum(struct hinic_rxq *rxq, u16 cons_idx, 944a61abb1SXue Chaojing struct sk_buff *skb) 954a61abb1SXue Chaojing { 964a61abb1SXue Chaojing struct net_device *netdev = rxq->netdev; 974a61abb1SXue Chaojing struct hinic_rq_cqe *cqe; 984a61abb1SXue Chaojing struct hinic_rq *rq; 994a61abb1SXue Chaojing u32 csum_err; 1004a61abb1SXue Chaojing u32 status; 1014a61abb1SXue Chaojing 1024a61abb1SXue Chaojing rq = rxq->rq; 1034a61abb1SXue Chaojing cqe = rq->cqe[cons_idx]; 1044a61abb1SXue Chaojing status = be32_to_cpu(cqe->status); 1054a61abb1SXue Chaojing csum_err = HINIC_RQ_CQE_STATUS_GET(status, CSUM_ERR); 1064a61abb1SXue Chaojing 1074a61abb1SXue Chaojing if (!(netdev->features & NETIF_F_RXCSUM)) 1084a61abb1SXue Chaojing return; 1094a61abb1SXue Chaojing 1104a61abb1SXue Chaojing if (!csum_err) 1114a61abb1SXue Chaojing skb->ip_summed = CHECKSUM_UNNECESSARY; 1124a61abb1SXue Chaojing else 1134a61abb1SXue Chaojing skb->ip_summed = CHECKSUM_NONE; 1144a61abb1SXue Chaojing } 115c3e79bafSAviad Krawczyk /** 116e2585ea7SAviad Krawczyk * rx_alloc_skb - allocate skb and map it to dma address 117e2585ea7SAviad Krawczyk * @rxq: rx queue 118e2585ea7SAviad Krawczyk * @dma_addr: returned dma address for the skb 119e2585ea7SAviad Krawczyk * 120e2585ea7SAviad Krawczyk * Return skb 121e2585ea7SAviad Krawczyk **/ 122e2585ea7SAviad Krawczyk static struct sk_buff *rx_alloc_skb(struct hinic_rxq *rxq, 123e2585ea7SAviad Krawczyk dma_addr_t *dma_addr) 124e2585ea7SAviad Krawczyk { 125e2585ea7SAviad Krawczyk struct hinic_dev *nic_dev = netdev_priv(rxq->netdev); 126e2585ea7SAviad Krawczyk struct hinic_hwdev *hwdev = nic_dev->hwdev; 127e2585ea7SAviad Krawczyk struct hinic_hwif *hwif = hwdev->hwif; 128e2585ea7SAviad Krawczyk struct pci_dev *pdev = hwif->pdev; 129e2585ea7SAviad Krawczyk struct sk_buff *skb; 130e2585ea7SAviad Krawczyk dma_addr_t addr; 131e2585ea7SAviad Krawczyk int err; 132e2585ea7SAviad Krawczyk 133e2585ea7SAviad Krawczyk skb = netdev_alloc_skb_ip_align(rxq->netdev, rxq->rq->buf_sz); 134e2585ea7SAviad Krawczyk if (!skb) { 135e2585ea7SAviad Krawczyk netdev_err(rxq->netdev, "Failed to allocate Rx SKB\n"); 136e2585ea7SAviad Krawczyk return NULL; 137e2585ea7SAviad Krawczyk } 138e2585ea7SAviad Krawczyk 139e2585ea7SAviad Krawczyk addr = dma_map_single(&pdev->dev, skb->data, rxq->rq->buf_sz, 140e2585ea7SAviad Krawczyk DMA_FROM_DEVICE); 141e2585ea7SAviad Krawczyk err = dma_mapping_error(&pdev->dev, addr); 142e2585ea7SAviad Krawczyk if (err) { 143e2585ea7SAviad Krawczyk dev_err(&pdev->dev, "Failed to map Rx DMA, err = %d\n", err); 144e2585ea7SAviad Krawczyk goto err_rx_map; 145e2585ea7SAviad Krawczyk } 146e2585ea7SAviad Krawczyk 147e2585ea7SAviad Krawczyk *dma_addr = addr; 148e2585ea7SAviad Krawczyk return skb; 149e2585ea7SAviad Krawczyk 150e2585ea7SAviad Krawczyk err_rx_map: 151e2585ea7SAviad Krawczyk dev_kfree_skb_any(skb); 152e2585ea7SAviad Krawczyk return NULL; 153e2585ea7SAviad Krawczyk } 154e2585ea7SAviad Krawczyk 155e2585ea7SAviad Krawczyk /** 156e2585ea7SAviad Krawczyk * rx_unmap_skb - unmap the dma address of the skb 157e2585ea7SAviad Krawczyk * @rxq: rx queue 158e2585ea7SAviad Krawczyk * @dma_addr: dma address of the skb 159e2585ea7SAviad Krawczyk **/ 160e2585ea7SAviad Krawczyk static void rx_unmap_skb(struct hinic_rxq *rxq, dma_addr_t dma_addr) 161e2585ea7SAviad Krawczyk { 162e2585ea7SAviad Krawczyk struct hinic_dev *nic_dev = netdev_priv(rxq->netdev); 163e2585ea7SAviad Krawczyk struct hinic_hwdev *hwdev = nic_dev->hwdev; 164e2585ea7SAviad Krawczyk struct hinic_hwif *hwif = hwdev->hwif; 165e2585ea7SAviad Krawczyk struct pci_dev *pdev = hwif->pdev; 166e2585ea7SAviad Krawczyk 167e2585ea7SAviad Krawczyk dma_unmap_single(&pdev->dev, dma_addr, rxq->rq->buf_sz, 168e2585ea7SAviad Krawczyk DMA_FROM_DEVICE); 169e2585ea7SAviad Krawczyk } 170e2585ea7SAviad Krawczyk 171e2585ea7SAviad Krawczyk /** 172e2585ea7SAviad Krawczyk * rx_free_skb - unmap and free skb 173e2585ea7SAviad Krawczyk * @rxq: rx queue 174e2585ea7SAviad Krawczyk * @skb: skb to free 175e2585ea7SAviad Krawczyk * @dma_addr: dma address of the skb 176e2585ea7SAviad Krawczyk **/ 177e2585ea7SAviad Krawczyk static void rx_free_skb(struct hinic_rxq *rxq, struct sk_buff *skb, 178e2585ea7SAviad Krawczyk dma_addr_t dma_addr) 179e2585ea7SAviad Krawczyk { 180e2585ea7SAviad Krawczyk rx_unmap_skb(rxq, dma_addr); 181e2585ea7SAviad Krawczyk dev_kfree_skb_any(skb); 182e2585ea7SAviad Krawczyk } 183e2585ea7SAviad Krawczyk 184e2585ea7SAviad Krawczyk /** 185e2585ea7SAviad Krawczyk * rx_alloc_pkts - allocate pkts in rx queue 186e2585ea7SAviad Krawczyk * @rxq: rx queue 187e2585ea7SAviad Krawczyk * 188e2585ea7SAviad Krawczyk * Return number of skbs allocated 189e2585ea7SAviad Krawczyk **/ 190e2585ea7SAviad Krawczyk static int rx_alloc_pkts(struct hinic_rxq *rxq) 191e2585ea7SAviad Krawczyk { 192e2585ea7SAviad Krawczyk struct hinic_dev *nic_dev = netdev_priv(rxq->netdev); 193e2585ea7SAviad Krawczyk struct hinic_rq_wqe *rq_wqe; 194e2585ea7SAviad Krawczyk unsigned int free_wqebbs; 195e2585ea7SAviad Krawczyk struct hinic_sge sge; 196e2585ea7SAviad Krawczyk dma_addr_t dma_addr; 197e2585ea7SAviad Krawczyk struct sk_buff *skb; 198e2585ea7SAviad Krawczyk u16 prod_idx; 199352f58b0SAviad Krawczyk int i; 200e2585ea7SAviad Krawczyk 201e2585ea7SAviad Krawczyk free_wqebbs = hinic_get_rq_free_wqebbs(rxq->rq); 202e2585ea7SAviad Krawczyk 203e2585ea7SAviad Krawczyk /* Limit the allocation chunks */ 204e2585ea7SAviad Krawczyk if (free_wqebbs > nic_dev->rx_weight) 205e2585ea7SAviad Krawczyk free_wqebbs = nic_dev->rx_weight; 206e2585ea7SAviad Krawczyk 207e2585ea7SAviad Krawczyk for (i = 0; i < free_wqebbs; i++) { 208e2585ea7SAviad Krawczyk skb = rx_alloc_skb(rxq, &dma_addr); 209e2585ea7SAviad Krawczyk if (!skb) { 210e2585ea7SAviad Krawczyk netdev_err(rxq->netdev, "Failed to alloc Rx skb\n"); 211e2585ea7SAviad Krawczyk goto skb_out; 212e2585ea7SAviad Krawczyk } 213e2585ea7SAviad Krawczyk 214e2585ea7SAviad Krawczyk hinic_set_sge(&sge, dma_addr, skb->len); 215e2585ea7SAviad Krawczyk 216e2585ea7SAviad Krawczyk rq_wqe = hinic_rq_get_wqe(rxq->rq, HINIC_RQ_WQE_SIZE, 217e2585ea7SAviad Krawczyk &prod_idx); 218e2585ea7SAviad Krawczyk if (!rq_wqe) { 219e2585ea7SAviad Krawczyk rx_free_skb(rxq, skb, dma_addr); 220e2585ea7SAviad Krawczyk goto skb_out; 221e2585ea7SAviad Krawczyk } 222e2585ea7SAviad Krawczyk 223e2585ea7SAviad Krawczyk hinic_rq_prepare_wqe(rxq->rq, prod_idx, rq_wqe, &sge); 224e2585ea7SAviad Krawczyk 225e2585ea7SAviad Krawczyk hinic_rq_write_wqe(rxq->rq, prod_idx, rq_wqe, skb); 226e2585ea7SAviad Krawczyk } 227e2585ea7SAviad Krawczyk 228e2585ea7SAviad Krawczyk skb_out: 229e2585ea7SAviad Krawczyk if (i) { 230e2585ea7SAviad Krawczyk wmb(); /* write all the wqes before update PI */ 231e2585ea7SAviad Krawczyk 232e2585ea7SAviad Krawczyk hinic_rq_update(rxq->rq, prod_idx); 233e2585ea7SAviad Krawczyk } 234e2585ea7SAviad Krawczyk 235e2585ea7SAviad Krawczyk return i; 236e2585ea7SAviad Krawczyk } 237e2585ea7SAviad Krawczyk 238e2585ea7SAviad Krawczyk /** 239e2585ea7SAviad Krawczyk * free_all_rx_skbs - free all skbs in rx queue 240e2585ea7SAviad Krawczyk * @rxq: rx queue 241e2585ea7SAviad Krawczyk **/ 242e2585ea7SAviad Krawczyk static void free_all_rx_skbs(struct hinic_rxq *rxq) 243e2585ea7SAviad Krawczyk { 244e2585ea7SAviad Krawczyk struct hinic_rq *rq = rxq->rq; 245e2585ea7SAviad Krawczyk struct hinic_hw_wqe *hw_wqe; 246e2585ea7SAviad Krawczyk struct hinic_sge sge; 247e2585ea7SAviad Krawczyk u16 ci; 248e2585ea7SAviad Krawczyk 249e2585ea7SAviad Krawczyk while ((hw_wqe = hinic_read_wqe(rq->wq, HINIC_RQ_WQE_SIZE, &ci))) { 250e2585ea7SAviad Krawczyk if (IS_ERR(hw_wqe)) 251e2585ea7SAviad Krawczyk break; 252e2585ea7SAviad Krawczyk 253e2585ea7SAviad Krawczyk hinic_rq_get_sge(rq, &hw_wqe->rq_wqe, ci, &sge); 254e2585ea7SAviad Krawczyk 255e2585ea7SAviad Krawczyk hinic_put_wqe(rq->wq, HINIC_RQ_WQE_SIZE); 256e2585ea7SAviad Krawczyk 257e2585ea7SAviad Krawczyk rx_free_skb(rxq, rq->saved_skb[ci], hinic_sge_to_dma(&sge)); 258e2585ea7SAviad Krawczyk } 259e2585ea7SAviad Krawczyk } 260e2585ea7SAviad Krawczyk 261e2585ea7SAviad Krawczyk /** 262e2585ea7SAviad Krawczyk * rx_recv_jumbo_pkt - Rx handler for jumbo pkt 263e2585ea7SAviad Krawczyk * @rxq: rx queue 264e2585ea7SAviad Krawczyk * @head_skb: the first skb in the list 265e2585ea7SAviad Krawczyk * @left_pkt_len: left size of the pkt exclude head skb 266e2585ea7SAviad Krawczyk * @ci: consumer index 267e2585ea7SAviad Krawczyk * 268e2585ea7SAviad Krawczyk * Return number of wqes that used for the left of the pkt 269e2585ea7SAviad Krawczyk **/ 270e2585ea7SAviad Krawczyk static int rx_recv_jumbo_pkt(struct hinic_rxq *rxq, struct sk_buff *head_skb, 271e2585ea7SAviad Krawczyk unsigned int left_pkt_len, u16 ci) 272e2585ea7SAviad Krawczyk { 273e2585ea7SAviad Krawczyk struct sk_buff *skb, *curr_skb = head_skb; 274e2585ea7SAviad Krawczyk struct hinic_rq_wqe *rq_wqe; 275e2585ea7SAviad Krawczyk unsigned int curr_len; 276e2585ea7SAviad Krawczyk struct hinic_sge sge; 277e2585ea7SAviad Krawczyk int num_wqes = 0; 278e2585ea7SAviad Krawczyk 279e2585ea7SAviad Krawczyk while (left_pkt_len > 0) { 280e2585ea7SAviad Krawczyk rq_wqe = hinic_rq_read_next_wqe(rxq->rq, HINIC_RQ_WQE_SIZE, 281e2585ea7SAviad Krawczyk &skb, &ci); 282e2585ea7SAviad Krawczyk 283e2585ea7SAviad Krawczyk num_wqes++; 284e2585ea7SAviad Krawczyk 285e2585ea7SAviad Krawczyk hinic_rq_get_sge(rxq->rq, rq_wqe, ci, &sge); 286e2585ea7SAviad Krawczyk 287e2585ea7SAviad Krawczyk rx_unmap_skb(rxq, hinic_sge_to_dma(&sge)); 288e2585ea7SAviad Krawczyk 289e2585ea7SAviad Krawczyk prefetch(skb->data); 290e2585ea7SAviad Krawczyk 291e2585ea7SAviad Krawczyk curr_len = (left_pkt_len > HINIC_RX_BUF_SZ) ? HINIC_RX_BUF_SZ : 292e2585ea7SAviad Krawczyk left_pkt_len; 293e2585ea7SAviad Krawczyk 294e2585ea7SAviad Krawczyk left_pkt_len -= curr_len; 295e2585ea7SAviad Krawczyk 296e2585ea7SAviad Krawczyk __skb_put(skb, curr_len); 297e2585ea7SAviad Krawczyk 298e2585ea7SAviad Krawczyk if (curr_skb == head_skb) 299e2585ea7SAviad Krawczyk skb_shinfo(head_skb)->frag_list = skb; 300e2585ea7SAviad Krawczyk else 301e2585ea7SAviad Krawczyk curr_skb->next = skb; 302e2585ea7SAviad Krawczyk 303e2585ea7SAviad Krawczyk head_skb->len += skb->len; 304e2585ea7SAviad Krawczyk head_skb->data_len += skb->len; 305e2585ea7SAviad Krawczyk head_skb->truesize += skb->truesize; 306e2585ea7SAviad Krawczyk 307e2585ea7SAviad Krawczyk curr_skb = skb; 308e2585ea7SAviad Krawczyk } 309e2585ea7SAviad Krawczyk 310e2585ea7SAviad Krawczyk return num_wqes; 311e2585ea7SAviad Krawczyk } 312e2585ea7SAviad Krawczyk 313e2585ea7SAviad Krawczyk /** 314e2585ea7SAviad Krawczyk * rxq_recv - Rx handler 315e2585ea7SAviad Krawczyk * @rxq: rx queue 316e2585ea7SAviad Krawczyk * @budget: maximum pkts to process 317e2585ea7SAviad Krawczyk * 318e2585ea7SAviad Krawczyk * Return number of pkts received 319e2585ea7SAviad Krawczyk **/ 320e2585ea7SAviad Krawczyk static int rxq_recv(struct hinic_rxq *rxq, int budget) 321e2585ea7SAviad Krawczyk { 322e2585ea7SAviad Krawczyk struct hinic_qp *qp = container_of(rxq->rq, struct hinic_qp, rq); 323e2585ea7SAviad Krawczyk u64 pkt_len = 0, rx_bytes = 0; 324e2585ea7SAviad Krawczyk struct hinic_rq_wqe *rq_wqe; 325e1a76515SXue Chaojing unsigned int free_wqebbs; 326e2585ea7SAviad Krawczyk int num_wqes, pkts = 0; 327e2585ea7SAviad Krawczyk struct hinic_sge sge; 328e2585ea7SAviad Krawczyk struct sk_buff *skb; 329e2585ea7SAviad Krawczyk u16 ci; 330e2585ea7SAviad Krawczyk 331e2585ea7SAviad Krawczyk while (pkts < budget) { 332e2585ea7SAviad Krawczyk num_wqes = 0; 333e2585ea7SAviad Krawczyk 334e2585ea7SAviad Krawczyk rq_wqe = hinic_rq_read_wqe(rxq->rq, HINIC_RQ_WQE_SIZE, &skb, 335e2585ea7SAviad Krawczyk &ci); 336e2585ea7SAviad Krawczyk if (!rq_wqe) 337e2585ea7SAviad Krawczyk break; 338e2585ea7SAviad Krawczyk 339e2585ea7SAviad Krawczyk hinic_rq_get_sge(rxq->rq, rq_wqe, ci, &sge); 340e2585ea7SAviad Krawczyk 341e2585ea7SAviad Krawczyk rx_unmap_skb(rxq, hinic_sge_to_dma(&sge)); 342e2585ea7SAviad Krawczyk 3434a61abb1SXue Chaojing rx_csum(rxq, ci, skb); 3444a61abb1SXue Chaojing 345e2585ea7SAviad Krawczyk prefetch(skb->data); 346e2585ea7SAviad Krawczyk 347e2585ea7SAviad Krawczyk pkt_len = sge.len; 348e2585ea7SAviad Krawczyk 349e2585ea7SAviad Krawczyk if (pkt_len <= HINIC_RX_BUF_SZ) { 350e2585ea7SAviad Krawczyk __skb_put(skb, pkt_len); 351e2585ea7SAviad Krawczyk } else { 352e2585ea7SAviad Krawczyk __skb_put(skb, HINIC_RX_BUF_SZ); 353e2585ea7SAviad Krawczyk num_wqes = rx_recv_jumbo_pkt(rxq, skb, pkt_len - 354e2585ea7SAviad Krawczyk HINIC_RX_BUF_SZ, ci); 355e2585ea7SAviad Krawczyk } 356e2585ea7SAviad Krawczyk 357e2585ea7SAviad Krawczyk hinic_rq_put_wqe(rxq->rq, ci, 358e2585ea7SAviad Krawczyk (num_wqes + 1) * HINIC_RQ_WQE_SIZE); 359e2585ea7SAviad Krawczyk 360e2585ea7SAviad Krawczyk skb_record_rx_queue(skb, qp->q_id); 361e2585ea7SAviad Krawczyk skb->protocol = eth_type_trans(skb, rxq->netdev); 362e2585ea7SAviad Krawczyk 363e2585ea7SAviad Krawczyk napi_gro_receive(&rxq->napi, skb); 364e2585ea7SAviad Krawczyk 365e2585ea7SAviad Krawczyk pkts++; 366e2585ea7SAviad Krawczyk rx_bytes += pkt_len; 367e2585ea7SAviad Krawczyk } 368e2585ea7SAviad Krawczyk 369e1a76515SXue Chaojing free_wqebbs = hinic_get_rq_free_wqebbs(rxq->rq); 370e1a76515SXue Chaojing if (free_wqebbs > HINIC_RX_BUFFER_WRITE) 371e1a76515SXue Chaojing rx_alloc_pkts(rxq); 372e2585ea7SAviad Krawczyk 373e2585ea7SAviad Krawczyk u64_stats_update_begin(&rxq->rxq_stats.syncp); 374e2585ea7SAviad Krawczyk rxq->rxq_stats.pkts += pkts; 375e2585ea7SAviad Krawczyk rxq->rxq_stats.bytes += rx_bytes; 376e2585ea7SAviad Krawczyk u64_stats_update_end(&rxq->rxq_stats.syncp); 377e2585ea7SAviad Krawczyk 378e2585ea7SAviad Krawczyk return pkts; 379e2585ea7SAviad Krawczyk } 380e2585ea7SAviad Krawczyk 381e2585ea7SAviad Krawczyk static int rx_poll(struct napi_struct *napi, int budget) 382e2585ea7SAviad Krawczyk { 383e2585ea7SAviad Krawczyk struct hinic_rxq *rxq = container_of(napi, struct hinic_rxq, napi); 384e2585ea7SAviad Krawczyk struct hinic_rq *rq = rxq->rq; 385e2585ea7SAviad Krawczyk int pkts; 386e2585ea7SAviad Krawczyk 387e2585ea7SAviad Krawczyk pkts = rxq_recv(rxq, budget); 388e2585ea7SAviad Krawczyk if (pkts >= budget) 389e2585ea7SAviad Krawczyk return budget; 390e2585ea7SAviad Krawczyk 391e2585ea7SAviad Krawczyk napi_complete(napi); 392e2585ea7SAviad Krawczyk enable_irq(rq->irq); 393e2585ea7SAviad Krawczyk return pkts; 394e2585ea7SAviad Krawczyk } 395e2585ea7SAviad Krawczyk 396e2585ea7SAviad Krawczyk static void rx_add_napi(struct hinic_rxq *rxq) 397e2585ea7SAviad Krawczyk { 398e2585ea7SAviad Krawczyk struct hinic_dev *nic_dev = netdev_priv(rxq->netdev); 399e2585ea7SAviad Krawczyk 400e2585ea7SAviad Krawczyk netif_napi_add(rxq->netdev, &rxq->napi, rx_poll, nic_dev->rx_weight); 401e2585ea7SAviad Krawczyk napi_enable(&rxq->napi); 402e2585ea7SAviad Krawczyk } 403e2585ea7SAviad Krawczyk 404e2585ea7SAviad Krawczyk static void rx_del_napi(struct hinic_rxq *rxq) 405e2585ea7SAviad Krawczyk { 406e2585ea7SAviad Krawczyk napi_disable(&rxq->napi); 407e2585ea7SAviad Krawczyk netif_napi_del(&rxq->napi); 408e2585ea7SAviad Krawczyk } 409e2585ea7SAviad Krawczyk 410e2585ea7SAviad Krawczyk static irqreturn_t rx_irq(int irq, void *data) 411e2585ea7SAviad Krawczyk { 412e2585ea7SAviad Krawczyk struct hinic_rxq *rxq = (struct hinic_rxq *)data; 413e2585ea7SAviad Krawczyk struct hinic_rq *rq = rxq->rq; 414e2585ea7SAviad Krawczyk struct hinic_dev *nic_dev; 415e2585ea7SAviad Krawczyk 416e2585ea7SAviad Krawczyk /* Disable the interrupt until napi will be completed */ 417e2585ea7SAviad Krawczyk disable_irq_nosync(rq->irq); 418e2585ea7SAviad Krawczyk 419e2585ea7SAviad Krawczyk nic_dev = netdev_priv(rxq->netdev); 420e2585ea7SAviad Krawczyk hinic_hwdev_msix_cnt_set(nic_dev->hwdev, rq->msix_entry); 421e2585ea7SAviad Krawczyk 422e2585ea7SAviad Krawczyk napi_schedule(&rxq->napi); 423e2585ea7SAviad Krawczyk return IRQ_HANDLED; 424e2585ea7SAviad Krawczyk } 425e2585ea7SAviad Krawczyk 426e2585ea7SAviad Krawczyk static int rx_request_irq(struct hinic_rxq *rxq) 427e2585ea7SAviad Krawczyk { 428e2585ea7SAviad Krawczyk struct hinic_dev *nic_dev = netdev_priv(rxq->netdev); 429e2585ea7SAviad Krawczyk struct hinic_hwdev *hwdev = nic_dev->hwdev; 430e2585ea7SAviad Krawczyk struct hinic_rq *rq = rxq->rq; 431352f58b0SAviad Krawczyk struct hinic_qp *qp; 432352f58b0SAviad Krawczyk struct cpumask mask; 433e2585ea7SAviad Krawczyk int err; 434e2585ea7SAviad Krawczyk 435e2585ea7SAviad Krawczyk rx_add_napi(rxq); 436e2585ea7SAviad Krawczyk 437e2585ea7SAviad Krawczyk hinic_hwdev_msix_set(hwdev, rq->msix_entry, 438e2585ea7SAviad Krawczyk RX_IRQ_NO_PENDING, RX_IRQ_NO_COALESC, 439e2585ea7SAviad Krawczyk RX_IRQ_NO_LLI_TIMER, RX_IRQ_NO_CREDIT, 440e2585ea7SAviad Krawczyk RX_IRQ_NO_RESEND_TIMER); 441e2585ea7SAviad Krawczyk 442e2585ea7SAviad Krawczyk err = request_irq(rq->irq, rx_irq, 0, rxq->irq_name, rxq); 443e2585ea7SAviad Krawczyk if (err) { 444e2585ea7SAviad Krawczyk rx_del_napi(rxq); 445e2585ea7SAviad Krawczyk return err; 446e2585ea7SAviad Krawczyk } 447e2585ea7SAviad Krawczyk 448352f58b0SAviad Krawczyk qp = container_of(rq, struct hinic_qp, rq); 449352f58b0SAviad Krawczyk cpumask_set_cpu(qp->q_id % num_online_cpus(), &mask); 450352f58b0SAviad Krawczyk return irq_set_affinity_hint(rq->irq, &mask); 451e2585ea7SAviad Krawczyk } 452e2585ea7SAviad Krawczyk 453e2585ea7SAviad Krawczyk static void rx_free_irq(struct hinic_rxq *rxq) 454e2585ea7SAviad Krawczyk { 455e2585ea7SAviad Krawczyk struct hinic_rq *rq = rxq->rq; 456e2585ea7SAviad Krawczyk 45782be2ab1SWei Yongjun irq_set_affinity_hint(rq->irq, NULL); 458e2585ea7SAviad Krawczyk free_irq(rq->irq, rxq); 459e2585ea7SAviad Krawczyk rx_del_napi(rxq); 460e2585ea7SAviad Krawczyk } 461e2585ea7SAviad Krawczyk 462e2585ea7SAviad Krawczyk /** 463c3e79bafSAviad Krawczyk * hinic_init_rxq - Initialize the Rx Queue 464c3e79bafSAviad Krawczyk * @rxq: Logical Rx Queue 465c3e79bafSAviad Krawczyk * @rq: Hardware Rx Queue to connect the Logical queue with 466c3e79bafSAviad Krawczyk * @netdev: network device to connect the Logical queue with 467c3e79bafSAviad Krawczyk * 468c3e79bafSAviad Krawczyk * Return 0 - Success, negative - Failure 469c3e79bafSAviad Krawczyk **/ 470c3e79bafSAviad Krawczyk int hinic_init_rxq(struct hinic_rxq *rxq, struct hinic_rq *rq, 471c3e79bafSAviad Krawczyk struct net_device *netdev) 472c3e79bafSAviad Krawczyk { 473e2585ea7SAviad Krawczyk struct hinic_qp *qp = container_of(rq, struct hinic_qp, rq); 474e2585ea7SAviad Krawczyk int err, pkts, irqname_len; 475e2585ea7SAviad Krawczyk 476c3e79bafSAviad Krawczyk rxq->netdev = netdev; 477c3e79bafSAviad Krawczyk rxq->rq = rq; 478c3e79bafSAviad Krawczyk 479c3e79bafSAviad Krawczyk rxq_stats_init(rxq); 480e2585ea7SAviad Krawczyk 481e2585ea7SAviad Krawczyk irqname_len = snprintf(NULL, 0, "hinic_rxq%d", qp->q_id) + 1; 482e2585ea7SAviad Krawczyk rxq->irq_name = devm_kzalloc(&netdev->dev, irqname_len, GFP_KERNEL); 483e2585ea7SAviad Krawczyk if (!rxq->irq_name) 484e2585ea7SAviad Krawczyk return -ENOMEM; 485e2585ea7SAviad Krawczyk 486e2585ea7SAviad Krawczyk sprintf(rxq->irq_name, "hinic_rxq%d", qp->q_id); 487e2585ea7SAviad Krawczyk 488e2585ea7SAviad Krawczyk pkts = rx_alloc_pkts(rxq); 489e2585ea7SAviad Krawczyk if (!pkts) { 490e2585ea7SAviad Krawczyk err = -ENOMEM; 491e2585ea7SAviad Krawczyk goto err_rx_pkts; 492e2585ea7SAviad Krawczyk } 493e2585ea7SAviad Krawczyk 494e2585ea7SAviad Krawczyk err = rx_request_irq(rxq); 495e2585ea7SAviad Krawczyk if (err) { 496e2585ea7SAviad Krawczyk netdev_err(netdev, "Failed to request Rx irq\n"); 497e2585ea7SAviad Krawczyk goto err_req_rx_irq; 498e2585ea7SAviad Krawczyk } 499e2585ea7SAviad Krawczyk 500c3e79bafSAviad Krawczyk return 0; 501e2585ea7SAviad Krawczyk 502e2585ea7SAviad Krawczyk err_req_rx_irq: 503e2585ea7SAviad Krawczyk err_rx_pkts: 504e2585ea7SAviad Krawczyk free_all_rx_skbs(rxq); 505e2585ea7SAviad Krawczyk devm_kfree(&netdev->dev, rxq->irq_name); 506e2585ea7SAviad Krawczyk return err; 507c3e79bafSAviad Krawczyk } 508c3e79bafSAviad Krawczyk 509c3e79bafSAviad Krawczyk /** 510c3e79bafSAviad Krawczyk * hinic_clean_rxq - Clean the Rx Queue 511c3e79bafSAviad Krawczyk * @rxq: Logical Rx Queue 512c3e79bafSAviad Krawczyk **/ 513c3e79bafSAviad Krawczyk void hinic_clean_rxq(struct hinic_rxq *rxq) 514c3e79bafSAviad Krawczyk { 515e2585ea7SAviad Krawczyk struct net_device *netdev = rxq->netdev; 516e2585ea7SAviad Krawczyk 517e2585ea7SAviad Krawczyk rx_free_irq(rxq); 518e2585ea7SAviad Krawczyk 519e2585ea7SAviad Krawczyk free_all_rx_skbs(rxq); 520e2585ea7SAviad Krawczyk devm_kfree(&netdev->dev, rxq->irq_name); 521c3e79bafSAviad Krawczyk } 522