10f3154e6SShannon Nelson // SPDX-License-Identifier: GPL-2.0 20f3154e6SShannon Nelson /* Copyright(c) 2017 - 2019 Pensando Systems, Inc */ 30f3154e6SShannon Nelson 40f3154e6SShannon Nelson #include <linux/ip.h> 50f3154e6SShannon Nelson #include <linux/ipv6.h> 60f3154e6SShannon Nelson #include <linux/if_vlan.h> 70f3154e6SShannon Nelson #include <net/ip6_checksum.h> 80f3154e6SShannon Nelson 90f3154e6SShannon Nelson #include "ionic.h" 100f3154e6SShannon Nelson #include "ionic_lif.h" 110f3154e6SShannon Nelson #include "ionic_txrx.h" 120f3154e6SShannon Nelson 135b3f3f2aSShannon Nelson static void ionic_rx_clean(struct ionic_queue *q, 145b3f3f2aSShannon Nelson struct ionic_desc_info *desc_info, 155b3f3f2aSShannon Nelson struct ionic_cq_info *cq_info, 165b3f3f2aSShannon Nelson void *cb_arg); 170f3154e6SShannon Nelson 18b14e4e95SShannon Nelson static bool ionic_rx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info); 19b14e4e95SShannon Nelson 20b14e4e95SShannon Nelson static bool ionic_tx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info); 21b14e4e95SShannon Nelson 220f3154e6SShannon Nelson static inline void ionic_txq_post(struct ionic_queue *q, bool ring_dbell, 230f3154e6SShannon Nelson ionic_desc_cb cb_func, void *cb_arg) 240f3154e6SShannon Nelson { 25f1d2e894SShannon Nelson DEBUG_STATS_TXQ_POST(q, ring_dbell); 260f3154e6SShannon Nelson 270f3154e6SShannon Nelson ionic_q_post(q, ring_dbell, cb_func, cb_arg); 280f3154e6SShannon Nelson } 290f3154e6SShannon Nelson 300f3154e6SShannon Nelson static inline void ionic_rxq_post(struct ionic_queue *q, bool ring_dbell, 310f3154e6SShannon Nelson ionic_desc_cb cb_func, void *cb_arg) 320f3154e6SShannon Nelson { 330f3154e6SShannon Nelson ionic_q_post(q, ring_dbell, cb_func, cb_arg); 340f3154e6SShannon Nelson 3534dec947SShannon Nelson DEBUG_STATS_RX_BUFF_CNT(q); 360f3154e6SShannon Nelson } 370f3154e6SShannon Nelson 380f3154e6SShannon Nelson static inline struct netdev_queue *q_to_ndq(struct ionic_queue *q) 390f3154e6SShannon Nelson { 400f3154e6SShannon Nelson return netdev_get_tx_queue(q->lif->netdev, q->index); 410f3154e6SShannon Nelson } 420f3154e6SShannon Nelson 4308f2e4b2SShannon Nelson static struct sk_buff *ionic_rx_skb_alloc(struct ionic_queue *q, 4408f2e4b2SShannon Nelson unsigned int len, bool frags) 450f3154e6SShannon Nelson { 4608f2e4b2SShannon Nelson struct ionic_lif *lif = q->lif; 4708f2e4b2SShannon Nelson struct ionic_rx_stats *stats; 4808f2e4b2SShannon Nelson struct net_device *netdev; 4908f2e4b2SShannon Nelson struct sk_buff *skb; 500f3154e6SShannon Nelson 5108f2e4b2SShannon Nelson netdev = lif->netdev; 5234dec947SShannon Nelson stats = &q->lif->rxqstats[q->index]; 530f3154e6SShannon Nelson 5408f2e4b2SShannon Nelson if (frags) 5508f2e4b2SShannon Nelson skb = napi_get_frags(&q_to_qcq(q)->napi); 5608f2e4b2SShannon Nelson else 5708f2e4b2SShannon Nelson skb = netdev_alloc_skb_ip_align(netdev, len); 5808f2e4b2SShannon Nelson 5908f2e4b2SShannon Nelson if (unlikely(!skb)) { 6008f2e4b2SShannon Nelson net_warn_ratelimited("%s: SKB alloc failed on %s!\n", 6108f2e4b2SShannon Nelson netdev->name, q->name); 6208f2e4b2SShannon Nelson stats->alloc_err++; 6308f2e4b2SShannon Nelson return NULL; 640f3154e6SShannon Nelson } 650f3154e6SShannon Nelson 6608f2e4b2SShannon Nelson return skb; 6708f2e4b2SShannon Nelson } 6808f2e4b2SShannon Nelson 6908f2e4b2SShannon Nelson static struct sk_buff *ionic_rx_frags(struct ionic_queue *q, 7008f2e4b2SShannon Nelson struct ionic_desc_info *desc_info, 7108f2e4b2SShannon Nelson struct ionic_cq_info *cq_info) 720f3154e6SShannon Nelson { 730f3154e6SShannon Nelson struct ionic_rxq_comp *comp = cq_info->cq_desc; 740f3154e6SShannon Nelson struct device *dev = q->lif->ionic->dev; 7508f2e4b2SShannon Nelson struct ionic_page_info *page_info; 7608f2e4b2SShannon Nelson struct sk_buff *skb; 7708f2e4b2SShannon Nelson unsigned int i; 7808f2e4b2SShannon Nelson u16 frag_len; 7908f2e4b2SShannon Nelson u16 len; 800f3154e6SShannon Nelson 8108f2e4b2SShannon Nelson page_info = &desc_info->pages[0]; 8208f2e4b2SShannon Nelson len = le16_to_cpu(comp->len); 8308f2e4b2SShannon Nelson 8408f2e4b2SShannon Nelson prefetch(page_address(page_info->page) + NET_IP_ALIGN); 8508f2e4b2SShannon Nelson 8608f2e4b2SShannon Nelson skb = ionic_rx_skb_alloc(q, len, true); 8708f2e4b2SShannon Nelson if (unlikely(!skb)) 8808f2e4b2SShannon Nelson return NULL; 8908f2e4b2SShannon Nelson 9008f2e4b2SShannon Nelson i = comp->num_sg_elems + 1; 9108f2e4b2SShannon Nelson do { 9208f2e4b2SShannon Nelson if (unlikely(!page_info->page)) { 9308f2e4b2SShannon Nelson struct napi_struct *napi = &q_to_qcq(q)->napi; 9408f2e4b2SShannon Nelson 9508f2e4b2SShannon Nelson napi->skb = NULL; 9608f2e4b2SShannon Nelson dev_kfree_skb(skb); 9708f2e4b2SShannon Nelson return NULL; 980f3154e6SShannon Nelson } 990f3154e6SShannon Nelson 10008f2e4b2SShannon Nelson frag_len = min(len, (u16)PAGE_SIZE); 10108f2e4b2SShannon Nelson len -= frag_len; 10208f2e4b2SShannon Nelson 10308f2e4b2SShannon Nelson dma_unmap_page(dev, dma_unmap_addr(page_info, dma_addr), 10408f2e4b2SShannon Nelson PAGE_SIZE, DMA_FROM_DEVICE); 10508f2e4b2SShannon Nelson skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 10608f2e4b2SShannon Nelson page_info->page, 0, frag_len, PAGE_SIZE); 10708f2e4b2SShannon Nelson page_info->page = NULL; 10808f2e4b2SShannon Nelson page_info++; 10908f2e4b2SShannon Nelson i--; 11008f2e4b2SShannon Nelson } while (i > 0); 11108f2e4b2SShannon Nelson 11208f2e4b2SShannon Nelson return skb; 1130f3154e6SShannon Nelson } 1140f3154e6SShannon Nelson 11508f2e4b2SShannon Nelson static struct sk_buff *ionic_rx_copybreak(struct ionic_queue *q, 11608f2e4b2SShannon Nelson struct ionic_desc_info *desc_info, 11708f2e4b2SShannon Nelson struct ionic_cq_info *cq_info) 11808f2e4b2SShannon Nelson { 11908f2e4b2SShannon Nelson struct ionic_rxq_comp *comp = cq_info->cq_desc; 12008f2e4b2SShannon Nelson struct device *dev = q->lif->ionic->dev; 12108f2e4b2SShannon Nelson struct ionic_page_info *page_info; 12208f2e4b2SShannon Nelson struct sk_buff *skb; 12308f2e4b2SShannon Nelson u16 len; 1240f3154e6SShannon Nelson 12508f2e4b2SShannon Nelson page_info = &desc_info->pages[0]; 12608f2e4b2SShannon Nelson len = le16_to_cpu(comp->len); 1270f3154e6SShannon Nelson 12808f2e4b2SShannon Nelson skb = ionic_rx_skb_alloc(q, len, false); 12908f2e4b2SShannon Nelson if (unlikely(!skb)) 13008f2e4b2SShannon Nelson return NULL; 1310f3154e6SShannon Nelson 13208f2e4b2SShannon Nelson if (unlikely(!page_info->page)) { 13308f2e4b2SShannon Nelson dev_kfree_skb(skb); 13408f2e4b2SShannon Nelson return NULL; 13508f2e4b2SShannon Nelson } 13608f2e4b2SShannon Nelson 13708f2e4b2SShannon Nelson dma_sync_single_for_cpu(dev, dma_unmap_addr(page_info, dma_addr), 13808f2e4b2SShannon Nelson len, DMA_FROM_DEVICE); 13908f2e4b2SShannon Nelson skb_copy_to_linear_data(skb, page_address(page_info->page), len); 14008f2e4b2SShannon Nelson dma_sync_single_for_device(dev, dma_unmap_addr(page_info, dma_addr), 14108f2e4b2SShannon Nelson len, DMA_FROM_DEVICE); 14208f2e4b2SShannon Nelson 14308f2e4b2SShannon Nelson skb_put(skb, len); 14408f2e4b2SShannon Nelson skb->protocol = eth_type_trans(skb, q->lif->netdev); 14508f2e4b2SShannon Nelson 14608f2e4b2SShannon Nelson return skb; 1470f3154e6SShannon Nelson } 1480f3154e6SShannon Nelson 1495b3f3f2aSShannon Nelson static void ionic_rx_clean(struct ionic_queue *q, 1505b3f3f2aSShannon Nelson struct ionic_desc_info *desc_info, 1515b3f3f2aSShannon Nelson struct ionic_cq_info *cq_info, 1525b3f3f2aSShannon Nelson void *cb_arg) 1530f3154e6SShannon Nelson { 1540f3154e6SShannon Nelson struct ionic_rxq_comp *comp = cq_info->cq_desc; 1550f3154e6SShannon Nelson struct ionic_qcq *qcq = q_to_qcq(q); 1560f3154e6SShannon Nelson struct ionic_rx_stats *stats; 1570f3154e6SShannon Nelson struct net_device *netdev; 15808f2e4b2SShannon Nelson struct sk_buff *skb; 1590f3154e6SShannon Nelson 1600f3154e6SShannon Nelson stats = q_to_rx_stats(q); 1610f3154e6SShannon Nelson netdev = q->lif->netdev; 1620f3154e6SShannon Nelson 16324cfa8c7SShannon Nelson if (comp->status) { 16424cfa8c7SShannon Nelson stats->dropped++; 1650f3154e6SShannon Nelson return; 16624cfa8c7SShannon Nelson } 1670f3154e6SShannon Nelson 1680f3154e6SShannon Nelson stats->pkts++; 1690f3154e6SShannon Nelson stats->bytes += le16_to_cpu(comp->len); 1700f3154e6SShannon Nelson 17108f2e4b2SShannon Nelson if (le16_to_cpu(comp->len) <= q->lif->rx_copybreak) 17208f2e4b2SShannon Nelson skb = ionic_rx_copybreak(q, desc_info, cq_info); 17308f2e4b2SShannon Nelson else 17408f2e4b2SShannon Nelson skb = ionic_rx_frags(q, desc_info, cq_info); 1750f3154e6SShannon Nelson 17624cfa8c7SShannon Nelson if (unlikely(!skb)) { 17724cfa8c7SShannon Nelson stats->dropped++; 17808f2e4b2SShannon Nelson return; 17924cfa8c7SShannon Nelson } 1800f3154e6SShannon Nelson 1810f3154e6SShannon Nelson skb_record_rx_queue(skb, q->index); 1820f3154e6SShannon Nelson 18308f2e4b2SShannon Nelson if (likely(netdev->features & NETIF_F_RXHASH)) { 1840f3154e6SShannon Nelson switch (comp->pkt_type_color & IONIC_RXQ_COMP_PKT_TYPE_MASK) { 1850f3154e6SShannon Nelson case IONIC_PKT_TYPE_IPV4: 1860f3154e6SShannon Nelson case IONIC_PKT_TYPE_IPV6: 1870f3154e6SShannon Nelson skb_set_hash(skb, le32_to_cpu(comp->rss_hash), 1880f3154e6SShannon Nelson PKT_HASH_TYPE_L3); 1890f3154e6SShannon Nelson break; 1900f3154e6SShannon Nelson case IONIC_PKT_TYPE_IPV4_TCP: 1910f3154e6SShannon Nelson case IONIC_PKT_TYPE_IPV6_TCP: 1920f3154e6SShannon Nelson case IONIC_PKT_TYPE_IPV4_UDP: 1930f3154e6SShannon Nelson case IONIC_PKT_TYPE_IPV6_UDP: 1940f3154e6SShannon Nelson skb_set_hash(skb, le32_to_cpu(comp->rss_hash), 1950f3154e6SShannon Nelson PKT_HASH_TYPE_L4); 1960f3154e6SShannon Nelson break; 1970f3154e6SShannon Nelson } 1980f3154e6SShannon Nelson } 1990f3154e6SShannon Nelson 20008f2e4b2SShannon Nelson if (likely(netdev->features & NETIF_F_RXCSUM)) { 2010f3154e6SShannon Nelson if (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_CALC) { 2020f3154e6SShannon Nelson skb->ip_summed = CHECKSUM_COMPLETE; 2030f3154e6SShannon Nelson skb->csum = (__wsum)le16_to_cpu(comp->csum); 2040f3154e6SShannon Nelson stats->csum_complete++; 2050f3154e6SShannon Nelson } 2060f3154e6SShannon Nelson } else { 2070f3154e6SShannon Nelson stats->csum_none++; 2080f3154e6SShannon Nelson } 2090f3154e6SShannon Nelson 21008f2e4b2SShannon Nelson if (unlikely((comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_TCP_BAD) || 2110f3154e6SShannon Nelson (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_UDP_BAD) || 21208f2e4b2SShannon Nelson (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_IP_BAD))) 2130f3154e6SShannon Nelson stats->csum_error++; 2140f3154e6SShannon Nelson 215f64e0c56SShannon Nelson if (likely(netdev->features & NETIF_F_HW_VLAN_CTAG_RX) && 216f64e0c56SShannon Nelson (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_VLAN)) { 2170f3154e6SShannon Nelson __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), 2180f3154e6SShannon Nelson le16_to_cpu(comp->vlan_tci)); 219f64e0c56SShannon Nelson stats->vlan_stripped++; 2200f3154e6SShannon Nelson } 2210f3154e6SShannon Nelson 22208f2e4b2SShannon Nelson if (le16_to_cpu(comp->len) <= q->lif->rx_copybreak) 2230f3154e6SShannon Nelson napi_gro_receive(&qcq->napi, skb); 22408f2e4b2SShannon Nelson else 22508f2e4b2SShannon Nelson napi_gro_frags(&qcq->napi); 2260f3154e6SShannon Nelson } 2270f3154e6SShannon Nelson 2280f3154e6SShannon Nelson static bool ionic_rx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info) 2290f3154e6SShannon Nelson { 2300f3154e6SShannon Nelson struct ionic_rxq_comp *comp = cq_info->cq_desc; 2310f3154e6SShannon Nelson struct ionic_queue *q = cq->bound_q; 2320f3154e6SShannon Nelson struct ionic_desc_info *desc_info; 2330f3154e6SShannon Nelson 2340f3154e6SShannon Nelson if (!color_match(comp->pkt_type_color, cq->done_color)) 2350f3154e6SShannon Nelson return false; 2360f3154e6SShannon Nelson 2370f3154e6SShannon Nelson /* check for empty queue */ 238f1d2e894SShannon Nelson if (q->tail_idx == q->head_idx) 2390f3154e6SShannon Nelson return false; 2400f3154e6SShannon Nelson 241339dcf7fSShannon Nelson if (q->tail_idx != le16_to_cpu(comp->comp_index)) 2420f3154e6SShannon Nelson return false; 2430f3154e6SShannon Nelson 244339dcf7fSShannon Nelson desc_info = &q->info[q->tail_idx]; 245f1d2e894SShannon Nelson q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); 2460f3154e6SShannon Nelson 2470f3154e6SShannon Nelson /* clean the related q entry, only one per qc completion */ 2480f3154e6SShannon Nelson ionic_rx_clean(q, desc_info, cq_info, desc_info->cb_arg); 2490f3154e6SShannon Nelson 2500f3154e6SShannon Nelson desc_info->cb = NULL; 2510f3154e6SShannon Nelson desc_info->cb_arg = NULL; 2520f3154e6SShannon Nelson 2530f3154e6SShannon Nelson return true; 2540f3154e6SShannon Nelson } 2550f3154e6SShannon Nelson 2560f3154e6SShannon Nelson void ionic_rx_flush(struct ionic_cq *cq) 2570f3154e6SShannon Nelson { 2580f3154e6SShannon Nelson struct ionic_dev *idev = &cq->lif->ionic->idev; 2590f3154e6SShannon Nelson u32 work_done; 2600f3154e6SShannon Nelson 261b14e4e95SShannon Nelson work_done = ionic_cq_service(cq, cq->num_descs, 262b14e4e95SShannon Nelson ionic_rx_service, NULL, NULL); 2630f3154e6SShannon Nelson 2640f3154e6SShannon Nelson if (work_done) 2650f3154e6SShannon Nelson ionic_intr_credits(idev->intr_ctrl, cq->bound_intr->index, 2660f3154e6SShannon Nelson work_done, IONIC_INTR_CRED_RESET_COALESCE); 2670f3154e6SShannon Nelson } 2680f3154e6SShannon Nelson 26963cd9083SShannon Nelson static int ionic_rx_page_alloc(struct ionic_queue *q, 27063cd9083SShannon Nelson struct ionic_page_info *page_info) 2710f3154e6SShannon Nelson { 2720f3154e6SShannon Nelson struct ionic_lif *lif = q->lif; 2730f3154e6SShannon Nelson struct ionic_rx_stats *stats; 2740f3154e6SShannon Nelson struct net_device *netdev; 2750f3154e6SShannon Nelson struct device *dev; 2760f3154e6SShannon Nelson 2770f3154e6SShannon Nelson netdev = lif->netdev; 2780f3154e6SShannon Nelson dev = lif->ionic->dev; 2790f3154e6SShannon Nelson stats = q_to_rx_stats(q); 28063cd9083SShannon Nelson 28163cd9083SShannon Nelson if (unlikely(!page_info)) { 28263cd9083SShannon Nelson net_err_ratelimited("%s: %s invalid page_info in alloc\n", 28363cd9083SShannon Nelson netdev->name, q->name); 28463cd9083SShannon Nelson return -EINVAL; 28563cd9083SShannon Nelson } 28663cd9083SShannon Nelson 28763cd9083SShannon Nelson page_info->page = dev_alloc_page(); 28863cd9083SShannon Nelson if (unlikely(!page_info->page)) { 28963cd9083SShannon Nelson net_err_ratelimited("%s: %s page alloc failed\n", 2900f3154e6SShannon Nelson netdev->name, q->name); 2910f3154e6SShannon Nelson stats->alloc_err++; 29263cd9083SShannon Nelson return -ENOMEM; 2930f3154e6SShannon Nelson } 2940f3154e6SShannon Nelson 29563cd9083SShannon Nelson page_info->dma_addr = dma_map_page(dev, page_info->page, 0, PAGE_SIZE, 29663cd9083SShannon Nelson DMA_FROM_DEVICE); 29763cd9083SShannon Nelson if (unlikely(dma_mapping_error(dev, page_info->dma_addr))) { 29863cd9083SShannon Nelson put_page(page_info->page); 29963cd9083SShannon Nelson page_info->dma_addr = 0; 30063cd9083SShannon Nelson page_info->page = NULL; 30163cd9083SShannon Nelson net_err_ratelimited("%s: %s dma map failed\n", 3020f3154e6SShannon Nelson netdev->name, q->name); 3030f3154e6SShannon Nelson stats->dma_map_err++; 30463cd9083SShannon Nelson return -EIO; 3050f3154e6SShannon Nelson } 3060f3154e6SShannon Nelson 30763cd9083SShannon Nelson return 0; 3080f3154e6SShannon Nelson } 3090f3154e6SShannon Nelson 31063cd9083SShannon Nelson static void ionic_rx_page_free(struct ionic_queue *q, 31163cd9083SShannon Nelson struct ionic_page_info *page_info) 31208f2e4b2SShannon Nelson { 31308f2e4b2SShannon Nelson struct ionic_lif *lif = q->lif; 31408f2e4b2SShannon Nelson struct net_device *netdev; 31508f2e4b2SShannon Nelson struct device *dev; 31608f2e4b2SShannon Nelson 31708f2e4b2SShannon Nelson netdev = lif->netdev; 31808f2e4b2SShannon Nelson dev = lif->ionic->dev; 31908f2e4b2SShannon Nelson 32063cd9083SShannon Nelson if (unlikely(!page_info)) { 32163cd9083SShannon Nelson net_err_ratelimited("%s: %s invalid page_info in free\n", 32208f2e4b2SShannon Nelson netdev->name, q->name); 32308f2e4b2SShannon Nelson return; 32408f2e4b2SShannon Nelson } 32508f2e4b2SShannon Nelson 32663cd9083SShannon Nelson if (unlikely(!page_info->page)) { 32763cd9083SShannon Nelson net_err_ratelimited("%s: %s invalid page in free\n", 32863cd9083SShannon Nelson netdev->name, q->name); 32963cd9083SShannon Nelson return; 33063cd9083SShannon Nelson } 33108f2e4b2SShannon Nelson 33263cd9083SShannon Nelson dma_unmap_page(dev, page_info->dma_addr, PAGE_SIZE, DMA_FROM_DEVICE); 33363cd9083SShannon Nelson 33463cd9083SShannon Nelson put_page(page_info->page); 33563cd9083SShannon Nelson page_info->dma_addr = 0; 33663cd9083SShannon Nelson page_info->page = NULL; 33708f2e4b2SShannon Nelson } 33808f2e4b2SShannon Nelson 3390f3154e6SShannon Nelson void ionic_rx_fill(struct ionic_queue *q) 3400f3154e6SShannon Nelson { 3410f3154e6SShannon Nelson struct net_device *netdev = q->lif->netdev; 34208f2e4b2SShannon Nelson struct ionic_desc_info *desc_info; 34308f2e4b2SShannon Nelson struct ionic_page_info *page_info; 34408f2e4b2SShannon Nelson struct ionic_rxq_sg_desc *sg_desc; 34508f2e4b2SShannon Nelson struct ionic_rxq_sg_elem *sg_elem; 3460f3154e6SShannon Nelson struct ionic_rxq_desc *desc; 347c37d6e3fSShannon Nelson unsigned int remain_len; 348c37d6e3fSShannon Nelson unsigned int seg_len; 34908f2e4b2SShannon Nelson unsigned int nfrags; 35008f2e4b2SShannon Nelson unsigned int i, j; 3510f3154e6SShannon Nelson unsigned int len; 3520f3154e6SShannon Nelson 3530f3154e6SShannon Nelson len = netdev->mtu + ETH_HLEN; 35408f2e4b2SShannon Nelson nfrags = round_up(len, PAGE_SIZE) / PAGE_SIZE; 3550f3154e6SShannon Nelson 3560f3154e6SShannon Nelson for (i = ionic_q_space_avail(q); i; i--) { 357c37d6e3fSShannon Nelson remain_len = len; 358f1d2e894SShannon Nelson desc_info = &q->info[q->head_idx]; 35908f2e4b2SShannon Nelson desc = desc_info->desc; 36008f2e4b2SShannon Nelson sg_desc = desc_info->sg_desc; 36108f2e4b2SShannon Nelson page_info = &desc_info->pages[0]; 3620f3154e6SShannon Nelson 36308f2e4b2SShannon Nelson if (page_info->page) { /* recycle the buffer */ 364155f15adSShannon Nelson ionic_rxq_post(q, false, ionic_rx_clean, NULL); 36508f2e4b2SShannon Nelson continue; 36608f2e4b2SShannon Nelson } 36708f2e4b2SShannon Nelson 36808f2e4b2SShannon Nelson /* fill main descriptor - pages[0] */ 36908f2e4b2SShannon Nelson desc->opcode = (nfrags > 1) ? IONIC_RXQ_DESC_OPCODE_SG : 37008f2e4b2SShannon Nelson IONIC_RXQ_DESC_OPCODE_SIMPLE; 37108f2e4b2SShannon Nelson desc_info->npages = nfrags; 37263cd9083SShannon Nelson if (unlikely(ionic_rx_page_alloc(q, page_info))) { 37308f2e4b2SShannon Nelson desc->addr = 0; 37408f2e4b2SShannon Nelson desc->len = 0; 37508f2e4b2SShannon Nelson return; 37608f2e4b2SShannon Nelson } 37708f2e4b2SShannon Nelson desc->addr = cpu_to_le64(page_info->dma_addr); 378c37d6e3fSShannon Nelson seg_len = min_t(unsigned int, PAGE_SIZE, len); 379c37d6e3fSShannon Nelson desc->len = cpu_to_le16(seg_len); 380c37d6e3fSShannon Nelson remain_len -= seg_len; 38108f2e4b2SShannon Nelson page_info++; 38208f2e4b2SShannon Nelson 38308f2e4b2SShannon Nelson /* fill sg descriptors - pages[1..n] */ 38408f2e4b2SShannon Nelson for (j = 0; j < nfrags - 1; j++) { 38508f2e4b2SShannon Nelson if (page_info->page) /* recycle the sg buffer */ 38608f2e4b2SShannon Nelson continue; 38708f2e4b2SShannon Nelson 38808f2e4b2SShannon Nelson sg_elem = &sg_desc->elems[j]; 38963cd9083SShannon Nelson if (unlikely(ionic_rx_page_alloc(q, page_info))) { 39008f2e4b2SShannon Nelson sg_elem->addr = 0; 39108f2e4b2SShannon Nelson sg_elem->len = 0; 39208f2e4b2SShannon Nelson return; 39308f2e4b2SShannon Nelson } 39408f2e4b2SShannon Nelson sg_elem->addr = cpu_to_le64(page_info->dma_addr); 395c37d6e3fSShannon Nelson seg_len = min_t(unsigned int, PAGE_SIZE, remain_len); 396c37d6e3fSShannon Nelson sg_elem->len = cpu_to_le16(seg_len); 397c37d6e3fSShannon Nelson remain_len -= seg_len; 39808f2e4b2SShannon Nelson page_info++; 39908f2e4b2SShannon Nelson } 4000f3154e6SShannon Nelson 401155f15adSShannon Nelson ionic_rxq_post(q, false, ionic_rx_clean, NULL); 4020f3154e6SShannon Nelson } 403155f15adSShannon Nelson 404155f15adSShannon Nelson ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type, 405f1d2e894SShannon Nelson q->dbval | q->head_idx); 4060f3154e6SShannon Nelson } 4070f3154e6SShannon Nelson 4080f3154e6SShannon Nelson static void ionic_rx_fill_cb(void *arg) 4090f3154e6SShannon Nelson { 4100f3154e6SShannon Nelson ionic_rx_fill(arg); 4110f3154e6SShannon Nelson } 4120f3154e6SShannon Nelson 4130f3154e6SShannon Nelson void ionic_rx_empty(struct ionic_queue *q) 4140f3154e6SShannon Nelson { 415f1d2e894SShannon Nelson struct ionic_desc_info *desc_info; 4160f3154e6SShannon Nelson struct ionic_rxq_desc *desc; 41708f2e4b2SShannon Nelson unsigned int i; 418f1d2e894SShannon Nelson u16 idx; 4190f3154e6SShannon Nelson 420f1d2e894SShannon Nelson idx = q->tail_idx; 421f1d2e894SShannon Nelson while (idx != q->head_idx) { 422f1d2e894SShannon Nelson desc_info = &q->info[idx]; 423f1d2e894SShannon Nelson desc = desc_info->desc; 42408f2e4b2SShannon Nelson desc->addr = 0; 42508f2e4b2SShannon Nelson desc->len = 0; 42608f2e4b2SShannon Nelson 42763cd9083SShannon Nelson for (i = 0; i < desc_info->npages; i++) 42863cd9083SShannon Nelson ionic_rx_page_free(q, &desc_info->pages[i]); 42908f2e4b2SShannon Nelson 430f1d2e894SShannon Nelson desc_info->cb_arg = NULL; 431f1d2e894SShannon Nelson idx = (idx + 1) & (q->num_descs - 1); 4320f3154e6SShannon Nelson } 4330f3154e6SShannon Nelson } 4340f3154e6SShannon Nelson 43504a83459SShannon Nelson static void ionic_dim_update(struct ionic_qcq *qcq) 43604a83459SShannon Nelson { 43704a83459SShannon Nelson struct dim_sample dim_sample; 43804a83459SShannon Nelson struct ionic_lif *lif; 43904a83459SShannon Nelson unsigned int qi; 44004a83459SShannon Nelson 44104a83459SShannon Nelson if (!qcq->intr.dim_coal_hw) 44204a83459SShannon Nelson return; 44304a83459SShannon Nelson 44404a83459SShannon Nelson lif = qcq->q.lif; 44504a83459SShannon Nelson qi = qcq->cq.bound_q->index; 44604a83459SShannon Nelson 44704a83459SShannon Nelson ionic_intr_coal_init(lif->ionic->idev.intr_ctrl, 44804a83459SShannon Nelson lif->rxqcqs[qi]->intr.index, 44904a83459SShannon Nelson qcq->intr.dim_coal_hw); 45004a83459SShannon Nelson 45104a83459SShannon Nelson dim_update_sample(qcq->cq.bound_intr->rearm_count, 45204a83459SShannon Nelson lif->txqstats[qi].pkts, 45304a83459SShannon Nelson lif->txqstats[qi].bytes, 45404a83459SShannon Nelson &dim_sample); 45504a83459SShannon Nelson 45604a83459SShannon Nelson net_dim(&qcq->dim, dim_sample); 45704a83459SShannon Nelson } 45804a83459SShannon Nelson 459fe8c30b5SShannon Nelson int ionic_tx_napi(struct napi_struct *napi, int budget) 460fe8c30b5SShannon Nelson { 461fe8c30b5SShannon Nelson struct ionic_qcq *qcq = napi_to_qcq(napi); 462fe8c30b5SShannon Nelson struct ionic_cq *cq = napi_to_cq(napi); 463fe8c30b5SShannon Nelson struct ionic_dev *idev; 464fe8c30b5SShannon Nelson struct ionic_lif *lif; 465fe8c30b5SShannon Nelson u32 work_done = 0; 466fe8c30b5SShannon Nelson u32 flags = 0; 467fe8c30b5SShannon Nelson 468fe8c30b5SShannon Nelson lif = cq->bound_q->lif; 469fe8c30b5SShannon Nelson idev = &lif->ionic->idev; 470fe8c30b5SShannon Nelson 471fe8c30b5SShannon Nelson work_done = ionic_cq_service(cq, budget, 472fe8c30b5SShannon Nelson ionic_tx_service, NULL, NULL); 473fe8c30b5SShannon Nelson 474fe8c30b5SShannon Nelson if (work_done < budget && napi_complete_done(napi, work_done)) { 47504a83459SShannon Nelson ionic_dim_update(qcq); 476fe8c30b5SShannon Nelson flags |= IONIC_INTR_CRED_UNMASK; 47704a83459SShannon Nelson cq->bound_intr->rearm_count++; 478fe8c30b5SShannon Nelson } 479fe8c30b5SShannon Nelson 480fe8c30b5SShannon Nelson if (work_done || flags) { 481fe8c30b5SShannon Nelson flags |= IONIC_INTR_CRED_RESET_COALESCE; 482fe8c30b5SShannon Nelson ionic_intr_credits(idev->intr_ctrl, 483fe8c30b5SShannon Nelson cq->bound_intr->index, 484fe8c30b5SShannon Nelson work_done, flags); 485fe8c30b5SShannon Nelson } 486fe8c30b5SShannon Nelson 487fe8c30b5SShannon Nelson DEBUG_STATS_NAPI_POLL(qcq, work_done); 488fe8c30b5SShannon Nelson 489fe8c30b5SShannon Nelson return work_done; 490fe8c30b5SShannon Nelson } 491fe8c30b5SShannon Nelson 4920f3154e6SShannon Nelson int ionic_rx_napi(struct napi_struct *napi, int budget) 4930f3154e6SShannon Nelson { 4940f3154e6SShannon Nelson struct ionic_qcq *qcq = napi_to_qcq(napi); 495fe8c30b5SShannon Nelson struct ionic_cq *cq = napi_to_cq(napi); 496fe8c30b5SShannon Nelson struct ionic_dev *idev; 497fe8c30b5SShannon Nelson struct ionic_lif *lif; 498fe8c30b5SShannon Nelson u32 work_done = 0; 499fe8c30b5SShannon Nelson u32 flags = 0; 500fe8c30b5SShannon Nelson 501fe8c30b5SShannon Nelson lif = cq->bound_q->lif; 502fe8c30b5SShannon Nelson idev = &lif->ionic->idev; 503fe8c30b5SShannon Nelson 504fe8c30b5SShannon Nelson work_done = ionic_cq_service(cq, budget, 505fe8c30b5SShannon Nelson ionic_rx_service, NULL, NULL); 506fe8c30b5SShannon Nelson 507fe8c30b5SShannon Nelson if (work_done) 508fe8c30b5SShannon Nelson ionic_rx_fill(cq->bound_q); 509fe8c30b5SShannon Nelson 510fe8c30b5SShannon Nelson if (work_done < budget && napi_complete_done(napi, work_done)) { 51104a83459SShannon Nelson ionic_dim_update(qcq); 512fe8c30b5SShannon Nelson flags |= IONIC_INTR_CRED_UNMASK; 51304a83459SShannon Nelson cq->bound_intr->rearm_count++; 514fe8c30b5SShannon Nelson } 515fe8c30b5SShannon Nelson 516fe8c30b5SShannon Nelson if (work_done || flags) { 517fe8c30b5SShannon Nelson flags |= IONIC_INTR_CRED_RESET_COALESCE; 518fe8c30b5SShannon Nelson ionic_intr_credits(idev->intr_ctrl, 519fe8c30b5SShannon Nelson cq->bound_intr->index, 520fe8c30b5SShannon Nelson work_done, flags); 521fe8c30b5SShannon Nelson } 522fe8c30b5SShannon Nelson 523fe8c30b5SShannon Nelson DEBUG_STATS_NAPI_POLL(qcq, work_done); 524fe8c30b5SShannon Nelson 525fe8c30b5SShannon Nelson return work_done; 526fe8c30b5SShannon Nelson } 527fe8c30b5SShannon Nelson 528fe8c30b5SShannon Nelson int ionic_txrx_napi(struct napi_struct *napi, int budget) 529fe8c30b5SShannon Nelson { 530fe8c30b5SShannon Nelson struct ionic_qcq *qcq = napi_to_qcq(napi); 5310f3154e6SShannon Nelson struct ionic_cq *rxcq = napi_to_cq(napi); 5320f3154e6SShannon Nelson unsigned int qi = rxcq->bound_q->index; 5330f3154e6SShannon Nelson struct ionic_dev *idev; 5340f3154e6SShannon Nelson struct ionic_lif *lif; 5350f3154e6SShannon Nelson struct ionic_cq *txcq; 536b14e4e95SShannon Nelson u32 rx_work_done = 0; 537b14e4e95SShannon Nelson u32 tx_work_done = 0; 5380f3154e6SShannon Nelson u32 flags = 0; 5390f3154e6SShannon Nelson 5400f3154e6SShannon Nelson lif = rxcq->bound_q->lif; 5410f3154e6SShannon Nelson idev = &lif->ionic->idev; 54234dec947SShannon Nelson txcq = &lif->txqcqs[qi]->cq; 5430f3154e6SShannon Nelson 544b14e4e95SShannon Nelson tx_work_done = ionic_cq_service(txcq, lif->tx_budget, 545b14e4e95SShannon Nelson ionic_tx_service, NULL, NULL); 5460f3154e6SShannon Nelson 547b14e4e95SShannon Nelson rx_work_done = ionic_cq_service(rxcq, budget, 548b14e4e95SShannon Nelson ionic_rx_service, NULL, NULL); 549b14e4e95SShannon Nelson if (rx_work_done) 5500f3154e6SShannon Nelson ionic_rx_fill_cb(rxcq->bound_q); 5510f3154e6SShannon Nelson 5529dda5110SShannon Nelson if (rx_work_done < budget && napi_complete_done(napi, rx_work_done)) { 55304a83459SShannon Nelson ionic_dim_update(qcq); 5540f3154e6SShannon Nelson flags |= IONIC_INTR_CRED_UNMASK; 55504a83459SShannon Nelson rxcq->bound_intr->rearm_count++; 5560f3154e6SShannon Nelson } 5570f3154e6SShannon Nelson 5589dda5110SShannon Nelson if (rx_work_done || flags) { 5590f3154e6SShannon Nelson flags |= IONIC_INTR_CRED_RESET_COALESCE; 5600f3154e6SShannon Nelson ionic_intr_credits(idev->intr_ctrl, rxcq->bound_intr->index, 561b14e4e95SShannon Nelson tx_work_done + rx_work_done, flags); 5620f3154e6SShannon Nelson } 5630f3154e6SShannon Nelson 564b14e4e95SShannon Nelson DEBUG_STATS_NAPI_POLL(qcq, rx_work_done); 565b14e4e95SShannon Nelson DEBUG_STATS_NAPI_POLL(qcq, tx_work_done); 5660f3154e6SShannon Nelson 5679dda5110SShannon Nelson return rx_work_done; 5680f3154e6SShannon Nelson } 5690f3154e6SShannon Nelson 5705b3f3f2aSShannon Nelson static dma_addr_t ionic_tx_map_single(struct ionic_queue *q, 5715b3f3f2aSShannon Nelson void *data, size_t len) 5720f3154e6SShannon Nelson { 5730f3154e6SShannon Nelson struct ionic_tx_stats *stats = q_to_tx_stats(q); 5740f3154e6SShannon Nelson struct device *dev = q->lif->ionic->dev; 5750f3154e6SShannon Nelson dma_addr_t dma_addr; 5760f3154e6SShannon Nelson 5770f3154e6SShannon Nelson dma_addr = dma_map_single(dev, data, len, DMA_TO_DEVICE); 5780f3154e6SShannon Nelson if (dma_mapping_error(dev, dma_addr)) { 5790f3154e6SShannon Nelson net_warn_ratelimited("%s: DMA single map failed on %s!\n", 5800f3154e6SShannon Nelson q->lif->netdev->name, q->name); 5810f3154e6SShannon Nelson stats->dma_map_err++; 5820f3154e6SShannon Nelson return 0; 5830f3154e6SShannon Nelson } 5840f3154e6SShannon Nelson return dma_addr; 5850f3154e6SShannon Nelson } 5860f3154e6SShannon Nelson 5875b3f3f2aSShannon Nelson static dma_addr_t ionic_tx_map_frag(struct ionic_queue *q, 5885b3f3f2aSShannon Nelson const skb_frag_t *frag, 5890f3154e6SShannon Nelson size_t offset, size_t len) 5900f3154e6SShannon Nelson { 5910f3154e6SShannon Nelson struct ionic_tx_stats *stats = q_to_tx_stats(q); 5920f3154e6SShannon Nelson struct device *dev = q->lif->ionic->dev; 5930f3154e6SShannon Nelson dma_addr_t dma_addr; 5940f3154e6SShannon Nelson 5950f3154e6SShannon Nelson dma_addr = skb_frag_dma_map(dev, frag, offset, len, DMA_TO_DEVICE); 5960f3154e6SShannon Nelson if (dma_mapping_error(dev, dma_addr)) { 5970f3154e6SShannon Nelson net_warn_ratelimited("%s: DMA frag map failed on %s!\n", 5980f3154e6SShannon Nelson q->lif->netdev->name, q->name); 5990f3154e6SShannon Nelson stats->dma_map_err++; 6000f3154e6SShannon Nelson } 6010f3154e6SShannon Nelson return dma_addr; 6020f3154e6SShannon Nelson } 6030f3154e6SShannon Nelson 6045b3f3f2aSShannon Nelson static void ionic_tx_clean(struct ionic_queue *q, 6055b3f3f2aSShannon Nelson struct ionic_desc_info *desc_info, 6065b3f3f2aSShannon Nelson struct ionic_cq_info *cq_info, 6075b3f3f2aSShannon Nelson void *cb_arg) 6080f3154e6SShannon Nelson { 6090f3154e6SShannon Nelson struct ionic_txq_sg_desc *sg_desc = desc_info->sg_desc; 6100f3154e6SShannon Nelson struct ionic_txq_sg_elem *elem = sg_desc->elems; 6110f3154e6SShannon Nelson struct ionic_tx_stats *stats = q_to_tx_stats(q); 6120f3154e6SShannon Nelson struct ionic_txq_desc *desc = desc_info->desc; 6130f3154e6SShannon Nelson struct device *dev = q->lif->ionic->dev; 6140f3154e6SShannon Nelson u8 opcode, flags, nsge; 6150f3154e6SShannon Nelson u16 queue_index; 6160f3154e6SShannon Nelson unsigned int i; 6170f3154e6SShannon Nelson u64 addr; 6180f3154e6SShannon Nelson 6190f3154e6SShannon Nelson decode_txq_desc_cmd(le64_to_cpu(desc->cmd), 6200f3154e6SShannon Nelson &opcode, &flags, &nsge, &addr); 6210f3154e6SShannon Nelson 6220f3154e6SShannon Nelson /* use unmap_single only if either this is not TSO, 6230f3154e6SShannon Nelson * or this is first descriptor of a TSO 6240f3154e6SShannon Nelson */ 6250f3154e6SShannon Nelson if (opcode != IONIC_TXQ_DESC_OPCODE_TSO || 6260f3154e6SShannon Nelson flags & IONIC_TXQ_DESC_FLAG_TSO_SOT) 6270f3154e6SShannon Nelson dma_unmap_single(dev, (dma_addr_t)addr, 6280f3154e6SShannon Nelson le16_to_cpu(desc->len), DMA_TO_DEVICE); 6290f3154e6SShannon Nelson else 6300f3154e6SShannon Nelson dma_unmap_page(dev, (dma_addr_t)addr, 6310f3154e6SShannon Nelson le16_to_cpu(desc->len), DMA_TO_DEVICE); 6320f3154e6SShannon Nelson 6330f3154e6SShannon Nelson for (i = 0; i < nsge; i++, elem++) 6340f3154e6SShannon Nelson dma_unmap_page(dev, (dma_addr_t)le64_to_cpu(elem->addr), 6350f3154e6SShannon Nelson le16_to_cpu(elem->len), DMA_TO_DEVICE); 6360f3154e6SShannon Nelson 6370f3154e6SShannon Nelson if (cb_arg) { 6380f3154e6SShannon Nelson struct sk_buff *skb = cb_arg; 6390f3154e6SShannon Nelson u32 len = skb->len; 6400f3154e6SShannon Nelson 6410f3154e6SShannon Nelson queue_index = skb_get_queue_mapping(skb); 6420f3154e6SShannon Nelson if (unlikely(__netif_subqueue_stopped(q->lif->netdev, 6430f3154e6SShannon Nelson queue_index))) { 6440f3154e6SShannon Nelson netif_wake_subqueue(q->lif->netdev, queue_index); 6450f3154e6SShannon Nelson q->wake++; 6460f3154e6SShannon Nelson } 6470f3154e6SShannon Nelson dev_kfree_skb_any(skb); 6480f3154e6SShannon Nelson stats->clean++; 6490f3154e6SShannon Nelson netdev_tx_completed_queue(q_to_ndq(q), 1, len); 6500f3154e6SShannon Nelson } 6510f3154e6SShannon Nelson } 6520f3154e6SShannon Nelson 653b14e4e95SShannon Nelson static bool ionic_tx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info) 6540f3154e6SShannon Nelson { 655b14e4e95SShannon Nelson struct ionic_txq_comp *comp = cq_info->cq_desc; 6560f3154e6SShannon Nelson struct ionic_queue *q = cq->bound_q; 6570f3154e6SShannon Nelson struct ionic_desc_info *desc_info; 658339dcf7fSShannon Nelson u16 index; 6590f3154e6SShannon Nelson 660b14e4e95SShannon Nelson if (!color_match(comp->color, cq->done_color)) 661b14e4e95SShannon Nelson return false; 6620f3154e6SShannon Nelson 6630f3154e6SShannon Nelson /* clean the related q entries, there could be 6640f3154e6SShannon Nelson * several q entries completed for each cq completion 6650f3154e6SShannon Nelson */ 6660f3154e6SShannon Nelson do { 667f1d2e894SShannon Nelson desc_info = &q->info[q->tail_idx]; 668339dcf7fSShannon Nelson index = q->tail_idx; 669f1d2e894SShannon Nelson q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); 670f1d2e894SShannon Nelson ionic_tx_clean(q, desc_info, cq_info, desc_info->cb_arg); 6710f3154e6SShannon Nelson desc_info->cb = NULL; 6720f3154e6SShannon Nelson desc_info->cb_arg = NULL; 673339dcf7fSShannon Nelson } while (index != le16_to_cpu(comp->comp_index)); 6740f3154e6SShannon Nelson 675b14e4e95SShannon Nelson return true; 6760f3154e6SShannon Nelson } 6770f3154e6SShannon Nelson 678b14e4e95SShannon Nelson void ionic_tx_flush(struct ionic_cq *cq) 679b14e4e95SShannon Nelson { 680b14e4e95SShannon Nelson struct ionic_dev *idev = &cq->lif->ionic->idev; 681b14e4e95SShannon Nelson u32 work_done; 682b14e4e95SShannon Nelson 683b14e4e95SShannon Nelson work_done = ionic_cq_service(cq, cq->num_descs, 684b14e4e95SShannon Nelson ionic_tx_service, NULL, NULL); 6850f3154e6SShannon Nelson if (work_done) 6860f3154e6SShannon Nelson ionic_intr_credits(idev->intr_ctrl, cq->bound_intr->index, 687b14e4e95SShannon Nelson work_done, IONIC_INTR_CRED_RESET_COALESCE); 6880f3154e6SShannon Nelson } 6890f3154e6SShannon Nelson 690f9c00e2cSShannon Nelson void ionic_tx_empty(struct ionic_queue *q) 691f9c00e2cSShannon Nelson { 692f9c00e2cSShannon Nelson struct ionic_desc_info *desc_info; 693f9c00e2cSShannon Nelson 694f9c00e2cSShannon Nelson /* walk the not completed tx entries, if any */ 695f1d2e894SShannon Nelson while (q->head_idx != q->tail_idx) { 696f1d2e894SShannon Nelson desc_info = &q->info[q->tail_idx]; 697f1d2e894SShannon Nelson q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); 698f9c00e2cSShannon Nelson ionic_tx_clean(q, desc_info, NULL, desc_info->cb_arg); 699f9c00e2cSShannon Nelson desc_info->cb = NULL; 700f9c00e2cSShannon Nelson desc_info->cb_arg = NULL; 701f9c00e2cSShannon Nelson } 702f9c00e2cSShannon Nelson } 703f9c00e2cSShannon Nelson 7040f3154e6SShannon Nelson static int ionic_tx_tcp_inner_pseudo_csum(struct sk_buff *skb) 7050f3154e6SShannon Nelson { 7060f3154e6SShannon Nelson int err; 7070f3154e6SShannon Nelson 7080f3154e6SShannon Nelson err = skb_cow_head(skb, 0); 7090f3154e6SShannon Nelson if (err) 7100f3154e6SShannon Nelson return err; 7110f3154e6SShannon Nelson 7120f3154e6SShannon Nelson if (skb->protocol == cpu_to_be16(ETH_P_IP)) { 7130f3154e6SShannon Nelson inner_ip_hdr(skb)->check = 0; 7140f3154e6SShannon Nelson inner_tcp_hdr(skb)->check = 7150f3154e6SShannon Nelson ~csum_tcpudp_magic(inner_ip_hdr(skb)->saddr, 7160f3154e6SShannon Nelson inner_ip_hdr(skb)->daddr, 7170f3154e6SShannon Nelson 0, IPPROTO_TCP, 0); 7180f3154e6SShannon Nelson } else if (skb->protocol == cpu_to_be16(ETH_P_IPV6)) { 7190f3154e6SShannon Nelson inner_tcp_hdr(skb)->check = 7200f3154e6SShannon Nelson ~csum_ipv6_magic(&inner_ipv6_hdr(skb)->saddr, 7210f3154e6SShannon Nelson &inner_ipv6_hdr(skb)->daddr, 7220f3154e6SShannon Nelson 0, IPPROTO_TCP, 0); 7230f3154e6SShannon Nelson } 7240f3154e6SShannon Nelson 7250f3154e6SShannon Nelson return 0; 7260f3154e6SShannon Nelson } 7270f3154e6SShannon Nelson 7280f3154e6SShannon Nelson static int ionic_tx_tcp_pseudo_csum(struct sk_buff *skb) 7290f3154e6SShannon Nelson { 7300f3154e6SShannon Nelson int err; 7310f3154e6SShannon Nelson 7320f3154e6SShannon Nelson err = skb_cow_head(skb, 0); 7330f3154e6SShannon Nelson if (err) 7340f3154e6SShannon Nelson return err; 7350f3154e6SShannon Nelson 7360f3154e6SShannon Nelson if (skb->protocol == cpu_to_be16(ETH_P_IP)) { 7370f3154e6SShannon Nelson ip_hdr(skb)->check = 0; 7380f3154e6SShannon Nelson tcp_hdr(skb)->check = 7390f3154e6SShannon Nelson ~csum_tcpudp_magic(ip_hdr(skb)->saddr, 7400f3154e6SShannon Nelson ip_hdr(skb)->daddr, 7410f3154e6SShannon Nelson 0, IPPROTO_TCP, 0); 7420f3154e6SShannon Nelson } else if (skb->protocol == cpu_to_be16(ETH_P_IPV6)) { 743fa6b8429SHeiner Kallweit tcp_v6_gso_csum_prep(skb); 7440f3154e6SShannon Nelson } 7450f3154e6SShannon Nelson 7460f3154e6SShannon Nelson return 0; 7470f3154e6SShannon Nelson } 7480f3154e6SShannon Nelson 7490f3154e6SShannon Nelson static void ionic_tx_tso_post(struct ionic_queue *q, struct ionic_txq_desc *desc, 7500f3154e6SShannon Nelson struct sk_buff *skb, 7510f3154e6SShannon Nelson dma_addr_t addr, u8 nsge, u16 len, 7520f3154e6SShannon Nelson unsigned int hdrlen, unsigned int mss, 7530f3154e6SShannon Nelson bool outer_csum, 7540f3154e6SShannon Nelson u16 vlan_tci, bool has_vlan, 7550f3154e6SShannon Nelson bool start, bool done) 7560f3154e6SShannon Nelson { 7570f3154e6SShannon Nelson u8 flags = 0; 7580f3154e6SShannon Nelson u64 cmd; 7590f3154e6SShannon Nelson 7600f3154e6SShannon Nelson flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0; 7610f3154e6SShannon Nelson flags |= outer_csum ? IONIC_TXQ_DESC_FLAG_ENCAP : 0; 7620f3154e6SShannon Nelson flags |= start ? IONIC_TXQ_DESC_FLAG_TSO_SOT : 0; 7630f3154e6SShannon Nelson flags |= done ? IONIC_TXQ_DESC_FLAG_TSO_EOT : 0; 7640f3154e6SShannon Nelson 7650f3154e6SShannon Nelson cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_TSO, flags, nsge, addr); 7660f3154e6SShannon Nelson desc->cmd = cpu_to_le64(cmd); 7670f3154e6SShannon Nelson desc->len = cpu_to_le16(len); 7680f3154e6SShannon Nelson desc->vlan_tci = cpu_to_le16(vlan_tci); 7690f3154e6SShannon Nelson desc->hdr_len = cpu_to_le16(hdrlen); 7700f3154e6SShannon Nelson desc->mss = cpu_to_le16(mss); 7710f3154e6SShannon Nelson 7720f3154e6SShannon Nelson if (done) { 7730f3154e6SShannon Nelson skb_tx_timestamp(skb); 7740f3154e6SShannon Nelson netdev_tx_sent_queue(q_to_ndq(q), skb->len); 7750f3154e6SShannon Nelson ionic_txq_post(q, !netdev_xmit_more(), ionic_tx_clean, skb); 7760f3154e6SShannon Nelson } else { 7770f3154e6SShannon Nelson ionic_txq_post(q, false, ionic_tx_clean, NULL); 7780f3154e6SShannon Nelson } 7790f3154e6SShannon Nelson } 7800f3154e6SShannon Nelson 7810f3154e6SShannon Nelson static struct ionic_txq_desc *ionic_tx_tso_next(struct ionic_queue *q, 7820f3154e6SShannon Nelson struct ionic_txq_sg_elem **elem) 7830f3154e6SShannon Nelson { 784f1d2e894SShannon Nelson struct ionic_txq_sg_desc *sg_desc = q->info[q->head_idx].txq_sg_desc; 785f1d2e894SShannon Nelson struct ionic_txq_desc *desc = q->info[q->head_idx].txq_desc; 7860f3154e6SShannon Nelson 7870f3154e6SShannon Nelson *elem = sg_desc->elems; 7880f3154e6SShannon Nelson return desc; 7890f3154e6SShannon Nelson } 7900f3154e6SShannon Nelson 7910f3154e6SShannon Nelson static int ionic_tx_tso(struct ionic_queue *q, struct sk_buff *skb) 7920f3154e6SShannon Nelson { 7930f3154e6SShannon Nelson struct ionic_tx_stats *stats = q_to_tx_stats(q); 794f1d2e894SShannon Nelson struct ionic_desc_info *rewind_desc_info; 7950f3154e6SShannon Nelson struct device *dev = q->lif->ionic->dev; 7960f3154e6SShannon Nelson struct ionic_txq_sg_elem *elem; 7970f3154e6SShannon Nelson struct ionic_txq_desc *desc; 7980f3154e6SShannon Nelson unsigned int frag_left = 0; 7990f3154e6SShannon Nelson unsigned int offset = 0; 800f1d2e894SShannon Nelson u16 abort = q->head_idx; 8010f3154e6SShannon Nelson unsigned int len_left; 8020f3154e6SShannon Nelson dma_addr_t desc_addr; 8030f3154e6SShannon Nelson unsigned int hdrlen; 8040f3154e6SShannon Nelson unsigned int nfrags; 8050f3154e6SShannon Nelson unsigned int seglen; 8060f3154e6SShannon Nelson u64 total_bytes = 0; 8070f3154e6SShannon Nelson u64 total_pkts = 0; 808f1d2e894SShannon Nelson u16 rewind = abort; 8090f3154e6SShannon Nelson unsigned int left; 8100f3154e6SShannon Nelson unsigned int len; 8110f3154e6SShannon Nelson unsigned int mss; 8120f3154e6SShannon Nelson skb_frag_t *frag; 8130f3154e6SShannon Nelson bool start, done; 8140f3154e6SShannon Nelson bool outer_csum; 8150f3154e6SShannon Nelson bool has_vlan; 8160f3154e6SShannon Nelson u16 desc_len; 8170f3154e6SShannon Nelson u8 desc_nsge; 8180f3154e6SShannon Nelson u16 vlan_tci; 8190f3154e6SShannon Nelson bool encap; 8200f3154e6SShannon Nelson int err; 8210f3154e6SShannon Nelson 8220f3154e6SShannon Nelson mss = skb_shinfo(skb)->gso_size; 8230f3154e6SShannon Nelson nfrags = skb_shinfo(skb)->nr_frags; 8240f3154e6SShannon Nelson len_left = skb->len - skb_headlen(skb); 8250f3154e6SShannon Nelson outer_csum = (skb_shinfo(skb)->gso_type & SKB_GSO_GRE_CSUM) || 8260f3154e6SShannon Nelson (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM); 8270f3154e6SShannon Nelson has_vlan = !!skb_vlan_tag_present(skb); 8280f3154e6SShannon Nelson vlan_tci = skb_vlan_tag_get(skb); 8290f3154e6SShannon Nelson encap = skb->encapsulation; 8300f3154e6SShannon Nelson 8310f3154e6SShannon Nelson /* Preload inner-most TCP csum field with IP pseudo hdr 8320f3154e6SShannon Nelson * calculated with IP length set to zero. HW will later 8330f3154e6SShannon Nelson * add in length to each TCP segment resulting from the TSO. 8340f3154e6SShannon Nelson */ 8350f3154e6SShannon Nelson 8360f3154e6SShannon Nelson if (encap) 8370f3154e6SShannon Nelson err = ionic_tx_tcp_inner_pseudo_csum(skb); 8380f3154e6SShannon Nelson else 8390f3154e6SShannon Nelson err = ionic_tx_tcp_pseudo_csum(skb); 8400f3154e6SShannon Nelson if (err) 8410f3154e6SShannon Nelson return err; 8420f3154e6SShannon Nelson 8430f3154e6SShannon Nelson if (encap) 8440f3154e6SShannon Nelson hdrlen = skb_inner_transport_header(skb) - skb->data + 8450f3154e6SShannon Nelson inner_tcp_hdrlen(skb); 8460f3154e6SShannon Nelson else 8470f3154e6SShannon Nelson hdrlen = skb_transport_offset(skb) + tcp_hdrlen(skb); 8480f3154e6SShannon Nelson 8490f3154e6SShannon Nelson seglen = hdrlen + mss; 8500f3154e6SShannon Nelson left = skb_headlen(skb); 8510f3154e6SShannon Nelson 8520f3154e6SShannon Nelson desc = ionic_tx_tso_next(q, &elem); 8530f3154e6SShannon Nelson start = true; 8540f3154e6SShannon Nelson 8550f3154e6SShannon Nelson /* Chop skb->data up into desc segments */ 8560f3154e6SShannon Nelson 8570f3154e6SShannon Nelson while (left > 0) { 8580f3154e6SShannon Nelson len = min(seglen, left); 8590f3154e6SShannon Nelson frag_left = seglen - len; 8600f3154e6SShannon Nelson desc_addr = ionic_tx_map_single(q, skb->data + offset, len); 8610f3154e6SShannon Nelson if (dma_mapping_error(dev, desc_addr)) 8620f3154e6SShannon Nelson goto err_out_abort; 8630f3154e6SShannon Nelson desc_len = len; 8640f3154e6SShannon Nelson desc_nsge = 0; 8650f3154e6SShannon Nelson left -= len; 8660f3154e6SShannon Nelson offset += len; 8670f3154e6SShannon Nelson if (nfrags > 0 && frag_left > 0) 8680f3154e6SShannon Nelson continue; 8690f3154e6SShannon Nelson done = (nfrags == 0 && left == 0); 8700f3154e6SShannon Nelson ionic_tx_tso_post(q, desc, skb, 8710f3154e6SShannon Nelson desc_addr, desc_nsge, desc_len, 8720f3154e6SShannon Nelson hdrlen, mss, 8730f3154e6SShannon Nelson outer_csum, 8740f3154e6SShannon Nelson vlan_tci, has_vlan, 8750f3154e6SShannon Nelson start, done); 8760f3154e6SShannon Nelson total_pkts++; 8770f3154e6SShannon Nelson total_bytes += start ? len : len + hdrlen; 8780f3154e6SShannon Nelson desc = ionic_tx_tso_next(q, &elem); 8790f3154e6SShannon Nelson start = false; 8800f3154e6SShannon Nelson seglen = mss; 8810f3154e6SShannon Nelson } 8820f3154e6SShannon Nelson 8830f3154e6SShannon Nelson /* Chop skb frags into desc segments */ 8840f3154e6SShannon Nelson 8850f3154e6SShannon Nelson for (frag = skb_shinfo(skb)->frags; len_left; frag++) { 8860f3154e6SShannon Nelson offset = 0; 8870f3154e6SShannon Nelson left = skb_frag_size(frag); 8880f3154e6SShannon Nelson len_left -= left; 8890f3154e6SShannon Nelson nfrags--; 8900f3154e6SShannon Nelson stats->frags++; 8910f3154e6SShannon Nelson 8920f3154e6SShannon Nelson while (left > 0) { 8930f3154e6SShannon Nelson if (frag_left > 0) { 8940f3154e6SShannon Nelson len = min(frag_left, left); 8950f3154e6SShannon Nelson frag_left -= len; 8960f3154e6SShannon Nelson elem->addr = 8970f3154e6SShannon Nelson cpu_to_le64(ionic_tx_map_frag(q, frag, 8980f3154e6SShannon Nelson offset, len)); 8990f3154e6SShannon Nelson if (dma_mapping_error(dev, elem->addr)) 9000f3154e6SShannon Nelson goto err_out_abort; 9010f3154e6SShannon Nelson elem->len = cpu_to_le16(len); 9020f3154e6SShannon Nelson elem++; 9030f3154e6SShannon Nelson desc_nsge++; 9040f3154e6SShannon Nelson left -= len; 9050f3154e6SShannon Nelson offset += len; 9060f3154e6SShannon Nelson if (nfrags > 0 && frag_left > 0) 9070f3154e6SShannon Nelson continue; 9080f3154e6SShannon Nelson done = (nfrags == 0 && left == 0); 9090f3154e6SShannon Nelson ionic_tx_tso_post(q, desc, skb, desc_addr, 9100f3154e6SShannon Nelson desc_nsge, desc_len, 9110f3154e6SShannon Nelson hdrlen, mss, outer_csum, 9120f3154e6SShannon Nelson vlan_tci, has_vlan, 9130f3154e6SShannon Nelson start, done); 9140f3154e6SShannon Nelson total_pkts++; 9150f3154e6SShannon Nelson total_bytes += start ? len : len + hdrlen; 9160f3154e6SShannon Nelson desc = ionic_tx_tso_next(q, &elem); 9170f3154e6SShannon Nelson start = false; 9180f3154e6SShannon Nelson } else { 9190f3154e6SShannon Nelson len = min(mss, left); 9200f3154e6SShannon Nelson frag_left = mss - len; 9210f3154e6SShannon Nelson desc_addr = ionic_tx_map_frag(q, frag, 9220f3154e6SShannon Nelson offset, len); 9230f3154e6SShannon Nelson if (dma_mapping_error(dev, desc_addr)) 9240f3154e6SShannon Nelson goto err_out_abort; 9250f3154e6SShannon Nelson desc_len = len; 9260f3154e6SShannon Nelson desc_nsge = 0; 9270f3154e6SShannon Nelson left -= len; 9280f3154e6SShannon Nelson offset += len; 9290f3154e6SShannon Nelson if (nfrags > 0 && frag_left > 0) 9300f3154e6SShannon Nelson continue; 9310f3154e6SShannon Nelson done = (nfrags == 0 && left == 0); 9320f3154e6SShannon Nelson ionic_tx_tso_post(q, desc, skb, desc_addr, 9330f3154e6SShannon Nelson desc_nsge, desc_len, 9340f3154e6SShannon Nelson hdrlen, mss, outer_csum, 9350f3154e6SShannon Nelson vlan_tci, has_vlan, 9360f3154e6SShannon Nelson start, done); 9370f3154e6SShannon Nelson total_pkts++; 9380f3154e6SShannon Nelson total_bytes += start ? len : len + hdrlen; 9390f3154e6SShannon Nelson desc = ionic_tx_tso_next(q, &elem); 9400f3154e6SShannon Nelson start = false; 9410f3154e6SShannon Nelson } 9420f3154e6SShannon Nelson } 9430f3154e6SShannon Nelson } 9440f3154e6SShannon Nelson 9450f3154e6SShannon Nelson stats->pkts += total_pkts; 9460f3154e6SShannon Nelson stats->bytes += total_bytes; 9470f3154e6SShannon Nelson stats->tso++; 948f64e0c56SShannon Nelson stats->tso_bytes += total_bytes; 9490f3154e6SShannon Nelson 9500f3154e6SShannon Nelson return 0; 9510f3154e6SShannon Nelson 9520f3154e6SShannon Nelson err_out_abort: 953f1d2e894SShannon Nelson while (rewind != q->head_idx) { 954f1d2e894SShannon Nelson rewind_desc_info = &q->info[rewind]; 955f1d2e894SShannon Nelson ionic_tx_clean(q, rewind_desc_info, NULL, NULL); 956f1d2e894SShannon Nelson rewind = (rewind + 1) & (q->num_descs - 1); 9570f3154e6SShannon Nelson } 958f1d2e894SShannon Nelson q->head_idx = abort; 9590f3154e6SShannon Nelson 9600f3154e6SShannon Nelson return -ENOMEM; 9610f3154e6SShannon Nelson } 9620f3154e6SShannon Nelson 9630f3154e6SShannon Nelson static int ionic_tx_calc_csum(struct ionic_queue *q, struct sk_buff *skb) 9640f3154e6SShannon Nelson { 965f1d2e894SShannon Nelson struct ionic_txq_desc *desc = q->info[q->head_idx].txq_desc; 9660f3154e6SShannon Nelson struct ionic_tx_stats *stats = q_to_tx_stats(q); 9670f3154e6SShannon Nelson struct device *dev = q->lif->ionic->dev; 9680f3154e6SShannon Nelson dma_addr_t dma_addr; 9690f3154e6SShannon Nelson bool has_vlan; 9700f3154e6SShannon Nelson u8 flags = 0; 9710f3154e6SShannon Nelson bool encap; 9720f3154e6SShannon Nelson u64 cmd; 9730f3154e6SShannon Nelson 9740f3154e6SShannon Nelson has_vlan = !!skb_vlan_tag_present(skb); 9750f3154e6SShannon Nelson encap = skb->encapsulation; 9760f3154e6SShannon Nelson 9770f3154e6SShannon Nelson dma_addr = ionic_tx_map_single(q, skb->data, skb_headlen(skb)); 9780f3154e6SShannon Nelson if (dma_mapping_error(dev, dma_addr)) 9790f3154e6SShannon Nelson return -ENOMEM; 9800f3154e6SShannon Nelson 9810f3154e6SShannon Nelson flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0; 9820f3154e6SShannon Nelson flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0; 9830f3154e6SShannon Nelson 9840f3154e6SShannon Nelson cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_PARTIAL, 9850f3154e6SShannon Nelson flags, skb_shinfo(skb)->nr_frags, dma_addr); 9860f3154e6SShannon Nelson desc->cmd = cpu_to_le64(cmd); 9870f3154e6SShannon Nelson desc->len = cpu_to_le16(skb_headlen(skb)); 9880f3154e6SShannon Nelson desc->csum_start = cpu_to_le16(skb_checksum_start_offset(skb)); 9890f3154e6SShannon Nelson desc->csum_offset = cpu_to_le16(skb->csum_offset); 990f64e0c56SShannon Nelson if (has_vlan) { 991f64e0c56SShannon Nelson desc->vlan_tci = cpu_to_le16(skb_vlan_tag_get(skb)); 992f64e0c56SShannon Nelson stats->vlan_inserted++; 993f64e0c56SShannon Nelson } 9940f3154e6SShannon Nelson 9950f3154e6SShannon Nelson if (skb->csum_not_inet) 9960f3154e6SShannon Nelson stats->crc32_csum++; 9970f3154e6SShannon Nelson else 9980f3154e6SShannon Nelson stats->csum++; 9990f3154e6SShannon Nelson 10000f3154e6SShannon Nelson return 0; 10010f3154e6SShannon Nelson } 10020f3154e6SShannon Nelson 10030f3154e6SShannon Nelson static int ionic_tx_calc_no_csum(struct ionic_queue *q, struct sk_buff *skb) 10040f3154e6SShannon Nelson { 1005f1d2e894SShannon Nelson struct ionic_txq_desc *desc = q->info[q->head_idx].txq_desc; 10060f3154e6SShannon Nelson struct ionic_tx_stats *stats = q_to_tx_stats(q); 10070f3154e6SShannon Nelson struct device *dev = q->lif->ionic->dev; 10080f3154e6SShannon Nelson dma_addr_t dma_addr; 10090f3154e6SShannon Nelson bool has_vlan; 10100f3154e6SShannon Nelson u8 flags = 0; 10110f3154e6SShannon Nelson bool encap; 10120f3154e6SShannon Nelson u64 cmd; 10130f3154e6SShannon Nelson 10140f3154e6SShannon Nelson has_vlan = !!skb_vlan_tag_present(skb); 10150f3154e6SShannon Nelson encap = skb->encapsulation; 10160f3154e6SShannon Nelson 10170f3154e6SShannon Nelson dma_addr = ionic_tx_map_single(q, skb->data, skb_headlen(skb)); 10180f3154e6SShannon Nelson if (dma_mapping_error(dev, dma_addr)) 10190f3154e6SShannon Nelson return -ENOMEM; 10200f3154e6SShannon Nelson 10210f3154e6SShannon Nelson flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0; 10220f3154e6SShannon Nelson flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0; 10230f3154e6SShannon Nelson 10240f3154e6SShannon Nelson cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_NONE, 10250f3154e6SShannon Nelson flags, skb_shinfo(skb)->nr_frags, dma_addr); 10260f3154e6SShannon Nelson desc->cmd = cpu_to_le64(cmd); 10270f3154e6SShannon Nelson desc->len = cpu_to_le16(skb_headlen(skb)); 1028f64e0c56SShannon Nelson if (has_vlan) { 10290f3154e6SShannon Nelson desc->vlan_tci = cpu_to_le16(skb_vlan_tag_get(skb)); 1030f64e0c56SShannon Nelson stats->vlan_inserted++; 1031f64e0c56SShannon Nelson } 10320f3154e6SShannon Nelson 1033f64e0c56SShannon Nelson stats->csum_none++; 10340f3154e6SShannon Nelson 10350f3154e6SShannon Nelson return 0; 10360f3154e6SShannon Nelson } 10370f3154e6SShannon Nelson 10380f3154e6SShannon Nelson static int ionic_tx_skb_frags(struct ionic_queue *q, struct sk_buff *skb) 10390f3154e6SShannon Nelson { 1040f1d2e894SShannon Nelson struct ionic_txq_sg_desc *sg_desc = q->info[q->head_idx].txq_sg_desc; 10410f3154e6SShannon Nelson unsigned int len_left = skb->len - skb_headlen(skb); 10420f3154e6SShannon Nelson struct ionic_txq_sg_elem *elem = sg_desc->elems; 10430f3154e6SShannon Nelson struct ionic_tx_stats *stats = q_to_tx_stats(q); 10440f3154e6SShannon Nelson struct device *dev = q->lif->ionic->dev; 10450f3154e6SShannon Nelson dma_addr_t dma_addr; 10460f3154e6SShannon Nelson skb_frag_t *frag; 10470f3154e6SShannon Nelson u16 len; 10480f3154e6SShannon Nelson 10490f3154e6SShannon Nelson for (frag = skb_shinfo(skb)->frags; len_left; frag++, elem++) { 10500f3154e6SShannon Nelson len = skb_frag_size(frag); 10510f3154e6SShannon Nelson elem->len = cpu_to_le16(len); 10520f3154e6SShannon Nelson dma_addr = ionic_tx_map_frag(q, frag, 0, len); 10530f3154e6SShannon Nelson if (dma_mapping_error(dev, dma_addr)) 10540f3154e6SShannon Nelson return -ENOMEM; 10550f3154e6SShannon Nelson elem->addr = cpu_to_le64(dma_addr); 10560f3154e6SShannon Nelson len_left -= len; 10570f3154e6SShannon Nelson stats->frags++; 10580f3154e6SShannon Nelson } 10590f3154e6SShannon Nelson 10600f3154e6SShannon Nelson return 0; 10610f3154e6SShannon Nelson } 10620f3154e6SShannon Nelson 10630f3154e6SShannon Nelson static int ionic_tx(struct ionic_queue *q, struct sk_buff *skb) 10640f3154e6SShannon Nelson { 10650f3154e6SShannon Nelson struct ionic_tx_stats *stats = q_to_tx_stats(q); 10660f3154e6SShannon Nelson int err; 10670f3154e6SShannon Nelson 10680f3154e6SShannon Nelson /* set up the initial descriptor */ 10690f3154e6SShannon Nelson if (skb->ip_summed == CHECKSUM_PARTIAL) 10700f3154e6SShannon Nelson err = ionic_tx_calc_csum(q, skb); 10710f3154e6SShannon Nelson else 10720f3154e6SShannon Nelson err = ionic_tx_calc_no_csum(q, skb); 10730f3154e6SShannon Nelson if (err) 10740f3154e6SShannon Nelson return err; 10750f3154e6SShannon Nelson 10760f3154e6SShannon Nelson /* add frags */ 10770f3154e6SShannon Nelson err = ionic_tx_skb_frags(q, skb); 10780f3154e6SShannon Nelson if (err) 10790f3154e6SShannon Nelson return err; 10800f3154e6SShannon Nelson 10810f3154e6SShannon Nelson skb_tx_timestamp(skb); 10820f3154e6SShannon Nelson stats->pkts++; 10830f3154e6SShannon Nelson stats->bytes += skb->len; 10840f3154e6SShannon Nelson 10850f3154e6SShannon Nelson netdev_tx_sent_queue(q_to_ndq(q), skb->len); 10860f3154e6SShannon Nelson ionic_txq_post(q, !netdev_xmit_more(), ionic_tx_clean, skb); 10870f3154e6SShannon Nelson 10880f3154e6SShannon Nelson return 0; 10890f3154e6SShannon Nelson } 10900f3154e6SShannon Nelson 10910f3154e6SShannon Nelson static int ionic_tx_descs_needed(struct ionic_queue *q, struct sk_buff *skb) 10920f3154e6SShannon Nelson { 10935b3f3f2aSShannon Nelson int sg_elems = q->lif->qtype_info[IONIC_QTYPE_TXQ].max_sg_elems; 10940f3154e6SShannon Nelson struct ionic_tx_stats *stats = q_to_tx_stats(q); 10950f3154e6SShannon Nelson int err; 10960f3154e6SShannon Nelson 10970f3154e6SShannon Nelson /* If TSO, need roundup(skb->len/mss) descs */ 10980f3154e6SShannon Nelson if (skb_is_gso(skb)) 10990f3154e6SShannon Nelson return (skb->len / skb_shinfo(skb)->gso_size) + 1; 11000f3154e6SShannon Nelson 11010f3154e6SShannon Nelson /* If non-TSO, just need 1 desc and nr_frags sg elems */ 11025b3f3f2aSShannon Nelson if (skb_shinfo(skb)->nr_frags <= sg_elems) 11030f3154e6SShannon Nelson return 1; 11040f3154e6SShannon Nelson 11050f3154e6SShannon Nelson /* Too many frags, so linearize */ 11060f3154e6SShannon Nelson err = skb_linearize(skb); 11070f3154e6SShannon Nelson if (err) 11080f3154e6SShannon Nelson return err; 11090f3154e6SShannon Nelson 11100f3154e6SShannon Nelson stats->linearize++; 11110f3154e6SShannon Nelson 11120f3154e6SShannon Nelson /* Need 1 desc and zero sg elems */ 11130f3154e6SShannon Nelson return 1; 11140f3154e6SShannon Nelson } 11150f3154e6SShannon Nelson 11160f3154e6SShannon Nelson static int ionic_maybe_stop_tx(struct ionic_queue *q, int ndescs) 11170f3154e6SShannon Nelson { 11180f3154e6SShannon Nelson int stopped = 0; 11190f3154e6SShannon Nelson 11200f3154e6SShannon Nelson if (unlikely(!ionic_q_has_space(q, ndescs))) { 11210f3154e6SShannon Nelson netif_stop_subqueue(q->lif->netdev, q->index); 11220f3154e6SShannon Nelson q->stop++; 11230f3154e6SShannon Nelson stopped = 1; 11240f3154e6SShannon Nelson 11250f3154e6SShannon Nelson /* Might race with ionic_tx_clean, check again */ 11260f3154e6SShannon Nelson smp_rmb(); 11270f3154e6SShannon Nelson if (ionic_q_has_space(q, ndescs)) { 11280f3154e6SShannon Nelson netif_wake_subqueue(q->lif->netdev, q->index); 11290f3154e6SShannon Nelson stopped = 0; 11300f3154e6SShannon Nelson } 11310f3154e6SShannon Nelson } 11320f3154e6SShannon Nelson 11330f3154e6SShannon Nelson return stopped; 11340f3154e6SShannon Nelson } 11350f3154e6SShannon Nelson 11360f3154e6SShannon Nelson netdev_tx_t ionic_start_xmit(struct sk_buff *skb, struct net_device *netdev) 11370f3154e6SShannon Nelson { 11380f3154e6SShannon Nelson u16 queue_index = skb_get_queue_mapping(skb); 11390f3154e6SShannon Nelson struct ionic_lif *lif = netdev_priv(netdev); 11400f3154e6SShannon Nelson struct ionic_queue *q; 11410f3154e6SShannon Nelson int ndescs; 11420f3154e6SShannon Nelson int err; 11430f3154e6SShannon Nelson 1144c6d3d73aSShannon Nelson if (unlikely(!test_bit(IONIC_LIF_F_UP, lif->state))) { 11450f3154e6SShannon Nelson dev_kfree_skb(skb); 11460f3154e6SShannon Nelson return NETDEV_TX_OK; 11470f3154e6SShannon Nelson } 11480f3154e6SShannon Nelson 114934dec947SShannon Nelson if (unlikely(queue_index >= lif->nxqs)) 11500f3154e6SShannon Nelson queue_index = 0; 115134dec947SShannon Nelson q = &lif->txqcqs[queue_index]->q; 11520f3154e6SShannon Nelson 11530f3154e6SShannon Nelson ndescs = ionic_tx_descs_needed(q, skb); 11540f3154e6SShannon Nelson if (ndescs < 0) 11550f3154e6SShannon Nelson goto err_out_drop; 11560f3154e6SShannon Nelson 11570f3154e6SShannon Nelson if (unlikely(ionic_maybe_stop_tx(q, ndescs))) 11580f3154e6SShannon Nelson return NETDEV_TX_BUSY; 11590f3154e6SShannon Nelson 11600f3154e6SShannon Nelson if (skb_is_gso(skb)) 11610f3154e6SShannon Nelson err = ionic_tx_tso(q, skb); 11620f3154e6SShannon Nelson else 11630f3154e6SShannon Nelson err = ionic_tx(q, skb); 11640f3154e6SShannon Nelson 11650f3154e6SShannon Nelson if (err) 11660f3154e6SShannon Nelson goto err_out_drop; 11670f3154e6SShannon Nelson 11680f3154e6SShannon Nelson /* Stop the queue if there aren't descriptors for the next packet. 11690f3154e6SShannon Nelson * Since our SG lists per descriptor take care of most of the possible 11700f3154e6SShannon Nelson * fragmentation, we don't need to have many descriptors available. 11710f3154e6SShannon Nelson */ 11720f3154e6SShannon Nelson ionic_maybe_stop_tx(q, 4); 11730f3154e6SShannon Nelson 11740f3154e6SShannon Nelson return NETDEV_TX_OK; 11750f3154e6SShannon Nelson 11760f3154e6SShannon Nelson err_out_drop: 11770f3154e6SShannon Nelson q->stop++; 11780f3154e6SShannon Nelson q->drop++; 11790f3154e6SShannon Nelson dev_kfree_skb(skb); 11800f3154e6SShannon Nelson return NETDEV_TX_OK; 11810f3154e6SShannon Nelson } 1182